├── .gitattributes
├── .github
├── ISSUE_TEMPLATE
│ ├── bug_report.md
│ └── feature_request.md
├── dependabot.yml
└── workflows
│ ├── codeql.yml
│ ├── dependency-review.yml
│ ├── release.yaml
│ └── test-and-coverage.yml
├── .gitignore
├── .goreleaser.yaml
├── .pre-commit-config.yaml
├── .pre-commit-hooks.yaml
├── .talismanrc
├── CODE_OF_CONDUCT.md
├── LICENSE
├── README.md
├── checksumcalculator
├── checksumcalculator.go
└── checksumcalculator_test.go
├── cmd
├── acceptance_test.go
├── checksum_cmd.go
├── checksum_cmd_test.go
├── pattern_cmd.go
├── pre_commit_hook.go
├── pre_push_hook.go
├── runner.go
├── scanner_cmd.go
├── scanner_cmd_test.go
├── talisman.go
├── talisman_internal_test.go
└── talisman_test.go
├── contributing.md
├── detector
├── chain.go
├── chain_test.go
├── detector
│ └── detector.go
├── filecontent
│ ├── base64_aggressive_detector.go
│ ├── base64_aggressive_detector_test.go
│ ├── base64_detector.go
│ ├── base64_detector_test.go
│ ├── filecontent_credit_card_detector.go
│ ├── filecontent_credit_card_detector_test.go
│ ├── filecontent_detector.go
│ ├── filecontent_detector_test.go
│ ├── hex_detector.go
│ ├── hex_detector_test.go
│ ├── shannon_entropy.go
│ ├── shannon_entropy_test.go
│ ├── word_check.go
│ ├── word_check_test.go
│ └── word_check_words_list.go
├── filename
│ ├── filename_detector.go
│ └── filename_detector_test.go
├── filesize
│ ├── filesize_detector.go
│ └── filesize_detector_test.go
├── helpers
│ ├── detection_results.go
│ ├── detection_results_test.go
│ ├── ignore_evaluator.go
│ └── ignore_evaluator_test.go
├── pattern
│ ├── match_pattern.go
│ ├── match_pattern_test.go
│ ├── pattern_detector.go
│ └── pattern_detector_test.go
└── severity
│ ├── pattern_severity.go
│ ├── severity.go
│ ├── severity_config.go
│ └── severity_test.go
├── examples
└── schema-store-talismanrc.json
├── git_testing
├── git_testing.go
└── git_testing_test.go
├── gitrepo
├── constants.go
├── git_readers.go
├── gitrepo.go
├── gitrepo_test.go
└── pixel.jpg
├── global_install_scripts
├── add_to_talismanignore.bash
├── add_to_talismanignore_in_git_repo.bash
├── install.bash
├── remove_hooks.bash
├── setup_talisman.bash
├── setup_talisman_hook_in_repo.bash
├── talisman_hook_script.bash
├── uninstall.bash
├── uninstall_git_repo_hook.bash
└── update_talisman.bash
├── go.mod
├── go.sum
├── install.sh
├── internal
└── mock
│ ├── checksumcalculator
│ └── checksumcalculator.go
│ ├── prompt
│ └── survey.go
│ └── utility
│ └── sha_256_hasher.go
├── mock_scripts
├── prompt
└── survey.go
├── report
└── report.go
├── scanner
├── scanner.go
└── scanner_test.go
├── talisman.png
├── talismanrc
├── rc_file.go
├── rc_file_test.go
├── scopes.go
├── talismanrc.go
├── talismanrc_test.go
├── types.go
├── types_test.go
└── util.go
├── test-install.sh
├── third-party
└── schema-store-talismanrc.json
└── utility
├── progress_bar.go
├── progress_bar_test.go
├── sha_256_hasher.go
├── sha_256_hasher_test.go
├── utility.go
└── utility_test.go
/.gitattributes:
--------------------------------------------------------------------------------
1 | * text=auto
2 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/bug_report.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Bug report
3 | about: Create a report to help us improve
4 | title: ''
5 | labels: ''
6 | assignees: ''
7 |
8 | ---
9 |
10 | **Describe the bug**
11 | A clear and concise description of what the bug is.
12 |
13 | **To Reproduce**
14 | Steps to reproduce the behavior:
15 | 1. Go to '...'
16 | 2. Click on '....'
17 | 3. Scroll down to '....'
18 | 4. See error
19 |
20 | **Expected behavior**
21 | A clear and concise description of what you expected to happen.
22 |
23 | **Screenshots**
24 | If applicable, add screenshots to help explain your problem.
25 |
26 | **Desktop (please complete the following information):**
27 | - OS: [e.g. iOS]
28 | - Browser [e.g. chrome, safari]
29 | - Version [e.g. 22]
30 |
31 | **Smartphone (please complete the following information):**
32 | - Device: [e.g. iPhone6]
33 | - OS: [e.g. iOS8.1]
34 | - Browser [e.g. stock browser, safari]
35 | - Version [e.g. 22]
36 |
37 | **Additional context**
38 | Add any other context about the problem here.
39 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/feature_request.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Feature request
3 | about: Suggest an idea/enhancement for this project
4 | title: ''
5 | labels: ''
6 | assignees: ''
7 |
8 | ---
9 |
10 | **Is your feature request related to a problem? Please describe.**
11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
12 |
13 | **Describe the solution you'd like**
14 | A clear and concise description of what you want to happen.
15 |
16 | **Describe alternatives you've considered**
17 | A clear and concise description of any alternative solutions or features you've considered.
18 |
19 | **Additional context**
20 | Add any other context or screenshots about the feature request here.
21 |
--------------------------------------------------------------------------------
/.github/dependabot.yml:
--------------------------------------------------------------------------------
1 | version: 2
2 | updates:
3 | - package-ecosystem: "gomod"
4 | directory: "/"
5 | schedule:
6 | interval: "monthly"
7 | - package-ecosystem: "github-actions"
8 | directory: "/"
9 | schedule:
10 | interval: "monthly"
11 |
--------------------------------------------------------------------------------
/.github/workflows/codeql.yml:
--------------------------------------------------------------------------------
1 | # For most projects, this workflow file will not need changing; you simply need
2 | # to commit it to your repository.
3 | #
4 | # You may wish to alter this file to override the set of languages analyzed,
5 | # or to provide custom queries or build logic.
6 | #
7 | # ******** NOTE ********
8 | # We have attempted to detect the languages in your repository. Please check
9 | # the `language` matrix defined below to confirm you have the correct set of
10 | # supported CodeQL languages.
11 | #
12 | name: "CodeQL"
13 |
14 | on:
15 | push:
16 | branches: [ "main" ]
17 | pull_request:
18 | # The branches below must be a subset of the branches above
19 | branches: [ "main" ]
20 | schedule:
21 | - cron: '17 22 * * 6'
22 |
23 | jobs:
24 | analyze:
25 | name: Analyze
26 | runs-on: ubuntu-latest
27 | permissions:
28 | actions: read
29 | contents: read
30 | security-events: write
31 |
32 | strategy:
33 | fail-fast: false
34 | matrix:
35 | language: [ 'go' ]
36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
38 |
39 | steps:
40 | - name: Checkout repository
41 | uses: actions/checkout@v4
42 |
43 | # Initializes the CodeQL tools for scanning.
44 | - name: Initialize CodeQL
45 | uses: github/codeql-action/init@v3
46 | with:
47 | languages: ${{ matrix.language }}
48 | # If you wish to specify custom queries, you can do so here or in a config file.
49 | # By default, queries listed here will override any specified in a config file.
50 | # Prefix the list here with "+" to use these queries and those in the config file.
51 |
52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
53 | # queries: security-extended,security-and-quality
54 |
55 |
56 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
57 | # If this step fails, then you should remove it and run the build manually (see below)
58 | - name: Autobuild
59 | uses: github/codeql-action/autobuild@v3
60 |
61 | # ℹ️ Command-line programs to run using the OS shell.
62 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
63 |
64 | # If the Autobuild fails above, remove it and uncomment the following three lines.
65 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
66 |
67 | # - run: |
68 | # echo "Run, Build Application using script"
69 | # ./location_of_script_within_repo/buildscript.sh
70 |
71 | - name: Perform CodeQL Analysis
72 | uses: github/codeql-action/analyze@v3
73 | with:
74 | category: "/language:${{matrix.language}}"
75 |
--------------------------------------------------------------------------------
/.github/workflows/dependency-review.yml:
--------------------------------------------------------------------------------
1 | # Dependency Review Action
2 | #
3 | # This Action will scan dependency manifest files that change as part of a Pull Request, surfacing known-vulnerable versions of the packages declared or updated in the PR. Once installed, if the workflow run is marked as required, PRs introducing known-vulnerable packages will be blocked from merging.
4 | #
5 | # Source repository: https://github.com/actions/dependency-review-action
6 | # Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement
7 | name: 'Dependency Review'
8 | on: [pull_request]
9 |
10 | permissions:
11 | contents: read
12 |
13 | jobs:
14 | dependency-review:
15 | runs-on: ubuntu-latest
16 | steps:
17 | - name: 'Checkout Repository'
18 | uses: actions/checkout@v4
19 | - name: 'Dependency Review'
20 | uses: actions/dependency-review-action@v4
21 |
--------------------------------------------------------------------------------
/.github/workflows/release.yaml:
--------------------------------------------------------------------------------
1 | name: Release
2 |
3 | on:
4 | push:
5 | tags:
6 | - 'v*'
7 |
8 | permissions:
9 | contents: write
10 | packages: write
11 |
12 | jobs:
13 | goreleaser:
14 | runs-on: ubuntu-latest
15 | steps:
16 | - name: Checkout
17 | uses: actions/checkout@v4
18 | with:
19 | fetch-depth: 0
20 | - name: Fetch all tags
21 | run: git fetch --force --tags
22 | - name: Set up Go
23 | uses: actions/setup-go@v5
24 | with:
25 | go-version-file: go.mod
26 | - name: Generate release notes
27 | id: release-notes
28 | uses: actions/github-script@v7
29 | with:
30 | result-encoding: string
31 | script: |
32 | return github.rest.repos.generateReleaseNotes({
33 | owner: context.repo.owner,
34 | repo: context.repo.repo,
35 | tag_name: "${{ github.ref_name }}",
36 | }).then(response => response.data.body)
37 | - name: Write release notes to file
38 | run: |
39 | cat << EOF > release-notes.md
40 | ${{ steps.release-notes.outputs.result }}
41 | EOF
42 | - name: Run GoReleaser
43 | uses: goreleaser/goreleaser-action@v6
44 | with:
45 | version: '~> v2'
46 | args: release --clean --release-notes release-notes.md
47 | env:
48 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
49 |
--------------------------------------------------------------------------------
/.github/workflows/test-and-coverage.yml:
--------------------------------------------------------------------------------
1 | name: Test
2 |
3 | on:
4 | push:
5 | branches: '**'
6 | pull_request:
7 | branches: [ main ]
8 |
9 | jobs:
10 | build:
11 | runs-on: ubuntu-latest
12 | steps:
13 | - uses: actions/checkout@v4
14 | - name: Set up Go
15 | uses: actions/setup-go@v5
16 | with:
17 | go-version-file: go.mod
18 | - name: Format
19 | run: |
20 | go fmt ./...
21 | git diff --no-patch --exit-code
22 | - name: Test
23 | run: |
24 | git config user.email "talisman-maintainers@thoughtworks.com"
25 | git config user.name "Talisman Maintainers"
26 | go test -covermode=count -v ./...
27 | - name: Install bash_unit
28 | run: bash <(curl -s https://raw.githubusercontent.com/pgrange/bash_unit/master/install.sh)
29 | - name: Test install script
30 | run: ./bash_unit test-install.sh
31 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | .vagrant
3 | _workspace
4 | *.swp
5 | talisman
6 | checksums
7 | vendor/
8 | detector/.talismanrc
9 | .vscode/**
10 | coverage.out
11 | coverage.txt
12 | .talismanrc
13 | talisman_reports/
14 | *.iml
15 | dist/
16 | *.out
17 | *.pprof
18 | *.mprof
19 | .envrc
20 | talisman_html_report
21 | .vscode
22 | .idea
23 |
24 | dist/
25 | out/
26 | .goreleaser-schema.json
27 |
28 | ### Go template
29 | # If you prefer the allow list template instead of the deny list, see community template:
30 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
31 | #
32 | # Binaries for programs and plugins
33 | *.exe
34 | *.exe~
35 | *.dll
36 | *.so
37 | *.dylib
38 |
39 | # Test binary, built with `go test -c`
40 | *.test
41 |
42 | # Output of the go coverage tool, specifically when used with LiteIDE
43 |
44 | # Dependency directories (remove the comment below to include it)
45 | # vendor/
46 |
47 | # Go workspace file
48 | go.work
49 |
50 | release-notes.md
51 |
--------------------------------------------------------------------------------
/.goreleaser.yaml:
--------------------------------------------------------------------------------
1 | version: 2
2 | project_name: talisman
3 |
4 | before:
5 | hooks:
6 | - go mod tidy
7 |
8 | builds:
9 | - env:
10 | - CGO_ENABLED=0
11 | goos:
12 | - linux
13 | - windows
14 | - darwin
15 |
16 | # Custom ldflags templates.
17 | # Default is `-s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} -X main.builtBy=goreleaser`.
18 | ldflags:
19 | - -s -w -X main.Version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}}
20 | main: ./cmd
21 |
22 | release:
23 | draft: true
24 | replace_existing_draft: true
25 |
26 | archives:
27 | - meta: false
28 | formats: binary
29 | wrap_in_directory: true
30 | name_template: "{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}"
31 |
32 | checksum:
33 | name_template: 'checksums'
34 | algorithm: sha256
35 |
36 | snapshot:
37 | version_template: "{{ incpatch .Version }}-next"
38 |
39 | changelog:
40 | sort: asc
41 | filters:
42 | exclude:
43 | - '^docs:'
44 | - '^test:'
45 |
--------------------------------------------------------------------------------
/.pre-commit-config.yaml:
--------------------------------------------------------------------------------
1 | repos:
2 | - repo: https://github.com/pre-commit/pre-commit-hooks
3 | rev: v5.0.0
4 | hooks:
5 | - id: trailing-whitespace
6 | - id: end-of-file-fixer
7 | - id: check-yaml
8 | - id: check-added-large-files
9 | - repo: local
10 | hooks:
11 | - id: go-format
12 | name: Go Format
13 | language: system
14 | entry: go fmt ./...
15 | types: [go]
16 | pass_filenames: false
17 | - id: go-test
18 | name: Go Tests
19 | language: system
20 | entry: go test ./...
21 | types: [go]
22 | pass_filenames: false
23 |
--------------------------------------------------------------------------------
/.pre-commit-hooks.yaml:
--------------------------------------------------------------------------------
1 | - id: talisman-commit
2 | name: talisman
3 | entry: cmd --githook pre-commit
4 | stages: [pre-commit]
5 | # talisman currently discovers files by itself and does not take them on the cli
6 | pass_filenames: false
7 | types: [text]
8 | language: golang
9 | language_version: 1.23.0
10 | minimum_pre_commit_version: 3.2.0
11 |
12 | - id: talisman-push
13 | name: talisman
14 | entry: cmd --githook pre-push
15 | stages: [pre-push]
16 | # talisman currently discovers files by itself and does not take them on the cli
17 | pass_filenames: false
18 | types: [text]
19 | language: golang
20 | language_version: 1.23.0
21 | minimum_pre_commit_version: 3.2.0
22 |
--------------------------------------------------------------------------------
/.talismanrc:
--------------------------------------------------------------------------------
1 | fileignoreconfig:
2 | - filename: README.md
3 | ignore_detectors:
4 | - filecontent
5 | - filename: build_binaries.bash
6 | checksum: fdc955c83e408178fcccd834c6db0b5622edb876ea9e72d25da8ef8ab1e87192
7 | - filename: checksumcalculator/checksumcalculator_test.go
8 | checksum: d74273fe8cb37c2c4a7469785eb21b4f1d23a3437927612f8a5964f5011f0998
9 | - filename: detector/detection_results_test.go
10 | checksum: 69fed055782cddfe0f0d23ea440cef9f9dd0b9e8a3c8a73856741bb26257b223
11 | ignore_detectors:
12 | - filecontent
13 | - filename: detector/filecontent/filecontent_detector_test.go
14 | checksum: 05d00bb99452d37ab45de28c9e074357e5be22a80d95dad1b2cdd01074b25b2a
15 | - filename: detector/filename/filename_detector.go
16 | checksum: 5404565683a7e812fa98ff2d14237c4d1ba7dc5b4aca2dd3ba663b33dc8ddae7
17 | - filename: detector/filename/filename_detector_test.go
18 | checksum: d04b9a2cc51e6835c1b3c13fc0872837b34bde2a5f2c070b1047b055299d5cf9
19 | - filename: detector/match_pattern_test.go
20 | checksum: d68aa0e06355e3b848941727d1fcb32cf47e3d615f9921f0db39855325010446
21 | - filename: detector/pattern/match_pattern_test.go
22 | checksum: c95b8106ced5ad34ec1d00773a05f8789715034a734197c93cdaa4ed5036c177
23 | - filename: detector/pattern/pattern_detector.go
24 | checksum: 78cddc944d4092ae2e88535d04f05281784848a990fb55a9d38339f29080a239
25 | - filename: detector/pattern/pattern_detector_test.go
26 | checksum: 4d70b790f28f2d23d506f808d489aa43f1efd2514549ae6a83a535e1223382e3
27 | - filename: detector/pattern_detector_test.go
28 | checksum: 4d70b790f28f2d23d506f808d489aa43f1efd2514549ae6a83a535e1223382e3
29 | - filename: detector/severity/severity_config.go
30 | checksum: a9b041a6f5a5c93146a79dfc7b3d2d26c4a6cf621aa0aa92db7e8c343c6926a5
31 | - filename: global_install_scripts/install.bash
32 | checksum: bf0da9b8b6f779f502564166323a903a49b56b9d8df2597729a5b96c8f066074
33 | - filename: install.sh
34 | checksum: c909f6a1caefba3f196d489f9262608044be596a44793c2173ec55b98ecec649
35 | - filename: talismanrc/talismanrc_test.go
36 | checksum: eab40d0745dc215267da86ef7f926be77c4d46e9a248dadbc7e52aa186e82853
37 | scopeconfig:
38 | - scope: go
39 | version: "1.0"
40 |
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | # Contributor Covenant Code of Conduct
2 |
3 | ## Our Pledge
4 |
5 | In the interest of fostering an open and welcoming environment, we as
6 | contributors and maintainers pledge to making participation in our project and
7 | our community a harassment-free experience for everyone, regardless of age, body
8 | size, disability, ethnicity, sex characteristics, gender identity and expression,
9 | level of experience, education, socio-economic status, nationality, personal
10 | appearance, race, religion, or sexual identity and orientation.
11 |
12 | ## Our Standards
13 |
14 | Examples of behavior that contributes to creating a positive environment
15 | include:
16 |
17 | * Using welcoming and inclusive language
18 | * Being respectful of differing viewpoints and experiences
19 | * Gracefully accepting constructive criticism
20 | * Focusing on what is best for the community
21 | * Showing empathy towards other community members
22 |
23 | Examples of unacceptable behavior by participants include:
24 |
25 | * The use of sexualized language or imagery and unwelcome sexual attention or
26 | advances
27 | * Trolling, insulting/derogatory comments, and personal or political attacks
28 | * Public or private harassment
29 | * Publishing others' private information, such as a physical or electronic
30 | address, without explicit permission
31 | * Other conduct which could reasonably be considered inappropriate in a
32 | professional setting
33 |
34 | ## Our Responsibilities
35 |
36 | Project maintainers are responsible for clarifying the standards of acceptable
37 | behavior and are expected to take appropriate and fair corrective action in
38 | response to any instances of unacceptable behavior.
39 |
40 | Project maintainers have the right and responsibility to remove, edit, or
41 | reject comments, commits, code, wiki edits, issues, and other contributions
42 | that are not aligned to this Code of Conduct, or to ban temporarily or
43 | permanently any contributor for other behaviors that they deem inappropriate,
44 | threatening, offensive, or harmful.
45 |
46 | ## Scope
47 |
48 | This Code of Conduct applies both within project spaces and in public spaces
49 | when an individual is representing the project or its community. Examples of
50 | representing a project or community include using an official project e-mail
51 | address, posting via an official social media account, or acting as an appointed
52 | representative at an online or offline event. Representation of a project may be
53 | further defined and clarified by project maintainers.
54 |
55 | ## Enforcement
56 |
57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be
58 | reported by contacting the project team at talisman-maintainers@thoughtworks.com. All
59 | complaints will be reviewed and investigated and will result in a response that
60 | is deemed necessary and appropriate to the circumstances. The project team is
61 | obligated to maintain confidentiality with regard to the reporter of an incident.
62 | Further details of specific enforcement policies may be posted separately.
63 |
64 | Project maintainers who do not follow or enforce the Code of Conduct in good
65 | faith may face temporary or permanent repercussions as determined by other
66 | members of the project's leadership.
67 |
68 | ## Attribution
69 |
70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
72 |
73 | [homepage]: https://www.contributor-covenant.org
74 |
75 | For answers to common questions about this code of conduct, see
76 | https://www.contributor-covenant.org/faq
77 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | The MIT License (MIT)
2 |
3 | Copyright (c) 2015 ThoughtWorks Inc.
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
23 |
--------------------------------------------------------------------------------
/checksumcalculator/checksumcalculator.go:
--------------------------------------------------------------------------------
1 | package checksumcalculator
2 |
3 | import (
4 | "strings"
5 | "talisman/gitrepo"
6 | "talisman/talismanrc"
7 | "talisman/utility"
8 | )
9 |
10 | type ChecksumCalculator interface {
11 | SuggestTalismanRC(fileNamePatterns []string) string
12 | CalculateCollectiveChecksumForPattern(fileNamePattern string) string
13 | }
14 |
15 | type checksumCalculator struct {
16 | allTrackedFiles []gitrepo.Addition
17 | hasher utility.SHA256Hasher
18 | }
19 |
20 | // NewChecksumCalculator returns new instance of the CheckSumDetector
21 | func NewChecksumCalculator(hasher utility.SHA256Hasher, gitAdditions []gitrepo.Addition) ChecksumCalculator {
22 | return &checksumCalculator{hasher: hasher, allTrackedFiles: gitAdditions}
23 | }
24 |
25 | // SuggestTalismanRC returns the suggestion for .talismanrc format
26 | func (cc *checksumCalculator) SuggestTalismanRC(fileNamePatterns []string) string {
27 | var fileIgnoreConfigs []talismanrc.FileIgnoreConfig
28 | result := strings.Builder{}
29 | for _, pattern := range fileNamePatterns {
30 | collectiveChecksum := cc.CalculateCollectiveChecksumForPattern(pattern)
31 | if collectiveChecksum != "" {
32 | fileIgnoreConfigs = append(fileIgnoreConfigs, talismanrc.IgnoreFileWithChecksum(pattern, collectiveChecksum))
33 | }
34 | }
35 | if len(fileIgnoreConfigs) != 0 {
36 | result.WriteString("\n\x1b[33m.talismanrc format for given file names / patterns\x1b[0m\n")
37 | result.Write([]byte(talismanrc.SuggestRCFor(fileIgnoreConfigs)))
38 | }
39 | return result.String()
40 | }
41 |
42 | // CalculateCollectiveChecksumForPattern calculates and returns the checksum for files matching the input pattern
43 | func (cc *checksumCalculator) CalculateCollectiveChecksumForPattern(fileNamePattern string) string {
44 | var patternPaths []string
45 | currentCollectiveChecksum := ""
46 | for _, file := range cc.allTrackedFiles {
47 | if file.Matches(fileNamePattern) {
48 | patternPaths = append(patternPaths, string(file.Path))
49 | }
50 | }
51 | // Calculate current collective checksum
52 | patternPaths = utility.UniqueItems(patternPaths)
53 | if len(patternPaths) != 0 {
54 | currentCollectiveChecksum = cc.hasher.CollectiveSHA256Hash(patternPaths)
55 | }
56 | return currentCollectiveChecksum
57 | }
58 |
--------------------------------------------------------------------------------
/checksumcalculator/checksumcalculator_test.go:
--------------------------------------------------------------------------------
1 | package checksumcalculator
2 |
3 | import (
4 | "talisman/gitrepo"
5 | "talisman/utility"
6 | "testing"
7 |
8 | "github.com/stretchr/testify/assert"
9 | )
10 |
11 | var defaultSHA256Hasher utility.SHA256Hasher
12 |
13 | func init() {
14 | defaultSHA256Hasher = utility.MakeHasher("default", ".")
15 | }
16 |
17 | func TestNewChecksumCalculator(t *testing.T) {
18 | t.Run("should return empty CollectiveChecksum when non existing file name pattern is sent", func(t *testing.T) {
19 | gitAdditions := []gitrepo.Addition{
20 | {
21 | Path: "GitRepoPath1",
22 | Name: "GitRepoName1",
23 | },
24 | }
25 | expectedCC := ""
26 | fileNamePattern := "*NonExistenceFileNamePattern"
27 | cc := NewChecksumCalculator(defaultSHA256Hasher, gitAdditions)
28 |
29 | actualCC := cc.CalculateCollectiveChecksumForPattern(fileNamePattern)
30 |
31 | assert.Equal(t, expectedCC, actualCC)
32 | })
33 |
34 | t.Run("should return CollectiveChecksum when existing file name pattern is sent", func(t *testing.T) {
35 | gitAdditions := []gitrepo.Addition{
36 | {
37 | Path: "GitRepoPath1/GitRepoName1",
38 | Name: "GitRepoName1",
39 | },
40 | }
41 | expectedCC := "19250a996e1200d33e91454bc662efae7682410e5347cfc56b0ff386dfbc10ae"
42 | fileNamePattern := "GitRepoPath1/"
43 | cc := NewChecksumCalculator(defaultSHA256Hasher, gitAdditions)
44 |
45 | actualCC := cc.CalculateCollectiveChecksumForPattern(fileNamePattern)
46 |
47 | assert.Equal(t, expectedCC, actualCC)
48 | })
49 |
50 | t.Run("should return the files own CollectiveChecksum when same file name is present in subfolders", func(t *testing.T) {
51 | gitAdditions := []gitrepo.Addition{
52 | {
53 | Path: "hello.txt",
54 | Name: "hello.txt",
55 | },
56 | {
57 | Path: "subfolder/hello.txt",
58 | Name: "hello.txt",
59 | },
60 | }
61 | hello_expectedCC := "9d30c2e4bcf181bba07374cc416f1892d89918038bce5172776475347c4d2d69"
62 | fileNamePattern1 := "hello.txt"
63 | cc1 := NewChecksumCalculator(defaultSHA256Hasher, gitAdditions)
64 | actualCC := cc1.CalculateCollectiveChecksumForPattern(fileNamePattern1)
65 | assert.Equal(t, hello_expectedCC, actualCC)
66 |
67 | subfolder_hello_expectedCC := "6c779c16bcc2e63c659be7649a531650210d6b96ae590a146f9ccdca383587f6"
68 | fileNamePattern2 := "subfolder/hello.txt"
69 | cc2 := NewChecksumCalculator(defaultSHA256Hasher, gitAdditions)
70 | subfolder_actualCC := cc2.CalculateCollectiveChecksumForPattern(fileNamePattern2)
71 | assert.Equal(t, subfolder_hello_expectedCC, subfolder_actualCC)
72 |
73 | all_txt_expectedCC := "aba77c9077539130e21a8f275fc5f1f43b7f0589e392bc89e4b96c578f0a9184"
74 | fileNamePattern3 := "*.txt"
75 | cc3 := NewChecksumCalculator(defaultSHA256Hasher, gitAdditions)
76 | all_txt_actualCC := cc3.CalculateCollectiveChecksumForPattern(fileNamePattern3)
77 | assert.Equal(t, all_txt_expectedCC, all_txt_actualCC)
78 | })
79 | }
80 |
81 | func TestDefaultChecksumCalculator_SuggestTalismanRC(t *testing.T) {
82 | t.Run("should return no suggestion for .talismanrc format when no matching file name patterns is sent", func(t *testing.T) {
83 | gitAdditions := []gitrepo.Addition{
84 | {
85 | Path: "GitRepoPath1",
86 | Name: "GitRepoName1",
87 | }, {
88 | Path: "GitRepoPath2",
89 | Name: "GitRepoName2",
90 | },
91 | }
92 | expectedCC := ""
93 | fileNamePatterns := []string{"*NonExistenceFileNamePattern1", "*NonExistenceFileNamePattern2", "*NonExistenceFileNamePattern3"}
94 | cc := NewChecksumCalculator(defaultSHA256Hasher, gitAdditions)
95 |
96 | actualCC := cc.SuggestTalismanRC(fileNamePatterns)
97 |
98 | assert.Equal(t, expectedCC, actualCC)
99 | })
100 |
101 | t.Run("should return suggestion for .talismanrc format when matching file name patterns is sent", func(t *testing.T) {
102 | gitAdditions := []gitrepo.Addition{
103 | {
104 | Path: "GitRepoPath1",
105 | Name: "GitRepoName1",
106 | }, {
107 | Path: "GitRepoPath2",
108 | Name: "GitRepoName2",
109 | },
110 | }
111 | expectedCC := "\n\x1b[33m.talismanrc format for given file names / patterns\x1b[0m\nfileignoreconfig:\n- filename: '*1'\n checksum: 54bbf09e5c906e2d7cc0808729f8120cfa3c4bad3fb6a85689ae23ca00e5a3c8\n- filename: Git*2\n checksum: d2103425cbcc10118556d7cd63dd198b3e771bcc0359d6b196ddafcf5b87dac5\nversion: \"1.0\"\n"
112 | fileNamePatterns := []string{"*1", "Git*2", "*NonExistenceFileNamePattern3"}
113 | cc := NewChecksumCalculator(defaultSHA256Hasher, gitAdditions)
114 |
115 | actualCC := cc.SuggestTalismanRC(fileNamePatterns)
116 |
117 | assert.Equal(t, expectedCC, actualCC)
118 | })
119 | }
120 |
--------------------------------------------------------------------------------
/cmd/checksum_cmd.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "talisman/checksumcalculator"
7 | "talisman/gitrepo"
8 | "talisman/utility"
9 |
10 | "github.com/sirupsen/logrus"
11 | )
12 |
13 | type ChecksumCmd struct {
14 | fileNamePatterns []string
15 | hasher utility.SHA256Hasher
16 | repoRoot string
17 | }
18 |
19 | func NewChecksumCmd(fileNamePatterns []string) *ChecksumCmd {
20 | wd, _ := os.Getwd()
21 | hasher := utility.MakeHasher("checksum", wd)
22 | return &ChecksumCmd{fileNamePatterns: fileNamePatterns, hasher: hasher, repoRoot: wd}
23 | }
24 |
25 | func (s *ChecksumCmd) Run() int {
26 | repo := gitrepo.RepoLocatedAt(s.repoRoot)
27 | if s.hasher == nil {
28 | logrus.Errorf("unable to start hasher")
29 | return EXIT_FAILURE
30 | }
31 |
32 | gitTrackedFilesAsAdditions := repo.TrackedFilesAsAdditions()
33 | gitTrackedFilesAsAdditions = append(gitTrackedFilesAsAdditions, repo.StagedAdditions()...)
34 |
35 | cc := checksumcalculator.NewChecksumCalculator(s.hasher, gitTrackedFilesAsAdditions)
36 | rcSuggestion := cc.SuggestTalismanRC(s.fileNamePatterns)
37 |
38 | if rcSuggestion != "" {
39 | fmt.Print(rcSuggestion)
40 | return EXIT_SUCCESS
41 | }
42 | return EXIT_FAILURE
43 | }
44 |
--------------------------------------------------------------------------------
/cmd/checksum_cmd_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "os"
5 | "talisman/git_testing"
6 | "testing"
7 |
8 | "github.com/stretchr/testify/assert"
9 | )
10 |
11 | func TestChecksumCalculatorShouldExitSuccess(t *testing.T) {
12 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) {
13 | git.SetupBaselineFiles("simple-file.txt")
14 | git.CreateFileWithContents("private.pem", "secret")
15 | git.CreateFileWithContents("another/private.pem", "secret")
16 | git.CreateFileWithContents("sample.txt", "password")
17 | os.Chdir(git.Root())
18 |
19 | checksumCmd := NewChecksumCmd([]string{"*.txt"})
20 | assert.Equal(t, 0, checksumCmd.Run(), "Expected run() to return 0 as given patterns are found and .talsimanrc is suggested")
21 | options.Checksum = ""
22 | })
23 | }
24 |
25 | func TestChecksumCalculatorShouldExitFailure(t *testing.T) {
26 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) {
27 | git.SetupBaselineFiles("simple-file.txt")
28 | git.CreateFileWithContents("private.pem", "secret")
29 | git.CreateFileWithContents("another/private.pem", "secret")
30 | git.CreateFileWithContents("sample.txt", "password")
31 | os.Chdir(git.Root())
32 |
33 | checksumCmd := NewChecksumCmd([]string{"*.java"})
34 | assert.Equal(t, 1, checksumCmd.Run(), "Expected run() to return 1 as given patterns are found and .talsimanrc is suggested")
35 | options.Checksum = ""
36 | })
37 | }
38 |
39 | func TestChecksumCalculatorShouldExitFailureWhenHasherIsEmpty(t *testing.T) {
40 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) {
41 | checksumCmd := ChecksumCmd{[]string{"*.java"}, nil, git.Root()}
42 | assert.Equal(t, 1, checksumCmd.Run(), "Expected run() to return 1 because hasher failed to start")
43 | })
44 | }
45 |
--------------------------------------------------------------------------------
/cmd/pattern_cmd.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "talisman/gitrepo"
5 | "talisman/utility"
6 |
7 | log "github.com/sirupsen/logrus"
8 |
9 | "github.com/bmatcuk/doublestar"
10 | )
11 |
12 | type PatternCmd struct {
13 | *runner
14 | }
15 |
16 | func NewPatternCmd(pattern string) *PatternCmd {
17 | var additions []gitrepo.Addition
18 |
19 | files, _ := doublestar.Glob(pattern)
20 | for _, file := range files {
21 | log.Debugf("reading file %s", file)
22 | data, err := utility.SafeReadFile(file)
23 |
24 | if err != nil {
25 | log.Warnf("Error reading file: %s. Skipping", file)
26 | continue
27 | }
28 |
29 | newAddition := gitrepo.NewAddition(file, data)
30 | additions = append(additions, newAddition)
31 | }
32 |
33 | return &PatternCmd{NewRunner(additions, "pattern")}
34 | }
35 |
--------------------------------------------------------------------------------
/cmd/pre_commit_hook.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "os"
5 |
6 | "talisman/gitrepo"
7 | )
8 |
9 | type PreCommitHook struct {
10 | runner
11 | }
12 |
13 | func NewPreCommitHook() *PreCommitHook {
14 | wd, _ := os.Getwd()
15 | repo := gitrepo.RepoLocatedAt(wd)
16 |
17 | return &PreCommitHook{*NewRunner(repo.GetDiffForStagedFiles(), PreCommit)}
18 | }
19 |
--------------------------------------------------------------------------------
/cmd/pre_push_hook.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bufio"
5 | "io"
6 | "os"
7 | "strings"
8 |
9 | log "github.com/sirupsen/logrus"
10 | "talisman/gitrepo"
11 | )
12 |
13 | const (
14 | //EmptySha represents the state of a brand new ref
15 | EmptySha string = "0000000000000000000000000000000000000000"
16 | //ShaId of the empty tree in Git
17 | EmptyTreeSha string = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
18 | )
19 |
20 | type PrePushHook struct {
21 | localRef, localCommit, remoteRef, remoteCommit string
22 | *runner
23 | }
24 |
25 | func NewPrePushHook(stdin io.Reader) *PrePushHook {
26 | localRef, localCommit, remoteRef, remoteCommit := readRefAndSha(stdin)
27 | prePushHook := &PrePushHook{
28 | localRef,
29 | localCommit,
30 | remoteRef,
31 | remoteCommit,
32 | NewRunner(nil, PrePush)}
33 | prePushHook.additions = prePushHook.getRepoAdditions()
34 | return prePushHook
35 | }
36 |
37 | // If the outgoing ref does not exist on the remote, all commits on the local ref will be checked
38 | // If the outgoing ref already exists, all additions in the range between "localSha" and "remoteSha" will be validated
39 | func (p *PrePushHook) getRepoAdditions() []gitrepo.Addition {
40 | if p.runningOnDeletedRef() {
41 | log.WithFields(log.Fields{
42 | "localRef": p.localRef,
43 | "localCommit": p.localCommit,
44 | "remoteRef": p.remoteRef,
45 | "remoteCommit": p.remoteCommit,
46 | }).Info("Running on a deleted ref. Nothing to verify as outgoing changes are all deletions.")
47 |
48 | return []gitrepo.Addition{}
49 | }
50 |
51 | if p.runningOnNewRef() {
52 | log.WithFields(log.Fields{
53 | "localRef": p.localRef,
54 | "localCommit": p.localCommit,
55 | "remoteRef": p.remoteRef,
56 | "remoteCommit": p.remoteCommit,
57 | }).Info("Running on a new ref. All changes in the ref will be verified.")
58 |
59 | return p.getRepoAdditionsFrom(EmptyTreeSha, p.localCommit)
60 | }
61 |
62 | log.WithFields(log.Fields{
63 | "localRef": p.localRef,
64 | "localCommit": p.localCommit,
65 | "remoteRef": p.remoteRef,
66 | "remoteCommit": p.remoteCommit,
67 | }).Info("Running on an existing ref. All changes in the commit range will be verified.")
68 |
69 | return p.getRepoAdditionsFrom(p.remoteCommit, p.localCommit)
70 | }
71 |
72 | func (p *PrePushHook) runningOnDeletedRef() bool {
73 | return p.localCommit == EmptySha
74 | }
75 |
76 | func (p *PrePushHook) runningOnNewRef() bool {
77 | return p.remoteCommit == EmptySha
78 | }
79 |
80 | func (p *PrePushHook) getRepoAdditionsFrom(oldCommit, newCommit string) []gitrepo.Addition {
81 | wd, _ := os.Getwd()
82 | repo := gitrepo.RepoLocatedAt(wd)
83 | return repo.AdditionsWithinRange(oldCommit, newCommit)
84 | }
85 |
86 | func readRefAndSha(file io.Reader) (string, string, string, string) {
87 | text, _ := bufio.NewReader(file).ReadString('\n')
88 | refsAndShas := strings.Split(strings.Trim(string(text), "\n"), " ")
89 | if len(refsAndShas) < 4 {
90 | return EmptySha, EmptySha, "", ""
91 | }
92 | return refsAndShas[0], refsAndShas[1], refsAndShas[2], refsAndShas[3]
93 | }
94 |
--------------------------------------------------------------------------------
/cmd/runner.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "talisman/detector"
7 | "talisman/detector/helpers"
8 | "talisman/detector/severity"
9 | "talisman/gitrepo"
10 | "talisman/prompt"
11 | "talisman/talismanrc"
12 | )
13 |
14 | // runner represents a single run of the validations for a given commit range
15 | type runner struct {
16 | additions []gitrepo.Addition
17 | results *helpers.DetectionResults
18 | mode string
19 | }
20 |
21 | // NewRunner returns a new runner.
22 | func NewRunner(additions []gitrepo.Addition, mode string) *runner {
23 | return &runner{
24 | additions: additions,
25 | results: helpers.NewDetectionResults(),
26 | mode: mode,
27 | }
28 | }
29 |
30 | // Run will validate the commit range for errors and return either COMPLETED_SUCCESSFULLY or COMPLETED_WITH_ERRORS
31 | func (r *runner) Run(tRC *talismanrc.TalismanRC, promptContext prompt.PromptContext) int {
32 | wd, _ := os.Getwd()
33 | repo := gitrepo.RepoLocatedAt(wd)
34 | ie := helpers.BuildIgnoreEvaluator(r.mode, tRC, repo)
35 |
36 | setCustomSeverities(tRC)
37 | additionsToScan := tRC.RemoveScopedFiles(r.additions)
38 |
39 | detector.DefaultChain(tRC, ie).Test(additionsToScan, tRC, r.results)
40 | r.printReport(promptContext)
41 | exitStatus := r.exitStatus()
42 | return exitStatus
43 | }
44 |
45 | func setCustomSeverities(tRC *talismanrc.TalismanRC) {
46 | for _, cs := range tRC.CustomSeverities {
47 | severity.SeverityConfiguration[cs.Detector] = cs.Severity
48 | }
49 | }
50 |
51 | func (r *runner) printReport(promptContext prompt.PromptContext) {
52 | if r.results.HasWarnings() {
53 | fmt.Println(r.results.ReportWarnings())
54 | }
55 | if r.results.HasIgnores() || r.results.HasFailures() {
56 | r.results.Report(promptContext, r.mode)
57 | }
58 | }
59 |
60 | func (r *runner) exitStatus() int {
61 | if r.results.HasFailures() {
62 | return EXIT_FAILURE
63 | }
64 | return EXIT_SUCCESS
65 | }
66 |
--------------------------------------------------------------------------------
/cmd/scanner_cmd.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "talisman/detector"
7 | "talisman/detector/helpers"
8 | "talisman/gitrepo"
9 | "talisman/report"
10 | "talisman/scanner"
11 | "talisman/talismanrc"
12 | "talisman/utility"
13 |
14 | logr "github.com/sirupsen/logrus"
15 | )
16 |
17 | const (
18 | SCAN_MODE = "scan"
19 | )
20 |
21 | type ScannerCmd struct {
22 | additions []gitrepo.Addition
23 | results *helpers.DetectionResults
24 | reportDirectory string
25 | ignoreEvaluator helpers.IgnoreEvaluator
26 | tRC *talismanrc.TalismanRC
27 | }
28 |
29 | // Run scans git commit history for potential secrets and returns 0 or 1 as exit code
30 | func (s *ScannerCmd) Run() int {
31 | fmt.Printf("\n\n")
32 | utility.CreateArt("Running Scan..")
33 |
34 | additionsToScan := s.tRC.RemoveScopedFiles(s.additions)
35 |
36 | detector.DefaultChain(s.tRC, s.ignoreEvaluator).Test(additionsToScan, s.tRC, s.results)
37 | reportsPath, err := report.GenerateReport(s.results, s.reportDirectory)
38 | if err != nil {
39 | logr.Errorf("error while generating report: %v", err)
40 | return EXIT_FAILURE
41 | }
42 |
43 | fmt.Printf("\nPlease check '%s' folder for the talisman scan report\n\n", reportsPath)
44 | return s.exitStatus()
45 | }
46 |
47 | func (s *ScannerCmd) exitStatus() int {
48 | if s.results.HasFailures() {
49 | return EXIT_FAILURE
50 | }
51 | return EXIT_SUCCESS
52 | }
53 |
54 | // NewScannerCmd Returns a new scanner command
55 | func NewScannerCmd(ignoreHistory bool, tRC *talismanrc.TalismanRC, reportDirectory string) *ScannerCmd {
56 | repoRoot, _ := os.Getwd()
57 | reader := gitrepo.NewBatchGitObjectHashReader(repoRoot)
58 | additions := scanner.GetAdditions(ignoreHistory, reader)
59 | ignoreEvaluator := helpers.ScanHistoryEvaluator()
60 | if ignoreHistory {
61 | ignoreEvaluator = helpers.BuildIgnoreEvaluator("default", tRC, gitrepo.RepoLocatedAt(repoRoot))
62 | }
63 | return &ScannerCmd{
64 | additions: additions,
65 | results: helpers.NewDetectionResults(),
66 | reportDirectory: reportDirectory,
67 | ignoreEvaluator: ignoreEvaluator,
68 | tRC: tRC,
69 | }
70 | }
71 |
--------------------------------------------------------------------------------
/cmd/scanner_cmd_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "os"
5 | "talisman/git_testing"
6 | "talisman/talismanrc"
7 | "testing"
8 |
9 | "github.com/stretchr/testify/assert"
10 | )
11 |
12 | func TestScannerCmdRunsSuccessfully(t *testing.T) {
13 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) {
14 | git.SetupBaselineFiles("simple-file")
15 | git.CreateFileWithContents("some-dir/should-be-included.txt", "safeContents")
16 | git.AddAndcommit("*", "Start of Scan")
17 | os.Chdir(git.Root())
18 |
19 | scannerCmd := NewScannerCmd(true, &talismanrc.TalismanRC{}, git.Root())
20 | scannerCmd.Run()
21 | assert.Equal(t, 0, scannerCmd.exitStatus(), "Expected ScannerCmd.exitStatus() to return 0 since no secret is found")
22 | })
23 | }
24 |
25 | func TestScannerCmdDetectsSecretAndFails(t *testing.T) {
26 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) {
27 | git.SetupBaselineFiles("simple-file")
28 | git.CreateFileWithContents("some-dir/file-with-secret.txt", awsAccessKeyIDExample)
29 | git.AddAndcommit("*", "Initial Commit")
30 | git.RemoveFile("some-dir/file-with-secret.txt")
31 | git.AddAndcommit("*", "Removed secret")
32 | git.CreateFileWithContents("some-dir/safe-file.txt", "safeContents")
33 | git.AddAndcommit("*", "Start of Scan")
34 | os.Chdir(git.Root())
35 |
36 | scannerCmd := NewScannerCmd(false, &talismanrc.TalismanRC{}, git.Root())
37 | scannerCmd.Run()
38 | assert.Equal(t, 1, scannerCmd.exitStatus(), "Expected ScannerCmd.exitStatus() to return 1 since secret present in history")
39 | })
40 | }
41 |
42 | func TestScannerCmdAddingSecretKeyShouldExitZeroIfFileIsWithinConfiguredScope(t *testing.T) {
43 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) {
44 | git.SetupBaselineFiles("simple-file")
45 | git.CreateFileWithContents("go.sum", awsAccessKeyIDExample)
46 | git.CreateFileWithContents("go.mod", awsAccessKeyIDExample)
47 | git.AddAndcommit("*", "go sum file")
48 | os.Chdir(git.Root())
49 |
50 | tRC := &talismanrc.TalismanRC{ScopeConfig: []talismanrc.ScopeConfig{{ScopeName: "go"}}}
51 | scannerCmd := NewScannerCmd(false, tRC, git.Root())
52 | scannerCmd.Run()
53 | assert.Equal(t, 0, scannerCmd.exitStatus(), "Expected ScannerCmd.exitStatus() to return 0 since no secret is found")
54 | })
55 | }
56 |
57 | func TestScannerCmdDetectsSecretAndIgnoresWhileRunningInIgnoreHistoryModeWithValidIgnoreConf(t *testing.T) {
58 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) {
59 | git.SetupBaselineFiles("simple-file")
60 | git.CreateFileWithContents("go.sum", awsAccessKeyIDExample)
61 | git.CreateFileWithContents("go.mod", awsAccessKeyIDExample)
62 | git.AddAndcommit("*", "go sum file")
63 | os.Chdir(git.Root())
64 |
65 | tRC := &talismanrc.TalismanRC{
66 | FileIgnoreConfig: []talismanrc.FileIgnoreConfig{
67 | {FileName: "go.sum", Checksum: "582093519ae682d5170aecc9b935af7e90ed528c577ecd2c9dd1fad8f4924ab9"},
68 | {FileName: "go.mod", Checksum: "8a03b9b61c505ace06d590d2b9b4f4b6fa70136e14c26875ced149180e00d1af"},
69 | }}
70 | scannerCmd := NewScannerCmd(true, tRC, git.Root())
71 | scannerCmd.Run()
72 | assert.Equal(t, 0, scannerCmd.exitStatus(), "Expected ScannerCmd.exitStatus() to return 0 since secrets file ignore is enabled")
73 | })
74 | }
75 |
76 | func TestScannerCmdDetectsSecretWhileRunningNormalScanMode(t *testing.T) {
77 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) {
78 | git.SetupBaselineFiles("simple-file")
79 | git.CreateFileWithContents("go.sum", awsAccessKeyIDExample)
80 | git.CreateFileWithContents("go.mod", awsAccessKeyIDExample)
81 | git.AddAndcommit("*", "go sum file")
82 | os.Chdir(git.Root())
83 |
84 | tRC := &talismanrc.TalismanRC{
85 | FileIgnoreConfig: []talismanrc.FileIgnoreConfig{
86 | {FileName: "go.sum", Checksum: "582093519ae682d5170aecc9b935af7e90ed528c577ecd2c9dd1fad8f4924ab9"},
87 | {FileName: "go.mod", Checksum: "8a03b9b61c505ace06d590d2b9b4f4b6fa70136e14c26875ced149180e00d1af"},
88 | }}
89 | scannerCmd := NewScannerCmd(false, tRC, git.Root())
90 | scannerCmd.Run()
91 | assert.Equal(t, 1, scannerCmd.exitStatus(), "Expected ScannerCmd.exitStatus() to return 1 because file ignore is disabled when scanning history")
92 | })
93 | }
94 |
--------------------------------------------------------------------------------
/cmd/talisman.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "io"
7 | "os"
8 | "runtime/pprof"
9 | "strings"
10 | "talisman/utility"
11 | "time"
12 |
13 | "talisman/prompt"
14 | "talisman/talismanrc"
15 |
16 | "runtime"
17 |
18 | log "github.com/sirupsen/logrus"
19 | "github.com/spf13/afero"
20 | flag "github.com/spf13/pflag"
21 | )
22 |
23 | var (
24 | showVersion bool
25 | //Version : Version of talisman
26 | Version = "Development Build"
27 | interactive bool
28 | talismanInput io.Reader
29 | )
30 |
31 | const (
32 | //PrePush : Const for name of pre-push hook
33 | PrePush = "pre-push"
34 | //PreCommit : Const for name of of pre-commit hook
35 | PreCommit = "pre-commit"
36 | //EXIT_SUCCESS : Const to indicate successful talisman invocation
37 | EXIT_SUCCESS = 0
38 | //EXIT_FAILURE : Const to indicate failed successful invocation
39 | EXIT_FAILURE = 1
40 | )
41 |
42 | var options struct {
43 | Debug bool
44 | LogLevel string
45 | GitHook string
46 | Pattern string
47 | Scan bool
48 | IgnoreHistory bool
49 | Checksum string
50 | ReportDirectory string
51 | ScanWithHtml bool
52 | ShouldProfile bool
53 | }
54 |
55 | //var options Options
56 |
57 | func init() {
58 | log.SetOutput(os.Stderr)
59 | talismanInput = os.Stdin
60 | flag.BoolVarP(&options.Debug,
61 | "debug", "d", false,
62 | "enable debug mode (warning: very verbose)")
63 | flag.StringVarP(&options.LogLevel,
64 | "loglevel", "l", "error",
65 | "set log level for talisman (allowed values: error|info|warn|debug, default: error)")
66 | flag.BoolVarP(&showVersion,
67 | "version", "v", false,
68 | "show current version of talisman")
69 | flag.StringVarP(&options.Pattern,
70 | "pattern", "p", "",
71 | "pattern (glob-like) of files to scan (ignores githooks)")
72 | flag.StringVarP(&options.GitHook,
73 | "githook", "g", PrePush,
74 | "either pre-push or pre-commit")
75 | flag.BoolVarP(&options.Scan,
76 | "scan", "s", false,
77 | "scanner scans the git commit history for potential secrets")
78 | flag.BoolVarP(&options.IgnoreHistory,
79 | "ignoreHistory", "^", false,
80 | "scanner scans all files on current head, will not scan through git commit history")
81 | flag.StringVarP(&options.Checksum,
82 | "checksum", "c", "",
83 | "checksum calculator calculates checksum and suggests .talismanrc entry")
84 | flag.StringVarP(&options.ReportDirectory,
85 | "reportDirectory", "r", "talisman_report",
86 | "directory where the scan report will be stored")
87 | flag.BoolVarP(&options.ScanWithHtml,
88 | "scanWithHtml", "w", false,
89 | "generate html report (**Make sure you have installed talisman_html_report to use this, as mentioned in talisman Readme**)")
90 | flag.BoolVarP(&interactive,
91 | "interactive", "i", false,
92 | "interactively update talismanrc (only makes sense with -g/--githook)")
93 | flag.BoolVarP(&options.ShouldProfile,
94 | "profile", "f", false,
95 | "profile cpu and memory usage of talisman")
96 | }
97 |
98 | func main() {
99 | flag.Parse()
100 |
101 | if flag.NFlag() == 0 {
102 | flag.PrintDefaults()
103 | os.Exit(EXIT_SUCCESS)
104 | }
105 |
106 | if showVersion {
107 | fmt.Printf("talisman %s\n", Version)
108 | os.Exit(EXIT_SUCCESS)
109 | }
110 |
111 | if options.GitHook != "" {
112 | if !(options.GitHook == PreCommit || options.GitHook == PrePush) {
113 | fmt.Println(fmt.Errorf("githook should be %s or %s, but got %s", PreCommit, PrePush, options.GitHook))
114 | os.Exit(EXIT_FAILURE)
115 | }
116 | }
117 |
118 | if options.ShouldProfile {
119 | stopProfFunc := setupProfiling()
120 | defer stopProfFunc()
121 | }
122 |
123 | promptContext := prompt.NewPromptContext(interactive, prompt.NewPrompt())
124 | os.Exit(run(promptContext))
125 | }
126 |
127 | func run(promptContext prompt.PromptContext) (returnCode int) {
128 | start := time.Now()
129 | defer func() { fmt.Printf("Talisman done in %v\n", time.Since(start)) }()
130 |
131 | if err := validateGitExecutable(afero.NewOsFs(), runtime.GOOS); err != nil {
132 | log.Errorf("error validating git executable: %v", err)
133 | return 1
134 | }
135 |
136 | setLogLevel()
137 |
138 | if options.GitHook == "" {
139 | options.GitHook = PrePush
140 | }
141 |
142 | optionsBytes, _ := json.Marshal(options)
143 | fields := make(map[string]interface{})
144 | _ = json.Unmarshal(optionsBytes, &fields)
145 | log.WithFields(fields).Debug("Talisman execution environment")
146 | defer utility.DestroyHashers()
147 | if options.Checksum != "" {
148 | log.Infof("Running %s patterns against checksum calculator", options.Checksum)
149 | return NewChecksumCmd(strings.Fields(options.Checksum)).Run()
150 | } else if options.Scan {
151 | log.Infof("Running scanner")
152 | talismanrc, err := talismanrc.Load()
153 | if err != nil {
154 | return EXIT_FAILURE
155 | }
156 | return NewScannerCmd(options.IgnoreHistory, talismanrc, options.ReportDirectory).Run()
157 | } else if options.ScanWithHtml {
158 | log.Infof("Running scanner with html report")
159 | talismanrc, err := talismanrc.Load()
160 | if err != nil {
161 | return EXIT_FAILURE
162 | }
163 | return NewScannerCmd(options.IgnoreHistory, talismanrc, "talisman_html_report").Run()
164 | } else if options.Pattern != "" {
165 | log.Infof("Running scan for %s", options.Pattern)
166 | talismanrc, err := talismanrc.Load()
167 | if err != nil {
168 | return EXIT_FAILURE
169 | }
170 | return NewPatternCmd(options.Pattern).Run(talismanrc, promptContext)
171 | } else if options.GitHook == PreCommit {
172 | log.Infof("Running %s hook", options.GitHook)
173 | talismanrc, err := talismanrc.Load()
174 | if err != nil {
175 | return EXIT_FAILURE
176 | }
177 | return NewPreCommitHook().Run(talismanrc, promptContext)
178 | } else {
179 | log.Infof("Running %s hook", options.GitHook)
180 | talismanrc, err := talismanrc.Load()
181 | if err != nil {
182 | return EXIT_FAILURE
183 | }
184 | return NewPrePushHook(talismanInput).Run(talismanrc, promptContext)
185 | }
186 | }
187 |
188 | func validateGitExecutable(fs afero.Fs, operatingSystem string) error {
189 | if operatingSystem == "windows" {
190 | extensions := strings.ToLower(os.Getenv("PATHEXT"))
191 | windowsExecutables := strings.Split(extensions, ";")
192 | for _, executable := range windowsExecutables {
193 | gitExecutable := fmt.Sprintf("git%s", executable)
194 | exists, err := afero.Exists(fs, gitExecutable)
195 | if err != nil {
196 | return err
197 | }
198 | if exists {
199 | return fmt.Errorf("not allowed to have git executable located in repository: %s", gitExecutable)
200 | }
201 | }
202 | }
203 | return nil
204 | }
205 |
206 | func setLogLevel() {
207 | switch options.LogLevel {
208 | case "info":
209 | log.SetLevel(log.InfoLevel)
210 | case "debug":
211 | log.SetLevel(log.DebugLevel)
212 | case "error":
213 | log.SetLevel(log.ErrorLevel)
214 | case "warn":
215 | log.SetLevel(log.WarnLevel)
216 | default:
217 | log.SetLevel(log.ErrorLevel)
218 | }
219 | if options.Debug {
220 | log.SetLevel(log.DebugLevel)
221 | }
222 | }
223 |
224 | func setupProfiling() func() {
225 | log.Info("Profiling initiated")
226 |
227 | cpuProfFile, err := os.Create("talisman.cpuprof")
228 | if err != nil {
229 | log.Fatalf("Unable to create cpu profiling output file talisman.cpuprof: %v", err)
230 | }
231 |
232 | memProfFile, err := os.Create("talisman.memprof")
233 | if err != nil {
234 | log.Fatalf("Unable to create memory profiling output file talisman.memprof: %v", err)
235 | }
236 |
237 | _ = pprof.StartCPUProfile(cpuProfFile)
238 | progEnded := false
239 |
240 | go func() {
241 | memProfTimer := time.NewTimer(500 * time.Millisecond)
242 |
243 | for !progEnded {
244 | <-memProfTimer.C
245 | _ = pprof.WriteHeapProfile(memProfFile)
246 | }
247 | _ = memProfFile.Close()
248 | }()
249 |
250 | return func() {
251 | progEnded = true
252 | pprof.StopCPUProfile()
253 | log.Info("Profiling completed")
254 | _ = cpuProfFile.Close()
255 | }
256 | }
257 |
--------------------------------------------------------------------------------
/cmd/talisman_internal_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/spf13/afero"
5 | "io/ioutil"
6 | "os"
7 | "testing"
8 |
9 | "github.com/stretchr/testify/assert"
10 | )
11 |
12 | func TestParsingShasFromStdIn(t *testing.T) {
13 | file, err := ioutil.TempFile(os.TempDir(), "mockStdin")
14 | if err != nil {
15 | panic(err)
16 | }
17 | defer os.Remove(file.Name())
18 | defer file.Close()
19 | file.WriteString("localRef localSha remoteRef remoteSha")
20 | file.Seek(0, 0)
21 |
22 | _, oldSha, _, newSha := readRefAndSha(file)
23 | assert.Equal(t, "localSha", oldSha, "oldSha did not equal 'localSha', got: %s", oldSha)
24 | assert.Equal(t, "remoteSha", newSha, "newSha did not equal 'remoteSha', got: %s", newSha)
25 | }
26 |
27 | func Test_validateGitExecutable(t *testing.T) {
28 | t.Run("given operating systems is windows", func(t *testing.T) {
29 |
30 | operatingSystem := "windows"
31 | os.Setenv("PATHEXT", ".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC")
32 |
33 | t.Run("should return error if git executable exists in current directory", func(t *testing.T) {
34 | fs := afero.NewMemMapFs()
35 | gitExecutable := "git.exe"
36 | afero.WriteFile(fs, gitExecutable, []byte("git executable"), 0700)
37 | err := validateGitExecutable(fs, operatingSystem)
38 | assert.EqualError(t, err, "not allowed to have git executable located in repository: git.exe")
39 | })
40 |
41 | t.Run("should return nil if git executable does not exist in current directory", func(t *testing.T) {
42 | err := validateGitExecutable(afero.NewMemMapFs(), operatingSystem)
43 | assert.NoError(t, err)
44 | })
45 |
46 | })
47 |
48 | t.Run("given operating systems is linux", func(t *testing.T) {
49 |
50 | operatingSystem := "linux"
51 |
52 | t.Run("should return nil if git executable exists in current directory", func(t *testing.T) {
53 | fs := afero.NewMemMapFs()
54 | gitExecutable := "git.exe"
55 | afero.WriteFile(fs, gitExecutable, []byte("git executable"), 0700)
56 | err := validateGitExecutable(fs, operatingSystem)
57 | assert.NoError(t, err)
58 | })
59 |
60 | t.Run("should return nil if git executable does not exist in current directory", func(t *testing.T) {
61 | err := validateGitExecutable(afero.NewMemMapFs(), operatingSystem)
62 | assert.NoError(t, err)
63 | })
64 |
65 | })
66 |
67 | }
68 |
--------------------------------------------------------------------------------
/cmd/talisman_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "testing"
6 |
7 | "github.com/sirupsen/logrus"
8 | "github.com/stretchr/testify/assert"
9 | )
10 |
11 | func Test_setLogLevel(t *testing.T) {
12 | levels := []string{"error", "warn", "info", "debug", "unknown"}
13 | expectedLogrusLevels := []logrus.Level{
14 | logrus.ErrorLevel, logrus.WarnLevel,
15 | logrus.InfoLevel, logrus.DebugLevel, logrus.ErrorLevel}
16 |
17 | for idx, level := range levels {
18 | options.LogLevel = level
19 | setLogLevel()
20 | assert.True(
21 | t,
22 | logrus.IsLevelEnabled(expectedLogrusLevels[idx]),
23 | fmt.Sprintf("expected level to be %v for options.LogLevel = %s", expectedLogrusLevels[idx], level))
24 |
25 | options.Debug = true
26 | setLogLevel()
27 | assert.True(
28 | t,
29 | logrus.IsLevelEnabled(logrus.DebugLevel),
30 | "expected level to be debug when options.Debug is set")
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/contributing.md:
--------------------------------------------------------------------------------
1 | # Contributing to Talisman
2 |
3 | By contributing to Talisman, you agree to abide by the [code of conduct](CODE_OF_CONDUCT.md).
4 |
5 | ## How to start contributing
6 |
7 | If you are not sure how to begin contributing to Talisman, have a look at the issues tagged under [good first issue](https://github.com/thoughtworks/talisman/labels/good%20first%20issue).
8 |
9 | ## Developing locally
10 |
11 | To contribute to Talisman, you need a working golang development environment.
12 | Check [this link](https://golang.org/doc/install) to help you get started.
13 |
14 | Once you have go installed and set up, clone the talisman repository. In your
15 | working copy, fetch the dependencies by having go mod fetch them for you:
16 |
17 | ```
18 | go mod vendor
19 | ```
20 |
21 | Run the tests:
22 |
23 | ```
24 | go test ./...
25 | ```
26 |
27 | Build talisman:
28 |
29 | ```
30 | go build -o dist/talisman -ldflags="-s -w" talisman/cmd
31 | ```
32 |
33 | To build for multiple platforms we use [GoReleaser](https://goreleaser.com/):
34 |
35 | ```
36 | goreleaser build --snapshot --clean
37 | ```
38 |
39 | ## Submitting a Pull Request
40 |
41 | To send in a pull request
42 |
43 | 1. Fork the repo.
44 | 2. Create a new feature branch based off the master branch.
45 | 3. Provide the commit message with the issue number and a proper description.
46 | 4. Ensure that all the tests pass.
47 | 5. Submit the pull request.
48 |
49 | ## Updating Talisman GitHub Pages
50 |
51 | 1. Checkout a new branch from gh-pages
52 | 2. Navigate to the docs/ folder and update the files
53 | 3. See instructions for checking locally [here](https://github.com/thoughtworks/talisman/blob/gh-pages/README.md).
54 | 4. Raise a pull request against the branch gh-pages
55 |
56 | ## Releasing
57 |
58 | 1. Tag the commit to be released with the next version according to
59 | [semver](https://semver.org/) conventions
60 | 2. Push the tag to trigger the GitHub Actions Release pipeline
61 | 3. Approve the [drafted GitHub Release](https://github.com/thoughtworks/talisman/releases)
62 |
63 | The latest version will now be accessible to anyone who builds their own
64 | binaries, downloads binaries directly from GitHub Releases or homebrew, or uses
65 | the install script from the website.
66 |
--------------------------------------------------------------------------------
/detector/chain.go:
--------------------------------------------------------------------------------
1 | package detector
2 |
3 | import (
4 | "os"
5 | "talisman/detector/detector"
6 | "talisman/detector/filecontent"
7 | "talisman/detector/filename"
8 | "talisman/detector/helpers"
9 | "talisman/detector/pattern"
10 | "talisman/gitrepo"
11 | "talisman/talismanrc"
12 | "talisman/utility"
13 |
14 | log "github.com/sirupsen/logrus"
15 | )
16 |
17 | // Chain represents a chain of Detectors.
18 | // It is itself a detector.
19 | type Chain struct {
20 | detectors []detector.Detector
21 | ignoreEvaluator helpers.IgnoreEvaluator
22 | }
23 |
24 | // NewChain returns an empty DetectorChain
25 | // It is itself a detector, but it tests nothing.
26 | func NewChain(ignoreEvaluator helpers.IgnoreEvaluator) *Chain {
27 | result := Chain{[]detector.Detector{}, ignoreEvaluator}
28 | return &result
29 | }
30 |
31 | // DefaultChain returns a DetectorChain with pre-configured detectors
32 | func DefaultChain(tRC *talismanrc.TalismanRC, ignoreEvaluator helpers.IgnoreEvaluator) *Chain {
33 | chain := NewChain(ignoreEvaluator)
34 | chain.AddDetector(filename.DefaultFileNameDetector(tRC.Threshold))
35 | chain.AddDetector(filecontent.NewFileContentDetector(tRC))
36 | chain.AddDetector(pattern.NewPatternDetector(tRC.CustomPatterns))
37 | return chain
38 | }
39 |
40 | // AddDetector adds the detector that is passed in to the chain
41 | func (dc *Chain) AddDetector(d detector.Detector) *Chain {
42 | dc.detectors = append(dc.detectors, d)
43 | return dc
44 | }
45 |
46 | // Test validates the additions against each detector in the chain.
47 | // The results are passed in from detector to detector and thus collect all errors from all detectors
48 | func (dc *Chain) Test(additions []gitrepo.Addition, talismanRC *talismanrc.TalismanRC, result *helpers.DetectionResults) {
49 | log.Printf("Number of files to scan: %d\n", len(additions))
50 | log.Printf("Number of detectors: %d\n", len(dc.detectors))
51 | total := len(additions) * len(dc.detectors)
52 | progressBar := utility.GetProgressBar(os.Stdout, "Talisman Scan")
53 | progressBar.Start(total)
54 | for _, v := range dc.detectors {
55 | v.Test(dc.ignoreEvaluator, additions, talismanRC, result, func() {
56 | progressBar.Increment()
57 | })
58 | }
59 | progressBar.Finish()
60 | }
61 |
--------------------------------------------------------------------------------
/detector/chain_test.go:
--------------------------------------------------------------------------------
1 | package detector
2 |
3 | import (
4 | "io/ioutil"
5 | "talisman/detector/filecontent"
6 | "talisman/detector/filename"
7 | "talisman/detector/helpers"
8 | "talisman/detector/pattern"
9 | "talisman/detector/severity"
10 | "talisman/gitrepo"
11 | "talisman/talismanrc"
12 | "testing"
13 |
14 | logr "github.com/sirupsen/logrus"
15 | "github.com/stretchr/testify/assert"
16 | )
17 |
18 | func init() {
19 | logr.SetOutput(ioutil.Discard)
20 | }
21 |
22 | type FailingDetection struct{}
23 |
24 | func (v FailingDetection) Test(comparator helpers.IgnoreEvaluator, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) {
25 | result.Fail("some_file", "filecontent", "FAILED BY DESIGN", []string{}, severity.Low)
26 | }
27 |
28 | type PassingDetection struct{}
29 |
30 | func (p PassingDetection) Test(comparator helpers.IgnoreEvaluator, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) {
31 | }
32 |
33 | func TestEmptyValidationChainPassesAllValidations(t *testing.T) {
34 | ie := helpers.BuildIgnoreEvaluator("pre-push", nil, gitrepo.RepoLocatedAt("."))
35 | v := NewChain(ie)
36 | results := helpers.NewDetectionResults()
37 | v.Test(nil, &talismanrc.TalismanRC{}, results)
38 | assert.False(t, results.HasFailures(), "Empty validation chain is expected to always pass")
39 | }
40 |
41 | func TestValidationChainWithFailingValidationAlwaysFails(t *testing.T) {
42 | ie := helpers.BuildIgnoreEvaluator("pre-push", nil, gitrepo.RepoLocatedAt("."))
43 | v := NewChain(ie)
44 | v.AddDetector(PassingDetection{})
45 | v.AddDetector(FailingDetection{})
46 | results := helpers.NewDetectionResults()
47 | v.Test(nil, &talismanrc.TalismanRC{}, results)
48 |
49 | assert.False(t, results.Successful(), "Expected validation chain with a failure to fail.")
50 | }
51 |
52 | func TestDefaultChainShouldCreateChainSpecifiedModeAndPresetDetectors(t *testing.T) {
53 | talismanRC := &talismanrc.TalismanRC{
54 | Threshold: severity.Medium,
55 | CustomPatterns: []talismanrc.PatternString{"AKIA*"},
56 | }
57 | ie := helpers.BuildIgnoreEvaluator("pre-push", talismanRC, gitrepo.RepoLocatedAt("."))
58 | v := DefaultChain(talismanRC, ie)
59 | assert.Equal(t, 3, len(v.detectors))
60 |
61 | defaultFileNameDetector := filename.DefaultFileNameDetector(talismanRC.Threshold)
62 | assert.Equal(t, defaultFileNameDetector, v.detectors[0])
63 |
64 | expectedFileContentDetector := filecontent.NewFileContentDetector(talismanRC)
65 | assert.Equal(t, expectedFileContentDetector, v.detectors[1])
66 |
67 | expectedPatternDetector := pattern.NewPatternDetector(talismanRC.CustomPatterns)
68 | assert.Equal(t, expectedPatternDetector, v.detectors[2])
69 | }
70 |
--------------------------------------------------------------------------------
/detector/detector/detector.go:
--------------------------------------------------------------------------------
1 | package detector
2 |
3 | import (
4 | "talisman/detector/helpers"
5 | "talisman/gitrepo"
6 | "talisman/talismanrc"
7 | )
8 |
9 | // Detector represents a single kind of test to be performed against a set of Additions
10 | // Detectors are expected to honor the ignores that are passed in and log them in the results
11 | // Detectors are expected to signal any errors to the results
12 | type Detector interface {
13 | Test(comparator helpers.IgnoreEvaluator, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func())
14 | }
15 |
--------------------------------------------------------------------------------
/detector/filecontent/base64_aggressive_detector.go:
--------------------------------------------------------------------------------
1 | package filecontent
2 |
3 | import (
4 | "encoding/base64"
5 | "strings"
6 | )
7 |
8 | var delimiters = []string{".", "-", "="}
9 |
10 | const aggressivenessThreshold = 15 //decreasing makes it more aggressive
11 |
12 | type Base64AggressiveDetector struct {
13 | }
14 |
15 | func (ac *Base64AggressiveDetector) Test(s string) string {
16 | for _, d := range delimiters {
17 | subStrings := strings.Split(s, d)
18 | res := checkEachSubString(subStrings)
19 | if res != "" {
20 | return res
21 | }
22 | }
23 | return ""
24 | }
25 |
26 | func decodeBase64(s string) string {
27 | if len(s) <= aggressivenessThreshold {
28 | return ""
29 | }
30 | _, err := base64.StdEncoding.DecodeString(s)
31 | if err != nil {
32 | return ""
33 | }
34 | return s
35 | }
36 |
37 | func checkEachSubString(subStrings []string) string {
38 | for _, sub := range subStrings {
39 | suspicious := decodeBase64(sub)
40 | if suspicious != "" {
41 | return suspicious
42 | }
43 | }
44 | return ""
45 | }
46 |
--------------------------------------------------------------------------------
/detector/filecontent/base64_aggressive_detector_test.go:
--------------------------------------------------------------------------------
1 | package filecontent
2 |
3 | import (
4 | "talisman/detector/helpers"
5 | "testing"
6 |
7 | "talisman/gitrepo"
8 | "talisman/talismanrc"
9 |
10 | "github.com/stretchr/testify/assert"
11 | )
12 |
13 | var _blankTalismanRC = &talismanrc.TalismanRC{}
14 | var dummyCompletionCallbackFunc = func() {}
15 | var aggressiveModeFileContentDetector = NewFileContentDetector(_blankTalismanRC).AggressiveMode()
16 |
17 | func TestShouldFlagPotentialAWSAccessKeysInAggressiveMode(t *testing.T) {
18 | const awsAccessKeyIDExample string = "AKIAIOSFODNN7EXAMPLE\n"
19 | results := helpers.NewDetectionResults()
20 | filename := "filename"
21 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(awsAccessKeyIDExample))}
22 |
23 | aggressiveModeFileContentDetector.
24 | Test(
25 | defaultIgnoreEvaluator,
26 | additions,
27 | _blankTalismanRC,
28 | results,
29 | dummyCompletionCallbackFunc)
30 |
31 | assert.True(t, results.HasFailures(), "Expected file to not contain base64 encoded texts.")
32 | }
33 |
34 | func TestShouldFlagPotentialAWSAccessKeysAtPropertyDefinitionInAggressiveMode(t *testing.T) {
35 | const awsAccessKeyIDExample string = "accessKey=AKIAIOSFODNN7EXAMPLE"
36 | results := helpers.NewDetectionResults()
37 | filename := "filename"
38 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(awsAccessKeyIDExample))}
39 |
40 | aggressiveModeFileContentDetector.
41 | Test(
42 | defaultIgnoreEvaluator,
43 | additions,
44 | _blankTalismanRC,
45 | results,
46 | dummyCompletionCallbackFunc)
47 |
48 | assert.True(t, results.HasFailures(), "Expected file to not contain base64 encoded texts.")
49 | }
50 |
51 | func TestShouldNotFlagPotentialSecretsWithinSafeJavaCodeEvenInAggressiveMode(t *testing.T) {
52 | const awsAccessKeyIDExample string = "public class HelloWorld {\r\n\r\n" +
53 | " public static void main(String[] args) {\r\n " +
54 | " // Prints \"Hello, World\" to the terminal window.\r\n " +
55 | " System.out.println(\"Hello, World\");\r\n " +
56 | " }\r\n\r\n" +
57 | "}"
58 | results := helpers.NewDetectionResults()
59 | filename := "filename"
60 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(awsAccessKeyIDExample))}
61 |
62 | aggressiveModeFileContentDetector.
63 | Test(
64 | defaultIgnoreEvaluator,
65 | additions,
66 | _blankTalismanRC,
67 | results,
68 | dummyCompletionCallbackFunc)
69 |
70 | assert.False(t, results.HasFailures(), "Expected file to not contain base64 encoded texts.")
71 | }
72 |
--------------------------------------------------------------------------------
/detector/filecontent/base64_detector.go:
--------------------------------------------------------------------------------
1 | package filecontent
2 |
3 | import (
4 | "math"
5 | "talisman/talismanrc"
6 |
7 | log "github.com/sirupsen/logrus"
8 | )
9 |
10 | const BASE64_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
11 | const BASE64_ENTROPY_THRESHOLD = 4.5
12 | const MIN_BASE64_SECRET_LENGTH = 20
13 |
14 | type Base64Detector struct {
15 | base64Map map[string]bool
16 | AggressiveDetector *Base64AggressiveDetector
17 | entropy *Entropy
18 | wordCheck *WordCheck
19 | base64EntropyThreshold float64
20 | }
21 |
22 | func NewBase64Detector(tRC *talismanrc.TalismanRC) *Base64Detector {
23 | bd := Base64Detector{}
24 | bd.initBase64Map()
25 | bd.AggressiveDetector = nil
26 |
27 | bd.base64EntropyThreshold = BASE64_ENTROPY_THRESHOLD
28 | if tRC.Experimental.Base64EntropyThreshold > 0.0 {
29 | bd.base64EntropyThreshold = tRC.Experimental.Base64EntropyThreshold
30 | log.Debugf("Setting b64 entropy threshold to %f", bd.base64EntropyThreshold)
31 | }
32 |
33 | bd.entropy = &Entropy{}
34 | return &bd
35 | }
36 |
37 | func (bd *Base64Detector) initBase64Map() {
38 | bd.base64Map = map[string]bool{}
39 | for i := 0; i < len(BASE64_CHARS); i++ {
40 | bd.base64Map[string(BASE64_CHARS[i])] = true
41 | }
42 | }
43 |
44 | func (bd *Base64Detector) CheckBase64Encoding(word string) string {
45 | entropyCandidates := bd.entropy.GetEntropyCandidatesWithinWord(word, MIN_BASE64_SECRET_LENGTH, bd.base64Map)
46 | for _, candidate := range entropyCandidates {
47 | entropy := bd.entropy.GetShannonEntropy(candidate, BASE64_CHARS)
48 | sliceLimit := int(math.Min(50, float64(len(candidate))))
49 | log.Debugf("Detected entropy for word %s = %f", candidate[0:sliceLimit], entropy)
50 | if entropy > bd.base64EntropyThreshold && !bd.wordCheck.containsWordsOnly(candidate) {
51 | return word
52 | }
53 | }
54 | if bd.AggressiveDetector != nil {
55 | return bd.AggressiveDetector.Test(word)
56 | }
57 | return ""
58 | }
59 |
--------------------------------------------------------------------------------
/detector/filecontent/base64_detector_test.go:
--------------------------------------------------------------------------------
1 | package filecontent
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/stretchr/testify/assert"
7 | )
8 |
9 | func TestBase64DetectorShouldNotDetectSafeText(t *testing.T) {
10 | s := "pretty safe"
11 | bd := Base64Detector{}
12 | bd.initBase64Map()
13 |
14 | res := bd.CheckBase64Encoding(s)
15 | assert.Equal(t, "", res)
16 | }
17 |
18 | func TestBase64DetectorShouldDetectBase64Text(t *testing.T) {
19 | s := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
20 | bd := Base64Detector{}
21 | bd.initBase64Map()
22 |
23 | res := bd.CheckBase64Encoding(s)
24 | assert.Equal(t, s, res)
25 | }
26 |
27 | func TestBase64DetectorShouldNotDetectLongMethodNamesEvenWithHighEntropy(t *testing.T) {
28 | s := "TestBase64DetectorShouldNotDetectLongMethodNamesEvenWithRidiculousHighEntropyWordsMightExist"
29 | bd := Base64Detector{}
30 | bd.initBase64Map()
31 |
32 | res := bd.CheckBase64Encoding(s)
33 | assert.Equal(t, "", res)
34 | }
35 |
--------------------------------------------------------------------------------
/detector/filecontent/filecontent_credit_card_detector.go:
--------------------------------------------------------------------------------
1 | package filecontent
2 |
3 | import "regexp"
4 |
5 | type CreditCardDetector struct {
6 | creditCardRegex []*regexp.Regexp
7 | }
8 |
9 | var (
10 | creditCardPatterns = []*regexp.Regexp{
11 | regexp.MustCompile(`(?:3[47][0-9]{13})`),
12 | regexp.MustCompile(`(?:3(?:0[0-5]|[68][0-9])[0-9]{11})`),
13 | regexp.MustCompile(`^65[4-9][0-9]{13}|64[4-9][0-9]{13}|6011[0-9]{12}|(622(?:12[6-9]|1[3-9][0-9]|[2-8][0-9][0-9]|9[01][0-9]|92[0-5])[0-9]{10})$`),
14 | regexp.MustCompile(`^(?:2131|1800|35\d{3})\d{11}$`),
15 | regexp.MustCompile(`^(5018|5020|5038|6304|6759|6761|6763)[0-9]{8,15}$`),
16 | regexp.MustCompile(`(?:(?:5[1-5][0-9]{2}|222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)[0-9]{12})`),
17 | regexp.MustCompile(`((?:4[0-9]{12})(?:[0-9]{3})?)`),
18 | }
19 | )
20 |
21 | func (detector CreditCardDetector) checkCreditCardNumber(content string) string {
22 | if !isLuhnNumber(content) {
23 | return ""
24 | }
25 | for _, regex := range detector.creditCardRegex {
26 | if regex.MatchString(content) {
27 | return content
28 | }
29 | }
30 | return ""
31 | }
32 |
33 | func NewCreditCardDetector() *CreditCardDetector {
34 | return &CreditCardDetector{creditCardPatterns}
35 | }
36 |
37 | func isLuhnNumber(content string) bool {
38 | var isAlternate bool
39 | var checksum int
40 |
41 | for position := len(content) - 1; position > -1; position-- {
42 | const ASCII_INDEX = 48
43 | digit := int(content[position] - ASCII_INDEX)
44 | if isAlternate {
45 | digit = digit * 2
46 | if digit > 9 {
47 | digit = (digit % 10) + 1
48 | }
49 | }
50 | isAlternate = !isAlternate
51 | checksum += digit
52 | }
53 | if checksum%10 == 0 {
54 | return true
55 | }
56 | return false
57 | }
58 |
--------------------------------------------------------------------------------
/detector/filecontent/filecontent_credit_card_detector_test.go:
--------------------------------------------------------------------------------
1 | package filecontent
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/stretchr/testify/assert"
7 | )
8 |
9 | func TestShouldReturnEmptyStringWhenCreditCardNumberDoesNotMatchAnyRegex(t *testing.T) {
10 | assert.Equal(t, "", NewCreditCardDetector().checkCreditCardNumber("1234123412341234"))
11 | }
12 |
13 | func TestShouldReturnEmptyStringWhenValidMasterCardPatternIsDetectedButNotValidLuhnNumber(t *testing.T) {
14 | assert.Equal(t, "", NewCreditCardDetector().checkCreditCardNumber("52222111111111111"))
15 | }
16 |
17 | func TestShouldReturnCardNumberWhenAmericanExpressCardNumberIsGiven(t *testing.T) {
18 | assert.Equal(t, "340000000000009", NewCreditCardDetector().checkCreditCardNumber("340000000000009"))
19 | }
20 |
21 | func TestShouldReturnCardNumberWhenDinersClubCardNumberIsGiven(t *testing.T) {
22 | assert.Equal(t, "30000000000004", NewCreditCardDetector().checkCreditCardNumber("30000000000004"))
23 | }
24 |
25 | func TestShouldReturnCardNumberWhenDiscoverCardNumberIsGiven(t *testing.T) {
26 | assert.Equal(t, "6011000000000004", NewCreditCardDetector().checkCreditCardNumber("6011000000000004"))
27 | }
28 |
29 | func TestShouldReturnCardNumberWhenJCBCardNumberIsGiven(t *testing.T) {
30 | assert.Equal(t, "3530111333300000", NewCreditCardDetector().checkCreditCardNumber("3530111333300000"))
31 | }
32 |
33 | func TestShouldReturnCardNumberWhenMaestroCardNumberIsGiven(t *testing.T) {
34 | assert.Equal(t, "6759649826438453", NewCreditCardDetector().checkCreditCardNumber("6759649826438453"))
35 | }
36 |
37 | func TestShouldReturnCardNumberWhenVisaCardNumberIsGiven(t *testing.T) {
38 | assert.Equal(t, "4111111111111111", NewCreditCardDetector().checkCreditCardNumber("4111111111111111"))
39 | }
40 |
--------------------------------------------------------------------------------
/detector/filecontent/filecontent_detector.go:
--------------------------------------------------------------------------------
1 | package filecontent
2 |
3 | import (
4 | "fmt"
5 | "regexp"
6 | "strings"
7 | "sync"
8 | "talisman/detector/helpers"
9 | "talisman/detector/severity"
10 | "talisman/gitrepo"
11 | "talisman/talismanrc"
12 |
13 | log "github.com/sirupsen/logrus"
14 | )
15 |
16 | type fn func(fc *FileContentDetector, word string) string
17 |
18 | type FileContentDetector struct {
19 | base64Detector *Base64Detector
20 | hexDetector *HexDetector
21 | creditCardDetector *CreditCardDetector
22 | base64EntropyThreshold float64
23 | }
24 |
25 | func NewFileContentDetector(tRC *talismanrc.TalismanRC) *FileContentDetector {
26 | fc := FileContentDetector{}
27 | fc.base64Detector = NewBase64Detector(tRC)
28 | fc.hexDetector = NewHexDetector()
29 | fc.creditCardDetector = NewCreditCardDetector()
30 | return &fc
31 | }
32 |
33 | func (fc *FileContentDetector) AggressiveMode() *FileContentDetector {
34 | fc.base64Detector.AggressiveDetector = &Base64AggressiveDetector{}
35 | return fc
36 | }
37 |
38 | type contentType int
39 |
40 | const (
41 | base64Content contentType = iota
42 | hexContent
43 | creditCardContent
44 | )
45 |
46 | func (ct contentType) getInfo() string {
47 | switch ct {
48 | case base64Content:
49 | return "Base64Detector: Failing file as it contains a base64 encoded text."
50 | case hexContent:
51 | return "HexDetector: Failing file as it contains a hex encoded text."
52 | case creditCardContent:
53 | return "CreditCardDetector: Failing file as it contains a potential credit card number."
54 | }
55 | return ""
56 | }
57 |
58 | func (ct contentType) getMessageFormat() string {
59 | switch ct {
60 | case base64Content:
61 | return "Expected file to not contain base64 encoded texts such as: %s"
62 | case hexContent:
63 | return "Expected file to not contain hex encoded texts such as: %s"
64 | case creditCardContent:
65 | return "Expected file to not contain credit card numbers such as: %s"
66 | }
67 |
68 | return ""
69 | }
70 |
71 | type content struct {
72 | name gitrepo.FileName
73 | path gitrepo.FilePath
74 | contentType contentType
75 | results []string
76 | severity severity.Severity
77 | }
78 |
79 | func (fc *FileContentDetector) Test(comparator helpers.IgnoreEvaluator, currentAdditions []gitrepo.Addition, talismanRC *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) {
80 | contentTypes := []struct {
81 | contentType
82 | fn
83 | severity severity.Severity
84 | }{
85 | {
86 | contentType: base64Content,
87 | fn: checkBase64,
88 | severity: severity.SeverityConfiguration["Base64Content"],
89 | },
90 | {
91 | contentType: hexContent,
92 | fn: checkHex,
93 | severity: severity.SeverityConfiguration["HexContent"],
94 | },
95 | {
96 | contentType: creditCardContent,
97 | fn: checkCreditCardNumber,
98 | severity: severity.SeverityConfiguration["CreditCardContent"],
99 | },
100 | }
101 | re := regexp.MustCompile(`(?i)checksum[ \t]*:[ \t]*[0-9a-fA-F]+`)
102 |
103 | contents := make(chan content, 512)
104 | ignoredFilePaths := make(chan gitrepo.FilePath, len(currentAdditions))
105 |
106 | waitGroup := &sync.WaitGroup{}
107 | waitGroup.Add(len(currentAdditions))
108 | for _, addition := range currentAdditions {
109 | go func(addition gitrepo.Addition) {
110 | defer waitGroup.Done()
111 | defer additionCompletionCallback()
112 | if comparator.ShouldIgnore(addition, "filecontent") {
113 | ignoredFilePaths <- addition.Path
114 | return
115 | }
116 |
117 | if string(addition.Name) == talismanrc.RCFileName {
118 | content := re.ReplaceAllString(string(addition.Data), "")
119 | data := []byte(content)
120 | addition.Data = data
121 | }
122 | addition.Data = []byte(talismanRC.RemoveAllowedPatterns(addition))
123 | for _, ct := range contentTypes {
124 | contents <- content{
125 | name: addition.Name,
126 | path: addition.Path,
127 | contentType: ct.contentType,
128 | results: fc.detectFile(addition.Data, ct.fn),
129 | severity: ct.severity,
130 | }
131 | }
132 | }(addition)
133 | }
134 | go func() {
135 | waitGroup.Wait()
136 | close(ignoredFilePaths)
137 | close(contents)
138 | }()
139 | for ignoredChanHasMore, contentChanHasMore := true, true; ignoredChanHasMore || contentChanHasMore; {
140 | select {
141 | case ignoredFilePath, hasMore := <-ignoredFilePaths:
142 | log.Debugf("Processing results for ignored file %v", ignoredFilePath)
143 | if !hasMore {
144 | ignoredChanHasMore = false
145 | continue
146 | }
147 | processIgnoredFilepath(ignoredFilePath, result)
148 | case c, hasMore := <-contents:
149 | log.Debugf("Processing results for file %v", c.path)
150 | if !hasMore {
151 | contentChanHasMore = false
152 | continue
153 | }
154 | processContent(c, talismanRC.Threshold, result)
155 | }
156 | }
157 | }
158 |
159 | func processIgnoredFilepath(path gitrepo.FilePath, result *helpers.DetectionResults) {
160 | log.WithFields(log.Fields{
161 | "filePath": path,
162 | }).Info("Ignoring addition as it was specified to be ignored.")
163 | result.Ignore(path, "filecontent")
164 | }
165 |
166 | func processContent(c content, threshold severity.Severity, result *helpers.DetectionResults) {
167 | for _, res := range c.results {
168 | if res != "" {
169 | log.WithFields(log.Fields{
170 | "filePath": c.path,
171 | }).Info(c.contentType.getInfo())
172 | if string(c.name) == talismanrc.RCFileName || !c.severity.ExceedsThreshold(threshold) {
173 | result.Warn(c.path, "filecontent", fmt.Sprintf(c.contentType.getMessageFormat(), formatForReporting(res)), []string{}, c.severity)
174 | } else {
175 | result.Fail(c.path, "filecontent", fmt.Sprintf(c.contentType.getMessageFormat(), formatForReporting(res)), []string{}, c.severity)
176 | }
177 | }
178 | }
179 | }
180 |
181 | func formatForReporting(input string) string {
182 | if len(input) > 50 {
183 | return input[:47] + "..."
184 | }
185 | return input
186 | }
187 |
188 | func (fc *FileContentDetector) detectFile(data []byte, getResult fn) []string {
189 | content := string(data)
190 | return fc.checkEachLine(content, getResult)
191 | }
192 |
193 | func (fc *FileContentDetector) checkEachLine(content string, getResult fn) []string {
194 | lines := strings.Split(content, "\n")
195 | res := []string{}
196 | for _, line := range lines {
197 | lineResult := fc.checkEachWord(line, getResult)
198 | if len(lineResult) > 0 {
199 | res = append(res, lineResult...)
200 | }
201 | }
202 | return res
203 | }
204 |
205 | func (fc *FileContentDetector) checkEachWord(line string, getResult fn) []string {
206 | words := strings.Fields(line)
207 | res := []string{}
208 | for _, word := range words {
209 | wordResult := getResult(fc, word)
210 | if wordResult != "" {
211 | res = append(res, wordResult)
212 | }
213 | }
214 | return res
215 | }
216 |
217 | func checkBase64(fc *FileContentDetector, word string) string {
218 | return fc.base64Detector.CheckBase64Encoding(word)
219 | }
220 |
221 | func checkCreditCardNumber(fc *FileContentDetector, word string) string {
222 | return fc.creditCardDetector.checkCreditCardNumber(word)
223 | }
224 |
225 | func checkHex(fc *FileContentDetector, word string) string {
226 | return fc.hexDetector.CheckHexEncoding(word)
227 | }
228 |
--------------------------------------------------------------------------------
/detector/filecontent/hex_detector.go:
--------------------------------------------------------------------------------
1 | package filecontent
2 |
3 | const HEX_CHARS = "1234567890abcdefABCDEF"
4 | const HEX_ENTROPY_THRESHOLD = 2.7
5 | const MIN_HEX_SECRET_LENGTH = 20
6 |
7 | type HexDetector struct {
8 | hexMap map[string]bool
9 | entropy *Entropy
10 | }
11 |
12 | func NewHexDetector() *HexDetector {
13 | bd := HexDetector{}
14 | bd.initHexMap()
15 | bd.entropy = &Entropy{}
16 | return &bd
17 | }
18 |
19 | func (hd *HexDetector) initHexMap() {
20 | hd.hexMap = map[string]bool{}
21 | for i := 0; i < len(HEX_CHARS); i++ {
22 | hd.hexMap[string(HEX_CHARS[i])] = true
23 | }
24 | }
25 |
26 | func (hd *HexDetector) CheckHexEncoding(word string) string {
27 | entropyCandidates := hd.entropy.GetEntropyCandidatesWithinWord(word, MIN_HEX_SECRET_LENGTH, hd.hexMap)
28 | for _, candidate := range entropyCandidates {
29 | entropy := hd.entropy.GetShannonEntropy(candidate, HEX_CHARS)
30 | if entropy > HEX_ENTROPY_THRESHOLD {
31 | return word
32 | }
33 | }
34 | return ""
35 | }
36 |
--------------------------------------------------------------------------------
/detector/filecontent/hex_detector_test.go:
--------------------------------------------------------------------------------
1 | package filecontent
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/stretchr/testify/assert"
7 | )
8 |
9 | func TestHexDetectorShouldNotDetectSafeText(t *testing.T) {
10 | s := "pretty safe"
11 | hd := HexDetector{}
12 | hd.initHexMap()
13 |
14 | res := hd.CheckHexEncoding(s)
15 | assert.Equal(t, "", res)
16 | }
17 |
18 | func TestHexDetectorShouldDetectBase64Text(t *testing.T) {
19 | s := "6A6176617375636B73676F726F636B7368616861"
20 | hd := HexDetector{}
21 | hd.initHexMap()
22 |
23 | res := hd.CheckHexEncoding(s)
24 | assert.Equal(t, s, res)
25 | }
26 |
--------------------------------------------------------------------------------
/detector/filecontent/shannon_entropy.go:
--------------------------------------------------------------------------------
1 | package filecontent
2 |
3 | import (
4 | "math"
5 | "strings"
6 | )
7 |
8 | type Entropy struct {
9 | }
10 |
11 | func (en *Entropy) GetShannonEntropy(str string, superSet string) float64 {
12 | if str == "" {
13 | return 0
14 | }
15 | entropy := 0.0
16 | for _, c := range superSet {
17 | p := float64(strings.Count(str, string(c))) / float64(len(str))
18 | if p > 0 {
19 | entropy -= p * math.Log2(p)
20 | }
21 | }
22 | return entropy
23 | }
24 |
25 | func (en *Entropy) GetEntropyCandidatesWithinWord(word string, minCandidateLength int, superSet map[string]bool) []string {
26 | candidates := []string{}
27 | count := 0
28 | subSet := ""
29 | if len(word) < minCandidateLength {
30 | return candidates
31 | }
32 | for _, c := range word {
33 | char := string(c)
34 | if superSet[char] {
35 | subSet += char
36 | count++
37 | } else {
38 | if count > minCandidateLength {
39 | candidates = append(candidates, subSet)
40 | }
41 | subSet = ""
42 | count = 0
43 | }
44 | }
45 | if count > minCandidateLength {
46 | candidates = append(candidates, subSet)
47 | }
48 | return candidates
49 | }
50 |
--------------------------------------------------------------------------------
/detector/filecontent/shannon_entropy_test.go:
--------------------------------------------------------------------------------
1 | package filecontent
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/stretchr/testify/assert"
7 | )
8 |
9 | func TestEntropyCandidatesShouldBeFoundForGivenSet(t *testing.T) {
10 | entropy := Entropy{}
11 | dc := Base64Detector{}
12 | dc.initBase64Map()
13 | candidatesWithinWord := entropy.GetEntropyCandidatesWithinWord("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", 20, dc.base64Map)
14 | assert.Equal(t, 1, len(candidatesWithinWord))
15 | }
16 |
17 | func TestEntropyCandidatesShouldBeEmptyForShorterWords(t *testing.T) {
18 | entropy := Entropy{}
19 | dc := Base64Detector{}
20 | dc.initBase64Map()
21 | candidatesWithinWord := entropy.GetEntropyCandidatesWithinWord("abc", 4, dc.base64Map)
22 | assert.Equal(t, 0, len(candidatesWithinWord))
23 | }
24 |
25 | func TestEntropyValueOfSecretShouldBeHigherThanFour(t *testing.T) {
26 | entropy := Entropy{}
27 | shannonEntropy := entropy.GetShannonEntropy("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", BASE64_CHARS)
28 | assert.True(t, 4 < shannonEntropy)
29 | }
30 |
31 | func TestEntropyValueOfEmptyStringShouldBeZero(t *testing.T) {
32 | entropy := Entropy{}
33 | shannonEntropy := entropy.GetShannonEntropy("", BASE64_CHARS)
34 | assert.Equal(t, float64(0), shannonEntropy)
35 | }
36 |
--------------------------------------------------------------------------------
/detector/filecontent/word_check.go:
--------------------------------------------------------------------------------
1 | package filecontent
2 |
3 | import (
4 | "bufio"
5 | log "github.com/sirupsen/logrus"
6 | "os"
7 | "strings"
8 | )
9 |
10 | type WordCheck struct {
11 | }
12 |
13 | const AVERAGE_LENGTH_OF_WORDS_IN_ENGLISH = 5 //See http://bit.ly/2qYFzFf for reference
14 |
15 | func (en *WordCheck) containsWordsOnly(text string) bool {
16 | text = strings.ToLower(text)
17 | file := &os.File{}
18 | defer file.Close()
19 | reader := bufio.NewReader(strings.NewReader(DictionaryWordsString))
20 | if reader == nil {
21 | return false
22 | }
23 | wordCount := howManyWordsExistInText(reader, text)
24 | if wordCount >= (len(text) / (AVERAGE_LENGTH_OF_WORDS_IN_ENGLISH)) {
25 | return true
26 | }
27 | return false
28 | }
29 |
30 | func howManyWordsExistInText(reader *bufio.Reader, text string) int {
31 | wordCount := 0
32 | for {
33 | word, err := reader.ReadString('\n')
34 | word = strings.Trim(word, "\n")
35 |
36 | if word != "" && len(word) > 2 && strings.Contains(text, word) {
37 | text = strings.Replace(text, word, "", 1) //already matched
38 | wordCount++
39 | }
40 |
41 | if err != nil { //EOF
42 | log.Debugf("[WordChecker]: Found %d words", wordCount)
43 | break
44 | }
45 | }
46 | return wordCount
47 | }
48 |
--------------------------------------------------------------------------------
/detector/filecontent/word_check_test.go:
--------------------------------------------------------------------------------
1 | package filecontent
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/stretchr/testify/assert"
7 | )
8 |
9 | func TestWordCheckWithWordsOnlyText(t *testing.T) {
10 | wc := WordCheck{}
11 | isWordsOnly := wc.containsWordsOnly("helloWorldGreetingsFromThoughtWorks")
12 | assert.True(t, isWordsOnly)
13 | }
14 |
15 | func TestWordCheckWithWordsOnlyLongText(t *testing.T) {
16 | wc := WordCheck{}
17 | isWordsOnly := wc.containsWordsOnly("TestBase64DetectorShouldNotDetectLongMethodNamesEvenWithRidiculousHighEntropyWordsMightExist")
18 | assert.True(t, isWordsOnly)
19 | }
20 |
21 | func TestWordCheckWithSingleWord(t *testing.T) {
22 | wc := WordCheck{}
23 | isWordsOnly := wc.containsWordsOnly("exception")
24 | assert.True(t, isWordsOnly)
25 | }
26 |
27 | func TestWordCheckWithSingleShortWord(t *testing.T) {
28 | wc := WordCheck{}
29 | isWordsOnly := wc.containsWordsOnly("for")
30 | assert.True(t, isWordsOnly)
31 | }
32 |
33 | func TestWordCheckWithSecret(t *testing.T) {
34 | wc := WordCheck{}
35 | isWordsOnly := wc.containsWordsOnly("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
36 | assert.False(t, isWordsOnly)
37 | }
38 |
39 | func TestWordCheckWithHex(t *testing.T) {
40 | wc := WordCheck{}
41 | isWordsOnly := wc.containsWordsOnly("68656C6C6F20776F726C6421")
42 | assert.False(t, isWordsOnly)
43 | }
44 |
45 | func TestWordCheckWithHalfWordsHalfHex(t *testing.T) {
46 | wc := WordCheck{}
47 | isWordsOnly := wc.containsWordsOnly("68656C6C6F20776F726C6421helloWorldGreetingsFromThoughtWorks")
48 | assert.False(t, isWordsOnly)
49 | }
50 |
51 | func TestWordCheckWithHalfWordsHalfSecret(t *testing.T) {
52 | wc := WordCheck{}
53 | isWordsOnly := wc.containsWordsOnly("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEYhelloWorldGreetingsFromThoughtWorks")
54 | assert.False(t, isWordsOnly)
55 | }
56 |
--------------------------------------------------------------------------------
/detector/filesize/filesize_detector.go:
--------------------------------------------------------------------------------
1 | package filesize
2 |
3 | import (
4 | "fmt"
5 | "talisman/detector/detector"
6 | "talisman/detector/helpers"
7 | "talisman/detector/severity"
8 | "talisman/gitrepo"
9 | "talisman/talismanrc"
10 |
11 | log "github.com/sirupsen/logrus"
12 | )
13 |
14 | type FileSizeDetector struct {
15 | size int
16 | }
17 |
18 | func NewFileSizeDetector(size int) detector.Detector {
19 | return FileSizeDetector{size}
20 | }
21 |
22 | func (fd FileSizeDetector) Test(comparator helpers.IgnoreEvaluator, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) {
23 | largeFileSizeSeverity := severity.SeverityConfiguration["LargeFileSize"]
24 | for _, addition := range currentAdditions {
25 | if comparator.ShouldIgnore(addition, "filesize") {
26 | log.WithFields(log.Fields{
27 | "filePath": addition.Path,
28 | }).Info("Ignoring addition as it was specified to be ignored.")
29 | result.Ignore(addition.Path, "filesize")
30 | additionCompletionCallback()
31 | continue
32 | }
33 | size := len(addition.Data)
34 | if size > fd.size {
35 | log.WithFields(log.Fields{
36 | "filePath": addition.Path,
37 | "fileSize": size,
38 | "maxSize": fd.size,
39 | }).Info("Failing file as it is larger than max allowed file size.")
40 | if largeFileSizeSeverity.ExceedsThreshold(ignoreConfig.Threshold) {
41 | result.Fail(addition.Path, "filesize", fmt.Sprintf("The file name %q with file size %d is larger than max allowed file size(%d)", addition.Path, size, fd.size), addition.Commits, largeFileSizeSeverity)
42 | } else {
43 | result.Warn(addition.Path, "filesize", fmt.Sprintf("The file name %q with file size %d is larger than max allowed file size(%d)", addition.Path, size, fd.size), addition.Commits, largeFileSizeSeverity)
44 | }
45 | }
46 | additionCompletionCallback()
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/detector/filesize/filesize_detector_test.go:
--------------------------------------------------------------------------------
1 | package filesize
2 |
3 | import (
4 | "talisman/detector/helpers"
5 | "talisman/detector/severity"
6 | "testing"
7 |
8 | "talisman/gitrepo"
9 | "talisman/talismanrc"
10 |
11 | "github.com/stretchr/testify/assert"
12 | )
13 |
14 | var talismanRC = &talismanrc.TalismanRC{}
15 | var defaultIgnoreEvaluator = helpers.BuildIgnoreEvaluator("default", talismanRC, gitrepo.RepoLocatedAt("."))
16 |
17 | func ignoreEvaluatorWithTalismanRC(tRC *talismanrc.TalismanRC) helpers.IgnoreEvaluator {
18 | return helpers.BuildIgnoreEvaluator("default", tRC, gitrepo.RepoLocatedAt("."))
19 | }
20 |
21 | func TestShouldFlagLargeFiles(t *testing.T) {
22 | results := helpers.NewDetectionResults()
23 | content := []byte("more than one byte")
24 | additions := []gitrepo.Addition{gitrepo.NewAddition("filename", content)}
25 | NewFileSizeDetector(2).Test(defaultIgnoreEvaluator, additions, talismanRC, results, func() {})
26 | assert.True(t, results.HasFailures(), "Expected file to fail the check against file size detector.")
27 | }
28 |
29 | func TestShouldNotFlagLargeFilesIfThresholdIsBelowSeverity(t *testing.T) {
30 | results := helpers.NewDetectionResults()
31 | content := []byte("more than one byte")
32 | talismanRCWithThreshold := &talismanrc.TalismanRC{Threshold: severity.High}
33 | additions := []gitrepo.Addition{gitrepo.NewAddition("filename", content)}
34 | NewFileSizeDetector(2).Test(defaultIgnoreEvaluator, additions, talismanRCWithThreshold, results, func() {})
35 | assert.False(t, results.HasFailures(), "Expected file to not fail the check against file size detector.")
36 | assert.True(t, results.HasWarnings(), "Expected file to have warnings against file size detector.")
37 | }
38 |
39 | func TestShouldNotFlagSmallFiles(t *testing.T) {
40 | results := helpers.NewDetectionResults()
41 | content := []byte("m")
42 | additions := []gitrepo.Addition{gitrepo.NewAddition("filename", content)}
43 | NewFileSizeDetector(2).Test(defaultIgnoreEvaluator, additions, talismanRC, results, func() {})
44 | assert.False(t, results.HasFailures(), "Expected file to not fail the check against file size detector.")
45 | }
46 |
47 | func TestShouldNotFlagIgnoredLargeFiles(t *testing.T) {
48 | results := helpers.NewDetectionResults()
49 | content := []byte("more than one byte")
50 |
51 | filename := "filename"
52 | fileIgnoreConfig := talismanrc.FileIgnoreConfig{
53 | FileName: filename,
54 | IgnoreDetectors: []string{"filesize"},
55 | }
56 | talismanRC := &talismanrc.TalismanRC{
57 | FileIgnoreConfig: []talismanrc.FileIgnoreConfig{fileIgnoreConfig},
58 | }
59 |
60 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
61 | NewFileSizeDetector(2).Test(ignoreEvaluatorWithTalismanRC(talismanRC), additions, talismanRC, results, func() {})
62 | assert.True(t, results.Successful(), "expected file %s to be ignored by file size detector", filename)
63 | }
64 |
--------------------------------------------------------------------------------
/detector/helpers/ignore_evaluator.go:
--------------------------------------------------------------------------------
1 | package helpers
2 |
3 | import (
4 | "os"
5 | "talisman/checksumcalculator"
6 | "talisman/gitrepo"
7 | "talisman/talismanrc"
8 | "talisman/utility"
9 | )
10 |
11 | type IgnoreEvaluator interface {
12 | ShouldIgnore(addition gitrepo.Addition, detectorType string) bool
13 | }
14 |
15 | type scanAllAdditions struct{}
16 |
17 | // Returns an IgnoreEvaluator that forces all files to be scanned, such as when scanning the history of a repo
18 | func ScanHistoryEvaluator() IgnoreEvaluator {
19 | return &scanAllAdditions{}
20 | }
21 |
22 | // Returns false so that all additions are scanned
23 | func (ie *scanAllAdditions) ShouldIgnore(gitrepo.Addition, string) bool {
24 | return false
25 | }
26 |
27 | type ignoreEvaluator struct {
28 | calculator checksumcalculator.ChecksumCalculator
29 | talismanRC *talismanrc.TalismanRC
30 | }
31 |
32 | // Returns an IgnoreEvaluator around the rules defined in the current .talismanrc file
33 | func BuildIgnoreEvaluator(hasherMode string, talismanRC *talismanrc.TalismanRC, repo gitrepo.GitRepo) IgnoreEvaluator {
34 | wd, _ := os.Getwd()
35 | hasher := utility.MakeHasher(hasherMode, wd)
36 | allTrackedFiles := append(repo.TrackedFilesAsAdditions(), repo.StagedAdditions()...)
37 | calculator := checksumcalculator.NewChecksumCalculator(hasher, allTrackedFiles)
38 | return &ignoreEvaluator{calculator: calculator, talismanRC: talismanRC}
39 | }
40 |
41 | // ShouldIgnore returns true if the talismanRC indicates that a Detector should ignore an Addition
42 | func (ie *ignoreEvaluator) ShouldIgnore(addition gitrepo.Addition, detectorType string) bool {
43 | return ie.talismanRC.Deny(addition, detectorType) || ie.isScanNotRequired(addition)
44 | }
45 |
46 | // isScanNotRequired returns true if an Addition's checksum matches one ignored by the .talismanrc file
47 | func (ie *ignoreEvaluator) isScanNotRequired(addition gitrepo.Addition) bool {
48 | for _, ignore := range ie.talismanRC.FileIgnoreConfig {
49 | if addition.Matches(ignore.GetFileName()) {
50 | currentCollectiveChecksum := ie.calculator.CalculateCollectiveChecksumForPattern(ignore.GetFileName())
51 | return ignore.ChecksumMatches(currentCollectiveChecksum)
52 | }
53 | }
54 | return false
55 | }
56 |
--------------------------------------------------------------------------------
/detector/helpers/ignore_evaluator_test.go:
--------------------------------------------------------------------------------
1 | package helpers
2 |
3 | import (
4 | "io"
5 | "talisman/gitrepo"
6 | mockchecksumcalculator "talisman/internal/mock/checksumcalculator"
7 | "talisman/talismanrc"
8 |
9 | "github.com/golang/mock/gomock"
10 | logr "github.com/sirupsen/logrus"
11 | "github.com/stretchr/testify/assert"
12 |
13 | "testing"
14 | )
15 |
16 | func init() {
17 | logr.SetOutput(io.Discard)
18 | }
19 | func TestIsScanNotRequired(t *testing.T) {
20 |
21 | t.Run("should return false if talismanrc is empty", func(t *testing.T) {
22 | ignoreConfig := &talismanrc.TalismanRC{
23 | FileIgnoreConfig: []talismanrc.FileIgnoreConfig{},
24 | }
25 | ie := ignoreEvaluator{nil, ignoreConfig}
26 | addition := gitrepo.Addition{Path: "some.txt"}
27 |
28 | required := ie.isScanNotRequired(addition)
29 |
30 | assert.False(t, required)
31 | })
32 |
33 | t.Run("should loop through talismanrc configs", func(t *testing.T) {
34 | ctrl := gomock.NewController(t)
35 | defer ctrl.Finish()
36 | checksumCalculator := mockchecksumcalculator.NewMockChecksumCalculator(ctrl)
37 | ignoreConfig := talismanrc.TalismanRC{
38 | FileIgnoreConfig: []talismanrc.FileIgnoreConfig{
39 | {
40 | FileName: "some.txt",
41 | Checksum: "sha1",
42 | },
43 | },
44 | }
45 | ie := ignoreEvaluator{calculator: checksumCalculator, talismanRC: &ignoreConfig}
46 | addition := gitrepo.Addition{Name: "some.txt", Path: "some.txt"}
47 | checksumCalculator.EXPECT().CalculateCollectiveChecksumForPattern("some.txt").Return("sha1")
48 |
49 | required := ie.isScanNotRequired(addition)
50 |
51 | assert.True(t, required)
52 | })
53 |
54 | }
55 |
56 | type sillyChecksumCalculator struct{}
57 |
58 | func (scc *sillyChecksumCalculator) CalculateCollectiveChecksumForPattern(fileNamePattern string) string {
59 | return "silly"
60 | }
61 | func (scc *sillyChecksumCalculator) SuggestTalismanRC(fileNamePatterns []string) string {
62 | return ""
63 | }
64 |
65 | func TestDeterminingFilesToIgnore(t *testing.T) {
66 | tRC := talismanrc.TalismanRC{
67 | FileIgnoreConfig: []talismanrc.FileIgnoreConfig{
68 | {
69 | FileName: "some.txt",
70 | Checksum: "silly",
71 | },
72 | {
73 | FileName: "other.txt",
74 | Checksum: "serious",
75 | },
76 | {
77 | FileName: "ignore-contents",
78 | IgnoreDetectors: []string{"filecontent"},
79 | },
80 | },
81 | }
82 | ie := ignoreEvaluator{&sillyChecksumCalculator{}, &tRC}
83 |
84 | t.Run("Should ignore file based on checksum", func(t *testing.T) {
85 | assert.True(t, ie.ShouldIgnore(gitrepo.Addition{Path: "some.txt"}, ""))
86 | })
87 |
88 | t.Run("Should not ignore file if checksum doesn't match", func(t *testing.T) {
89 | assert.False(t, ie.ShouldIgnore(gitrepo.Addition{Path: "other.txt"}, ""))
90 | })
91 |
92 | t.Run("Should ignore if detector is disabled for file", func(t *testing.T) {
93 | assert.True(t, ie.ShouldIgnore(gitrepo.Addition{Path: "ignore-contents"}, "filecontent"))
94 | })
95 |
96 | t.Run("Should not ignore if a different detector is disabled for file", func(t *testing.T) {
97 | assert.False(t, ie.ShouldIgnore(gitrepo.Addition{Path: "ignore-contents"}, "filename"))
98 | })
99 | }
100 |
101 | func TestNeverIgnoreFilesForHistory(t *testing.T) {
102 | scanAllEvaluator := ScanHistoryEvaluator()
103 | assert.False(t, scanAllEvaluator.ShouldIgnore(gitrepo.Addition{Name: "any-file"}, "any_detector"))
104 | }
105 |
--------------------------------------------------------------------------------
/detector/pattern/match_pattern.go:
--------------------------------------------------------------------------------
1 | package pattern
2 |
3 | import (
4 | "fmt"
5 | "regexp"
6 | "talisman/detector/severity"
7 | "talisman/talismanrc"
8 |
9 | "github.com/sirupsen/logrus"
10 | )
11 |
12 | type PatternMatcher struct {
13 | regexes []*severity.PatternSeverity
14 | }
15 |
16 | type DetectionsWithSeverity struct {
17 | detections []string
18 | severity severity.Severity
19 | }
20 |
21 | func (pm *PatternMatcher) check(content string, thresholdValue severity.Severity) []DetectionsWithSeverity {
22 | var detectionsWithSeverity []DetectionsWithSeverity
23 | for _, pattern := range pm.regexes {
24 | var detected []string
25 | regex := pattern.Pattern
26 | logrus.Debugf("checking for pattern %v", regex)
27 | matches := regex.FindAllString(content, -1)
28 | if matches != nil {
29 | detected = append(detected, matches...)
30 | detectionsWithSeverity = append(detectionsWithSeverity, DetectionsWithSeverity{detections: detected, severity: pattern.Severity})
31 | }
32 | }
33 | return detectionsWithSeverity
34 | }
35 |
36 | func (pm *PatternMatcher) add(ps talismanrc.PatternString) {
37 | re, err := regexp.Compile(fmt.Sprintf("(%s)", string(ps)))
38 | if err != nil {
39 | logrus.Warnf("ignoring invalid pattern '%s'", ps)
40 | return
41 | }
42 | logrus.Infof("added custom pattern '%s' with high severity", ps)
43 | pm.regexes = append(pm.regexes, &severity.PatternSeverity{Pattern: re, Severity: severity.SeverityConfiguration["CustomPattern"]})
44 | }
45 |
46 | func NewPatternMatcher(patterns []*severity.PatternSeverity) *PatternMatcher {
47 | return &PatternMatcher{patterns}
48 | }
49 |
--------------------------------------------------------------------------------
/detector/pattern/match_pattern_test.go:
--------------------------------------------------------------------------------
1 | package pattern
2 |
3 | import (
4 | "io/ioutil"
5 | "regexp"
6 | "talisman/detector/severity"
7 | "talisman/talismanrc"
8 | "testing"
9 |
10 | logr "github.com/sirupsen/logrus"
11 | "github.com/stretchr/testify/assert"
12 | )
13 |
14 | func init() {
15 | logr.SetOutput(ioutil.Discard)
16 | }
17 |
18 | var (
19 | testRegexpPasswordPattern = `(?i)(['"_]?password['"]? *[:=][^,;]{8,})`
20 | testRegexpPassword = regexp.MustCompile(testRegexpPasswordPattern)
21 | testRegexpPwPattern = `(?i)(['"_]?pw['"]? *[:=][^,;]{8,})`
22 | testRegexpPw = regexp.MustCompile(testRegexpPwPattern)
23 | )
24 |
25 | func TestShouldReturnEmptyStringWhenDoesNotMatchAnyRegex(t *testing.T) {
26 | detections := NewPatternMatcher([]*severity.PatternSeverity{{Pattern: testRegexpPassword, Severity: severity.Low}}).check("safeString", severity.Low)
27 | assert.Equal(t, []DetectionsWithSeverity(nil), detections)
28 | }
29 |
30 | func TestShouldReturnStringWhenMatchedPasswordPattern(t *testing.T) {
31 | detections1 := NewPatternMatcher([]*severity.PatternSeverity{{Pattern: testRegexpPassword, Severity: severity.Low}}).check("password\" : 123456789", severity.Low)
32 | detections2 := NewPatternMatcher([]*severity.PatternSeverity{{Pattern: testRegexpPw, Severity: severity.Medium}}).check("pw\" : 123456789", severity.Low)
33 | assert.Equal(t, []DetectionsWithSeverity{{detections: []string{"password\" : 123456789"}, severity: severity.Low}}, detections1)
34 | assert.Equal(t, []DetectionsWithSeverity{{detections: []string{"pw\" : 123456789"}, severity: severity.Medium}}, detections2)
35 | }
36 |
37 | func TestShouldAddGoodPatternWithHighToMatcher(t *testing.T) {
38 | pm := NewPatternMatcher([]*severity.PatternSeverity{})
39 | pm.add(talismanrc.PatternString(testRegexpPwPattern))
40 | detections := pm.check("pw\" : 123456789", severity.Low)
41 | assert.Equal(t, []DetectionsWithSeverity{{detections: []string{"pw\" : 123456789"}, severity: severity.High}}, detections)
42 | }
43 |
44 | func TestShouldNotAddBadPatternToMatcher(t *testing.T) {
45 | pm := NewPatternMatcher([]*severity.PatternSeverity{})
46 | pm.add(`*a(crappy|regex`)
47 | assert.Equal(t, 0, len(pm.regexes))
48 | }
49 |
--------------------------------------------------------------------------------
/detector/pattern/pattern_detector.go:
--------------------------------------------------------------------------------
1 | package pattern
2 |
3 | import (
4 | "fmt"
5 | "regexp"
6 | "sync"
7 | "talisman/detector/helpers"
8 | "talisman/detector/severity"
9 | "talisman/gitrepo"
10 | "talisman/talismanrc"
11 |
12 | log "github.com/sirupsen/logrus"
13 | )
14 |
15 | type PatternDetector struct {
16 | secretsPattern *PatternMatcher
17 | }
18 |
19 | var (
20 | detectorPatterns = []*severity.PatternSeverity{
21 | {Pattern: regexp.MustCompile(`(?i)((.*)(password|passphrase|secret|key|pwd|pword|pass)(.*) *[:=>,][^,;\n]{8,})`), Severity: severity.SeverityConfiguration["PasswordPhrasePattern"]},
22 | {Pattern: regexp.MustCompile(`(?i)((:)(password|passphrase|secret|key|pwd|pword|pass)(.*) *[ ][^,;\n]{8,})`), Severity: severity.SeverityConfiguration["PasswordPhrasePattern"]},
23 | {Pattern: regexp.MustCompile(`(?i)(['"_]?pw['"]? *[:=][^,;\n]{8,})`), Severity: severity.SeverityConfiguration["PasswordPhrasePattern"]},
24 | {Pattern: regexp.MustCompile(`(?i)(\S*)`), Severity: severity.SeverityConfiguration["ConsumerKeyPattern"]},
25 | {Pattern: regexp.MustCompile(`(?i)(\S*)`), Severity: severity.SeverityConfiguration["ConsumerSecretParrern"]},
26 | {Pattern: regexp.MustCompile(`(?i)(AWS[ \w]+key[ \w]+[:=])`), Severity: severity.SeverityConfiguration["AWSKeyPattern"]},
27 | {Pattern: regexp.MustCompile(`(?i)(AWS[ \w]+secret[ \w]+[:=])`), Severity: severity.SeverityConfiguration["AWSSecretPattern"]},
28 | {Pattern: regexp.MustCompile(`(?s)(BEGIN RSA PRIVATE KEY.*END RSA PRIVATE KEY)`), Severity: severity.SeverityConfiguration["RSAKeyPattern"]},
29 | }
30 | )
31 |
32 | type match struct {
33 | name gitrepo.FileName
34 | path gitrepo.FilePath
35 | commits []string
36 | detections []DetectionsWithSeverity
37 | }
38 |
39 | // Test tests the contents of the Additions to ensure that they don't look suspicious
40 | func (detector PatternDetector) Test(comparator helpers.IgnoreEvaluator, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) {
41 | matches := make(chan match, 512)
42 | ignoredFilePaths := make(chan gitrepo.FilePath, 512)
43 | waitGroup := &sync.WaitGroup{}
44 | waitGroup.Add(len(currentAdditions))
45 | for _, addition := range currentAdditions {
46 | go func(addition gitrepo.Addition) {
47 | defer waitGroup.Done()
48 | defer additionCompletionCallback()
49 | if comparator.ShouldIgnore(addition, "filecontent") {
50 | ignoredFilePaths <- addition.Path
51 | return
52 | }
53 | detections := detector.secretsPattern.check(ignoreConfig.RemoveAllowedPatterns(addition), ignoreConfig.Threshold)
54 | matches <- match{name: addition.Name, path: addition.Path, detections: detections, commits: addition.Commits}
55 | }(addition)
56 | }
57 | go func() {
58 | waitGroup.Wait()
59 | close(matches)
60 | close(ignoredFilePaths)
61 | }()
62 | for ignoredChanHasMore, matchChanHasMore := true, true; ignoredChanHasMore || matchChanHasMore; {
63 | select {
64 | case match, hasMore := <-matches:
65 | if !hasMore {
66 | matchChanHasMore = false
67 | continue
68 | }
69 | detector.processMatch(match, result, ignoreConfig.Threshold)
70 | case ignore, hasMore := <-ignoredFilePaths:
71 | if !hasMore {
72 | ignoredChanHasMore = false
73 | continue
74 | }
75 | detector.processIgnore(ignore, result)
76 | }
77 | }
78 | }
79 |
80 | func (detector PatternDetector) processIgnore(ignoredFilePath gitrepo.FilePath, result *helpers.DetectionResults) {
81 | log.WithFields(log.Fields{
82 | "filePath": ignoredFilePath,
83 | }).Info("Ignoring addition as it was specified to be ignored.")
84 | result.Ignore(ignoredFilePath, "filecontent")
85 | }
86 |
87 | func (detector PatternDetector) processMatch(match match, result *helpers.DetectionResults, threshold severity.Severity) {
88 | for _, detectionWithSeverity := range match.detections {
89 | for _, detection := range detectionWithSeverity.detections {
90 | if detection != "" {
91 | if string(match.name) == talismanrc.RCFileName || !detectionWithSeverity.severity.ExceedsThreshold(threshold) {
92 | log.WithFields(log.Fields{
93 | "filePath": match.path,
94 | "pattern": detection,
95 | }).Warn("Warning file as it matched pattern.")
96 | result.Warn(match.path, "filecontent", fmt.Sprintf("Potential secret pattern : %s", detection), match.commits, detectionWithSeverity.severity)
97 | } else {
98 | log.WithFields(log.Fields{
99 | "filePath": match.path,
100 | "pattern": detection,
101 | }).Info("Failing file as it matched pattern.")
102 | result.Fail(match.path, "filecontent", fmt.Sprintf("Potential secret pattern : %s", detection), match.commits, detectionWithSeverity.severity)
103 | }
104 | }
105 | }
106 | }
107 | }
108 |
109 | // NewPatternDetector returns a PatternDetector that tests Additions against the pre-configured patterns
110 | func NewPatternDetector(custom []talismanrc.PatternString) *PatternDetector {
111 | matcher := NewPatternMatcher(detectorPatterns)
112 | for _, pattern := range custom {
113 | matcher.add(pattern)
114 | }
115 | return &PatternDetector{matcher}
116 | }
117 |
--------------------------------------------------------------------------------
/detector/pattern/pattern_detector_test.go:
--------------------------------------------------------------------------------
1 | package pattern
2 |
3 | import (
4 | "regexp"
5 | "strings"
6 | "talisman/detector/helpers"
7 | "talisman/detector/severity"
8 | "talisman/gitrepo"
9 | "talisman/talismanrc"
10 | "testing"
11 |
12 | "github.com/stretchr/testify/assert"
13 | )
14 |
15 | var talismanRC = &talismanrc.TalismanRC{}
16 | var defaultIgnoreEvaluator = helpers.BuildIgnoreEvaluator("default", talismanRC, gitrepo.RepoLocatedAt("."))
17 | var dummyCallback = func() {}
18 |
19 | var (
20 | customPatterns []talismanrc.PatternString
21 | )
22 |
23 | func ignoreEvaluatorWithTalismanRC(tRC *talismanrc.TalismanRC) helpers.IgnoreEvaluator {
24 | return helpers.BuildIgnoreEvaluator("default", tRC, gitrepo.RepoLocatedAt("."))
25 | }
26 |
27 | func TestShouldDetectPasswordPatterns(t *testing.T) {
28 | filename := "secret.txt"
29 | values := [7]string{"password", "secret", "key", "pwd", "pass", "pword", "passphrase"}
30 | for i := 0; i < len(values); i++ {
31 | shouldPassDetectionOfSecretPattern(filename, []byte(strings.ToTitle(values[i])+":UnsafeString"), t)
32 | shouldPassDetectionOfSecretPattern(filename, []byte(values[i]+"=UnsafeString"), t)
33 | shouldPassDetectionOfSecretPattern(filename, []byte("."+values[i]+"=randomStringGoesHere}"), t)
34 | shouldPassDetectionOfSecretPattern(filename, []byte(":"+values[i]+" randomStringGoesHere"), t)
35 | shouldPassDetectionOfSecretPattern(filename, []byte(values[i]+" ,\"randomStringGoesHere\""), t)
36 | shouldPassDetectionOfSecretPattern(filename, []byte("'"+values[i]+"' ,\"randomStringGoesHere\""), t)
37 | shouldPassDetectionOfSecretPattern(filename, []byte("\""+values[i]+"\" ,\"randomStringGoesHere\""), t)
38 | shouldPassDetectionOfSecretPattern(filename,
39 | []byte("\"SERVER_"+strings.ToUpper(values[i])+"\" : UnsafeString"),
40 | t)
41 | shouldPassDetectionOfSecretPattern(filename, []byte(values[i]+"2-string : UnsafeString"), t)
42 | shouldPassDetectionOfSecretPattern(filename,
43 | []byte("<"+values[i]+" data=123> randomStringGoesHere "+values[i]+">"),
44 | t)
45 | shouldPassDetectionOfSecretPattern(filename,
46 | []byte(" randomStringGoesHere "),
47 | t)
48 | }
49 |
50 | shouldPassDetectionOfSecretPattern(filename, []byte("\"pw\" : UnsafeString"), t)
51 | shouldPassDetectionOfSecretPattern(filename, []byte("Pw=UnsafeString"), t)
52 |
53 | shouldPassDetectionOfSecretPattern(filename, []byte("alksjdhfkjaklsdhflk12345adskjf"), t)
54 | shouldPassDetectionOfSecretPattern(filename, []byte("AWS key :"), t)
55 | shouldPassDetectionOfSecretPattern(filename, []byte(`BEGIN RSA PRIVATE KEY-----
56 | aghjdjadslgjagsfjlsgjalsgjaghjldasja
57 | -----END RSA PRIVATE KEY`), t)
58 | shouldPassDetectionOfSecretPattern(filename, []byte(`PWD=appropriate`), t)
59 | shouldPassDetectionOfSecretPattern(filename, []byte(`pass=appropriate`), t)
60 | shouldPassDetectionOfSecretPattern(filename, []byte(`adminpwd=appropriate`), t)
61 |
62 | shouldFailDetectionOfSecretPattern(filename, []byte("\"pAsSWoRD\" :1234567"), t)
63 | shouldFailDetectionOfSecretPattern(filename, []byte(`setPassword("12345678")`), t)
64 | shouldFailDetectionOfSecretPattern(filename, []byte(`setenv(password,123456)`), t)
65 | shouldFailDetectionOfSecretPattern(filename, []byte(`random=12345678)`), t)
66 | }
67 |
68 | func TestShouldIgnorePasswordPatternsIfChecksumMatches(t *testing.T) {
69 | results := helpers.NewDetectionResults()
70 | content := []byte("\"password\" : UnsafePassword")
71 | filename := "secret.txt"
72 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
73 | fileIgnoreConfig := talismanrc.FileIgnoreConfig{
74 | FileName: filename,
75 | Checksum: "833b6c24c8c2c5c7e1663226dc401b29c005492dc76a1150fc0e0f07f29d4cc3",
76 | IgnoreDetectors: []string{"filecontent"},
77 | AllowedPatterns: []string{}}
78 | ignores := &talismanrc.TalismanRC{FileIgnoreConfig: []talismanrc.FileIgnoreConfig{fileIgnoreConfig}}
79 |
80 | NewPatternDetector(customPatterns).Test(ignoreEvaluatorWithTalismanRC(ignores), additions, ignores, results, dummyCallback)
81 |
82 | assert.True(t, results.Successful(), "Expected file %s to be ignored because checksum matches", filename)
83 | }
84 |
85 | func TestShouldIgnoreAllowedPattern(t *testing.T) {
86 | results := helpers.NewDetectionResults()
87 | content := []byte("\"key\" : \"This is an allowed keyword\"\npassword=y0uw1lln3v3rgu3ssmyP@55w0rd")
88 | filename := "allowed_pattern.txt"
89 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
90 | fileIgnoreConfig := talismanrc.FileIgnoreConfig{
91 | FileName: filename, Checksum: "",
92 | IgnoreDetectors: []string{},
93 | AllowedPatterns: []string{"key"}}
94 | ignores := &talismanrc.TalismanRC{
95 | FileIgnoreConfig: []talismanrc.FileIgnoreConfig{fileIgnoreConfig},
96 | AllowedPatterns: []*talismanrc.Pattern{{Regexp: regexp.MustCompile("password")}}}
97 |
98 | NewPatternDetector(customPatterns).Test(defaultIgnoreEvaluator, additions, ignores, results, dummyCallback)
99 |
100 | assert.True(t,
101 | results.Successful(),
102 | "Expected keywords %v %v to be ignored by Talisman", fileIgnoreConfig.AllowedPatterns, ignores.AllowedPatterns)
103 | }
104 | func TestShouldOnlyWarnSecretPatternIfBelowThreshold(t *testing.T) {
105 | results := helpers.NewDetectionResults()
106 | content := []byte(`password=UnsafeString`)
107 | filename := "secret.txt"
108 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
109 | talismanRCWithThreshold := &talismanrc.TalismanRC{Threshold: severity.High}
110 |
111 | NewPatternDetector(customPatterns).Test(ignoreEvaluatorWithTalismanRC(talismanRCWithThreshold), additions, talismanRCWithThreshold, results, dummyCallback)
112 |
113 | assert.False(t, results.HasFailures(), "Expected file %s to not have failures", filename)
114 | assert.True(t, results.HasWarnings(), "Expected file %s to have warnings", filename)
115 | }
116 |
117 | func DetectionOfSecretPattern(filename string, content []byte) (*helpers.DetectionResults, []gitrepo.Addition, string) {
118 | results := helpers.NewDetectionResults()
119 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
120 | NewPatternDetector(customPatterns).Test(defaultIgnoreEvaluator, additions, talismanRC, results, dummyCallback)
121 | expected := "Potential secret pattern : " + string(content)
122 | return results, additions, expected
123 | }
124 |
125 | func shouldPassDetectionOfSecretPattern(filename string, content []byte, t *testing.T) {
126 | results, additions, expected := DetectionOfSecretPattern(filename, content)
127 | assert.Equal(t, expected, getFailureMessage(results, additions))
128 | assert.Len(t, results.Results, 1)
129 | }
130 |
131 | func shouldFailDetectionOfSecretPattern(filename string, content []byte, t *testing.T) {
132 | results, additions, expected := DetectionOfSecretPattern(filename, content)
133 | assert.NotEqual(t, expected, getFailureMessage(results, additions))
134 | assert.Len(t, results.Results, 0)
135 | }
136 |
137 | func getFailureMessage(results *helpers.DetectionResults, additions []gitrepo.Addition) string {
138 | failureMessages := []string{}
139 | for _, failureDetails := range results.GetFailures(additions[0].Path) {
140 | failureMessages = append(failureMessages, failureDetails.Message)
141 | }
142 | if len(failureMessages) == 0 {
143 | return ""
144 | }
145 | return failureMessages[0]
146 | }
147 |
--------------------------------------------------------------------------------
/detector/severity/pattern_severity.go:
--------------------------------------------------------------------------------
1 | package severity
2 |
3 | import (
4 | "regexp"
5 | )
6 |
7 | type PatternSeverity struct {
8 | Pattern *regexp.Regexp
9 | Severity Severity
10 | }
11 |
--------------------------------------------------------------------------------
/detector/severity/severity.go:
--------------------------------------------------------------------------------
1 | package severity
2 |
3 | import (
4 | "fmt"
5 | "strings"
6 | )
7 |
8 | var severityMap = map[Severity]string{
9 | Low: "low",
10 | Medium: "medium",
11 | High: "high",
12 | }
13 |
14 | func String(severity Severity) string {
15 | return severityMap[severity]
16 | }
17 |
18 | func FromString(severity string) (Severity, error) {
19 | severityInLowerCase := strings.ToLower(severity)
20 | for k, v := range severityMap {
21 | if v == severityInLowerCase {
22 | return k, nil
23 | }
24 | }
25 | return 0, fmt.Errorf("unknown severity %v", severity)
26 | }
27 |
28 | type Severity int
29 |
30 | func (s Severity) String() string {
31 | return String(s)
32 | }
33 |
34 | func (s Severity) ExceedsThreshold(threshold Severity) bool {
35 | return s >= threshold
36 | }
37 |
38 | func (s *Severity) UnmarshalYAML(get func(interface{}) error) error {
39 | in := ""
40 | err := get(&in)
41 | if err != nil {
42 | return fmt.Errorf("Severity.Umarshal error: %v\n", err)
43 | }
44 | *s, err = FromString(in)
45 | return err
46 | }
47 |
48 | func (s Severity) MarshalYAML() (interface{}, error) {
49 | return String(s), nil
50 | }
51 |
52 | func (s *Severity) UnmarshalJSON(input []byte) error {
53 | v, err := FromString(string(input))
54 | if err != nil {
55 | return err
56 | }
57 | *s = v
58 | return nil
59 | }
60 |
61 | func (s Severity) MarshalJSON() ([]byte, error) {
62 | bytes := []byte(fmt.Sprintf("\"%s\"", String(s)))
63 | return bytes, nil
64 | }
65 |
66 | const (
67 | Low = Severity(iota + 1)
68 | Medium
69 | High
70 | )
71 |
--------------------------------------------------------------------------------
/detector/severity/severity_config.go:
--------------------------------------------------------------------------------
1 | package severity
2 |
3 | var SeverityConfiguration = map[string]Severity{
4 | "ConsumerKeyPattern": High,
5 | "ConsumerSecretParrern": High,
6 | "AWSKeyPattern": High,
7 | "AWSSecretPattern": High,
8 | "RSAKeyPattern": High,
9 | "DSAFile": High,
10 | "PrivateKeyFile": High,
11 | "PemFile": High,
12 | "PpkFile": High,
13 | "SecretToken": High,
14 | "KeyPairFile": High,
15 | "CustomPattern": High,
16 | "PKCSFile": High,
17 | "PFXFile": High,
18 | "P12File": High,
19 | "NetrcFile": High,
20 | "RSAFile": High,
21 | "KeyChainFile": High,
22 | "KeyStoreFile": High,
23 | "OauthTokenFile": High,
24 | "HTPASSWDFile": High,
25 | "TunnelBlockFile": High,
26 | "CredentialsXML": High,
27 | "JenkinsPublishOverSSHFile": High,
28 | "Base64Content": High,
29 | "HexContent": High,
30 | "s3Config": Medium,
31 | "OpenVPNFile": Medium,
32 | "DatabaseYml": Medium,
33 | "ShellHistory": Low,
34 | "ASCFile": Low,
35 | "KDBFile": Low,
36 | "AgileKeyChainFile": Low,
37 | "PubXML": Low,
38 | "GitRobRC": Low,
39 | "ShellRC": Low,
40 | "CreditCardContent": Low,
41 | "ShellProfile": Low,
42 | "ShellAlias": Low,
43 | "OmniAuth": Low,
44 | "CarrierWaveRB": Low,
45 | "SchemaRB": Low,
46 | "PythonSettings": Low,
47 | "PhpConfig": Low,
48 | "PhpLocalSettings": Low,
49 | "EnvFile": Low,
50 | "BDumpFile": Low,
51 | "BSQLFile": Low,
52 | "PasswordFile": Low,
53 | "BackupFile": Low,
54 | "LogFile": Low,
55 | "KWallet": Low,
56 | "GNUCash": Low,
57 | "PasswordPhrasePattern": Low,
58 | "LargeFileSize": Low,
59 | }
60 |
--------------------------------------------------------------------------------
/detector/severity/severity_test.go:
--------------------------------------------------------------------------------
1 | package severity
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/stretchr/testify/assert"
7 | )
8 |
9 | func TestShouldReturnSeverityStringForDefinedSeverity(t *testing.T) {
10 | assert.Equal(t, String(Low), "low")
11 | assert.Equal(t, String(Medium), "medium")
12 | assert.Equal(t, String(High), "high")
13 | }
14 | func TestShouldReturnEmptyForInvalidSeverity(t *testing.T) {
15 | assert.Equal(t, String(10), "")
16 | }
17 |
18 | func TestShouldReturnSeverityForDefinedStrings(t *testing.T) {
19 | severityValue, _ := FromString("low")
20 | assert.Equal(t, severityValue, Low)
21 | severityValue, _ = FromString("mEDIum")
22 | assert.Equal(t, severityValue, Medium)
23 | severityValue, _ = FromString("HIGH")
24 | assert.Equal(t, severityValue, High)
25 | }
26 |
27 | func TestShouldReturnSeverityZeroWithErrorForUnknownStrings(t *testing.T) {
28 | severityValue, err := FromString("FakeSeverity")
29 | assert.Equal(t, Severity(0), severityValue)
30 | assert.Error(t, err)
31 | assert.Equal(t, "unknown severity FakeSeverity", err.Error())
32 | }
33 |
--------------------------------------------------------------------------------
/examples/schema-store-talismanrc.json:
--------------------------------------------------------------------------------
1 | {
2 | "$schema": "http://json-schema.org/draft-07/schema#",
3 | "$id": "https://github.com/thoughtworks/talisman/talismanrc",
4 | "title": "schema for .talismanrc",
5 | "type": "object",
6 | "additionalProperties": false,
7 | "fileMatch": [
8 | ".talismanrc"
9 | ],
10 | "properties": {
11 | "fileignoreconfig": {
12 | "type": "array",
13 | "items": {
14 | "type": "object",
15 | "properties": {
16 | "filename": {
17 | "type": "string",
18 | "description": "Fully qualified filename"
19 | },
20 | "checksum": {
21 | "type": "string",
22 | "description": "This field should always have the value specified by Talisman message"
23 | },
24 | "ignore_detectors": {
25 | "type": "array",
26 | "description": "Disable specific detectors for a particular file",
27 | "items": {
28 | "type": "string",
29 | "enum": ["filecontent", "filename", "filesize"]
30 | }
31 | },
32 | "allowed_patterns": {
33 | "type": "array",
34 | "description": "Keywords to ignore to reduce the number of false positives",
35 | "items": {
36 | "type": "string"
37 | }
38 | }
39 | },
40 | "required": ["filename"]
41 | }
42 | },
43 | "scopeconfig": {
44 | "type": "array",
45 | "description": "Talisman is configured to ignore certain files based on the specified scopes",
46 | "items": {
47 | "type": "object",
48 | "properties": {
49 | "scope": {
50 | "type": "string"
51 | }
52 | },
53 | "required": ["scope"]
54 | }
55 | },
56 | "allowed_patterns": {
57 | "type": "array",
58 | "description": "Keywords to ignore to reduce the number of false positives",
59 | "items": {
60 | "type": "string"
61 | }
62 | },
63 | "custom_patterns": {
64 | "type": "array",
65 | "description": "You can specify custom regex patterns to look for in the current repository",
66 | "items": {
67 | "type": "string"
68 | }
69 | },
70 | "custom_severities": {
71 | "type": "array",
72 | "description": "Custom detectors severities",
73 | "items": {
74 | "type": "object",
75 | "properties": {
76 | "detector": {
77 | "type": "string"
78 | },
79 | "severity": {
80 | "type": "string",
81 | "enum": ["low", "medium", "high"]
82 | }
83 | },
84 | "required": ["detector", "severity"]
85 | }
86 | },
87 | "threshold": {
88 | "type": "string",
89 | "description": "Default minimal threshold",
90 | "enum": ["low", "medium", "high"]
91 | },
92 | "version": {
93 | "type": "string",
94 | "description": ".talismanrc version"
95 | }
96 | },
97 | "required": []
98 | }
99 |
--------------------------------------------------------------------------------
/git_testing/git_testing.go:
--------------------------------------------------------------------------------
1 | package git_testing
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "os/exec"
7 | "path/filepath"
8 | "strings"
9 |
10 | lorem "github.com/drhodes/golorem"
11 | "github.com/sirupsen/logrus"
12 | "github.com/spf13/afero"
13 | )
14 |
15 | var Logger *logrus.Entry
16 |
17 | // GitTesting provides an API for manipulating a git repository during tests
18 | type GitTesting struct {
19 | root string
20 | }
21 |
22 | type GitOperation func(*GitTesting)
23 |
24 | // DoInTempGitRepo initializes a temporary git repository and executes the provided GitOperation in it
25 | func DoInTempGitRepo(gitOperation GitOperation) {
26 | gt := Init()
27 | defer gt.Clean()
28 | gitOperation(gt)
29 | }
30 |
31 | // Init creates a GitTesting based in a temporary directory
32 | func Init() *GitTesting {
33 | fs := afero.NewMemMapFs()
34 | path, _ := afero.TempDir(fs, afero.GetTempDir(fs, "talisman-test"), "")
35 | return initAt(path)
36 | }
37 |
38 | // initAt creates a GitTesting based at the specified path
39 | func initAt(gitRoot string) *GitTesting {
40 | os.MkdirAll(gitRoot, 0777)
41 | testingRepo := &GitTesting{gitRoot}
42 | output := testingRepo.execCommand("git", "init", ".")
43 | logrus.Debugf("Git init result %v", string(output))
44 | testingRepo.execCommand("git", "config", "user.email", "talisman-test-user@example.com")
45 | testingRepo.execCommand("git", "config", "user.name", "Talisman Test User")
46 | testingRepo.execCommand("git", "config", "commit.gpgsign", "false")
47 | testingRepo.removeHooks()
48 | return testingRepo
49 | }
50 |
51 | // Clean removes the directory containing the git repository represented by a GitTesting
52 | func (git *GitTesting) Clean() {
53 | os.RemoveAll(git.root)
54 | }
55 |
56 | func (git *GitTesting) SetupBaselineFiles(filenames ...string) {
57 | Logger.Debugf("Creating %v in %s\n", filenames, git.root)
58 | for _, filename := range filenames {
59 | git.CreateFileWithContents(filename, lorem.Sentence(8, 10), lorem.Sentence(8, 10))
60 | }
61 | git.AddAndcommit("*", "initial commit")
62 | }
63 |
64 | func (git *GitTesting) EarliestCommit() string {
65 | return git.execCommand("git", "rev-list", "--max-parents=0", "HEAD")
66 | }
67 |
68 | func (git *GitTesting) LatestCommit() string {
69 | return git.execCommand("git", "rev-parse", "HEAD")
70 | }
71 |
72 | func (git *GitTesting) CreateFileWithContents(filePath string, contents ...string) string {
73 | git.doInGitRoot(func() {
74 | os.MkdirAll(filepath.Dir(filePath), 0700)
75 | f, err := os.Create(filePath)
76 | git.die(fmt.Sprintf("when creating file %s", filePath), err)
77 | defer f.Close()
78 | for _, line := range contents {
79 | f.WriteString(line)
80 | }
81 | })
82 | return filePath
83 | }
84 |
85 | func (git *GitTesting) OverwriteFileContent(filePath string, contents ...string) {
86 | git.doInGitRoot(func() {
87 | os.MkdirAll(filepath.Dir(filePath), 0770)
88 | f, err := os.Create(filePath)
89 | git.die(fmt.Sprintf("when overwriting file %s", filePath), err)
90 | defer f.Close()
91 | for _, line := range contents {
92 | f.WriteString(line)
93 | }
94 | f.Sync()
95 | })
96 | }
97 |
98 | func (git *GitTesting) AppendFileContent(filePath string, contents ...string) {
99 | git.doInGitRoot(func() {
100 | os.MkdirAll(filepath.Dir(filePath), 0770)
101 | f, err := os.OpenFile(filePath, os.O_RDWR|os.O_APPEND, 0660)
102 | git.die(fmt.Sprintf("when appending file %s", filePath), err)
103 | defer f.Close()
104 | for _, line := range contents {
105 | f.WriteString(line)
106 | }
107 | f.Sync()
108 | })
109 | }
110 |
111 | func (git *GitTesting) RemoveFile(filename string) {
112 | git.doInGitRoot(func() {
113 | os.Remove(filename)
114 | })
115 | }
116 |
117 | func (git *GitTesting) FileContents(filePath string) []byte {
118 | var result []byte
119 | var err error
120 | git.doInGitRoot(func() {
121 | result, err = os.ReadFile(filePath)
122 | git.die(fmt.Sprintf("when reading file %s", filePath), err)
123 | })
124 | return result
125 | }
126 |
127 | func (git *GitTesting) AddAndcommit(fileName string, message string) {
128 | git.Add(fileName)
129 | git.Commit(fileName, message)
130 | }
131 |
132 | func (git *GitTesting) Add(fileName string) {
133 | git.execCommand("git", "add", fileName)
134 | }
135 |
136 | func (git *GitTesting) Commit(fileName string, message string) {
137 | git.execCommand("git", "commit", "-m", message)
138 | }
139 |
140 | // GetBlobDetails returns git blob details for a path
141 | func (git *GitTesting) GetBlobDetails(fileName string) string {
142 | var output []byte
143 | objectHashAndFilename := ""
144 | git.doInGitRoot(func() {
145 | fmt.Println("hello")
146 | result := exec.Command("git", "rev-list", "--objects", "--all")
147 | output, _ = result.Output()
148 | objects := strings.Split(string(output), "\n")
149 | for _, object := range objects {
150 | objectDetails := strings.Split(object, " ")
151 | if len(objectDetails) == 2 && objectDetails[1] == fileName {
152 | objectHashAndFilename = object
153 | return
154 | }
155 | }
156 | })
157 | return objectHashAndFilename
158 | }
159 |
160 | // execCommand executes a command with given arguments in the git repo directory
161 | func (git *GitTesting) execCommand(commandName string, args ...string) string {
162 | var output []byte
163 | git.doInGitRoot(func() {
164 | result := exec.Command(commandName, args...)
165 | var err error
166 | output, err = result.Output()
167 | summaryMessage := fmt.Sprintf("Command: %s %v\nWorkingDirectory: %s\nOutput %s\nError: %v", commandName, args, git.root, string(output), err)
168 | git.die(summaryMessage, err)
169 | Logger.Debug(summaryMessage)
170 | })
171 | if len(output) > 0 {
172 | return strings.Trim(string(output), "\n")
173 | }
174 | return ""
175 | }
176 |
177 | func (git *GitTesting) die(msg string, err error) {
178 | if err != nil {
179 | Logger.Debugf(msg)
180 | panic(msg)
181 | }
182 | }
183 |
184 | func (git *GitTesting) doInGitRoot(operation func()) {
185 | wd, _ := os.Getwd()
186 | os.Chdir(git.root)
187 | defer func() { os.Chdir(wd) }()
188 | operation()
189 | }
190 |
191 | // Root returns the root directory of the git-testing repo
192 | func (git *GitTesting) Root() string {
193 | return git.root
194 | }
195 |
196 | // removeHooks removes all file-system hooks from git-test repo.
197 | // We do this to prevent any user-installed hooks from interfering with tests.
198 | func (git *GitTesting) removeHooks() {
199 | git.execCommand("rm", "-rf", ".git/hooks/")
200 | }
201 |
--------------------------------------------------------------------------------
/git_testing/git_testing_test.go:
--------------------------------------------------------------------------------
1 | package git_testing
2 |
3 | import (
4 | "os"
5 | "os/exec"
6 | "path"
7 | "path/filepath"
8 | "regexp"
9 | "strings"
10 | "testing"
11 |
12 | "github.com/sirupsen/logrus"
13 | "github.com/stretchr/testify/assert"
14 | )
15 |
16 | var logger *logrus.Entry
17 |
18 | func init() {
19 | Logger = logrus.WithField("Environment", "Debug")
20 | Logger.Debug("GitTesting test started")
21 | logrus.SetOutput(os.Stderr)
22 | logger = Logger
23 | }
24 |
25 | func TestInitializingANewRepoSetsUpFolderAndGitStructures(t *testing.T) {
26 | DoInTempGitRepo(func(repo *GitTesting) {
27 | assert.True(t, exists(repo.root), "GitTesting initialization should create the directory structure required")
28 | assert.True(t, isGitRepo(repo.root), "Repo root does not contain the .git folder")
29 | })
30 | }
31 |
32 | func TestSettingUpBaselineFilesSetsUpACommitInRepo(t *testing.T) {
33 | DoInTempGitRepo(func(repo *GitTesting) {
34 | repo.SetupBaselineFiles("a.txt", filepath.Join("alice", "bob", "b.txt"))
35 | verifyPresenceOfGitRepoWithCommits(t, 1, repo.root)
36 | })
37 | }
38 |
39 | func TestEditingFilesInARepoWorks(t *testing.T) {
40 | DoInTempGitRepo(func(repo *GitTesting) {
41 | repo.SetupBaselineFiles("a.txt", filepath.Join("alice", "bob", "b.txt"))
42 | repo.AppendFileContent("a.txt", "\nmonkey see.\n", "monkey do.")
43 | content := repo.FileContents("a.txt")
44 | assert.True(t, strings.HasSuffix(string(content), "monkey see.\nmonkey do."))
45 | repo.AddAndcommit("a.txt", "modified content")
46 | verifyPresenceOfGitRepoWithCommits(t, 2, repo.root)
47 | })
48 | }
49 |
50 | func TestRemovingFilesInARepoWorks(t *testing.T) {
51 | DoInTempGitRepo(func(repo *GitTesting) {
52 | repo.SetupBaselineFiles("a.txt", filepath.Join("alice", "bob", "b.txt"))
53 | repo.RemoveFile("a.txt")
54 | assert.False(t, exists(filepath.Join("data", "testLocation1", "a.txt")), "Unexpected. Deleted file a.txt still exists inside the repo")
55 | repo.AddAndcommit("a.txt", "removed it")
56 | verifyPresenceOfGitRepoWithCommits(t, 2, repo.root)
57 | })
58 | }
59 |
60 | func TestEarliestCommits(t *testing.T) {
61 | DoInTempGitRepo(func(repo *GitTesting) {
62 | repo.SetupBaselineFiles("a.txt")
63 | initialCommit := repo.EarliestCommit()
64 | repo.AppendFileContent("a.txt", "\nmonkey see.\n", "monkey do.")
65 | repo.AddAndcommit("a.txt", "modified content")
66 | assert.Equal(t, initialCommit, repo.EarliestCommit(), "First commit is not expected to change on repo modifications")
67 | })
68 | }
69 |
70 | func TestLatestCommits(t *testing.T) {
71 | DoInTempGitRepo(func(repo *GitTesting) {
72 | repo.SetupBaselineFiles("a.txt")
73 | repo.AppendFileContent("a.txt", "\nmonkey see.\n", "monkey do.")
74 | repo.AddAndcommit("a.txt", "modified content")
75 | repo.AppendFileContent("a.txt", "\nline n-1.\n", "line n.")
76 | repo.AddAndcommit("a.txt", "more modified content")
77 | assert.NotEqual(t, repo.EarliestCommit(), repo.LatestCommit()) //bad test.
78 | })
79 | }
80 |
81 | func verifyPresenceOfGitRepoWithCommits(t *testing.T, expectedCommitCount int, repoLocation string) {
82 | wd, _ := os.Getwd()
83 | os.Chdir(repoLocation)
84 | defer func() { os.Chdir(wd) }()
85 |
86 | cmd := exec.Command("git", "log", "--pretty=short")
87 | o, err := cmd.CombinedOutput()
88 | dieOnError(err)
89 | matches := regExp("(?m)^commit\\s[a-z0-9]+\\s+.*$").FindAllString(string(o), -1)
90 | assert.Len(t, matches, expectedCommitCount, "Repo root does not contain exactly %d commits.", expectedCommitCount)
91 | }
92 |
93 | func regExp(pattern string) *regexp.Regexp {
94 | return regexp.MustCompile(pattern)
95 | }
96 |
97 | func isGitRepo(loc string) bool {
98 | return exists(path.Join(loc, ".git"))
99 | }
100 |
101 | func exists(path string) bool {
102 | _, err := os.Stat(path)
103 | if (err != nil) && (os.IsNotExist(err)) {
104 | return false
105 | } else if err != nil {
106 | dieOnError(err)
107 | return true
108 | } else {
109 | return true
110 | }
111 | }
112 |
113 | func dieOnError(err error) {
114 | if err != nil {
115 | panic(err)
116 | }
117 | }
118 |
--------------------------------------------------------------------------------
/gitrepo/constants.go:
--------------------------------------------------------------------------------
1 | package gitrepo
2 |
3 | const (
4 | //GIT_STAGED_PREFIX is used to read a staged git file
5 | GIT_STAGED_PREFIX = ""
6 |
7 | //GIT_STAGED_PREFIX is used to read a commited git file
8 | GIT_HEAD_PREFIX = "HEAD"
9 | )
10 |
--------------------------------------------------------------------------------
/gitrepo/git_readers.go:
--------------------------------------------------------------------------------
1 | package gitrepo
2 |
3 | import (
4 | "bufio"
5 | "fmt"
6 | "io"
7 | "os/exec"
8 | "strconv"
9 |
10 | "github.com/sirupsen/logrus"
11 | )
12 |
13 | type ReadFunc func(string) ([]byte, error)
14 | type BatchReader interface {
15 | Start() error
16 | Read(string) ([]byte, error)
17 | Shutdown() error
18 | }
19 |
20 | type BatchGitObjectReader struct {
21 | repo *GitRepo
22 | cmd *exec.Cmd
23 | inputWriter *bufio.Writer
24 | outputReader *bufio.Reader
25 | read ReadFunc
26 | }
27 |
28 | func (bgor *BatchGitObjectReader) Start() error {
29 | return bgor.cmd.Start()
30 | }
31 |
32 | func (bgor *BatchGitObjectReader) Shutdown() error {
33 | return bgor.cmd.Process.Kill()
34 | }
35 |
36 | func (bgor *BatchGitObjectReader) Read(expr string) ([]byte, error) {
37 | return bgor.read(expr)
38 | }
39 |
40 | func newBatchGitObjectReader(root string) *BatchGitObjectReader {
41 | repo := &GitRepo{root}
42 | cmd := repo.makeRepoCommand("git", "cat-file", "--batch=%(objectsize)")
43 | inputPipe, err := cmd.StdinPipe()
44 | if err != nil {
45 | logrus.Fatalf("error creating stdin pipe for batch git file reader subprocess: %v", err)
46 | }
47 | outputPipe, err := cmd.StdoutPipe()
48 | if err != nil {
49 | logrus.Fatalf("error creating stdout pipe for batch git file reader subprocess: %v", err)
50 | }
51 | batchReader := BatchGitObjectReader{
52 | repo: repo,
53 | cmd: cmd,
54 | inputWriter: bufio.NewWriter(inputPipe),
55 | outputReader: bufio.NewReader(outputPipe),
56 | }
57 | return &batchReader
58 | }
59 |
60 | func NewBatchGitHeadPathReader(root string) BatchReader {
61 | bgor := newBatchGitObjectReader(root)
62 | bgor.read = bgor.makePathReader(GIT_HEAD_PREFIX)
63 | return bgor
64 | }
65 |
66 | func NewBatchGitStagedPathReader(root string) BatchReader {
67 | bgor := newBatchGitObjectReader(root)
68 | bgor.read = bgor.makePathReader(GIT_STAGED_PREFIX)
69 | return bgor
70 | }
71 |
72 | func NewBatchGitObjectHashReader(root string) BatchReader {
73 | bgor := newBatchGitObjectReader(root)
74 | bgor.read = bgor.makeObjectHashReader()
75 | return bgor
76 | }
77 |
78 | type gitCatFileReadResult struct {
79 | contents []byte
80 | err error
81 | }
82 |
83 | func (bgor *BatchGitObjectReader) makePathReader(prefix string) ReadFunc {
84 | pathChan := make(chan ([]byte))
85 | resultsChan := make(chan (gitCatFileReadResult))
86 | go bgor.doCatFile(pathChan, resultsChan)
87 | return func(path string) ([]byte, error) {
88 | pathChan <- []byte(prefix + ":" + path)
89 | result := <-resultsChan
90 | return result.contents, result.err
91 | }
92 | }
93 |
94 | func (bgor *BatchGitObjectReader) makeObjectHashReader() ReadFunc {
95 | objectHashChan := make(chan ([]byte))
96 | resultsChan := make(chan (gitCatFileReadResult))
97 | go bgor.doCatFile(objectHashChan, resultsChan)
98 | return func(objectHash string) ([]byte, error) {
99 | objectHashChan <- []byte(objectHash)
100 | result := <-resultsChan
101 | return result.contents, result.err
102 | }
103 | }
104 |
105 | func (bgor *BatchGitObjectReader) doCatFile(gitExpressionChan chan ([]byte), resultsChan chan (gitCatFileReadResult)) {
106 | for {
107 | gitExpression := <-gitExpressionChan
108 | gitExpression = append(gitExpression, '\n')
109 | //Write file-path expression to process input
110 | bgor.inputWriter.Write(gitExpression)
111 | bgor.inputWriter.Flush()
112 |
113 | //Read line containing filesize from process output
114 | filesizeBytes, err := bgor.outputReader.ReadBytes('\n')
115 | if err != nil {
116 | logrus.Errorf("error reading filesize: %v", err)
117 | resultsChan <- gitCatFileReadResult{[]byte{}, err}
118 | continue
119 | }
120 | filesize, err := strconv.Atoi(string(filesizeBytes[:len(filesizeBytes)-1]))
121 | if err != nil {
122 | logrus.Errorf("error parsing filesize: %v", err)
123 | resultsChan <- gitCatFileReadResult{[]byte{}, err}
124 | continue
125 | }
126 | logrus.Debugf("Git Batch Reader: FilePath: %v, Size:%v", gitExpression, filesize)
127 |
128 | //Read file contents upto filesize bytes from process output
129 | fileBytes := make([]byte, filesize)
130 | n, err := io.ReadFull(bgor.outputReader, fileBytes)
131 | if n != filesize || err != nil {
132 | logrus.Errorf("error reading exactly %v bytes of %v: %v (read %v bytes)", filesize, gitExpression, err, n)
133 | resultsChan <- gitCatFileReadResult{[]byte{}, err}
134 | continue
135 | }
136 |
137 | //Read and discard trailing newline (not doing this will cause errors going forward)
138 | b, err := bgor.outputReader.ReadByte()
139 | if err != nil || b != '\n' {
140 | resultsChan <- gitCatFileReadResult{[]byte{}, fmt.Errorf("error discarding trailing newline : %v trailing byte: %v", err, b)}
141 | continue
142 | }
143 |
144 | resultsChan <- gitCatFileReadResult{fileBytes, nil}
145 | }
146 | }
147 |
--------------------------------------------------------------------------------
/gitrepo/pixel.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/thoughtworks/talisman/12fab7055d7e640d5dae43209bafc48bf5ef1fd7/gitrepo/pixel.jpg
--------------------------------------------------------------------------------
/global_install_scripts/add_to_talismanignore.bash:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -euo pipefail
3 | shopt -s extglob
4 |
5 | DEBUG=${DEBUG:-''}
6 |
7 | function run() {
8 | function echo_debug() {
9 | [[ -z "${DEBUG}" ]] && return
10 | echo -ne $(tput setaf 3) >&2
11 | echo "$1" >&2
12 | echo -ne $(tput sgr0) >&2
13 | }
14 | export -f echo_debug
15 |
16 | INSTALL_ORG_REPO=${INSTALL_ORG_REPO:-'thoughtworks/talisman'}
17 | SCRIPT_BASE="https://raw.githubusercontent.com/${INSTALL_ORG_REPO}/master/global_install_scripts"
18 |
19 | TEMP_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'talisman_uninstall')
20 | trap "rm -r ${TEMP_DIR}" EXIT
21 | chmod 0700 ${TEMP_DIR}
22 |
23 | ADD_AN_IGNORE_SCRIPT=${TEMP_DIR}/add_to_talismanignore_in_git_repo.bash
24 |
25 | function get_dependent_scripts() {
26 | echo_debug "getting ${SCRIPT_BASE}/add_to_talismanignore_in_git_repo.bash via curl"
27 | curl --silent "${SCRIPT_BASE}/add_to_talismanignore_in_git_repo.bash" >${ADD_AN_IGNORE_SCRIPT}
28 | chmod +x ${ADD_AN_IGNORE_SCRIPT}
29 | }
30 |
31 | get_dependent_scripts
32 |
33 | echo "Adding pattern to .talismanignore recursively in git repos"
34 | read -p "Please enter pattern to add to .talismanignore (enter to abort): " IGNORE_PATTERN
35 | [[ -n $IGNORE_PATTERN ]] || exit 1
36 |
37 | read -e -p "Please enter root directory to search for git repos (Default: ${HOME}): " SEARCH_ROOT
38 | SEARCH_ROOT=${SEARCH_ROOT:-$HOME}
39 | SEARCH_CMD="find"
40 | EXTRA_SEARCH_OPTS=""
41 | echo -e "\tSearching ${SEARCH_ROOT} for git repositories"
42 |
43 | SUDO_PREFIX=""
44 | if [[ "${SEARCH_ROOT}" == "/" ]]; then
45 | echo -e "\tPlease enter your password when prompted to enable script to search as root user:"
46 | SUDO_PREFIX="sudo"
47 | EXTRA_SEARCH_OPTS="-xdev \( -path '/private/var' -prune \) -o"
48 | fi
49 |
50 | CMD_STRING="${SUDO_PREFIX} ${SEARCH_CMD} ${SEARCH_ROOT} ${EXTRA_SEARCH_OPTS} -name .git -type d -exec ${ADD_AN_IGNORE_SCRIPT} {} ${IGNORE_PATTERN} \;"
51 | echo_debug "EXECUTING: ${CMD_STRING}"
52 | eval "${CMD_STRING}"
53 | }
54 |
55 | run $0 $@
56 |
--------------------------------------------------------------------------------
/global_install_scripts/add_to_talismanignore_in_git_repo.bash:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -euo pipefail
3 |
4 | GIT_REPO_DOT_GIT=$1
5 | IGNORE_PATTERN=$2
6 |
7 | function echo_debug() {
8 | [[ -z "${DEBUG}" ]] && return
9 | echo -ne $(tput setaf 3) >&2
10 | echo "$1" >&2
11 | echo -ne $(tput sgr0) >&2
12 | }
13 |
14 | function echo_success {
15 | echo -ne $(tput setaf 2)
16 | echo "$1" >&2
17 | echo -ne $(tput sgr0)
18 | }
19 |
20 | echo_debug "Adding $IGNORE_PATTERN to ${GIT_REPO_DOT_GIT}/../.talismanignore"
21 | echo $IGNORE_PATTERN >>${GIT_REPO_DOT_GIT}/../.talismanignore && echo_success "Added $IGNORE_PATTERN to ${GIT_REPO_DOT_GIT}/../.talismanignore"
22 |
--------------------------------------------------------------------------------
/global_install_scripts/remove_hooks.bash:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -euo pipefail
3 | shopt -s extglob
4 |
5 | DEBUG=${DEBUG:-''}
6 |
7 | declare HOOK_SCRIPT='pre-commit' # TODO: need ability to uninstall pre-push hook as well.
8 | if [[ $# -gt 0 && $1 =~ pre-push.* ]]; then
9 | HOOK_SCRIPT='pre-push'
10 | fi
11 |
12 | function run() {
13 | # Arguments: $1 = 'pre-commit' or 'pre-push'. whether to set talisman up as pre-commit or pre-push hook : TODO: not implemented yet
14 | # Environment variables:
15 | # DEBUG="any-non-emply-value": verbose output for debugging the script
16 | # INSTALL_ORG_REPO="..." : the github org/repo to install from (default thoughtworks/talisman)
17 | #
18 | # Download the script needed for uninstalling the repo level hooks
19 | # For each git repo found in the search root (default $HOME)
20 | # Run the repo level uninstall hook. This will remove the symlink from .git/hooks/pre- to the central $TALISMAN_SETUP_DIR
21 | # The script will only remove talisman hook, not a pre-commit.com script or some other non-talisman hook
22 | # Write exceptions to a file for manual action
23 | # Look in the uninstall_git_repo_hook.bash script for more details on what it does
24 | # Remove the talisman_hook_script in .git-template/hooks/pre-
25 | # Remove talisman binary and talisman_hook_script from $TALISMAN_SETUP_DIR ($HOME/.talisman/bin)
26 |
27 | function echo_error() {
28 | echo -ne $(tput setaf 1) >&2
29 | echo "$1" >&2
30 | echo -ne $(tput sgr0) >&2
31 | }
32 | export -f echo_error
33 |
34 | function echo_debug() {
35 | [[ -z "${DEBUG}" ]] && return
36 | echo -ne $(tput setaf 3) >&2
37 | echo "$1" >&2
38 | echo -ne $(tput sgr0) >&2
39 | }
40 | export -f echo_debug
41 |
42 | function echo_success {
43 | echo -ne $(tput setaf 2)
44 | echo "$1" >&2
45 | echo -ne $(tput sgr0)
46 | }
47 | export -f echo_success
48 |
49 | TALISMAN_SETUP_DIR="/usr/local/Cellar/talisman"
50 | TALISMAN_HOOK_SCRIPT_DIR=${HOME}/".talisman"
51 | TEMPLATE_DIR=$(git config --global init.templatedir) || true
52 | INSTALL_ORG_REPO=${INSTALL_ORG_REPO:-'thoughtworks/talisman'}
53 | SCRIPT_BASE="https://raw.githubusercontent.com/${INSTALL_ORG_REPO}/master/global_install_scripts"
54 |
55 | TEMP_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'talisman_uninstall')
56 | trap "rm -r ${TEMP_DIR}" EXIT
57 | chmod 0700 ${TEMP_DIR}
58 |
59 | DELETE_REPO_HOOK_SCRIPT=${TEMP_DIR}/uninstall_git_repo_hook.bash
60 | function get_dependent_scripts() {
61 | curl --silent "${SCRIPT_BASE}/uninstall_git_repo_hook.bash" >${DELETE_REPO_HOOK_SCRIPT}
62 | chmod +x ${DELETE_REPO_HOOK_SCRIPT}
63 | }
64 |
65 | function remove_git_talisman_hooks() {
66 | if [[ ! -x ${DELETE_REPO_HOOK_SCRIPT} ]]; then
67 | echo_error "Couldn't find executable script ${DELETE_REPO_HOOK_SCRIPT}"
68 | exit 1
69 | fi
70 |
71 | echo "Removing talisman hooks recursively in git repos"
72 | read -e -p "Please enter root directory to search for git repos (Default: ${HOME}): " SEARCH_ROOT
73 | SEARCH_ROOT=${SEARCH_ROOT:-$HOME}
74 | SEARCH_CMD="find"
75 | EXTRA_SEARCH_OPTS=""
76 | echo -e "\tSearching ${SEARCH_ROOT} for git repositories"
77 |
78 | SUDO_PREFIX=""
79 | if [[ "${SEARCH_ROOT}" == "/" ]]; then
80 | echo -e "\tPlease enter your password when prompted to enable script to search as root user:"
81 | SUDO_PREFIX="sudo"
82 | EXTRA_SEARCH_OPTS="-xdev \( -path '/private/var' -prune \) -o"
83 | fi
84 | EXCEPTIONS_FILE=${TEMP_DIR}/repos_with_multiple_hooks.paths
85 | touch ${EXCEPTIONS_FILE}
86 |
87 | TALISMAN_HOOK_SCRIPT_PATH=${TALISMAN_HOOK_SCRIPT_DIR}/talisman_hook_script
88 | CMD_STRING="${SUDO_PREFIX} ${SEARCH_CMD} ${SEARCH_ROOT} ${EXTRA_SEARCH_OPTS} -name .git -type d -exec ${DELETE_REPO_HOOK_SCRIPT} ${TALISMAN_HOOK_SCRIPT_PATH} ${EXCEPTIONS_FILE} {} ${HOOK_SCRIPT} \;"
89 | echo_debug "EXECUTING: ${CMD_STRING}"
90 | eval "${CMD_STRING}" || true
91 |
92 | NUMBER_OF_EXCEPTION_REPOS=$(cat ${EXCEPTIONS_FILE} | wc -l)
93 |
94 | if [ ${NUMBER_OF_EXCEPTION_REPOS} -gt 0 ]; then
95 | EXCEPTIONS_FILE_HOME_PATH="${HOME}/repos_to_remove_talisman_from.paths"
96 | mv ${EXCEPTIONS_FILE} ${EXCEPTIONS_FILE_HOME_PATH}
97 | echo_error ""
98 | echo_error "Please see ${EXCEPTIONS_FILE_HOME_PATH} for a list of repositories"
99 | echo_error "that talisman couldn't be automatically removed from"
100 | echo_error "This is likely because these repos are using pre-commit (https://pre-commit.com)"
101 | echo_error "Remove lines related to talisman from the .pre-commit-config.yaml manually"
102 | fi
103 | }
104 |
105 | get_dependent_scripts
106 | remove_git_talisman_hooks
107 |
108 | echo_debug "Removing talisman hooks from .git-template"
109 | echo_debug "${TEMPLATE_DIR}/hooks/${HOOK_SCRIPT}"
110 | if [[ -n $TEMPLATE_DIR && -e ${TEMPLATE_DIR}/hooks/${HOOK_SCRIPT} &&
111 | ${TALISMAN_HOOK_SCRIPT_DIR}/talisman_hook_script -ef ${TEMPLATE_DIR}/hooks/${HOOK_SCRIPT} ]]; then
112 | rm -f "${TEMPLATE_DIR}/hooks/${HOOK_SCRIPT}" &&
113 | echo_success "Removed ${HOOK_SCRIPT} from ${TEMPLATE_DIR}"
114 | fi
115 |
116 | echo_debug "Removing talisman hook script from ${TALISMAN_HOOK_SCRIPT_DIR}"
117 | rm -rf $TALISMAN_HOOK_SCRIPT_DIR &&
118 | echo_success "Removed talisman hook script from ${TALISMAN_HOOK_SCRIPT_DIR}"
119 |
120 | if [ -n "${TALISMAN_HOME:-}" ]; then
121 | echo_error "Pleasee run \"brew uninstall talisman\" to remove the talisman binary"
122 | echo "Please remember to remove TALISMAN_HOME from your environment variables"
123 | fi
124 | }
125 |
126 | run $0 $@
127 |
--------------------------------------------------------------------------------
/global_install_scripts/setup_talisman_hook_in_repo.bash:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -euo pipefail
3 |
4 | IFS=$'\n'
5 | function run() {
6 |
7 | TALISMAN_HOOK_SCRIPT_PATH=$1
8 | EXCEPTIONS_FILE=$2
9 | DOT_GIT_DIR=$3
10 | HOOK_SCRIPT=$4
11 |
12 | function echo_error() {
13 | echo -ne $(tput setaf 1) >&2
14 | echo "$1" >&2
15 | echo -ne $(tput sgr0) >&2
16 | }
17 |
18 | function echo_debug() {
19 | [[ -z "${DEBUG}" ]] && return
20 | echo -ne $(tput setaf 3) >&2
21 | echo "$1" >&2
22 | echo -ne $(tput sgr0) >&2
23 | }
24 |
25 | function echo_success {
26 | echo -ne $(tput setaf 2)
27 | echo "$1" >&2
28 | echo -ne $(tput sgr0)
29 | }
30 |
31 | REPO_HOOK_SCRIPT=${DOT_GIT_DIR}/hooks/${HOOK_SCRIPT}
32 | #check if a hook already exists
33 | if [ -e "${REPO_HOOK_SCRIPT}" ]; then
34 | #check if already hooked up to talisman
35 | if [ "${REPO_HOOK_SCRIPT}" -ef "${TALISMAN_HOOK_SCRIPT_PATH}" ]; then
36 | echo_success "Talisman already setup in ${REPO_HOOK_SCRIPT}"
37 | else
38 | if [ -e "${DOT_GIT_DIR}/../.pre-commit-config.yaml" ]; then
39 | echo_error "Pre-existing pre-commit.com hook detected in ${DOT_GIT_DIR}/hooks"
40 | fi
41 | echo ${DOT_GIT_DIR} | sed 's#/.git$##' >>${EXCEPTIONS_FILE}
42 | fi
43 | else
44 | echo "Setting up ${HOOK_SCRIPT} hook in ${DOT_GIT_DIR}/hooks"
45 | mkdir -p ${DOT_GIT_DIR}/hooks || (echo_error "Could not create hooks directory" && return)
46 | LN_FLAGS="-sf"
47 | [ -n "true" ] && LN_FLAGS="${LN_FLAGS}v"
48 | OS=$(uname -s)
49 | case $OS in
50 | "MINGW32_NT-10.0-WOW" | "MINGW64_NT-10.0")
51 | DOT_GIT_DIR_WIN=$(sed -e 's/\/\([a-z]\)\//\1:\\/' -e 's/\//\\/g' <<<"$DOT_GIT_DIR")
52 | TALISMAN_HOOK_SCRIPT_PATH_WIN=$(sed -e 's/\/\([a-z]\)\//\1:\\/' -e 's/\//\\/g' <<<"$TALISMAN_HOOK_SCRIPT_PATH")
53 | cmd <<<"mklink /H "${DOT_GIT_DIR_WIN}\\hooks\\${HOOK_SCRIPT}" "${TALISMAN_HOOK_SCRIPT_PATH_WIN}"" >/dev/null
54 | ;;
55 | *)
56 | ln ${LN_FLAGS} ${TALISMAN_HOOK_SCRIPT_PATH} ${DOT_GIT_DIR}/hooks/${HOOK_SCRIPT}
57 | ;;
58 | esac
59 |
60 | echo_success "DONE"
61 | fi
62 | }
63 |
64 | run $@
65 |
--------------------------------------------------------------------------------
/global_install_scripts/talisman_hook_script.bash:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | shopt -s extglob
3 |
4 | # set TALISMAN_DEBUG="some-non-empty-value" in the env to get verbose output when the hook or talisman is running
5 | function echo_debug() {
6 | MSG="$@"
7 | [[ -n "${TALISMAN_DEBUG}" ]] && echo "${MSG}"
8 | }
9 |
10 | function echo_warning() {
11 | echo -ne $(tput setaf 3) >&2
12 | echo "$1" >&2
13 | echo -ne $(tput sgr0) >&2
14 | }
15 |
16 | function echo_error() {
17 | echo -ne $(tput setaf 1) >&2
18 | echo "$1" >&2
19 | echo -ne $(tput sgr0) >&2
20 | }
21 |
22 | function echo_success() {
23 | echo -ne $(tput setaf 2)
24 | echo "$1" >&2
25 | echo -ne $(tput sgr0)
26 | }
27 |
28 | function toLower() {
29 | echo "$1" | awk '{print tolower($0)}'
30 | }
31 |
32 | declare HOOKNAME="pre-commit"
33 | NAME=$(basename $0)
34 | ORG_REPO=${ORG_REPO:-'thoughtworks/talisman'}
35 |
36 | # given the various symlinks, this script may be invoked as
37 | # 'pre-commit', 'pre-push', 'talisman_hook_script pre-commit' or 'talisman_hook_script pre-push'
38 | case "$NAME" in
39 | pre-commit* | pre-push*) HOOKNAME="${NAME}" ;;
40 | talisman_hook_script)
41 | if [[ $# -gt 0 && $1 =~ pre-push.* ]]; then
42 | HOOKNAME="pre-push"
43 | fi
44 | ;;
45 | *)
46 | echo "Unexpected invocation. Please check invocation name and parameters"
47 | exit 1
48 | ;;
49 | esac
50 |
51 | TALISMAN_UPGRADE_CONNECT_TIMEOUT=${TALISMAN_UPGRADE_CONNECT_TIMEOUT:-10}
52 | function check_and_upgrade_talisman_binary() {
53 | if [[ -n "${TALISMAN_HOME:-}" && "$TALISMAN_SKIP_UPGRADE" != "true" ]]; then
54 | LATEST_VERSION=$(curl --connect-timeout $TALISMAN_UPGRADE_CONNECT_TIMEOUT -Is https://github.com/${ORG_REPO}/releases/latest | grep -iE "^location:" | grep -o '[^/]\+$' | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+')
55 | CURRENT_VERSION=$(${TALISMAN_BINARY} --version | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+')
56 | if [ -z "$LATEST_VERSION" ]; then
57 | echo_warning "Failed to retrieve latest Talisman version, skipping update."
58 | elif [ "$LATEST_VERSION" != "$CURRENT_VERSION" ]; then
59 | echo ""
60 | echo_warning "Your version of Talisman is outdated. Updating Talisman to v${LATEST_VERSION}"
61 | curl --silent https://raw.githubusercontent.com/${ORG_REPO}/master/global_install_scripts/update_talisman.bash >/tmp/update_talisman.bash && /bin/bash /tmp/update_talisman.bash
62 | else
63 | echo_debug "Talisman version up-to-date, skipping update"
64 | fi
65 | fi
66 | }
67 |
68 | check_and_upgrade_talisman_binary
69 | # Here HOOKNAME should be either 'pre-commit' (default) or 'pre-push'
70 | echo_debug "Firing ${HOOKNAME} hook"
71 |
72 | # Don't run talisman checks in a git repo, if we find a .talisman_skip or .talisman_skip.pre- file in the repo
73 | if [[ -f .talisman_skip || -f .talisman_skip.${HOOKNAME} ]]; then
74 | echo_debug "Found skip file. Not performing checks"
75 | exit 0
76 | fi
77 |
78 | TALISMAN_DEBUG="$(toLower "${TALISMAN_DEBUG}")"
79 | DEBUG_OPTS=""
80 | [[ "${TALISMAN_DEBUG}" == "true" ]] && DEBUG_OPTS="-d"
81 |
82 | TALISMAN_INTERACTIVE="$(toLower "${TALISMAN_INTERACTIVE}")"
83 | INTERACTIVE=""
84 | if [ "${TALISMAN_INTERACTIVE}" == "true" ]; then
85 | INTERACTIVE="-i"
86 | [[ "${HOOKNAME}" == "pre-commit" ]] && exec to the central $TALISMAN_SETUP_DIR
21 | # The script will only remove talisman hook, not a pre-commit.com script or some other non-talisman hook
22 | # Write exceptions to a file for manual action
23 | # Look in the uninstall_git_repo_hook.bash script for more details on what it does
24 | # Remove the talisman_hook_script in .git-template/hooks/pre-
25 | # Remove talisman binary and talisman_hook_script from $TALISMAN_SETUP_DIR ($HOME/.talisman/bin)
26 |
27 | function echo_error() {
28 | echo -ne $(tput setaf 1) >&2
29 | echo "$1" >&2
30 | echo -ne $(tput sgr0) >&2
31 | }
32 | export -f echo_error
33 |
34 | function echo_debug() {
35 | [[ -z "${DEBUG}" ]] && return
36 | echo -ne $(tput setaf 3) >&2
37 | echo "$1" >&2
38 | echo -ne $(tput sgr0) >&2
39 | }
40 | export -f echo_debug
41 |
42 | function echo_success() {
43 | echo -ne $(tput setaf 2)
44 | echo "$1" >&2
45 | echo -ne $(tput sgr0)
46 | }
47 | export -f echo_success
48 |
49 | TALISMAN_SETUP_DIR=${HOME}/.talisman/bin
50 | TEMPLATE_DIR=$(git config --global init.templatedir) || true
51 | INSTALL_ORG_REPO=${INSTALL_ORG_REPO:-'thoughtworks/talisman'}
52 | SCRIPT_BASE="https://raw.githubusercontent.com/${INSTALL_ORG_REPO}/master/global_install_scripts"
53 |
54 | TEMP_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'talisman_uninstall')
55 | trap "rm -r ${TEMP_DIR}" EXIT
56 | chmod 0700 ${TEMP_DIR}
57 |
58 | DELETE_REPO_HOOK_SCRIPT=${TEMP_DIR}/uninstall_git_repo_hook.bash
59 | function get_dependent_scripts() {
60 | curl --silent "${SCRIPT_BASE}/uninstall_git_repo_hook.bash" >${DELETE_REPO_HOOK_SCRIPT}
61 | chmod +x ${DELETE_REPO_HOOK_SCRIPT}
62 | }
63 |
64 | function remove_git_talisman_hooks() {
65 | if [[ ! -x ${DELETE_REPO_HOOK_SCRIPT} ]]; then
66 | echo_error "Couldn't find executable script ${DELETE_REPO_HOOK_SCRIPT}"
67 | exit 1
68 | fi
69 |
70 | echo "Removing talisman hooks recursively in git repos"
71 | read -e -p "Please enter root directory to search for git repos (Default: ${HOME}): " SEARCH_ROOT
72 | SEARCH_ROOT=${SEARCH_ROOT:-$HOME}
73 | SEARCH_CMD="find"
74 | EXTRA_SEARCH_OPTS=""
75 | echo -e "\tSearching ${SEARCH_ROOT} for git repositories"
76 |
77 | SUDO_PREFIX=""
78 | if [[ "${SEARCH_ROOT}" == "/" ]]; then
79 | echo -e "\tPlease enter your password when prompted to enable script to search as root user:"
80 | SUDO_PREFIX="sudo"
81 | EXTRA_SEARCH_OPTS="-xdev \( -path '/private/var' -prune \) -o"
82 | fi
83 | EXCEPTIONS_FILE=${TEMP_DIR}/repos_with_multiple_hooks.paths
84 | touch ${EXCEPTIONS_FILE}
85 |
86 | TALISMAN_PATH=${TALISMAN_SETUP_DIR}/talisman_hook_script
87 | CMD_STRING="${SUDO_PREFIX} ${SEARCH_CMD} ${SEARCH_ROOT} ${EXTRA_SEARCH_OPTS} -name .git -type d -exec ${DELETE_REPO_HOOK_SCRIPT} ${TALISMAN_PATH} ${EXCEPTIONS_FILE} {} ${HOOK_SCRIPT} \;"
88 | echo_debug "EXECUTING: ${CMD_STRING}"
89 | eval "${CMD_STRING}" || true
90 |
91 | NUMBER_OF_EXCEPTION_REPOS=$(cat ${EXCEPTIONS_FILE} | wc -l)
92 |
93 | if [ ${NUMBER_OF_EXCEPTION_REPOS} -gt 0 ]; then
94 | EXCEPTIONS_FILE_HOME_PATH="${HOME}/repos_to_remove_talisman_from.paths"
95 | mv ${EXCEPTIONS_FILE} ${EXCEPTIONS_FILE_HOME_PATH}
96 | echo_error ""
97 | echo_error "Please see ${EXCEPTIONS_FILE_HOME_PATH} for a list of repositories"
98 | echo_error "that talisman couldn't be automatically removed from"
99 | echo_error "This is likely because these repos are using pre-commit (https://pre-commit.com)"
100 | echo_error "Remove lines related to talisman from the .pre-commit-config.yaml manually"
101 | fi
102 | }
103 |
104 | function remove_talisman_env_variables() {
105 | FILE_PATH="$1"
106 | if [ -f $FILE_PATH ] && grep -q "TALISMAN_HOME" $FILE_PATH; then
107 | sed -i'-talisman.bak' '/# >>> talisman >>>/,/# <<< talisman <<&2
6 | echo "$1" >&2
7 | echo -ne $(tput sgr0) >&2
8 | }
9 |
10 | function echo_debug() {
11 | [[ -z "${DEBUG}" ]] && return
12 | echo -ne $(tput setaf 3) >&2
13 | echo "$1" >&2
14 | echo -ne $(tput sgr0) >&2
15 | }
16 |
17 | function echo_success {
18 | echo -ne $(tput setaf 2)
19 | echo "$1" >&2
20 | echo -ne $(tput sgr0)
21 | }
22 |
23 | TALISMAN_PATH=$1
24 | EXCEPTIONS_FILE=$2
25 | DOT_GIT_DIR=$3
26 | HOOK_SCRIPT=$4
27 |
28 | REPO_HOOK_SCRIPT=${DOT_GIT_DIR}/hooks/${HOOK_SCRIPT}
29 | echo_debug "Processing hook: ${REPO_HOOK_SCRIPT}"
30 |
31 | if [[ ! -e "${REPO_HOOK_SCRIPT}" ]]; then
32 | echo_success "No ${REPO_HOOK_SCRIPT}, nothing to do"
33 | exit 0
34 | fi
35 |
36 | # remove script if it is symlinked to talisman
37 | if [ "${REPO_HOOK_SCRIPT}" -ef "${TALISMAN_PATH}" ]; then
38 | rm ${REPO_HOOK_SCRIPT} && echo_success "Removed ${REPO_HOOK_SCRIPT}"
39 | exit 0
40 | fi
41 |
42 | if [ -e "${DOT_GIT_DIR}/../.pre-commit-config.yaml" ]; then
43 | # check if the .pre-commit-config contains "talisman", if so ask them to remove it manually
44 | echo_error "Pre-existing pre-commit.com hook detected in ${DOT_GIT_DIR}/hooks"
45 | fi
46 | echo ${DOT_GIT_DIR} | sed 's#/.git$##' >>$EXCEPTIONS_FILE
47 | }
48 |
49 | run $@
50 |
--------------------------------------------------------------------------------
/global_install_scripts/update_talisman.bash:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -euo pipefail
3 | shopt -s extglob
4 |
5 | DEBUG=${DEBUG:-''}
6 | FORCE_DOWNLOAD=${FORCE_DOWNLOAD:-''}
7 | declare UPDATE_TYPE=""
8 | if [[ $# -gt 0 && $1 =~ talisman-binary.* ]]; then
9 | UPDATE_TYPE='talisman-binary'
10 | fi
11 |
12 | function run() {
13 |
14 | # Download appropriate (appropriate = based on OS and ARCH) talisman binary from github
15 | # Copy the talisman binary to $TALISMAN_SETUP_DIR ($HOME/.talisman/bin)
16 |
17 | declare TALISMAN_BINARY_NAME
18 |
19 | IFS=$'\n'
20 | VERSION=${VERSION:-'latest'}
21 | INSTALL_ORG_REPO=${INSTALL_ORG_REPO:-'thoughtworks/talisman'}
22 |
23 | TALISMAN_SETUP_DIR=${HOME}/.talisman/bin # location of central install: talisman binary and hook script
24 |
25 | TEMP_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'talisman_setup')
26 | #trap "rm -r ${TEMP_DIR}" EXIT
27 | chmod 0700 ${TEMP_DIR}
28 |
29 | function echo_error() {
30 | echo -ne $(tput setaf 1) >&2
31 | echo "$1" >&2
32 | echo -ne $(tput sgr0) >&2
33 | }
34 | export -f echo_error
35 |
36 | function echo_debug() {
37 | [[ -z "${DEBUG}" ]] && return
38 | echo -ne $(tput setaf 3) >&2
39 | echo "$1" >&2
40 | echo -ne $(tput sgr0) >&2
41 | }
42 | export -f echo_debug
43 |
44 | function echo_success() {
45 | echo -ne $(tput setaf 2)
46 | echo "$1" >&2
47 | echo -ne $(tput sgr0)
48 | }
49 | export -f echo_success
50 |
51 |
52 | function operating_system() {
53 | OS=$(uname -s)
54 | case $OS in
55 | "Linux")
56 | echo "linux"
57 | ;;
58 | "Darwin")
59 | echo "darwin"
60 | ;;
61 | MINGW32_NT-10.0-WOW*)
62 | echo "windows"
63 | ;;
64 | MINGW64_NT-10.0*)
65 | echo "windows"
66 | ;;
67 | MSYS_NT-*)
68 | echo "windows"
69 | ;;
70 | *)
71 | echo_error "Talisman currently only supports Windows, Linux and MacOS(darwin) systems."
72 | echo_error "If this is a problem for you, please open an issue: https://github.com/${INSTALL_ORG_REPO}/issues/new"
73 | exit $E_UNSUPPORTED_ARCH
74 | ;;
75 | esac
76 | }
77 |
78 | function set_talisman_binary_name() {
79 | # based on OS (linux/darwin) and ARCH(32/64 bit)
80 | echo_debug "Running set_talisman_binary_name"
81 | declare OS
82 | OS=$(operating_system)
83 | ARCH=$(uname -m)
84 | case $ARCH in
85 | "x86_64")
86 | OS="${OS}_amd64"
87 | ;;
88 | "i686" | "i386")
89 | OS="${OS}_386"
90 | ;;
91 | "arm64")
92 | OS="${OS}_arm64"
93 | ;;
94 | *)
95 | echo_error "Talisman currently only supports x86 and x86_64 architectures."
96 | echo_error "If this is a problem for you, please open an issue: https://github.com/${INSTALL_ORG_REPO}/issues/new"
97 | exit $E_UNSUPPORTED_ARCH
98 | ;;
99 | esac
100 |
101 | TALISMAN_BINARY_NAME="talisman_${OS}"
102 | if [[ $OS == *"windows"* ]]; then
103 | TALISMAN_BINARY_NAME="${TALISMAN_BINARY_NAME}.exe"
104 | fi
105 | }
106 |
107 | function download() {
108 | echo_debug "Running download()"
109 | OBJECT=$1
110 | DOWNLOAD_URL=$(curl -Ls https://api.github.com/repos/"$INSTALL_ORG_REPO"/releases/latest |
111 | grep download_url | awk '{print $2}' | tr -d '"' | grep "$OBJECT")
112 |
113 | echo_debug "Downloading ${OBJECT} from ${DOWNLOAD_URL}"
114 | curl --location --silent ${DOWNLOAD_URL} >${TEMP_DIR}/${OBJECT}
115 | }
116 |
117 | function verify_checksum() {
118 | FILE_NAME=$1
119 | CHECKSUM_FILE_NAME='checksums'
120 | echo_debug "Verifying checksum for ${FILE_NAME}"
121 | download ${CHECKSUM_FILE_NAME}
122 |
123 | pushd ${TEMP_DIR} >/dev/null 2>&1
124 | grep ${TALISMAN_BINARY_NAME} ${CHECKSUM_FILE_NAME} >${CHECKSUM_FILE_NAME}.single
125 |
126 | if ! command -v shasum &> /dev/null; then
127 | sha256sum -c ${CHECKSUM_FILE_NAME}.single
128 | else
129 | shasum -a 256 -c ${CHECKSUM_FILE_NAME}.single
130 | fi
131 | popd >/dev/null 2>&1
132 | echo_debug "Checksum verification successful!"
133 | echo
134 | }
135 |
136 | function download_talisman_binary() {
137 | #download talisman binary
138 | echo_debug "Running download_talisman_binary"
139 | download ${TALISMAN_BINARY_NAME}
140 | verify_checksum ${TALISMAN_BINARY_NAME}
141 | }
142 |
143 | function verify_binary_is_working() {
144 | chmod +x ${TEMP_DIR}/${TALISMAN_BINARY_NAME}
145 | if ! ${TEMP_DIR}/${TALISMAN_BINARY_NAME} --version; then
146 | echo_error "Binary is not working, SKIPPING UPGRADE, Please open issue on github with your OS name and version"
147 | exit 0
148 | fi
149 | }
150 |
151 | function download_talisman_hook_script() {
152 | echo_debug "Running download_talisman_hook_script"
153 | curl --silent https://raw.githubusercontent.com/${INSTALL_ORG_REPO}/master/global_install_scripts/talisman_hook_script.bash >${TEMP_DIR}/talisman_hook_script
154 | }
155 |
156 | function setup_talisman() {
157 | # copy talisman binary from TEMP folder to the central location
158 | rm -f ${TALISMAN_SETUP_DIR}/${TALISMAN_BINARY_NAME}
159 | cp ${TEMP_DIR}/${TALISMAN_BINARY_NAME} ${TALISMAN_SETUP_DIR}
160 | chmod +x ${TALISMAN_SETUP_DIR}/${TALISMAN_BINARY_NAME}
161 | echo_success "Talisman binary updated successfully!"
162 | }
163 |
164 | function setup_talisman_hook_script() {
165 | BINARY_PATH=${TALISMAN_SETUP_DIR}/${TALISMAN_BINARY_NAME}
166 | rm -f ${TALISMAN_SETUP_DIR}/talisman_hook_script
167 | sed "s@\${TALISMAN_BINARY}@$BINARY_PATH@g" ${TEMP_DIR}/talisman_hook_script >${TALISMAN_SETUP_DIR}/talisman_hook_script
168 | chmod +x ${TALISMAN_SETUP_DIR}/talisman_hook_script
169 | echo_success "Talisman hook script updated successfully!"
170 | }
171 |
172 | function set_talisman_env_variables_properly() {
173 | FILE_PATH="$1"
174 | if [ -f $FILE_PATH ] && grep -q "TALISMAN_HOME" $FILE_PATH; then
175 | if ! grep -q ">>> talisman >>>" $FILE_PATH; then
176 | sed -i'-talisman.bak' '/TALISMAN_HOME/d' $FILE_PATH
177 | echo -e "\n" >>${ENV_FILE}
178 | echo "# >>> talisman >>>" >>$FILE_PATH
179 | echo "# Below environment variables should not be modified unless you know what you are doing" >>$FILE_PATH
180 | echo "export TALISMAN_HOME=${TALISMAN_SETUP_DIR}" >>$FILE_PATH
181 | echo "alias talisman=\$TALISMAN_HOME/${TALISMAN_BINARY_NAME}" >>$FILE_PATH
182 | echo "# <<< talisman <<<" >>$FILE_PATH
183 | fi
184 | fi
185 | }
186 |
187 | set_talisman_binary_name
188 | echo "Downloading latest talisman binary..."
189 | download_talisman_binary
190 | verify_binary_is_working
191 | setup_talisman
192 | if [ -z "$UPDATE_TYPE" ]; then
193 | echo "Downloading latest talisman hook script..."
194 | download_talisman_hook_script
195 | setup_talisman_hook_script
196 | fi
197 | # Correcting talisman env variables if they are not in proper format
198 | if [ -n "${TALISMAN_HOME:-}" ]; then
199 | set_talisman_env_variables_properly ~/.bashrc
200 | set_talisman_env_variables_properly ~/.bash_profile
201 | set_talisman_env_variables_properly ~/.profile
202 | fi
203 |
204 | }
205 |
206 | run $0 $@
207 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module talisman
2 |
3 | require (
4 | github.com/AlecAivazis/survey/v2 v2.3.7
5 | github.com/bmatcuk/doublestar v1.3.4
6 | github.com/cheggaaa/pb/v3 v3.1.7
7 | github.com/common-nighthawk/go-figure v0.0.0-20200609044655-c4b36f998cf2
8 | github.com/drhodes/golorem v0.0.0-20160418191928-ecccc744c2d9
9 | github.com/fatih/color v1.18.0
10 | github.com/golang/mock v1.6.0
11 | github.com/olekukonko/tablewriter v0.0.5
12 | github.com/sirupsen/logrus v1.9.3
13 | github.com/spf13/afero v1.14.0
14 | github.com/spf13/pflag v1.0.6
15 | github.com/stretchr/testify v1.10.0
16 | gopkg.in/yaml.v2 v2.4.0
17 | )
18 |
19 | require (
20 | github.com/VividCortex/ewma v1.2.0 // indirect
21 | github.com/davecgh/go-spew v1.1.1 // indirect
22 | github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
23 | github.com/mattn/go-colorable v0.1.14 // indirect
24 | github.com/mattn/go-isatty v0.0.20 // indirect
25 | github.com/mattn/go-runewidth v0.0.16 // indirect
26 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
27 | github.com/pmezard/go-difflib v1.0.0 // indirect
28 | github.com/rivo/uniseg v0.4.7 // indirect
29 | golang.org/x/sys v0.30.0 // indirect
30 | golang.org/x/term v0.29.0 // indirect
31 | golang.org/x/text v0.23.0 // indirect
32 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
33 | gopkg.in/yaml.v3 v3.0.1 // indirect
34 | )
35 |
36 | go 1.23.0
37 |
38 | toolchain go1.24.2
39 |
--------------------------------------------------------------------------------
/install.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -euo pipefail
3 |
4 | declare BINARY_NAME
5 |
6 | E_UNSUPPORTED_ARCH=5
7 | CHECKSUM_FILE_NAME='checksums'
8 |
9 | DEBUG=${DEBUG:-''}
10 | VERSION=${VERSION:-'latest'}
11 | INSTALL_ORG_REPO=${INSTALL_ORG_REPO:-'thoughtworks/talisman'}
12 | INSTALL_LOCATION=${INSTALL_LOCATION:-'/usr/local/bin'}
13 |
14 | function echo_error() {
15 | echo -ne "$(tput setaf 1)" >&2
16 | echo "$1" >&2
17 | echo -ne "$(tput sgr0)" >&2
18 | }
19 |
20 | function echo_debug() {
21 | [[ -z "$DEBUG" ]] && return
22 | echo -ne "$(tput setaf 3)" >&2
23 | echo "$1" >&2
24 | echo -ne "$(tput sgr0)" >&2
25 | }
26 |
27 | function echo_success() {
28 | echo -ne "$(tput setaf 2)"
29 | echo "$1" >&2
30 | echo -ne "$(tput sgr0)"
31 | }
32 |
33 | function operating_system() {
34 | OS=$(uname -s)
35 | case $OS in
36 | "Linux")
37 | echo "linux"
38 | ;;
39 | "Darwin")
40 | echo "darwin"
41 | ;;
42 | MINGW32_NT-* | MINGW64_NT-* | MSYS_NT-*)
43 | echo "windows"
44 | ;;
45 | *)
46 | echo_error "Talisman currently only supports Windows, Linux, and MacOS (darwin) systems."
47 | echo_error "If this is a problem for you, please open an issue: https://github.com/$INSTALL_ORG_REPO/issues/new"
48 | exit $E_UNSUPPORTED_ARCH
49 | ;;
50 | esac
51 | }
52 |
53 | function architecture() {
54 | ARCH=$(uname -m)
55 | case $ARCH in
56 | "x86_64")
57 | echo "amd64"
58 | ;;
59 | "i686" | "i386")
60 | echo "386"
61 | ;;
62 | "arm64" | "aarch64")
63 | echo "arm64"
64 | ;;
65 | *)
66 | echo_error "Talisman currently only supports x86 and x86_64 and arm64 architectures."
67 | echo_error "If this is a problem for you, please open an issue: https://github.com/$INSTALL_ORG_REPO/issues/new"
68 | exit $E_UNSUPPORTED_ARCH
69 | ;;
70 | esac
71 | }
72 |
73 | function set_binary_name() {
74 | BINARY_NAME="talisman_$(operating_system)_$(architecture)"
75 | if [ "$(operating_system)" = "windows" ]; then
76 | BINARY_NAME="$BINARY_NAME.exe"
77 | fi
78 | echo_success "Selected $BINARY_NAME"
79 | }
80 |
81 | function download() {
82 | if [ "$VERSION" != "latest" ]; then
83 | VERSION="tags/$VERSION"
84 | fi
85 |
86 | ASSETS=$(curl -Ls https://api.github.com/repos/"$INSTALL_ORG_REPO"/releases/"$VERSION" |
87 | grep download_url | awk '{print $2}' | tr -d '"')
88 | BINARY_URL=$(echo "$ASSETS" | grep "$BINARY_NAME")
89 | CHECKSUM_URL=$(echo "$ASSETS" | grep $CHECKSUM_FILE_NAME)
90 | echo_debug "Downloading $BINARY_NAME and from $BINARY_URL"
91 | curl --location --silent "$BINARY_URL" >"$TEMP_DIR/$BINARY_NAME"
92 | echo_debug "Downloading $CHECKSUM_FILE_NAME and from $CHECKSUM_URL"
93 | curl --location --silent "$CHECKSUM_URL" >"$TEMP_DIR/$CHECKSUM_FILE_NAME"
94 | echo_success "Downloaded talisman binary and checksums"
95 | }
96 |
97 | function verify_checksum() {
98 | pushd "$TEMP_DIR" >/dev/null 2>&1
99 |
100 | if ! command -v shasum &>/dev/null; then
101 | sha256sum --ignore-missing -c $CHECKSUM_FILE_NAME
102 | else
103 | shasum -a 256 --ignore-missing -c $CHECKSUM_FILE_NAME
104 | fi
105 |
106 | popd >/dev/null 2>&1
107 | echo_success "Checksum OK"
108 | }
109 |
110 | function install() {
111 | if (touch "$INSTALL_LOCATION/$BINARY_NAME" &>/dev/null); then
112 | cp "$TEMP_DIR/$BINARY_NAME" "$INSTALL_LOCATION/$BINARY_NAME"
113 | chmod +x "$INSTALL_LOCATION/$BINARY_NAME"
114 | ln -s "$INSTALL_LOCATION/$BINARY_NAME" "$INSTALL_LOCATION/talisman"
115 | elif (which sudo &>/dev/null); then
116 | sudo cp "$TEMP_DIR/$BINARY_NAME" "$INSTALL_LOCATION/$BINARY_NAME"
117 | sudo chmod +x "$INSTALL_LOCATION/$BINARY_NAME"
118 | sudo ln -s "$INSTALL_LOCATION/$BINARY_NAME" "$INSTALL_LOCATION/talisman"
119 | else
120 | echo_error "Insufficient permission to install to $INSTALL_LOCATION"
121 | exit 126
122 | fi
123 | }
124 |
125 | function run() {
126 | TEMP_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'talisman_setup')
127 | # shellcheck disable=SC2064
128 | trap "rm -r $TEMP_DIR" EXIT
129 | chmod 0700 "$TEMP_DIR"
130 |
131 | if [ ! -d "$INSTALL_LOCATION" ]; then
132 | echo_error "$INSTALL_LOCATION is not a directory!"
133 | exit 1
134 | fi
135 |
136 | set_binary_name
137 | download
138 | verify_checksum
139 | install
140 | }
141 |
142 | run
143 |
--------------------------------------------------------------------------------
/internal/mock/checksumcalculator/checksumcalculator.go:
--------------------------------------------------------------------------------
1 | // Code generated by MockGen. DO NOT EDIT.
2 | // Source: checksumcalculator/checksumcalculator.go
3 |
4 | // Package mock is a generated GoMock package.
5 | package mock
6 |
7 | import (
8 | reflect "reflect"
9 |
10 | gomock "github.com/golang/mock/gomock"
11 | )
12 |
13 | // MockChecksumCalculator is a mock of ChecksumCalculator interface.
14 | type MockChecksumCalculator struct {
15 | ctrl *gomock.Controller
16 | recorder *MockChecksumCalculatorMockRecorder
17 | }
18 |
19 | // MockChecksumCalculatorMockRecorder is the mock recorder for MockChecksumCalculator.
20 | type MockChecksumCalculatorMockRecorder struct {
21 | mock *MockChecksumCalculator
22 | }
23 |
24 | // NewMockChecksumCalculator creates a new mock instance.
25 | func NewMockChecksumCalculator(ctrl *gomock.Controller) *MockChecksumCalculator {
26 | mock := &MockChecksumCalculator{ctrl: ctrl}
27 | mock.recorder = &MockChecksumCalculatorMockRecorder{mock}
28 | return mock
29 | }
30 |
31 | // EXPECT returns an object that allows the caller to indicate expected use.
32 | func (m *MockChecksumCalculator) EXPECT() *MockChecksumCalculatorMockRecorder {
33 | return m.recorder
34 | }
35 |
36 | // CalculateCollectiveChecksumForPattern mocks base method.
37 | func (m *MockChecksumCalculator) CalculateCollectiveChecksumForPattern(fileNamePattern string) string {
38 | m.ctrl.T.Helper()
39 | ret := m.ctrl.Call(m, "CalculateCollectiveChecksumForPattern", fileNamePattern)
40 | ret0, _ := ret[0].(string)
41 | return ret0
42 | }
43 |
44 | // CalculateCollectiveChecksumForPattern indicates an expected call of CalculateCollectiveChecksumForPattern.
45 | func (mr *MockChecksumCalculatorMockRecorder) CalculateCollectiveChecksumForPattern(fileNamePattern interface{}) *gomock.Call {
46 | mr.mock.ctrl.T.Helper()
47 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CalculateCollectiveChecksumForPattern", reflect.TypeOf((*MockChecksumCalculator)(nil).CalculateCollectiveChecksumForPattern), fileNamePattern)
48 | }
49 |
50 | // SuggestTalismanRC mocks base method.
51 | func (m *MockChecksumCalculator) SuggestTalismanRC(fileNamePatterns []string) string {
52 | m.ctrl.T.Helper()
53 | ret := m.ctrl.Call(m, "SuggestTalismanRC", fileNamePatterns)
54 | ret0, _ := ret[0].(string)
55 | return ret0
56 | }
57 |
58 | // SuggestTalismanRC indicates an expected call of SuggestTalismanRC.
59 | func (mr *MockChecksumCalculatorMockRecorder) SuggestTalismanRC(fileNamePatterns interface{}) *gomock.Call {
60 | mr.mock.ctrl.T.Helper()
61 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SuggestTalismanRC", reflect.TypeOf((*MockChecksumCalculator)(nil).SuggestTalismanRC), fileNamePatterns)
62 | }
63 |
--------------------------------------------------------------------------------
/internal/mock/prompt/survey.go:
--------------------------------------------------------------------------------
1 | // Code generated by MockGen. DO NOT EDIT.
2 | // Source: prompt/survey.go
3 |
4 | // Package mock is a generated GoMock package.
5 | package mock
6 |
7 | import (
8 | reflect "reflect"
9 |
10 | gomock "github.com/golang/mock/gomock"
11 | )
12 |
13 | // MockPrompt is a mock of Prompt interface.
14 | type MockPrompt struct {
15 | ctrl *gomock.Controller
16 | recorder *MockPromptMockRecorder
17 | }
18 |
19 | // MockPromptMockRecorder is the mock recorder for MockPrompt.
20 | type MockPromptMockRecorder struct {
21 | mock *MockPrompt
22 | }
23 |
24 | // NewMockPrompt creates a new mock instance.
25 | func NewMockPrompt(ctrl *gomock.Controller) *MockPrompt {
26 | mock := &MockPrompt{ctrl: ctrl}
27 | mock.recorder = &MockPromptMockRecorder{mock}
28 | return mock
29 | }
30 |
31 | // EXPECT returns an object that allows the caller to indicate expected use.
32 | func (m *MockPrompt) EXPECT() *MockPromptMockRecorder {
33 | return m.recorder
34 | }
35 |
36 | // Confirm mocks base method.
37 | func (m *MockPrompt) Confirm(arg0 string) bool {
38 | m.ctrl.T.Helper()
39 | ret := m.ctrl.Call(m, "Confirm", arg0)
40 | ret0, _ := ret[0].(bool)
41 | return ret0
42 | }
43 |
44 | // Confirm indicates an expected call of Confirm.
45 | func (mr *MockPromptMockRecorder) Confirm(arg0 interface{}) *gomock.Call {
46 | mr.mock.ctrl.T.Helper()
47 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Confirm", reflect.TypeOf((*MockPrompt)(nil).Confirm), arg0)
48 | }
49 |
--------------------------------------------------------------------------------
/internal/mock/utility/sha_256_hasher.go:
--------------------------------------------------------------------------------
1 | // Code generated by MockGen. DO NOT EDIT.
2 | // Source: utility/sha_256_hasher.go
3 |
4 | // Package mock is a generated GoMock package.
5 | package mock
6 |
7 | import (
8 | reflect "reflect"
9 |
10 | gomock "github.com/golang/mock/gomock"
11 | )
12 |
13 | // MockSHA256Hasher is a mock of SHA256Hasher interface.
14 | type MockSHA256Hasher struct {
15 | ctrl *gomock.Controller
16 | recorder *MockSHA256HasherMockRecorder
17 | }
18 |
19 | // MockSHA256HasherMockRecorder is the mock recorder for MockSHA256Hasher.
20 | type MockSHA256HasherMockRecorder struct {
21 | mock *MockSHA256Hasher
22 | }
23 |
24 | // NewMockSHA256Hasher creates a new mock instance.
25 | func NewMockSHA256Hasher(ctrl *gomock.Controller) *MockSHA256Hasher {
26 | mock := &MockSHA256Hasher{ctrl: ctrl}
27 | mock.recorder = &MockSHA256HasherMockRecorder{mock}
28 | return mock
29 | }
30 |
31 | // EXPECT returns an object that allows the caller to indicate expected use.
32 | func (m *MockSHA256Hasher) EXPECT() *MockSHA256HasherMockRecorder {
33 | return m.recorder
34 | }
35 |
36 | // CollectiveSHA256Hash mocks base method.
37 | func (m *MockSHA256Hasher) CollectiveSHA256Hash(paths []string) string {
38 | m.ctrl.T.Helper()
39 | ret := m.ctrl.Call(m, "CollectiveSHA256Hash", paths)
40 | ret0, _ := ret[0].(string)
41 | return ret0
42 | }
43 |
44 | // CollectiveSHA256Hash indicates an expected call of CollectiveSHA256Hash.
45 | func (mr *MockSHA256HasherMockRecorder) CollectiveSHA256Hash(paths interface{}) *gomock.Call {
46 | mr.mock.ctrl.T.Helper()
47 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CollectiveSHA256Hash", reflect.TypeOf((*MockSHA256Hasher)(nil).CollectiveSHA256Hash), paths)
48 | }
49 |
50 | // Shutdown mocks base method.
51 | func (m *MockSHA256Hasher) Shutdown() error {
52 | m.ctrl.T.Helper()
53 | ret := m.ctrl.Call(m, "Shutdown")
54 | ret0, _ := ret[0].(error)
55 | return ret0
56 | }
57 |
58 | // Shutdown indicates an expected call of Shutdown.
59 | func (mr *MockSHA256HasherMockRecorder) Shutdown() *gomock.Call {
60 | mr.mock.ctrl.T.Helper()
61 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Shutdown", reflect.TypeOf((*MockSHA256Hasher)(nil).Shutdown))
62 | }
63 |
64 | // Start mocks base method.
65 | func (m *MockSHA256Hasher) Start() error {
66 | m.ctrl.T.Helper()
67 | ret := m.ctrl.Call(m, "Start")
68 | ret0, _ := ret[0].(error)
69 | return ret0
70 | }
71 |
72 | // Start indicates an expected call of Start.
73 | func (mr *MockSHA256HasherMockRecorder) Start() *gomock.Call {
74 | mr.mock.ctrl.T.Helper()
75 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockSHA256Hasher)(nil).Start))
76 | }
77 |
--------------------------------------------------------------------------------
/mock_scripts:
--------------------------------------------------------------------------------
1 | #! /usr/bin/env bash
2 |
3 | install_mockgen() {
4 | pushd ..
5 | go install github.com/golang/mock/mockgen@latest
6 | popd
7 | }
8 |
9 | generate_mock() {
10 | FILE=$1
11 | mockgen -destination internal/mock/$FILE -package mock -source $FILE
12 | }
13 |
14 | set -ex
15 | #Ensure mockgen exists
16 | which mockgen || install_mockgen
17 | mkdir -p internal/mock
18 |
19 | # Mock for survey package
20 |
21 | generate_mock prompt/survey.go
22 | generate_mock checksumcalculator/checksumcalculator.go
23 | generate_mock utility/sha_256_hasher.go
--------------------------------------------------------------------------------
/prompt/survey.go:
--------------------------------------------------------------------------------
1 | package prompt
2 |
3 | //go:generate mockgen -destination ../internal/mock/prompt/survey.go -package mock -source survey.go
4 |
5 | import (
6 | "github.com/AlecAivazis/survey/v2"
7 | "log"
8 | )
9 |
10 | type Prompt interface {
11 | Confirm(string) bool
12 | }
13 |
14 | func NewPrompt() Prompt {
15 | return prompt{}
16 | }
17 |
18 | type prompt struct{}
19 |
20 | func NewPromptContext(interactive bool, prompt Prompt) PromptContext {
21 | return PromptContext{
22 | Interactive: interactive,
23 | Prompt: prompt,
24 | }
25 | }
26 |
27 | type PromptContext struct {
28 | Interactive bool
29 | Prompt Prompt
30 | }
31 |
32 | func (p prompt) Confirm(message string) bool {
33 | if message == "" {
34 | message = "Confirm?"
35 | }
36 |
37 | confirmPrompt := &survey.Confirm{
38 | Default: false,
39 | Message: message,
40 | }
41 |
42 | confirmation := false
43 | err := survey.AskOne(confirmPrompt, &confirmation)
44 | if err != nil {
45 | log.Printf("error occurred when getting input from user: %s", err)
46 | return false
47 | }
48 |
49 | return confirmation
50 | }
51 |
--------------------------------------------------------------------------------
/report/report.go:
--------------------------------------------------------------------------------
1 | package report
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "log"
7 | "os"
8 | "os/user"
9 | "path/filepath"
10 | "talisman/detector/helpers"
11 | "talisman/utility"
12 |
13 | "github.com/fatih/color"
14 | )
15 |
16 | const jsonFileName string = "report.json"
17 | const htmlReportDir string = "talisman_html_report"
18 | const jsonReportDir string = "talisman_html_report"
19 |
20 | // GenerateReport generates a talisman scan report in html format
21 | func GenerateReport(r *helpers.DetectionResults, directory string) (string, error) {
22 | var jsonFilePath string
23 | var homeDir string
24 | var baseReportDirPath string
25 |
26 | usr, err := user.Current()
27 | if err != nil {
28 | return "", fmt.Errorf("error getting current user: %v", err.Error())
29 | }
30 | homeDir = usr.HomeDir
31 | path := jsonReportDir
32 | if directory == htmlReportDir {
33 | path = directory
34 | baseReportDirPath = filepath.Join(homeDir, ".talisman", htmlReportDir)
35 | jsonFilePath = filepath.Join(path, "/data", jsonFileName)
36 | err = utility.Dir(baseReportDirPath, htmlReportDir)
37 | if err != nil {
38 | generateErrorMsg()
39 | return "", fmt.Errorf("error copying reports: %v", err)
40 | }
41 | } else {
42 | path = filepath.Join(directory, "talisman_reports")
43 | _ = os.RemoveAll(path)
44 | path = filepath.Join(path, "data")
45 | jsonFilePath = filepath.Join(path, jsonFileName)
46 | }
47 |
48 | err = os.MkdirAll(path, 0755)
49 | if err != nil {
50 | return "", fmt.Errorf("error creating path %s: %v", path, err)
51 | }
52 |
53 | _, err = generateAndWriteToFile(r, jsonFilePath)
54 | if err != nil {
55 | return "", err
56 | }
57 | return path, nil
58 | }
59 |
60 | func generateAndWriteToFile(r *helpers.DetectionResults, jsonFilePath string) (path string, err error) {
61 | jsonFile, err := os.Create(jsonFilePath)
62 | defer func() {
63 | if err = jsonFile.Close(); err != nil {
64 | err = fmt.Errorf("error closing file %s: %v %#v", jsonFilePath, err, err)
65 | }
66 | }()
67 |
68 | if err != nil {
69 | return "", fmt.Errorf("error creating file %s: %v", jsonFilePath, err)
70 | }
71 |
72 | jsonString, err := json.Marshal(r)
73 | if err != nil {
74 | return "", fmt.Errorf("error while rendering report json: %v : %#v", err, err)
75 | }
76 | _, err = jsonFile.Write(jsonString)
77 | if err != nil {
78 | return "", fmt.Errorf("error while writing report json to file: %v %#v", err, err)
79 | }
80 | return jsonFilePath, nil
81 | }
82 |
83 | func generateErrorMsg() {
84 | color.HiMagenta("\nLooks like you are using 'talisman --scanWithHtml' for scanning.")
85 | color.HiMagenta("But it appears that you have not installed Talisman Html Report")
86 | color.HiMagenta("Please go through Talisman Readme and make sure you install the same from:")
87 | color.Yellow("\nhttps://github.com/jaydeepc/talisman-html-report")
88 | color.Cyan("\nOR use 'talisman --scan' if you want the JSON report alone\n")
89 | fmt.Printf("\n")
90 | color.Red("Failed: Unable to perform Scan")
91 | fmt.Printf("\n")
92 | log.Fatalln("Run Status: Failed")
93 | }
94 |
--------------------------------------------------------------------------------
/scanner/scanner.go:
--------------------------------------------------------------------------------
1 | package scanner
2 |
3 | import (
4 | "log"
5 | "os"
6 | "os/exec"
7 | "strings"
8 | "talisman/gitrepo"
9 | "talisman/utility"
10 |
11 | "github.com/sirupsen/logrus"
12 | )
13 |
14 | type blobDetails struct {
15 | hash, filePath string
16 | }
17 |
18 | // BlobsInCommits is a map of blob and list of the commits the blobs is present in.
19 | type BlobsInCommits struct {
20 | commits map[blobDetails][]string
21 | }
22 |
23 | // GetAdditions will get all the additions for entire git history
24 | func GetAdditions(ignoreHistory bool, br gitrepo.BatchReader) []gitrepo.Addition {
25 | blobsInCommits := getBlobsInCommit(ignoreHistory)
26 | var additions []gitrepo.Addition
27 | err := br.Start()
28 | if err != nil {
29 | logrus.Errorf("error creating file reader %v", err)
30 | }
31 | defer func() {
32 | err = br.Shutdown()
33 | if err != nil {
34 | logrus.Errorf("error creating file reader %v", err)
35 | }
36 | }()
37 |
38 | for blob := range blobsInCommits.commits {
39 | contents, _ := br.Read(blob.hash)
40 | newAddition := gitrepo.NewScannerAddition(blob.filePath, blobsInCommits.commits[blob], contents)
41 | additions = append(additions, newAddition)
42 | }
43 |
44 | return additions
45 | }
46 |
47 | func getBlobsInCommit(ignoreHistory bool) BlobsInCommits {
48 | progressBar := utility.GetProgressBar(os.Stdout, "Talisman Fetch Blobs")
49 | commits := getAllCommits(ignoreHistory)
50 | progressBar.Start(len(commits) - 1)
51 | blobsInCommits := newBlobsInCommit()
52 | result := make(chan []string, len(commits))
53 | for _, commit := range commits {
54 | go putBlobsInChannel(commit, result)
55 | }
56 | for i := 1; i < len(commits); i++ {
57 | progressBar.Increment()
58 | getBlobsFromChannel(blobsInCommits, result)
59 | }
60 | progressBar.Finish()
61 | return blobsInCommits
62 | }
63 |
64 | func putBlobsInChannel(commit string, result chan []string) {
65 | if commit != "" {
66 | blobDetailsBytes, _ := exec.Command("git", "ls-tree", "-r", commit).CombinedOutput()
67 | blobDetailsList := strings.Split(string(blobDetailsBytes), "\n")
68 | blobDetailsList = append(blobDetailsList, commit)
69 | result <- blobDetailsList
70 | }
71 | }
72 |
73 | func getBlobsFromChannel(blobsInCommits BlobsInCommits, result chan []string) {
74 | blobEntries := <-result
75 | commit := blobEntries[len(blobEntries)-1]
76 | for _, blobEntry := range blobEntries[:len(blobEntries)-1] {
77 | if blobEntry != "" {
78 | blobHashAndName := strings.Split(strings.Split(blobEntry, " ")[2], "\t")
79 | blob := blobDetails{hash: blobHashAndName[0], filePath: blobHashAndName[1]}
80 | blobsInCommits.commits[blob] = append(blobsInCommits.commits[blob], commit)
81 | }
82 | }
83 | }
84 |
85 | func getAllCommits(ignoreHistory bool) []string {
86 | commitRange := "--all"
87 | if ignoreHistory {
88 | commitRange = "--max-count=1"
89 | }
90 | out, err := exec.Command("git", "log", commitRange, "--pretty=%H").CombinedOutput()
91 | if err != nil {
92 | log.Fatal(err)
93 | }
94 | return strings.Split(string(out), "\n")
95 | }
96 |
97 | func newBlobsInCommit() BlobsInCommits {
98 | commits := make(map[blobDetails][]string)
99 | return BlobsInCommits{commits: commits}
100 | }
101 |
--------------------------------------------------------------------------------
/scanner/scanner_test.go:
--------------------------------------------------------------------------------
1 | package scanner
2 |
3 | import (
4 | "io/ioutil"
5 | "testing"
6 |
7 | logr "github.com/sirupsen/logrus"
8 | "github.com/stretchr/testify/assert"
9 | )
10 |
11 | func init() {
12 | logr.SetOutput(ioutil.Discard)
13 | }
14 |
15 | func Test_getBlobsFromChannel(t *testing.T) {
16 | ch := make(chan []string)
17 | go func() {
18 | ch <- []string{
19 | "100644 blob 351324aa7b3c66043e484c2f2c7b7f1842152f35 .gitignore",
20 | "100644 blob 8715df9907604c8ee8fc5e377821817f84f014fa .pre-commit-hooks.yaml",
21 | "commitSha",
22 | }
23 | }()
24 | blobsInCommits := BlobsInCommits{commits: map[blobDetails][]string{}}
25 | getBlobsFromChannel(blobsInCommits, ch)
26 |
27 | commits := blobsInCommits.commits
28 | assert.Len(t, commits, 2)
29 | assert.Equal(t, []string{"commitSha"}, commits[blobDetails{"351324aa7b3c66043e484c2f2c7b7f1842152f35", ".gitignore"}])
30 | assert.Equal(t, []string{"commitSha"}, commits[blobDetails{"8715df9907604c8ee8fc5e377821817f84f014fa", ".pre-commit-hooks.yaml"}])
31 | }
32 |
--------------------------------------------------------------------------------
/talisman.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/thoughtworks/talisman/12fab7055d7e640d5dae43209bafc48bf5ef1fd7/talisman.png
--------------------------------------------------------------------------------
/talismanrc/rc_file.go:
--------------------------------------------------------------------------------
1 | package talismanrc
2 |
3 | import (
4 | "fmt"
5 |
6 | logr "github.com/sirupsen/logrus"
7 |
8 | "github.com/spf13/afero"
9 | "gopkg.in/yaml.v2"
10 | )
11 |
12 | const (
13 | // RCFileName represents the name of default file in which all the ignore patterns are configured in new version
14 | RCFileName = ".talismanrc"
15 | DefaultRCVersion = "1.0"
16 | )
17 |
18 | var (
19 | fs = afero.NewOsFs()
20 | )
21 |
22 | // Load creates a TalismanRC struct based on a .talismanrc file, if present
23 | func Load() (*TalismanRC, error) {
24 | fileContents, err := afero.ReadFile(fs, RCFileName)
25 | if err != nil {
26 | // File does not exist or is not readable, proceed as if there is no .talismanrc
27 | fileContents = []byte{}
28 | }
29 | return talismanRCFromYaml(fileContents)
30 | }
31 |
32 | func talismanRCFromYaml(fileContents []byte) (*TalismanRC, error) {
33 | talismanRCFromFile := TalismanRC{}
34 | err := yaml.Unmarshal(fileContents, &talismanRCFromFile)
35 | if err != nil {
36 | logr.Errorf("Unable to parse .talismanrc : %v", err)
37 | fmt.Println(fmt.Errorf("\n\x1b[1m\x1b[31mUnable to parse .talismanrc %s. Please ensure it is following the right YAML structure\x1b[0m\x1b[0m", err))
38 | return &TalismanRC{}, err
39 | }
40 | if talismanRCFromFile.Version == "" {
41 | talismanRCFromFile.Version = DefaultRCVersion
42 | }
43 | return &talismanRCFromFile, nil
44 | }
45 |
46 | func (tRC *TalismanRC) saveToFile() {
47 | ignoreEntries, _ := yaml.Marshal(&tRC)
48 | err := afero.WriteFile(fs, RCFileName, ignoreEntries, 0644)
49 | if err != nil {
50 | logr.Errorf("error writing to %s: %s", RCFileName, err)
51 | }
52 | }
53 |
54 | func SetFs__(_fs afero.Fs) {
55 | fs = _fs
56 | }
57 |
--------------------------------------------------------------------------------
/talismanrc/rc_file_test.go:
--------------------------------------------------------------------------------
1 | package talismanrc
2 |
3 | import (
4 | "regexp"
5 | "talisman/detector/severity"
6 | "testing"
7 |
8 | "github.com/spf13/afero"
9 | "github.com/stretchr/testify/assert"
10 | )
11 |
12 | func TestLoadingFromFile(t *testing.T) {
13 | fs := afero.NewMemMapFs()
14 | SetFs__(fs)
15 |
16 | t.Run("Creates an empty TalismanRC if .talismanrc file doesn't exist", func(t *testing.T) {
17 | talismanRCFileExists, _ := afero.Exists(fs, RCFileName)
18 | assert.False(t, talismanRCFileExists, ".talismanrc file should NOT exist for this test!")
19 | emptyRC, err := Load()
20 | assert.NoError(t, err, "Should not error if there is a problem reading the file")
21 | assert.Equal(t, &TalismanRC{Version: DefaultRCVersion}, emptyRC)
22 | })
23 |
24 | t.Run("Loads all valid TalismanRC fields", func(t *testing.T) {
25 | err := afero.WriteFile(fs, RCFileName, []byte(fullyConfiguredTalismanRC), 0666)
26 | assert.NoError(t, err, "Problem setting up test .talismanrc?")
27 |
28 | talismanRCFromFile, _ := Load()
29 | expectedTalismanRC := &TalismanRC{
30 | FileIgnoreConfig: []FileIgnoreConfig{
31 | {FileName: "existing.pem", Checksum: "123444ddssa75333b25b6275f97680604add51b84eb8f4a3b9dcbbc652e6f27ac"}},
32 | ScopeConfig: []ScopeConfig{{"go"}},
33 | AllowedPatterns: []*Pattern{
34 | {regexp.MustCompile("this-is-okay")},
35 | {regexp.MustCompile("key={listOfThings.id}")}},
36 | CustomPatterns: []PatternString{"this-isn't-okay"},
37 | Threshold: severity.Medium,
38 | CustomSeverities: []CustomSeverityConfig{
39 | {Detector: "HexContent", Severity: severity.Low}},
40 | Experimental: ExperimentalConfig{Base64EntropyThreshold: 4.7},
41 | Version: "1.0",
42 | }
43 | assert.Equal(t, expectedTalismanRC, talismanRCFromFile)
44 | })
45 | }
46 |
47 | func TestWritingToFile(t *testing.T) {
48 | tRC := &TalismanRC{Version: DefaultRCVersion}
49 | fs := afero.NewMemMapFs()
50 | SetFs__(fs)
51 |
52 | t.Run("When there is no .talismanrc file", func(t *testing.T) {
53 | talismanRCFileExists, _ := afero.Exists(fs, RCFileName)
54 | assert.False(t, talismanRCFileExists, "Problem setting up tests")
55 | tRC.saveToFile()
56 | talismanRCFileExists, _ = afero.Exists(fs, RCFileName)
57 | assert.True(t, talismanRCFileExists, "Should have created a new .talismanrc file")
58 | fileContents, _ := afero.ReadFile(fs, RCFileName)
59 | assert.Equal(t, "version: \"1.0\"\n", string(fileContents))
60 | })
61 |
62 | t.Run("When there already is a .talismanrc file", func(t *testing.T) {
63 | err := afero.WriteFile(fs, RCFileName, []byte("Some existing content to overwrite"), 0666)
64 | assert.NoError(t, err, "Problem setting up tests")
65 | tRC.saveToFile()
66 | talismanRCFileExists, _ := afero.Exists(fs, RCFileName)
67 | assert.True(t, talismanRCFileExists, "Should have created a new .talismanrc file")
68 | fileContents, _ := afero.ReadFile(fs, RCFileName)
69 | assert.Equal(t, "version: \"1.0\"\n", string(fileContents))
70 | })
71 | }
72 |
73 | func TestUnmarshalsValidYaml(t *testing.T) {
74 | t.Run("Should not fail as long as the yaml structure is correct", func(t *testing.T) {
75 | fileContents := []byte(`
76 | ---
77 | fileignoreconfig:
78 | - filename: testfile_1.yml
79 | checksum: file1_checksum
80 | custom_patterns:
81 | - 'pwd_[a-z]{8,20}'`)
82 |
83 | rc, err := talismanRCFromYaml(fileContents)
84 | assert.Nil(t, err, "Should successfully unmarshal valid yaml")
85 | assert.Equal(t, 1, len(rc.FileIgnoreConfig))
86 | assert.Equal(t, 1, len(rc.CustomPatterns))
87 | })
88 |
89 | t.Run("Should read multiple entries in rc file correctly", func(t *testing.T) {
90 | fileContent := []byte(`
91 | fileignoreconfig:
92 | - filename: testfile_1.yml
93 | checksum: file1_checksum
94 | - filename: testfile_2.yml
95 | checksum: file2_checksum
96 | - filename: testfile_3.yml
97 | checksum: file3_checksum`)
98 |
99 | rc, _ := talismanRCFromYaml(fileContent)
100 | assert.Equal(t, 3, len(rc.FileIgnoreConfig))
101 |
102 | assert.Equal(t, rc.FileIgnoreConfig[0].GetFileName(), "testfile_1.yml")
103 | assert.True(t, rc.FileIgnoreConfig[0].ChecksumMatches("file1_checksum"))
104 | assert.Equal(t, rc.FileIgnoreConfig[1].GetFileName(), "testfile_2.yml")
105 | assert.True(t, rc.FileIgnoreConfig[1].ChecksumMatches("file2_checksum"))
106 | assert.Equal(t, rc.FileIgnoreConfig[2].GetFileName(), "testfile_3.yml")
107 | assert.True(t, rc.FileIgnoreConfig[2].ChecksumMatches("file3_checksum"))
108 | })
109 |
110 | t.Run("Should read severity level", func(t *testing.T) {
111 | talismanRCContents := []byte("threshold: high")
112 | persistedTalismanrc, _ := talismanRCFromYaml(talismanRCContents)
113 | assert.Equal(t, persistedTalismanrc.Threshold, severity.High)
114 | })
115 |
116 | t.Run("Should read custom severities", func(t *testing.T) {
117 | talismanRCContents := []byte(`
118 | custom_severities:
119 | - detector: Base64Content
120 | severity: low
121 | `)
122 | talismanRC, _ := talismanRCFromYaml(talismanRCContents)
123 | assert.Equal(t, talismanRC.CustomSeverities, []CustomSeverityConfig{{Detector: "Base64Content", Severity: severity.Low}})
124 | })
125 | }
126 |
127 | func TestShouldIgnoreUnformattedFiles(t *testing.T) {
128 | for _, s := range []string{"#", "#monkey", "# this monkey likes bananas "} {
129 | fileContents := []byte(s)
130 | talismanRC, err := talismanRCFromYaml(fileContents)
131 | assert.Nil(t, err, "Should successfully unmarshal commented yaml")
132 | assert.Equal(t, &TalismanRC{Version: "1.0"}, talismanRC, "Expected commented line '%s' to result in an empty TalismanRC")
133 | }
134 | }
135 |
--------------------------------------------------------------------------------
/talismanrc/scopes.go:
--------------------------------------------------------------------------------
1 | package talismanrc
2 |
3 | // Mapping of language scopes to names of files that should be ignored anywhere in a repository
4 | var knownScopes = map[string][]string{
5 | "node": {"pnpm-lock.yaml", "yarn.lock", "package-lock.json"},
6 | "go": {"makefile", "go.mod", "go.sum", "Gopkg.toml", "Gopkg.lock", "glide.yaml", "glide.lock"},
7 | "images": {"*.jpeg", "*.jpg", "*.png", "*.tiff", "*.bmp"},
8 | "bazel": {"*.bzl"},
9 | "terraform": {".terraform.lock.hcl"},
10 | "php": {"composer.lock"},
11 | "python": {"poetry.lock", "Pipfile.lock", "requirements.txt", "uv.lock"},
12 | }
13 |
--------------------------------------------------------------------------------
/talismanrc/talismanrc.go:
--------------------------------------------------------------------------------
1 | package talismanrc
2 |
3 | import (
4 | "sort"
5 |
6 | logr "github.com/sirupsen/logrus"
7 | "gopkg.in/yaml.v2"
8 |
9 | "talisman/detector/severity"
10 |
11 | "talisman/gitrepo"
12 | )
13 |
14 | type TalismanRC struct {
15 | FileIgnoreConfig []FileIgnoreConfig `yaml:"fileignoreconfig,omitempty"`
16 | ScopeConfig []ScopeConfig `yaml:"scopeconfig,omitempty"`
17 | CustomPatterns []PatternString `yaml:"custom_patterns,omitempty"`
18 | CustomSeverities []CustomSeverityConfig `yaml:"custom_severities,omitempty"`
19 | AllowedPatterns []*Pattern `yaml:"allowed_patterns,omitempty"`
20 | Experimental ExperimentalConfig `yaml:"experimental,omitempty"`
21 | Threshold severity.Severity `yaml:"threshold,omitempty"`
22 | Version string `yaml:"version"`
23 | }
24 |
25 | // SuggestRCFor returns a string representation of a .talismanrc for the specified FileIgnoreConfigs
26 | func SuggestRCFor(configs []FileIgnoreConfig) string {
27 | tRC := TalismanRC{FileIgnoreConfig: configs, Version: DefaultRCVersion}
28 | result, _ := yaml.Marshal(tRC)
29 |
30 | return string(result)
31 | }
32 |
33 | // RemoveScopedFiles removes scope files from additions
34 | func (tRC *TalismanRC) RemoveScopedFiles(additions []gitrepo.Addition) []gitrepo.Addition {
35 | var applicableScopeFileNames []string
36 | if tRC.ScopeConfig != nil {
37 | for _, scope := range tRC.ScopeConfig {
38 | if len(knownScopes[scope.ScopeName]) > 0 {
39 | applicableScopeFileNames = append(applicableScopeFileNames, knownScopes[scope.ScopeName]...)
40 | }
41 | }
42 | }
43 | var result []gitrepo.Addition
44 | for _, addition := range additions {
45 | isFilePresentInScope := false
46 | for _, fileName := range applicableScopeFileNames {
47 | if addition.NameMatches(fileName) {
48 | isFilePresentInScope = true
49 | break
50 | }
51 | }
52 | if !isFilePresentInScope {
53 | result = append(result, addition)
54 | }
55 | }
56 | return result
57 | }
58 |
59 | // AddIgnores inserts the specified FileIgnoreConfigs to an existing .talismanrc file, or creates one if it doesn't exist.
60 | func (tRC *TalismanRC) AddIgnores(entriesToAdd []FileIgnoreConfig) {
61 | if len(entriesToAdd) > 0 {
62 | logr.Debugf("Adding entries: %v", entriesToAdd)
63 | tRC.FileIgnoreConfig = combineFileIgnores(tRC.FileIgnoreConfig, entriesToAdd)
64 | tRC.saveToFile()
65 | }
66 | }
67 |
68 | func combineFileIgnores(exsiting, incoming []FileIgnoreConfig) []FileIgnoreConfig {
69 | existingMap := make(map[string]FileIgnoreConfig)
70 | for _, fIC := range exsiting {
71 | existingMap[fIC.FileName] = fIC
72 | }
73 | for _, fIC := range incoming {
74 | existingMap[fIC.FileName] = fIC
75 | }
76 | result := make([]FileIgnoreConfig, len(existingMap))
77 | resultKeys := make([]string, len(existingMap))
78 | index := 0
79 | //sort keys in alphabetical order
80 | for k := range existingMap {
81 | resultKeys[index] = k
82 | index++
83 | }
84 | sort.Strings(resultKeys)
85 | //add result entries based on sorted keys
86 | index = 0
87 | for _, k := range resultKeys {
88 | result[index] = existingMap[k]
89 | index++
90 | }
91 | return result
92 | }
93 |
94 | // RemoveAllowedPatterns removes globally- and per-file allowed patterns from an Addition
95 | func (tRC *TalismanRC) RemoveAllowedPatterns(addition gitrepo.Addition) string {
96 | // Processing global allowed patterns
97 | for _, pattern := range tRC.AllowedPatterns {
98 | addition.Data = pattern.ReplaceAll(addition.Data, []byte(""))
99 | }
100 |
101 | // Processing allowed patterns based on file path
102 | for _, ignoreConfig := range tRC.FileIgnoreConfig {
103 | if addition.Matches(ignoreConfig.GetFileName()) {
104 | for _, pattern := range ignoreConfig.GetAllowedPatterns() {
105 | addition.Data = pattern.ReplaceAll(addition.Data, []byte(""))
106 | }
107 | }
108 | }
109 | return string(addition.Data)
110 | }
111 |
112 | // Deny answers true if the Addition should NOT be checked by the specified detector
113 | func (tRC *TalismanRC) Deny(addition gitrepo.Addition, detectorName string) bool {
114 | for _, pattern := range tRC.effectiveRules(detectorName) {
115 | if addition.Matches(pattern) {
116 | return true
117 | }
118 | }
119 | return false
120 | }
121 |
122 | // Accept answers true if the Addition should be checked by the specified detector
123 | func (tRC *TalismanRC) Accept(addition gitrepo.Addition, detectorName string) bool {
124 | return !tRC.Deny(addition, detectorName)
125 | }
126 |
127 | func (tRC *TalismanRC) effectiveRules(detectorName string) []string {
128 | var result []string
129 | for _, ignore := range tRC.FileIgnoreConfig {
130 | if ignore.isEffective(detectorName) {
131 | result = append(result, ignore.GetFileName())
132 | }
133 | }
134 | return result
135 | }
136 |
--------------------------------------------------------------------------------
/talismanrc/types.go:
--------------------------------------------------------------------------------
1 | package talismanrc
2 |
3 | import (
4 | "regexp"
5 | "talisman/detector/severity"
6 |
7 | logr "github.com/sirupsen/logrus"
8 | )
9 |
10 | type PatternString string
11 |
12 | type Pattern struct {
13 | *regexp.Regexp
14 | }
15 |
16 | func (p Pattern) MarshalYAML() (interface{}, error) {
17 | return p.String(), nil
18 | }
19 |
20 | func (p *Pattern) UnmarshalYAML(unmarshal func(interface{}) error) error {
21 | var s string
22 | err := unmarshal(&s)
23 | if err != nil {
24 | logr.Errorf("Pattern.UmarshalYAML error: %v", err)
25 | return err
26 | }
27 | *p = Pattern{regexp.MustCompile(s)}
28 | return nil
29 | }
30 |
31 | type CustomSeverityConfig struct {
32 | Detector string `yaml:"detector"`
33 | Severity severity.Severity `yaml:"severity"`
34 | }
35 |
36 | type FileIgnoreConfig struct {
37 | FileName string `yaml:"filename"`
38 | Checksum string `yaml:"checksum,omitempty"`
39 | IgnoreDetectors []string `yaml:"ignore_detectors,omitempty"`
40 | AllowedPatterns []string `yaml:"allowed_patterns,omitempty"`
41 |
42 | compiledPatterns []*regexp.Regexp
43 | }
44 |
45 | func (i *FileIgnoreConfig) isEffective(detectorName string) bool {
46 | return !isEmptyString(i.FileName) &&
47 | contains(i.IgnoreDetectors, detectorName)
48 | }
49 |
50 | func (i *FileIgnoreConfig) GetFileName() string {
51 | return i.FileName
52 | }
53 |
54 | func (i *FileIgnoreConfig) ChecksumMatches(incomingChecksum string) bool {
55 | return i.Checksum == incomingChecksum
56 | }
57 |
58 | func (i *FileIgnoreConfig) GetAllowedPatterns() []*regexp.Regexp {
59 | if i.compiledPatterns == nil {
60 | i.compiledPatterns = make([]*regexp.Regexp, len(i.AllowedPatterns))
61 | for idx, p := range i.AllowedPatterns {
62 | i.compiledPatterns[idx] = regexp.MustCompile(p)
63 | }
64 | }
65 | return i.compiledPatterns
66 | }
67 |
68 | func IgnoreFileWithChecksum(filename, checksum string) FileIgnoreConfig {
69 | return FileIgnoreConfig{FileName: filename, Checksum: checksum}
70 | }
71 |
72 | type ScopeConfig struct {
73 | ScopeName string `yaml:"scope"`
74 | }
75 |
76 | type ExperimentalConfig struct {
77 | Base64EntropyThreshold float64 `yaml:"base64EntropyThreshold,omitempty"`
78 | }
79 |
--------------------------------------------------------------------------------
/talismanrc/types_test.go:
--------------------------------------------------------------------------------
1 | package talismanrc
2 |
3 | import (
4 | "io"
5 | "regexp"
6 | "strings"
7 | "testing"
8 |
9 | logr "github.com/sirupsen/logrus"
10 | "github.com/stretchr/testify/assert"
11 | "gopkg.in/yaml.v2"
12 | )
13 |
14 | func init() {
15 | logr.SetOutput(io.Discard)
16 | }
17 |
18 | func TestCustomMarshalling(t *testing.T) {
19 | t.Run("Can unmarshal yaml into a Pattern struct", func(t *testing.T) {
20 | savedPattern := []byte("text-pattern")
21 | fromText := Pattern{}
22 | err := yaml.Unmarshal(savedPattern, &fromText)
23 | assert.Nil(t, err, "Should have unmarshalled %s into a Pattern", savedPattern)
24 | assert.Equal(t, Pattern{regexp.MustCompile(string(savedPattern))}, fromText)
25 | })
26 |
27 | t.Run("Can marshal a Pattern struct into yaml", func(t *testing.T) {
28 | pattern := Pattern{regexp.MustCompile("pattern")}
29 | str, err := yaml.Marshal(pattern)
30 | assert.Nil(t, err, "Should have marshalled %v into a string of yaml", pattern)
31 | assert.Equal(t, pattern.String(), strings.TrimSpace(string(str)))
32 | })
33 | }
34 |
35 | func TestFileIgnoreConfig(t *testing.T) {
36 | t.Run("Checksum matching", func(t *testing.T) {
37 | fileIgnoreConfig := &FileIgnoreConfig{
38 | FileName: "some_filename",
39 | Checksum: "some_checksum",
40 | IgnoreDetectors: nil,
41 | AllowedPatterns: nil,
42 | }
43 |
44 | assert.True(t, fileIgnoreConfig.ChecksumMatches("some_checksum"))
45 | assert.False(t, fileIgnoreConfig.ChecksumMatches("some_other_checksum"))
46 | })
47 |
48 | t.Run("Compiles regexes for patterns as needed", func(t *testing.T) {
49 | fileIgnoreConfig := &FileIgnoreConfig{
50 | FileName: "some_filename",
51 | Checksum: "some_checksum",
52 | IgnoreDetectors: nil,
53 | AllowedPatterns: nil,
54 | }
55 |
56 | //No allowed patterns specified
57 | allowedPatterns := fileIgnoreConfig.GetAllowedPatterns()
58 | assert.Equal(t, 0, len(allowedPatterns))
59 |
60 | fileIgnoreConfig.compiledPatterns = nil
61 | fileIgnoreConfig.AllowedPatterns = []string{"[Ff]ile[nN]ame"}
62 | allowedPatterns = fileIgnoreConfig.GetAllowedPatterns()
63 | assert.Equal(t, 1, len(allowedPatterns))
64 | assert.Regexp(t, allowedPatterns[0], "fileName")
65 | })
66 | }
67 |
--------------------------------------------------------------------------------
/talismanrc/util.go:
--------------------------------------------------------------------------------
1 | package talismanrc
2 |
3 | import "regexp"
4 |
5 | var emptyStringPattern = regexp.MustCompile(`^\s*$`)
6 |
7 | func isEmptyString(str string) bool {
8 | return emptyStringPattern.MatchString(str)
9 | }
10 |
11 | func contains(s []string, e string) bool {
12 | for _, a := range s {
13 | if a == e {
14 | return true
15 | }
16 | }
17 | return false
18 | }
19 |
--------------------------------------------------------------------------------
/test-install.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | _linux_uname() {
4 | if [ "${FAKE_PARAMS[0]}" = "-m" ]; then
5 | echo "x86_64"
6 | else
7 | echo "Linux"
8 | fi
9 | }
10 | export -f _linux_uname
11 |
12 | _windows_uname() {
13 | if [ "${FAKE_PARAMS[0]}" = "-m" ]; then
14 | echo "i686"
15 | else
16 | echo "MINGW64_NT-10.0-19045"
17 | fi
18 | }
19 | export -f _windows_uname
20 |
21 | _mac_uname() {
22 | if [ "${FAKE_PARAMS[0]}" = "-m" ]; then
23 | echo "aarch64"
24 | else
25 | echo "Darwin"
26 | fi
27 | }
28 | export -f _mac_uname
29 |
30 | _curl_spy() {
31 | echo "${FAKE_PARAMS[@]}" >>"$1"/_curl_args
32 | echo 'download_url: talisman_linux_amd64checksums'
33 | }
34 | export -f _curl_spy
35 |
36 | setup() {
37 | temp=$(mktemp -d)
38 | fake uname _linux_uname
39 | fake curl "_curl_spy $temp"
40 | fake shasum true
41 | fake tput true
42 | }
43 |
44 | teardown() {
45 | rm -rf "$temp"
46 | }
47 |
48 | test_installs_without_sudo() {
49 | fake sudo 'echo "expected no sudo" && exit 1'
50 | INSTALL_LOCATION=$temp ./install.sh
51 | assert "test -x $temp/talisman_linux_amd64" "Should install file with executable mode"
52 | assert_matches "$temp/talisman_linux_amd64" "$(readlink "$temp/talisman")" "Should create a link"
53 | }
54 |
55 | test_installs_with_sudo_if_available() {
56 | fake touch 'echo "Permission denied" && exit 1'
57 | fake which 'echo "sudo installed" && exit 0'
58 | # shellcheck disable=SC2016
59 | fake sudo 'bash -c "${FAKE_PARAMS[*]}"'
60 | INSTALL_LOCATION=$temp ./install.sh
61 | assert "test -x $temp/talisman_linux_amd64" "Should install file with executable mode"
62 | assert_matches "$temp/talisman_linux_amd64" "$(readlink "$temp/talisman")" "Should create a link"
63 | }
64 |
65 | test_errors_if_unable_to_install() {
66 | fake touch 'echo "Permission denied" && exit 1'
67 | fake which 'echo "sudo not installed" && exit 1'
68 | assert_status_code 126 "INSTALL_LOCATION=$temp ./install.sh"
69 | }
70 |
71 | test_errors_if_no_install_location() {
72 | assert_status_code 1 "INSTALL_LOCATION=/does/not/exist ./install.sh"
73 | }
74 |
75 | test_defaults_to_installing_latest_release() {
76 | INSTALL_LOCATION=$temp ./install.sh
77 | requested_release=$(head -n 1 "$temp"/_curl_args | awk '{print $2}')
78 | assert_matches ".*releases/latest$" "$requested_release" "Should install latest release if no version specified"
79 | }
80 |
81 | test_installing_specific_version() {
82 | VERSION=v1.64.0 INSTALL_LOCATION=$temp ./install.sh
83 | requested_release=$(head -n 1 "$temp"/_curl_args | awk '{print $2}')
84 | assert_matches ".*releases/tags/v1.64.0$" "$requested_release" "Should install specified version"
85 | }
86 |
87 | test_mac_arm_binary_name() {
88 | fake uname _mac_uname
89 | fake curl 'echo "download_url: talisman_darwin_arm64checksums"'
90 | INSTALL_LOCATION=$temp ./install.sh
91 | assert "test -x $temp/talisman_darwin_arm64" "Should install file with executable mode"
92 | assert_matches "$temp/talisman_darwin_arm64" "$(readlink "$temp/talisman")" "Should create a link"
93 | }
94 |
95 | test_windows_binary_name() {
96 | fake uname _windows_uname
97 | fake curl 'echo "download_url: talisman_windows_386.exechecksums"'
98 | INSTALL_LOCATION=$temp ./install.sh
99 | assert "test -x $temp/talisman_windows_386.exe" "Should install file with executable mode"
100 | assert_matches "$temp/talisman_windows_386.exe" "$(readlink "$temp/talisman")" "Should create a link"
101 | }
102 |
103 | test_pre_commit_golang_version() {
104 | version_in_go_mod=$(grep '^go ' go.mod | awk '{print $2}')
105 | grep 'language_version:' .pre-commit-hooks.yaml | awk '{print $2}' | while read -r version_in_pre_commit; do
106 | assert_matches "$version_in_go_mod" "$version_in_pre_commit" "pre-commit-hooks should specify same golang version as go.mod"
107 | done
108 | }
109 |
--------------------------------------------------------------------------------
/third-party/schema-store-talismanrc.json:
--------------------------------------------------------------------------------
1 | {
2 | "$schema": "http://json-schema.org/draft-07/schema#",
3 | "$id": "https://github.com/thoughtworks/talisman/talismanrc",
4 | "title": "schema for .talismanrc",
5 | "type": "object",
6 | "additionalProperties": false,
7 | "fileMatch": [
8 | ".talismanrc"
9 | ],
10 | "properties": {
11 | "fileignoreconfig": {
12 | "type": "array",
13 | "items": {
14 | "type": "object",
15 | "properties": {
16 | "filename": {
17 | "type": "string",
18 | "description": "Fully qualified filename"
19 | },
20 | "checksum": {
21 | "type": "string",
22 | "description": "This field should always have the value specified by Talisman message"
23 | },
24 | "ignore_detectors": {
25 | "type": "array",
26 | "description": "Disable specific detectors for a particular file",
27 | "items": {
28 | "type": "string",
29 | "enum": ["filecontent", "filename", "filesize"]
30 | }
31 | },
32 | "allowed_patterns": {
33 | "type": "array",
34 | "description": "Keywords to ignore to reduce the number of false positives",
35 | "items": {
36 | "type": "string"
37 | }
38 | }
39 | },
40 | "required": ["filename"]
41 | }
42 | },
43 | "scopeconfig": {
44 | "type": "array",
45 | "description": "Talisman is configured to ignore certain files based on the specified scopes",
46 | "items": {
47 | "type": "object",
48 | "properties": {
49 | "scope": {
50 | "type": "string"
51 | }
52 | },
53 | "required": ["scope"]
54 | }
55 | },
56 | "allowed_patterns": {
57 | "type": "array",
58 | "description": "Keywords to ignore to reduce the number of false positives",
59 | "items": {
60 | "type": "string"
61 | }
62 | },
63 | "custom_patterns": {
64 | "type": "array",
65 | "description": "You can specify custom regex patterns to look for in the current repository",
66 | "items": {
67 | "type": "string"
68 | }
69 | },
70 | "custom_severities": {
71 | "type": "array",
72 | "description": "Custom detectors severities",
73 | "items": {
74 | "type": "object",
75 | "properties": {
76 | "detector": {
77 | "type": "string"
78 | },
79 | "severity": {
80 | "type": "string",
81 | "enum": ["low", "medium", "high"]
82 | }
83 | },
84 | "required": ["detector", "severity"]
85 | }
86 | },
87 | "threshold": {
88 | "type": "string",
89 | "description": "Default minimal threshold",
90 | "enum": ["low", "medium", "high"]
91 | },
92 | "version": {
93 | "type": "string",
94 | "description": ".talismanrc version"
95 | }
96 | },
97 | "required": []
98 | }
99 |
--------------------------------------------------------------------------------
/utility/progress_bar.go:
--------------------------------------------------------------------------------
1 | package utility
2 |
3 | import (
4 | "fmt"
5 | "os"
6 |
7 | "github.com/cheggaaa/pb/v3"
8 | )
9 |
10 | func GetProgressBar(out *os.File, title string) progressBar {
11 | if isTerminal(out) {
12 | return &defaultProgressBar{title: title}
13 | } else {
14 | return &noOpProgressBar{}
15 | }
16 | }
17 |
18 | func isTerminal(out *os.File) bool {
19 | fileInfo, _ := out.Stat()
20 | return (fileInfo.Mode() & os.ModeCharDevice) != 0
21 | }
22 |
23 | type progressBar interface {
24 | Start(int)
25 | Increment()
26 | Finish()
27 | }
28 |
29 | type noOpProgressBar struct {
30 | }
31 |
32 | func (d *noOpProgressBar) Start(int) {}
33 |
34 | func (d *noOpProgressBar) Increment() {}
35 |
36 | func (d *noOpProgressBar) Finish() {}
37 |
38 | type defaultProgressBar struct {
39 | bar *pb.ProgressBar
40 | title string
41 | }
42 |
43 | func (d *defaultProgressBar) Start(total int) {
44 | template := fmt.Sprintf(`{{ red "%s:" }} {{counters .}} {{ bar . "<" "-" (cycle . "↖" "↗" "↘" "↙" ) "." ">"}} {{percent . | rndcolor }} {{green}} {{blue}}`, d.title)
45 | bar := pb.ProgressBarTemplate(template).New(total)
46 | bar.Set(pb.Terminal, true)
47 | d.bar = bar.Start()
48 | }
49 |
50 | func (d *defaultProgressBar) Increment() {
51 | d.bar.Increment()
52 | }
53 |
54 | func (d *defaultProgressBar) Finish() {
55 | d.bar.Finish()
56 | }
57 |
--------------------------------------------------------------------------------
/utility/progress_bar_test.go:
--------------------------------------------------------------------------------
1 | package utility
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/stretchr/testify/assert"
7 | )
8 |
9 | func TestDefaultProgressBar(t *testing.T) {
10 | t.Run("Start should start inner progress bar", func(t *testing.T) {
11 | progressBar := &defaultProgressBar{}
12 |
13 | progressBar.Start(10)
14 |
15 | assert.True(t, progressBar.bar.IsStarted())
16 | })
17 |
18 | t.Run("Increment should update inner progress bar", func(t *testing.T) {
19 | progressBar := &defaultProgressBar{}
20 |
21 | progressBar.Start(10)
22 | progressBar.Increment()
23 |
24 | assert.Equal(t, int64(1), progressBar.bar.Current())
25 | })
26 |
27 | t.Run("Finish should finish progress bar", func(t *testing.T) {
28 | progressBar := &defaultProgressBar{}
29 |
30 | progressBar.Start(2)
31 | progressBar.Increment()
32 | progressBar.Increment()
33 | progressBar.Finish()
34 |
35 | assert.False(t, progressBar.bar.IsStarted())
36 | })
37 | }
38 |
--------------------------------------------------------------------------------
/utility/sha_256_hasher.go:
--------------------------------------------------------------------------------
1 | package utility
2 |
3 | import (
4 | "crypto/sha256"
5 | "encoding/hex"
6 | "talisman/gitrepo"
7 |
8 | "github.com/sirupsen/logrus"
9 | )
10 |
11 | type SHA256Hasher interface {
12 | CollectiveSHA256Hash(paths []string) string
13 | Start() error
14 | Shutdown() error
15 | }
16 |
17 | type DefaultSHA256Hasher struct{}
18 |
19 | // CollectiveSHA256Hash return collective sha256 hash of the passed paths
20 | func (*DefaultSHA256Hasher) CollectiveSHA256Hash(paths []string) string {
21 | return collectiveSHA256Hash(paths, SafeReadFile)
22 | }
23 |
24 | func (*DefaultSHA256Hasher) Start() error { return nil }
25 | func (*DefaultSHA256Hasher) Shutdown() error { return nil }
26 |
27 | type gitBatchSHA256Hasher struct {
28 | br gitrepo.BatchReader
29 | }
30 |
31 | func (g *gitBatchSHA256Hasher) CollectiveSHA256Hash(paths []string) string {
32 | return collectiveSHA256Hash(paths, g.br.Read)
33 | }
34 |
35 | func (g *gitBatchSHA256Hasher) Start() error {
36 | return g.br.Start()
37 | }
38 |
39 | func (g *gitBatchSHA256Hasher) Shutdown() error {
40 | return g.br.Shutdown()
41 | }
42 |
43 | func hashByte(contentPtr *[]byte) string {
44 | contents := *contentPtr
45 | hasher := sha256.New()
46 | hasher.Write(contents)
47 | return hex.EncodeToString(hasher.Sum(nil))
48 | }
49 |
50 | func collectiveSHA256Hash(paths []string, FileReader func(string) ([]byte, error)) string {
51 | var finHash = ""
52 | for _, path := range paths {
53 | sbyte := []byte(finHash)
54 | concatBytes := hashByte(&sbyte)
55 | nameByte := []byte(path)
56 | nameHash := hashByte(&nameByte)
57 | fileBytes, _ := FileReader(path)
58 | fileHash := hashByte(&fileBytes)
59 | finHash = concatBytes + fileHash + nameHash
60 | }
61 | c := []byte(finHash)
62 | m := hashByte(&c)
63 | return m
64 | }
65 |
66 | var hashers = make(map[string]SHA256Hasher)
67 |
68 | // MakeHasher returns a SHA256 file/object hasher based on mode and a repo root
69 | func MakeHasher(mode string, root string) SHA256Hasher {
70 | if hashers[mode] != nil {
71 | return hashers[mode]
72 | }
73 | switch mode {
74 | case "pre-push":
75 | hashers[mode] = &gitBatchSHA256Hasher{gitrepo.NewBatchGitHeadPathReader(root)}
76 | case "pre-commit":
77 | hashers[mode] = &gitBatchSHA256Hasher{gitrepo.NewBatchGitStagedPathReader(root)}
78 | case "scan":
79 | hashers[mode] = &gitBatchSHA256Hasher{gitrepo.NewBatchGitObjectHashReader(root)}
80 | case "pattern":
81 | hashers[mode] = &DefaultSHA256Hasher{}
82 | case "checksum":
83 | hashers[mode] = &gitBatchSHA256Hasher{gitrepo.NewBatchGitStagedPathReader(root)}
84 | case "default":
85 | hashers[mode] = &DefaultSHA256Hasher{}
86 | }
87 | err := hashers[mode].Start()
88 | if err != nil {
89 | logrus.Errorf("unable to start hasher: %v", err)
90 | return nil
91 | }
92 | return hashers[mode]
93 | }
94 |
95 | func DestroyHashers() {
96 | for _, hasher := range hashers {
97 | hasher.Shutdown()
98 | }
99 | hashers = make(map[string]SHA256Hasher)
100 | }
101 |
--------------------------------------------------------------------------------
/utility/sha_256_hasher_test.go:
--------------------------------------------------------------------------------
1 | package utility
2 |
3 | import (
4 | "github.com/stretchr/testify/assert"
5 | "testing"
6 | )
7 |
8 | func TestShouldReturnCorrectFileHash(t *testing.T) {
9 | hasher := DefaultSHA256Hasher{}
10 | checksumSomeFile := hasher.CollectiveSHA256Hash([]string{"some_file.pem"})
11 | checksumTestSomeFile := hasher.CollectiveSHA256Hash([]string{"test/some_file.pem"})
12 | assert.Equal(t, checksumSomeFile, "87139cc4d975333b25b6275f97680604add51b84eb8f4a3b9dcbbc652e6f27ac", "Should be equal to some_file.pem hash value")
13 | assert.Equal(t, checksumTestSomeFile, "25bd31a28bf9d4e06327f1c4a5cab2260574ae508803f66adcc393350e994866", "Should be equal to test/some_file.pem hash value")
14 | }
15 |
16 | func TestShouldReturnEmptyFileHashWhenNoPathsPassed(t *testing.T) {
17 | hasher := DefaultSHA256Hasher{}
18 | checksum := hasher.CollectiveSHA256Hash([]string{})
19 | assert.Equal(t, checksum, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "Should be equal to empty hash value when no paths passed")
20 | }
21 |
--------------------------------------------------------------------------------
/utility/utility.go:
--------------------------------------------------------------------------------
1 | package utility
2 |
3 | import (
4 | "fmt"
5 | "io"
6 | "io/ioutil"
7 | "os"
8 | "path"
9 |
10 | log "github.com/sirupsen/logrus"
11 |
12 | figure "github.com/common-nighthawk/go-figure"
13 | )
14 |
15 | // UniqueItems returns the array of strings containing unique items
16 | func UniqueItems(stringSlice []string) []string {
17 | keys := make(map[string]bool)
18 | list := []string{}
19 | for _, entry := range stringSlice {
20 | if _, value := keys[entry]; !value {
21 | keys[entry] = true
22 | list = append(list, entry)
23 | }
24 | }
25 | return list
26 | }
27 |
28 | // Creates art for console output
29 | func CreateArt(msg string) {
30 | myFigure := figure.NewFigure(msg, "basic", true)
31 | myFigure.Print()
32 | }
33 |
34 | // Copies Files and Directories from source to destination
35 | func File(src, dst string) error {
36 | var err error
37 | var srcfd *os.File
38 | var dstfd *os.File
39 | var srcinfo os.FileInfo
40 |
41 | if srcfd, err = os.Open(src); err != nil {
42 | return err
43 | }
44 | defer srcfd.Close()
45 |
46 | if dstfd, err = os.Create(dst); err != nil {
47 | return err
48 | }
49 | defer dstfd.Close()
50 |
51 | if _, err = io.Copy(dstfd, srcfd); err != nil {
52 | return err
53 | }
54 | if srcinfo, err = os.Stat(src); err != nil {
55 | return err
56 | }
57 | return os.Chmod(dst, srcinfo.Mode())
58 | }
59 |
60 | func Dir(src string, dst string) error {
61 | var err error
62 | var fds []os.FileInfo
63 | var srcinfo os.FileInfo
64 |
65 | if srcinfo, err = os.Stat(src); err != nil {
66 | return err
67 |
68 | }
69 |
70 | if err = os.MkdirAll(dst, srcinfo.Mode()); err != nil {
71 | return err
72 | }
73 |
74 | if fds, err = ioutil.ReadDir(src); err != nil {
75 | return err
76 | }
77 | for _, fd := range fds {
78 | srcfp := path.Join(src, fd.Name())
79 | dstfp := path.Join(dst, fd.Name())
80 |
81 | if fd.IsDir() {
82 | if err = Dir(srcfp, dstfp); err != nil {
83 | fmt.Println(err)
84 | }
85 | } else {
86 | if err = File(srcfp, dstfp); err != nil {
87 | fmt.Println(err)
88 | }
89 | }
90 | }
91 | return nil
92 | }
93 |
94 | func IsFileSymlink(path string) bool {
95 | fileMetadata, err := os.Lstat(path)
96 | if err != nil {
97 | return false
98 | }
99 | return fileMetadata.Mode()&os.ModeSymlink != 0
100 | }
101 |
102 | func SafeReadFile(path string) ([]byte, error) {
103 | if IsFileSymlink(path) {
104 | log.Debug("Symlink was detected! Not following symlink ", path)
105 | return []byte{}, nil
106 | }
107 | return ioutil.ReadFile(path)
108 | }
109 |
--------------------------------------------------------------------------------
/utility/utility_test.go:
--------------------------------------------------------------------------------
1 | package utility
2 |
3 | import (
4 | "github.com/stretchr/testify/assert"
5 | "io/ioutil"
6 | "os"
7 | "testing"
8 | )
9 |
10 | func TestShouldReadNormalFileCorrectly(t *testing.T) {
11 | tempFile, _ := ioutil.TempFile(os.TempDir(), "somefile")
12 | dataToBeWrittenInFile := []byte{0, 1, 2, 3}
13 | tempFile.Write(dataToBeWrittenInFile)
14 | tempFile.Close()
15 |
16 | readDataFromFileUsingIoutilDotReadFile, _ := ioutil.ReadFile(tempFile.Name())
17 | readDataFromFileUsingSafeFileRead, _ := SafeReadFile(tempFile.Name())
18 | os.Remove(tempFile.Name())
19 |
20 | assert.Equal(t, readDataFromFileUsingIoutilDotReadFile, dataToBeWrittenInFile)
21 | assert.Equal(t, readDataFromFileUsingSafeFileRead, dataToBeWrittenInFile)
22 | }
23 |
24 | func TestShouldNotReadSymbolicLinkTargetFile(t *testing.T) {
25 | tempFile, _ := ioutil.TempFile(os.TempDir(), "somefile")
26 | dataToBeWrittenInFile := []byte{0, 1, 2, 3}
27 | tempFile.Write(dataToBeWrittenInFile)
28 | tempFile.Close()
29 | symlinkFileName := tempFile.Name() + "symlink"
30 | os.Symlink(tempFile.Name(), symlinkFileName)
31 |
32 | readDataFromSymlinkUsingIoutilDotReadFile, _ := ioutil.ReadFile(symlinkFileName)
33 | readDataFromSymlinkUsingSafeFileRead, _ := SafeReadFile(symlinkFileName)
34 | os.Remove(symlinkFileName)
35 | os.Remove(tempFile.Name())
36 |
37 | assert.Equal(t, readDataFromSymlinkUsingIoutilDotReadFile, dataToBeWrittenInFile)
38 | assert.Equal(t, readDataFromSymlinkUsingSafeFileRead, []byte{})
39 | }
40 |
--------------------------------------------------------------------------------