├── .coveragerc ├── .gitignore ├── .pre-commit-config.yaml ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README.md ├── credentials.sample.json ├── detect_secrets_server ├── __init__.py ├── __main__.py ├── actions │ ├── __init__.py │ ├── initialize.py │ ├── install.py │ ├── list.py │ └── scan.py ├── adhoc │ ├── __init__.py │ └── github │ │ ├── __init__.py │ │ └── webhook.py ├── constants.py ├── core │ ├── __init__.py │ └── usage │ │ ├── __init__.py │ │ ├── add.py │ │ ├── common │ │ ├── __init__.py │ │ ├── hooks.py │ │ ├── install.py │ │ ├── options.py │ │ ├── output.py │ │ ├── storage.py │ │ └── validators.py │ │ ├── install.py │ │ ├── list.py │ │ ├── parser.py │ │ ├── s3.py │ │ └── scan.py ├── hooks │ ├── __init__.py │ ├── base.py │ ├── external.py │ ├── pysensu_yelp.py │ └── stdout.py ├── repos │ ├── __init__.py │ ├── base_tracked_repo.py │ ├── factory.py │ ├── local_tracked_repo.py │ └── s3_tracked_repo.py ├── storage │ ├── __init__.py │ ├── base.py │ ├── core │ │ ├── __init__.py │ │ └── git.py │ ├── file.py │ └── s3.py └── util │ ├── __init__.py │ └── version.py ├── examples ├── aws_credentials.json ├── config.yaml ├── pysensu.config.yaml ├── repos.yaml ├── s3.yaml └── standalone_hook.py ├── requirements-dev-minimal.txt ├── requirements-dev.txt ├── scripts └── uploader.sh ├── setup.cfg ├── setup.py ├── test_data └── sample.diff ├── testing ├── __init__.py ├── base_usage_test.py ├── factories.py ├── github │ ├── commit_comment.json │ ├── issue_comment.json │ ├── issues.json │ ├── pull_request.json │ └── pull_request_review_comment.json ├── mocks.py └── util.py ├── tests ├── actions │ ├── __init__.py │ ├── conftest.py │ ├── initialize_test.py │ ├── install_test.py │ └── scan_test.py ├── adhoc │ └── github │ │ └── github_webhook_test.py ├── conftest.py ├── core │ └── usage │ │ ├── add_test.py │ │ ├── output_test.py │ │ ├── parser_test.py │ │ ├── s3_test.py │ │ └── scan_test.py ├── main_test.py ├── repos │ ├── __init__.py │ ├── base_tracked_repo_test.py │ └── s3_tracked_repo_test.py └── storage │ ├── __init__.py │ ├── base_test.py │ ├── file_test.py │ └── s3_test.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = . 4 | omit = 5 | .tox/* 6 | /tmp* 7 | setup.py 8 | 9 | # These are solely output plugins, and make it hard to really test 10 | # without mocking the world 11 | detect_secrets_server/hooks/* 12 | 13 | venv/* 14 | 15 | [report] 16 | skip_covered = True 17 | exclude_lines = 18 | # Don't complain if non-runnable code isn't run: 19 | ^if __name__ == ['"]__main__['"]:$ 20 | 21 | # Need to redefine this, as per documentation 22 | pragma: no cover 23 | 24 | # Don't complain if performing logic for cross-version functionality 25 | ^\s*except ImportError:\b 26 | 27 | # Don't complain if tests don't hit defensive assertion code: 28 | ^\s*raise AssertionError\b 29 | ^\s*raise NotImplementedError\b 30 | ^\s*return NotImplemented\b 31 | ^\s*raise$ 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | *.sw[op] 3 | 4 | .coverage 5 | *.egg-info 6 | .tox 7 | venv 8 | /tmp 9 | 10 | 11 | .*ignore 12 | !.gitignore 13 | 14 | .pysensu.config.yaml 15 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | - repo: https://github.com/pre-commit/pre-commit-hooks 2 | rev: v2.5.0 3 | hooks: 4 | - id: trailing-whitespace 5 | - id: end-of-file-fixer 6 | - id: check-docstring-first 7 | - id: debug-statements 8 | - id: name-tests-test 9 | exclude: tests/util 10 | - id: check-ast 11 | - repo: https://github.com/pre-commit/mirrors-autopep8 12 | rev: v1.5 13 | hooks: 14 | - id: autopep8 15 | - repo: https://gitlab.com/pycqa/flake8 16 | rev: 3.7.9 17 | hooks: 18 | - id: flake8 19 | args: ['--ignore=E501,W503,W504'] 20 | - repo: https://github.com/asottile/reorder_python_imports 21 | rev: v1.9.0 22 | hooks: 23 | - id: reorder-python-imports 24 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | matrix: 3 | include: 4 | - env: TOXENV=py35 5 | python: 3.5 6 | - env: TOXENV=py36 7 | python: 3.6 8 | - env: TOXENV=py37 9 | python: 3.7 10 | dist: xenial # required for Python >= 3.7 (travis-ci/travis-ci#9069) 11 | - env: TOXENV=pypy3 12 | python: pypy3 13 | install: 14 | - pip install tox 15 | script: make test 16 | cache: 17 | directories: 18 | - $HOME/.cache/pre-commit 19 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # What's New 2 | 3 | Thanks to all our contributors, users, and the many people that make 4 | `detect-secrets-server` possible! :heart: 5 | 6 | If you love `detect-secrets-server`, please star our project on GitHub to show 7 | your support! :star: 8 | 9 | 27 | 28 | ### v0.3.2 29 | ##### August 28th, 2020 30 | 31 | #### :snake: Miscellaneous 32 | 33 | * Add ability to pass arbritrary arguments to `detect-secrets` ([#65]) 34 | 35 | [#65]: https://github.com/Yelp/detect-secrets-server/pull/65 36 | 37 | 38 | 39 | ### v0.3.1 40 | ##### August 26th, 2020 41 | 42 | #### :snake: Miscellaneous 43 | 44 | * Update development environment and testing setup ([#63]) 45 | 46 | [#63]: https://github.com/Yelp/detect-secrets-server/pull/63 47 | 48 | 49 | 50 | ### v0.3.0 51 | ##### August 26th, 2020 52 | 53 | #### :boom: Breaking Changes 54 | 55 | * Drop support for Python 3 ([#51]) 56 | 57 | #### :tada: New Features 58 | 59 | * Add a GitHub webhook scanner ([#56]) 60 | 61 | [#51]: https://github.com/Yelp/detect-secrets-server/pull/51 62 | [#56]: https://github.com/Yelp/detect-secrets-server/pull/56 63 | 64 | 65 | 66 | ### v0.2.20 67 | ##### February 10th, 2020 68 | 69 | #### :snake: Miscellaneous 70 | 71 | * Bumped detect-secrets version from [0.12.7](https://github.com/Yelp/detect-secrets/blob/master/CHANGELOG.md#v0127) to [0.13.0](https://github.com/Yelp/detect-secrets/blob/master/CHANGELOG.md#v0130) 72 | 73 | ### v0.2.19 74 | ##### February 10th, 2020 75 | 76 | #### :bug: Bugfixes 77 | 78 | * [Exception handling on retrieving baseline](https://github.com/Yelp/detect-secrets-server/commit/8371f54193a353b94acd578065c45dc3c839ebf1) 79 | 80 | 81 | 82 | ### v0.2.18 83 | ##### February 4th, 2020 84 | 85 | #### :tada: New Features 86 | 87 | * [Add ability to scan repo HEAD](https://github.com/Yelp/detect-secrets-server/commit/17e57e7ce3e60772cec96f3ad424d74684b3fdaf) 88 | 89 | 90 | 91 | ### v0.2.17 92 | ##### October 15th, 2019 93 | 94 | #### :bug: Bugfixes 95 | 96 | * Fixed a bug where our cron functionality didn't handle a custom root directory ([#36], thanks [@gsoyka]) 97 | 98 | [#36]: https://github.com/Yelp/detect-secrets-server/pull/36 99 | 100 | 101 | 102 | ### v0.2.16 103 | ##### October 2nd, 2019 104 | 105 | #### :tada: New Features 106 | 107 | * [Added handling of local bare repos](https://github.com/Yelp/detect-secrets-server/commit/99d4a1e384c07a77a2d32fd9febe8c897d94c922) 108 | 109 | 110 | 111 | ### v0.2.15 112 | ##### September 30th, 2019 113 | 114 | #### :bug: Bugfixes 115 | 116 | * Fixed a bug where we were would crash with a `OSError: [Errno 7] Argument list too long` if there were too many files in the git diff ([#35]) 117 | 118 | [#35]: https://github.com/Yelp/detect-secrets-server/pull/35 119 | 120 | 121 | 122 | ### v0.2.14 123 | ##### September 19th, 2019 124 | 125 | #### :tada: New Features 126 | 127 | * Added an `--always-run-output-hook` flag ([#34], thanks [@mindfunk]) 128 | 129 | #### :bug: Bugfixes 130 | 131 | * [Fixed a bug where we were never skipping ignored file extensions](https://github.com/Yelp/detect-secrets-server/commit/1c0d5120b979d68f357eb473bf476a66b4899ce9) 132 | 133 | [#34]: https://github.com/Yelp/detect-secrets-server/pull/34 134 | 135 | 136 | 137 | ### v0.2.13 138 | ##### September 16th, 2019 139 | 140 | #### :snake: Miscellaneous 141 | 142 | - Bumped the detect-secrets from version [`v0.12.2`](https://github.com/Yelp/detect-secrets/blob/master/CHANGELOG.md#v0122) to [`v0.12.6`](https://github.com/Yelp/detect-secrets/blob/master/CHANGELOG.md#v0126) 143 | 144 | 145 | 146 | ### v0.2.12 147 | ##### June 4th, 2019 148 | 149 | #### :bug: Bugfixes 150 | 151 | * Fixed a **very important bug** where we were not fetching changes for non-local repositories ([#30], thanks [@chetmancini], [@akshayatplivo], [@ajchida], [@rameshkumar-a])) 152 | * [Fixed a `UnidiffParseError: Hunk is shorter than expected` crash](https://github.com/Yelp/detect-secrets-server/pull/30/commits/bc0170045e3778446c0d68fb19b0dc58543602c2) 153 | 154 | #### :art: Display Changes 155 | 156 | * [Added a helpful error message for when a user tries to use S3 features without boto3 installed](https://github.com/Yelp/detect-secrets-server/commit/15525d4eb35dcd1b79e458cdf360ab9f5a77957c) 157 | 158 | [#30]: https://github.com/Yelp/detect-secrets-server/pull/30 159 | [@chetmancini]: https://github.com/chetmancini 160 | [@akshayatplivo]: https://github.com/akshayatplivo 161 | [@ajchida]: https://github.com/ajchida 162 | [@rameshkumar-a]: https://github.com/rameshkumar-a 163 | 164 | 165 | 166 | ### v0.2.11 167 | ##### March 21st, 2019 168 | 169 | #### :tada: New Features 170 | 171 | * [Bumped version of `detect-secrets`](https://github.com/Yelp/detect-secrets-server/commit/bfe7295b3681f0fe9d6d4652fa9437aab5e2e664) from [`v0.12.0`](https://github.com/Yelp/detect-secrets/blob/master/CHANGELOG.md#v0120) to [v0.12.2](https://github.com/Yelp/detect-secrets/blob/master/CHANGELOG.md#v0122), primarily to improve performance 172 | 173 | 174 | 175 | ### v0.2.10 176 | ##### March 14th, 2019 177 | 178 | #### :bug: Bugfixes 179 | 180 | * Fixed a bug where we were not assigning the commit of found secrets to HEAD ([#27]) 181 | 182 | [#27]: https://github.com/Yelp/detect-secrets-server/pull/27 183 | 184 | 185 | 186 | ### v0.2.9 187 | ##### March 14th, 2019 188 | 189 | #### :snake: Miscellaneous 190 | 191 | * [Fixed](https://github.com/Yelp/detect-secrets-server/commit/472ba87ecc220be96f10477914b09da159d9bc04) an [internal issue](https://github.com/Yelp/venv-update) 192 | 193 | 194 | 195 | ### v0.2.8 196 | ##### March 14th, 2019 197 | 198 | #### :bug: Bugfixes 199 | 200 | * Fixed a bug where we were `git fetch`ing for local git repositories ([#26]) 201 | 202 | [#26]: https://github.com/Yelp/detect-secrets-server/pull/26 203 | 204 | 205 | 206 | ### v0.2.7 207 | ##### March 13th, 2019 208 | 209 | #### :tada: New Features 210 | 211 | * Added a `--diff-filter` optimization, so we only scan added, copied or modified files ([#22]) 212 | 213 | [#22]: https://github.com/Yelp/detect-secrets-server/pull/22 214 | 215 | #### :bug: Bugfixes 216 | 217 | * Fixed a bug where, `scan` on bare repositories gave a `Your local changes to the following files would be overwritten by merge:` error ([#23]) 218 | 219 | [#23]: https://github.com/Yelp/detect-secrets-server/pull/23 220 | 221 | 222 | 223 | ### v0.2.6 224 | ##### February 12th, 2019 225 | 226 | #### :bug: Bugfixes 227 | 228 | * [Fixed a bug where we were using an older version of `detect-secrets` in our `requirements-dev` `txt` files](https://github.com/Yelp/detect-secrets-server/commit/0ff9f095167e5090a8ebba1ddc4e7317b3c23800) 229 | 230 | 231 | 232 | ### v0.2.5 233 | ##### February 12th, 2019 234 | 235 | #### :tada: New Features 236 | 237 | * Added `--exclude-files` and `--exclude-lines` args to scan ([#18]) 238 | * Added git commit to secrets before calling `output_hook.alert` ([#15]) 239 | 240 | [#15]: https://github.com/Yelp/detect-secrets-server/pull/15 241 | 242 | #### :boom: Breaking Changes 243 | 244 | * Started to ignore the `exclude_regex` in repo metadata when scanning as a short-term solution for [Issue 17](https://github.com/Yelp/detect-secrets-server/issues/17) ([#18]) 245 | 246 | [#18]: https://github.com/Yelp/detect-secrets-server/pull/18 247 | 248 | 249 | 250 | ### v0.2.4 251 | ##### January 14th, 2019 252 | 253 | #### :bug: Bugfixes 254 | 255 | * `add` and `scan` now handle non-SSH urls for git cloning. See 256 | [Issue 13](https://github.com/Yelp/detect-secrets-server/issues/13) for more details. 257 | 258 | 259 | 260 | ### v0.2.2 261 | ##### January 11th, 2019 262 | 263 | #### :tada: New Features 264 | 265 | * Bumped version of `detect-secrets` to 0.11.4, so that we can leverage the 266 | new `AWSKeyDetector` and the `KeywordDetector`. 267 | 268 | 269 | 270 | ### v0.2.1 271 | ##### January 10th, 2019 272 | 273 | #### :tada: New Features 274 | 275 | * Added support for delegating state management to output hooks, using the 276 | flag `--always-update-state`. 277 | 278 | 279 | 280 | ### v0.2.0 281 | ##### January 09th, 2019 282 | 283 | #### :boom: Breaking Changes 284 | 285 | * All previous config files' format has been changed, for better usability 286 | (and reducing the need to supply multiple config files during a single 287 | invocation). Be sure to check out some examples in 288 | [examples/](https://github.com/Yelp/detect-secrets-server/tree/master/examples) 289 | 290 | * The CLI API has also been changed, to support better usability. Check out 291 | how to use the new commands with `-h`. 292 | 293 | #### :tada: New Features 294 | 295 | * **Actually** works with the latest version of `detect-secrets`. 296 | 297 | * New `--output-hook` functionality, to specify arbitrary scripts for handling 298 | alerts. This should make it easier, so users aren't forced into using pysensu. 299 | 300 | * `detect-secrets-server list` supports a convenient way to list all tracked 301 | repositories. 302 | 303 | * `detect-secrets-server install` is a modular way to connect tracked repositories 304 | with a system that runs `detect-secrets-server scan` on a regular basis. 305 | Currently, the only supported method is `cron`. 306 | 307 | #### :mega: Release Highlights 308 | 309 | * Minimal dependencies! Previously, you had to install boto3, even if you weren't 310 | using the S3 storage option. Now, only install what you need, based on your 311 | unique setup. 312 | 313 | * Introduction of the `Storage` class abstraction. This separates the management 314 | of tracked repositories (git cloning, baseline comparisons) with the method of 315 | storing server metadata, for cleaner code, decoupled architecture, and 316 | modularity. 317 | 318 | 319 | 320 | # Special thanks to our awesome contributors! :clap: 321 | 322 | - [@gsoyka] 323 | - [@mindfunk] 324 | 325 | [@gsoyka]: https://github.com/gsoyka 326 | [@mindfunk]: https://github.com/mindfunk 327 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2017-2018 Yelp Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: minimal 2 | minimal: setup 3 | 4 | .PHONY: setup 5 | setup: 6 | tox -e venv 7 | 8 | .PHONY: install-hooks 9 | install-hooks: 10 | tox -e pre-commit -- install -f --install-hooks 11 | 12 | .PHONY: test 13 | test: 14 | tox 15 | 16 | .PHONY: clean 17 | clean: 18 | find . -name '*.pyc' -delete 19 | find . -name '__pycache__' -delete 20 | rm -rf .tox 21 | #rm -rf venv 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/Yelp/detect-secrets-server.svg?branch=master)](https://travis-ci.org/Yelp/detect-secrets-server) 2 | [![PyPI version](https://badge.fury.io/py/detect-secrets-server.svg)](https://badge.fury.io/py/detect-secrets-server) 3 | 4 | # detect-secrets-server 5 | 6 | > :warning: **Yelp has stopped active development** for this repository, for the foreseeable 7 | > future (2021-04-12). 8 | > 9 | > The upstreamed `detect-secrets` package underwent some major improvements to the tool, 10 | > launching their official public version (v1.0), and fundamentally changing how secrets were 11 | > scanned and processed. With these changes, Yelp has found success with integrating the tool 12 | > directly with [Github webhooks](https://docs.github.com/en/developers/webhooks-and-events/webhook-events-and-payloads#pull_request), transforming the server-side component from a 13 | > scheduled batch job, to an on-demand diff scan. 14 | > 15 | > This repository has not been updated to be compatible with the new version, nor do we know 16 | > when we might be able to do so. 17 | > 18 | > This solution may still be desirable for you, if you want to keep track of multiple repositories, 19 | > and you don't have access to some feed of pull requests. However, we encourage you to check out 20 | > some of the new features in `detect-secrets` v1, as you may find the [new 21 | > architecture](https://github.com/Yelp/detect-secrets/blob/master/docs/design.md) may support 22 | > your use case, through a much more simple solution. 23 | 24 | ## About 25 | 26 | `detect-secrets-server` is the server-side counterpart to [`detect-secrets`]( 27 | https://github.com/Yelp/detect-secrets), that can be used to detect secrets retroactively. 28 | While `detect-secrets` is a fantastic tool to self-identify secrets in your codebase and 29 | prevent them from entering, it is ultimately a client-side protection and can be easily 30 | bypassed. 31 | 32 | Adding a 33 | [pre-receive hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#_code_pre_receive_code) 34 | would also fail to be effective, due to the nuanced nature of `detect-secrets`. 35 | If you're preventing any potential secrets at a commit level, you may block developers 36 | due to false positives. 37 | 38 | Therefore, `detect-secrets-server` accomplishes several things: 39 | 40 | 1. **Tracks** multiple repositories and maintains its own state of known secrets, 41 | 2. Periodically **scans** tracked repositories for any new incoming secrets, and 42 | 3. Sends **alerts** when it finds secrets in new commits. 43 | 44 | ## Example Usage 45 | 46 | ### Quick Start 47 | 48 | ``` 49 | $ pip install detect-secrets-server[cron] 50 | $ detect-secrets-server add git@github.com:yelp/detect-secrets 51 | $ detect-secrets-server install cron 52 | ``` 53 | 54 | This will add `detect-secrets` as a tracked repository, and install it to the 55 | current user's crontab so that it will periodically scan for updates. 56 | 57 | ### Manually Scanning a Repository 58 | 59 | Once you have a tracked repository, you can scan it as follows: 60 | 61 | ``` 62 | $ detect-secrets-server scan yelp/detect-secrets 63 | ``` 64 | 65 | ### Adding a Local Repository 66 | 67 | Instead of having `detect-secrets-server` clone git repositories on your behalf, you can 68 | have it point to locally managed repositories. This is especially handy when testing 69 | `detect-secrets-server`. 70 | 71 | ``` 72 | ~/pg/detect-secrets-server $ detect-secrets-server add ../detect-secrets --local 73 | ~/pg/detect-secrets-server $ cd ../detect-secrets 74 | ~/pg/detect-secrets $ echo "'$(echo "asdf" | shasum -a 256 | cut -d ' ' -f 1)'" >> detect_secrets/pre_commit_hook.py 75 | ~/pg/detect-secrets $ git add -u; git commit -m 'test'; cd ../detect-secrets-server 76 | ~/pg/detect-secrets-server $ detect-secrets-server scan ../detect-secrets --local 77 | ``` 78 | 79 | ### Adding Multiple Repositories at Once 80 | 81 | To track multiple repositories at once, you can specify a config file when adding tracked 82 | repositories. 83 | 84 | ``` 85 | $ detect-secrets-server add examples/repos.yaml --config 86 | ``` 87 | 88 | The following keys are accepted in this config file: 89 | 90 | ``` 91 | repos.yaml 92 | |- tracked # This is a list of repositories that will be tracked 93 | ``` 94 | 95 | Tracked repository options are as follows: 96 | 97 | | attribute | description 98 | | -------------- | ----------- 99 | | repo | git URL or local file path to clone (**required**). 100 | | crontab | [crontab syntax](https://crontab.guru/) of how often to run a scan for this repo. 101 | | sha | The commit hash to start scanning from. If not provided, will use HEAD. 102 | | storage | Either one of the following: (`file`, `s3`). Determines where to store metadata. Defaults to `file`. 103 | | is\_local\_repo| True/False depending on if the repo is already on the filesystem. Defaults to False. 104 | | plugins | Individual repository plugin settings, to override default values. 105 | | baseline | The filename to parse the detect-secrets baseline from. 106 | | exclude\_regex | Per repo regex for excluding files from scan. 107 | 108 | Be sure to check out `examples/repos.yaml` for an reference. 109 | 110 | ## Configuration Options 111 | 112 | ### Plugins Options 113 | 114 | There are several ways to manage the various `detect-secrets` plugins for your 115 | individual tracked repositories. 116 | 117 | By default, all repositories will inherit the default values as prescribed by 118 | `detect-secrets`. These can be overridden with the same CLI flags as you would 119 | for `detect-secrets` (e.g. `--hex-limit 5`, `--no-private-key-scan`). 120 | 121 | If you choose to use a config file to add multiple repositories at once, you can 122 | specify all the plugins' options that you want to customize under the `plugins` 123 | key. Each key is the **name** of the plugin, and its values are the keyword 124 | arguments that it accepts. 125 | 126 | Note that any plugin not explicitly mentioned will use default values. If you 127 | explicitly want to disable a given plugin for a given repository, simply set its 128 | value to `False`. 129 | 130 | For example, in `examples/repos.yaml`, we have the following plugin configuration: 131 | 132 | ``` 133 | plugins: 134 | Base64HighEntropyString: 135 | base64_limit: 4 136 | PrivateKeyDetector: False 137 | ``` 138 | 139 | This will initialize plugins as follows: 140 | 141 | * Base64HighEntropyString: 4 (explicitly set) 142 | * BasicAuthDetector: enabled (enabled by default) 143 | * HexHighEntropyString: 3 (default limits) 144 | * PrivateKeyDetector: disabled (explicitly disabled) 145 | 146 | ### Storage Options 147 | 148 | `detect-secrets-server` stores state through metadata it keeps for the repositories 149 | it tracks. You can configure a variety of different storage options for this using 150 | the `--storage` option, including: 151 | 152 | #### file 153 | 154 | The most basic version is file-based storage. Metadata is stored in a directory structure 155 | under your configured root directory (`--root-dir`, defaults to `~/.detect-secrets-server`). 156 | 157 | #### s3 158 | 159 | If you want to store metadata as files in Amazon S3, you can do so too. Be sure to pip install 160 | the `boto3` library, and specify the additional S3 config options necessary. 161 | 162 | ``` 163 | s3 storage settings: 164 | Configure options for using Amazon S3 as a storage option. 165 | 166 | --s3-credentials-file FILENAME 167 | Specify keys for storing files on S3. 168 | --s3-bucket BUCKET_NAME 169 | Specify which bucket to perform S3 operations on. 170 | --s3-prefix PREFIX Specify the path prefix within the S3 bucket. 171 | --s3-config CONFIG_FILE 172 | Specify config file for all S3 config options. 173 | ``` 174 | 175 | You can also specify a config file instead, with `--s3-config`. For example, the following 176 | invocations are equivalent: 177 | 178 | ``` 179 | $ detect-secrets-server add git@github.com:yelp/detect-secrets --storage s3 \ 180 | --s3-credentials-file examples/aws_credentials.json \ 181 | --s3-bucket my-bucket-in-us-east-1 \ 182 | --s3-prefix secret_detector/tracked_repos 183 | ``` 184 | 185 | and 186 | 187 | ``` 188 | $ detect-secrets-server add git@github.com:yelp/detect-secrets --storage s3 \ 189 | --s3-config examples/s3.yaml 190 | ``` 191 | 192 | ### Alerting Options 193 | 194 | You are able to configure `detect-secrets-server` to alert you through a variety of ways 195 | when it detects a secret. These include: 196 | 197 | #### Adhoc Script 198 | 199 | When you specify an executable file with `--output-hook`, this file will run upon secret 200 | detection. Using `examples/standalone_hook.py` as an example, the output would look 201 | something like: 202 | 203 | ``` 204 | repo: yelp/detect-secrets 205 | { 206 | "detect_secrets/pre_commit_hook.py": [ 207 | { 208 | "author": "aaronloo", 209 | "hashed_secret": "7cec71eb6b597e71690378dfb169169a283f2e94", 210 | "line_number": 1, 211 | "type": "Hex High Entropy String" 212 | } 213 | ] 214 | } 215 | ``` 216 | 217 | #### pysensu 218 | 219 | We support 220 | [PySensu alerting](http://pysensu-yelp.readthedocs.io/en/latest/#pysensu_yelp.send_event) 221 | as well, so check out those docs if you want to configure your Sensu alerts. 222 | 223 | You can invoke this like the following: 224 | 225 | ``` 226 | $ detect-secrets-server scan yelp/detect-secrets \ 227 | --output-hook pysensu \ 228 | --output-config examples/pysensu.config.yaml 229 | ``` 230 | -------------------------------------------------------------------------------- /credentials.sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "accessKeyId": "", 3 | "secretAccessKey": "", 4 | "region": "us-east-1" 5 | } 6 | -------------------------------------------------------------------------------- /detect_secrets_server/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.3.2' 2 | -------------------------------------------------------------------------------- /detect_secrets_server/__main__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from detect_secrets.core.log import log 4 | 5 | from detect_secrets_server import actions 6 | from detect_secrets_server.core.usage.parser import ServerParserBuilder 7 | 8 | 9 | def parse_args(argv): 10 | return ServerParserBuilder().parse_args(argv) 11 | 12 | 13 | def main(argv=None): 14 | if argv is None: # pragma: no cover 15 | argv = sys.argv[1:] 16 | 17 | if len(argv) == 0: # pragma: no cover 18 | argv.append('-h') 19 | 20 | args = parse_args(argv) 21 | if args.verbose: # pragma: no cover 22 | log.set_debug_level(args.verbose) 23 | 24 | if args.action == 'add': 25 | if getattr(args, 'config', False): 26 | actions.initialize(args) 27 | else: 28 | actions.add_repo(args) 29 | 30 | elif args.action == 'install': 31 | actions.install_mapper(args) 32 | 33 | elif args.action == 'list': 34 | actions.display_tracked_repositories(args) 35 | 36 | elif args.action == 'scan': 37 | return actions.scan_repo(args) 38 | 39 | return 0 40 | 41 | 42 | if __name__ == '__main__': 43 | sys.exit(main()) 44 | -------------------------------------------------------------------------------- /detect_secrets_server/actions/__init__.py: -------------------------------------------------------------------------------- 1 | from .initialize import add_repo # noqa: F401 2 | from .initialize import initialize # noqa: F401 3 | from .install import install_mapper # noqa: F401 4 | from .list import display_tracked_repositories # noqa: F401 5 | from .scan import scan_repo # noqa: F401 6 | -------------------------------------------------------------------------------- /detect_secrets_server/actions/initialize.py: -------------------------------------------------------------------------------- 1 | from detect_secrets_server.repos.base_tracked_repo import OverrideLevel 2 | from detect_secrets_server.repos.factory import tracked_repo_factory 3 | 4 | 5 | def add_repo(args): 6 | """Sets up an individual repository for tracking.""" 7 | repo = _create_single_tracked_repo( 8 | repo=args.repo, 9 | 10 | # Will be updated to HEAD upon first update 11 | sha='', 12 | 13 | crontab=args.crontab, 14 | plugins=args.plugins, 15 | rootdir=args.root_dir, 16 | baseline_filename=args.baseline, 17 | exclude_regex=args.exclude_regex, 18 | 19 | is_local=args.local, 20 | s3_config=args.s3_config if args.storage == 's3' else None, 21 | ) 22 | 23 | _clone_and_save_repo(repo) 24 | 25 | 26 | def initialize(args): 27 | """Initializes a list of repositories from a specified repos.yaml, 28 | and returns the commands to add to your crontab. 29 | 30 | Sets up local file storage for tracking repositories. 31 | """ 32 | tracked_repos = [ 33 | _create_single_tracked_repo( 34 | repo=repo['repo'], 35 | crontab=repo['crontab'], 36 | sha=repo['sha'], 37 | plugins=repo['plugins'], 38 | baseline_filename=repo['baseline'], 39 | exclude_regex=repo['exclude_regex'], 40 | 41 | is_local=repo.get('is_local_repo', False), 42 | s3_config=args.s3_config if repo['storage'] == 's3' else None, 43 | 44 | rootdir=args.root_dir, 45 | ) 46 | for repo in args.repo 47 | ] 48 | 49 | for repo in tracked_repos: 50 | _clone_and_save_repo(repo) 51 | 52 | 53 | def _create_single_tracked_repo( 54 | repo, 55 | sha, 56 | crontab, 57 | plugins, 58 | rootdir, 59 | baseline_filename, 60 | exclude_regex, 61 | is_local, 62 | s3_config, 63 | ): 64 | """ 65 | These are REQUIRED arguments: 66 | :type repo: str 67 | :param repo: The url to clone, in `git clone ` 68 | 69 | :type sha: str 70 | :param sha: Last commit hash scanned 71 | 72 | :type crontab: str 73 | :param crontab: crontab syntax, denoting frequency of repo scan 74 | 75 | These arguments can have global defaults: 76 | :type plugins: dict 77 | :param plugins: mapping of plugin classnames to initialization values 78 | 79 | :type baseline_filename: str 80 | :param baseline_filename: repo-specific filename of baseline file 81 | 82 | Optional arguments include: 83 | :type rootdir: str 84 | :param rootdir: location of where you want to clone the repo for 85 | local storage 86 | 87 | :type exclude_regex: str 88 | :param exclude_regex: filenames that match this regex will be excluded from 89 | scanning. 90 | 91 | :type is_local: bool 92 | :param is_local: indicates that repository is locally stored (no need to 93 | git clone) 94 | 95 | :type s3_config: dict 96 | :param s3_config: files generated to save state will be synced with Amazon S3. 97 | """ 98 | repo_class = tracked_repo_factory( 99 | is_local, 100 | bool(s3_config), 101 | ) 102 | 103 | return repo_class( 104 | repo=repo, 105 | sha=sha, 106 | crontab=crontab, 107 | 108 | plugins=plugins, 109 | rootdir=rootdir, 110 | baseline_filename=baseline_filename, 111 | exclude_regex=exclude_regex, 112 | 113 | s3_config=s3_config, 114 | ) 115 | 116 | 117 | def _clone_and_save_repo(repo): 118 | """ 119 | :type repo: BaseTrackedRepo 120 | :param repo: repo to clone (if appropriate) and save 121 | """ 122 | # Clone repo, if needed. 123 | repo.storage.clone() 124 | 125 | # Make the last_commit_hash of repo point to HEAD 126 | if not repo.last_commit_hash: 127 | repo.update() 128 | return repo.save(OverrideLevel.ALWAYS) 129 | 130 | # Save the last_commit_hash, if we have nothing on file already 131 | return repo.save(OverrideLevel.NEVER) 132 | -------------------------------------------------------------------------------- /detect_secrets_server/actions/install.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | 3 | from crontab import CronTab 4 | 5 | from .list import list_tracked_repositories 6 | 7 | 8 | def install_mapper(args): 9 | mapping = { 10 | 'cron': _install_cron, 11 | } 12 | 13 | mapping[args.method](args) 14 | 15 | 16 | def _install_cron(args): 17 | # Get user's current crontab, so as not to override it 18 | old_content = [] 19 | cron = CronTab(user=True) 20 | with tempfile.NamedTemporaryFile() as temp: 21 | cron.write(temp.name) 22 | 23 | # Ignore all previous entries 24 | for line in temp.readlines(): 25 | line = line.decode() 26 | if line and 'detect-secrets-server' not in line: 27 | old_content.append(line.strip()) 28 | 29 | # Create jobs from tracked repositories 30 | jobs = [] 31 | for repo, is_local in list_tracked_repositories(args): 32 | command = '{} detect-secrets-server scan {}'.format( 33 | repo['crontab'], 34 | repo['repo'] 35 | ) 36 | if is_local: 37 | command += ' --local' 38 | if args.root_dir: 39 | command += ' --root-dir {}'.format(args.root_dir) 40 | if args.output_hook_command: 41 | command += ' {}'.format(args.output_hook_command) 42 | jobs.append(command.strip()) 43 | 44 | # Construct new crontab 45 | content = '\n'.join(jobs) 46 | if old_content: 47 | content = '{}\n\n{}'.format( 48 | '\n'.join(old_content), 49 | content, 50 | ) 51 | 52 | cron = CronTab( 53 | tab=content, 54 | user=True, 55 | ) 56 | cron.write_to_user(user=True) 57 | -------------------------------------------------------------------------------- /detect_secrets_server/actions/list.py: -------------------------------------------------------------------------------- 1 | from detect_secrets_server.storage.file import FileStorageWithLocalGit 2 | from detect_secrets_server.storage.s3 import S3Storage 3 | 4 | 5 | def display_tracked_repositories(args): 6 | for repo, is_local in list_tracked_repositories(args): 7 | if is_local is None or args.local == is_local: 8 | print(repo['repo']) 9 | 10 | 11 | def list_tracked_repositories(args): 12 | mapping = { 13 | 's3': lambda args: S3Storage(args.root_dir, args.s3_config), 14 | 15 | # Using the local version, since the local version includes the non-local one. 16 | 'file': lambda args: FileStorageWithLocalGit(args.root_dir), 17 | } 18 | 19 | return mapping[args.storage](args).get_tracked_repositories() 20 | -------------------------------------------------------------------------------- /detect_secrets_server/actions/scan.py: -------------------------------------------------------------------------------- 1 | from detect_secrets.core.log import log 2 | 3 | from detect_secrets_server.actions.initialize import _clone_and_save_repo 4 | from detect_secrets_server.repos.base_tracked_repo import OverrideLevel 5 | from detect_secrets_server.repos.factory import tracked_repo_factory 6 | 7 | try: 8 | FileNotFoundError 9 | except NameError: # pragma: no cover 10 | FileNotFoundError = IOError 11 | 12 | 13 | def scan_repo(args): 14 | """Returns 0 on success""" 15 | try: 16 | repo = tracked_repo_factory( 17 | args.local, 18 | bool(getattr(args, 's3_config', None)), 19 | ).load_from_file( 20 | args.repo, 21 | args.root_dir, 22 | s3_config=getattr(args, 's3_config', None), 23 | ) 24 | except FileNotFoundError: 25 | log.error('Unable to find repo: %s', args.repo) 26 | return 1 27 | 28 | # if last_commit_hash is empty, re-clone and see if there's an initial commit hash 29 | if repo.last_commit_hash is None: 30 | _clone_and_save_repo(repo) 31 | 32 | secrets = repo.scan( 33 | exclude_files_regex=args.exclude_files, 34 | exclude_lines_regex=args.exclude_lines, 35 | scan_head=args.scan_head, 36 | ) 37 | 38 | if (len(secrets.data) > 0) or args.always_run_output_hook: 39 | _alert_on_secrets_found(repo, secrets.json(), args.output_hook) 40 | 41 | if args.always_update_state or ( 42 | (len(secrets.data) == 0) 43 | and 44 | (not args.dry_run) 45 | and 46 | (not args.scan_head) 47 | ): 48 | _update_tracked_repo(repo) 49 | 50 | return 0 51 | 52 | 53 | def _update_tracked_repo(repo): 54 | """Save and update records, since the latest scan indicates that the 55 | most recent commit is clean. 56 | """ 57 | log.info('No secrets found for %s', repo.name) 58 | 59 | repo.update() 60 | repo.save(OverrideLevel.ALWAYS) 61 | 62 | 63 | def _alert_on_secrets_found(repo, secrets, output_hook): 64 | """ 65 | :type repo: detect_secrets_server.repos.base_tracked_repo.BaseTrackedRepo 66 | 67 | :type secrets: dict 68 | :param secrets: output of 69 | detect_secrets.core.secrets_collection.SecretsCollection.json() 70 | 71 | :type output_hook: detect_secrets_server.hooks.base.BaseHook 72 | """ 73 | log.error('Secrets found in %s', repo.name) 74 | 75 | _set_authors_for_found_secrets(repo, secrets) 76 | 77 | output_hook.alert(repo.name, secrets) 78 | 79 | 80 | def _set_authors_for_found_secrets(repo, secrets): 81 | """Use git blame to try and identify the user who committed the 82 | potential secret. This allows us to follow up with a specific user if 83 | a secret is found. 84 | 85 | Modifies secrets in-place. 86 | """ 87 | for filename in secrets: 88 | for potential_secret_dict in secrets[filename]: 89 | blame_info = repo.storage.get_blame( 90 | filename, 91 | potential_secret_dict['line_number'], 92 | ) 93 | potential_secret_dict['author'] = ( 94 | _extract_user_from_git_blame_info(blame_info) 95 | ) 96 | # Set commit as current head when found, not when secret was added 97 | potential_secret_dict['commit'] = repo.storage.get_last_commit_hash() 98 | 99 | 100 | def _extract_user_from_git_blame_info(info): 101 | """As this tool is meant to be used in an enterprise setting, we assume 102 | that the email address of the committer uniquely identifies a given user. 103 | 104 | This function extracts that information. 105 | """ 106 | info = info.split() 107 | 108 | index_of_mail = info.index('author-mail') 109 | email = info[index_of_mail + 1] # Eg. `` 110 | index_of_at_symbol = email.index('@') 111 | 112 | # This will skip the prefix `<`, and extract the user up to the `@` sign. 113 | return email[1:index_of_at_symbol] 114 | -------------------------------------------------------------------------------- /detect_secrets_server/adhoc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/detect-secrets-server/f2a708fa1a4628a7a93039cae92ef90e8b94d9db/detect_secrets_server/adhoc/__init__.py -------------------------------------------------------------------------------- /detect_secrets_server/adhoc/github/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/detect-secrets-server/f2a708fa1a4628a7a93039cae92ef90e8b94d9db/detect_secrets_server/adhoc/github/__init__.py -------------------------------------------------------------------------------- /detect_secrets_server/adhoc/github/webhook.py: -------------------------------------------------------------------------------- 1 | """ 2 | For organizations that integrate with Github, they have the ability to setup 3 | a webhook to receive events for all repos under the entire organization. In 4 | such cases, this script allows you to scan other fields in which secrets may 5 | be leaked, rather than just focusing on secrets in code. 6 | """ 7 | import io 8 | try: 9 | from contextlib import redirect_stdout 10 | except ImportError: # pragma: no cover 11 | import sys 12 | from contextlib import contextmanager 13 | 14 | @contextmanager 15 | def redirect_stdout(target): 16 | original = sys.stdout 17 | sys.stdout = target 18 | yield 19 | sys.stdout = original 20 | 21 | from detect_secrets.main import main as run_detect_secrets 22 | 23 | 24 | def scan_for_secrets(event_type, body, *args): 25 | """ 26 | :type event_type: str 27 | :param event_type: a full list can be found 28 | https://developer.github.com/v3/activity/events/types/ 29 | :type body: Dict[str, Any] 30 | :param body: a GitHub webhook event 31 | :type args: List 32 | :param args: parameters to pass to detect-secrets directly 33 | 34 | :rtype: Optional[str] 35 | :returns: link to field with leaked secret 36 | """ 37 | mapping = { 38 | 'commit_comment': _parse_comment, 39 | 'issue_comment': _parse_comment, 40 | 'pull_request_review_comment': _parse_comment, 41 | 'issues': _parse_issue, 42 | 'pull_request': _parse_pull_request, 43 | 44 | # NOTE: We're currently ignoring `project*` events, because we don't use 45 | # it. Pull requests welcome! 46 | } 47 | try: 48 | payload, attribution_link = mapping[event_type](body) 49 | except KeyError: 50 | # Not an applicable event. 51 | return None 52 | 53 | f = io.StringIO() 54 | with redirect_stdout(f): 55 | run_detect_secrets([ 56 | 'scan', 57 | '--string', payload, 58 | *args, 59 | ]) 60 | 61 | has_results = any([ 62 | line 63 | for line in f.getvalue().splitlines() 64 | 65 | # NOTE: Expected format: ': [True/False]' 66 | if 'True' in line.split(':')[1] 67 | ]) 68 | 69 | return attribution_link if has_results else None 70 | 71 | 72 | def _parse_comment(body): 73 | """ 74 | :type body: Dict[str, Any] 75 | :rtype: Tuple[str, str] 76 | """ 77 | if body.get('action', 'created') == 'deleted': 78 | # This indicates that this is not an applicable event. 79 | raise KeyError 80 | 81 | return ( 82 | body['comment']['body'], 83 | body['comment']['html_url'], 84 | ) 85 | 86 | 87 | def _parse_issue(body): 88 | """ 89 | :type body: Dict[str, Any] 90 | :rtype: Tuple[str, str] 91 | """ 92 | if body['action'] not in {'opened', 'edited', }: 93 | # This indicates that this is not an applicable event. 94 | raise KeyError 95 | 96 | # NOTE: Explicitly ignoring the issue "title" here, because 97 | # I trust developers enough (hopefully, not famous last words). 98 | # I think a secret in the title falls under the same threat 99 | # vector as a secret in the labels. 100 | return ( 101 | body['issue']['body'], 102 | body['issue']['html_url'], 103 | ) 104 | 105 | 106 | def _parse_pull_request(body): 107 | """ 108 | :type body: Dict[str, Any] 109 | :rtype: Tuple[str, str] 110 | """ 111 | if body['action'] not in {'opened', 'edited', }: 112 | # This indicates that this is not an applicable event. 113 | raise KeyError 114 | 115 | return ( 116 | body['pull_request']['body'], 117 | body['pull_request']['html_url'], 118 | ) 119 | -------------------------------------------------------------------------------- /detect_secrets_server/constants.py: -------------------------------------------------------------------------------- 1 | # We don't scan files with these extensions. 2 | # NOTE: We might be able to do this better with 3 | # `subprocess.check_output(['file', filename])` 4 | # and look for "ASCII text", but that might be more expensive. 5 | # 6 | # Definitely something to look into, if this list gets unruly long. 7 | IGNORED_FILE_EXTENSIONS = set( 8 | ( 9 | '.' + extension 10 | for extension in ( 11 | '7z', 12 | 'bmp', 13 | 'bz2', 14 | 'dmg', 15 | 'exe', 16 | 'gif', 17 | 'gz', 18 | 'ico', 19 | 'jar', 20 | 'jpg', 21 | 'jpeg', 22 | 'png', 23 | 'rar', 24 | 'realm', 25 | 's7z', 26 | 'svg', 27 | 'tar', 28 | 'tif', 29 | 'tiff', 30 | 'webp', 31 | 'zip', 32 | ) 33 | ) 34 | ) 35 | -------------------------------------------------------------------------------- /detect_secrets_server/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/detect-secrets-server/f2a708fa1a4628a7a93039cae92ef90e8b94d9db/detect_secrets_server/core/__init__.py -------------------------------------------------------------------------------- /detect_secrets_server/core/usage/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/detect-secrets-server/f2a708fa1a4628a7a93039cae92ef90e8b94d9db/detect_secrets_server/core/usage/__init__.py -------------------------------------------------------------------------------- /detect_secrets_server/core/usage/add.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | from crontab import CronSlices 5 | from detect_secrets.core.log import log 6 | from detect_secrets.core.usage import PluginOptions 7 | 8 | from .common.options import CommonOptions 9 | from .common.validators import config_file 10 | from .common.validators import is_git_url 11 | from .common.validators import is_valid_file 12 | 13 | 14 | class AddOptions(CommonOptions): 15 | """Describes how to use this tool for initialization of tracked repos.""" 16 | 17 | def __init__(self, subparser): 18 | super(AddOptions, self).__init__(subparser, 'add') 19 | 20 | def add_arguments(self): 21 | self.parser.add_argument( 22 | 'repo', 23 | nargs=1, 24 | help=( 25 | 'Track a repository specifying a git URL (that you would ' 26 | '`git clone`). Newly tracked repositories will store HEAD ' 27 | 'as the last scanned commit hash.' 28 | ), 29 | ) 30 | 31 | self.add_local_flag()\ 32 | ._add_config_flag_argument()\ 33 | ._add_initialize_options()\ 34 | ._add_crontab_argument() 35 | PluginOptions(self.parser).add_arguments() 36 | 37 | return self 38 | 39 | def _add_config_flag_argument(self): 40 | """Supports the use of a config file to initialize tracked repos.""" 41 | self.parser.add_argument( 42 | '--config', 43 | action='store_true', 44 | help=( 45 | 'Indicates that the repo argument is a config file, ' 46 | 'that should be used for initializing tracked repositories.' 47 | ), 48 | ) 49 | 50 | return self 51 | 52 | def _add_crontab_argument(self): 53 | self.parser.add_argument( 54 | '--crontab', 55 | nargs=1, 56 | default=['0 0 * * *'], 57 | type=_is_valid_crontab, 58 | help='Indicates the frequency which the repository should be scanned.', 59 | ) 60 | 61 | return self 62 | 63 | def _add_initialize_options(self): 64 | """Users can also configure their options on the command line, 65 | as compared to only specifying it through a config file. 66 | """ 67 | parser = self.parser.add_argument_group( 68 | title='settings', 69 | ) 70 | 71 | parser.add_argument( 72 | '--baseline', 73 | type=str, 74 | nargs=1, 75 | help=( 76 | 'Specify a default baseline filename to look for, within ' 77 | 'each tracked repository.' 78 | ), 79 | ) 80 | 81 | parser.add_argument( 82 | '--exclude-regex', 83 | type=str, 84 | nargs=1, 85 | help=( 86 | 'This regex will be added to repo metadata files when' 87 | 'adding a repository or overriding an existing one.' 88 | ), 89 | metavar='REGEX', 90 | ) 91 | 92 | return self 93 | 94 | @staticmethod 95 | def consolidate_args(args): 96 | """Validation and appropriate formatting of args.repo""" 97 | args.repo = args.repo[0] 98 | if args.local and args.config: 99 | raise argparse.ArgumentTypeError( 100 | 'Can\'t use --config with --local.', 101 | ) 102 | 103 | if args.config: 104 | try: 105 | args.repo = config_file(args.repo)['tracked'] 106 | except KeyError: 107 | raise argparse.ArgumentTypeError( 108 | 'Invalid config file format.' 109 | ) 110 | 111 | _consolidate_config_file_plugin_options(args) 112 | elif args.local: 113 | is_valid_file(args.repo) 114 | args.repo = os.path.abspath(args.repo) 115 | else: 116 | is_git_url(args.repo) 117 | 118 | _consolidate_initialize_args(args) 119 | 120 | # This needs to be run *after* args.repo is consolidated, so S3Options 121 | # can work properly. 122 | CommonOptions.consolidate_args(args) 123 | PluginOptions.consolidate_args(args) 124 | 125 | 126 | def _consolidate_initialize_args(args): 127 | if args.baseline: 128 | args.baseline = args.baseline[0] 129 | 130 | if args.exclude_regex: 131 | args.exclude_regex = args.exclude_regex[0] 132 | 133 | if args.crontab: 134 | args.crontab = args.crontab[0] 135 | 136 | 137 | def _consolidate_config_file_plugin_options(args): 138 | """ 139 | There are three ways to configure plugin options (in order of priority): 140 | 1. command line 141 | 2. config file 142 | 3. default values 143 | 144 | This overrides config file values with specified command line values, 145 | if appropriate. This is also mostly based off PluginOptions.consolidate_args 146 | """ 147 | # Collect CLI specified arguments 148 | disabled_plugins = [] 149 | cli_options = {} 150 | known_plugins = set() 151 | for plugin in PluginOptions.all_plugins: 152 | known_plugins.add(plugin.classname) 153 | 154 | arg_name = PluginOptions._convert_flag_text_to_argument_name( 155 | plugin.disable_flag_text, 156 | ) 157 | 158 | is_disabled = getattr(args, arg_name) 159 | if is_disabled: 160 | disabled_plugins.append(plugin.classname) 161 | continue 162 | 163 | specified_values = {} 164 | for arg in plugin.related_args: 165 | arg_name = PluginOptions._convert_flag_text_to_argument_name(arg[0]) 166 | specified_value = getattr(args, arg_name) 167 | if specified_value: 168 | specified_values[arg_name] = specified_value 169 | 170 | if specified_values: 171 | cli_options[plugin.classname] = specified_values 172 | 173 | repos = [] 174 | 175 | # Apply it to all tracked repos 176 | for tracked_repo in args.repo: 177 | if _should_discard_tracked_repo_in_config(tracked_repo): 178 | continue 179 | 180 | if 'plugins' not in tracked_repo: 181 | repos.append(tracked_repo) 182 | continue 183 | 184 | # Remove unknown plugins 185 | unknown_plugins = filter( 186 | lambda name: name not in known_plugins, 187 | tracked_repo['plugins'].keys(), 188 | ) 189 | for plugin_name in list(unknown_plugins): 190 | del tracked_repo['plugins'][plugin_name] 191 | 192 | # CLI overrides 193 | for plugin_classname, values in cli_options.items(): 194 | if plugin_classname not in tracked_repo['plugins']: 195 | tracked_repo['plugins'][plugin_classname] = {} 196 | 197 | for key, value in values.items(): 198 | tracked_repo['plugins'][plugin_classname][key] = value 199 | 200 | # Apply disabled plugins after setting them, to avoid strange use case 201 | # where user sets both the disable flag, and the custom value flag. 202 | # This menas that disabled plugins carry more weight than custom values. 203 | for disabled_plugin in disabled_plugins: 204 | if disabled_plugin in tracked_repo['plugins']: 205 | del tracked_repo['plugins'][disabled_plugin] 206 | 207 | repos.append(tracked_repo) 208 | 209 | args.repo = repos 210 | 211 | 212 | def _should_discard_tracked_repo_in_config(tracked_repo): 213 | try: 214 | if tracked_repo.get('is_local_repo', False): 215 | is_valid_file(tracked_repo['repo']) 216 | else: 217 | is_git_url(tracked_repo['repo']) 218 | 219 | return False 220 | except argparse.ArgumentTypeError as e: 221 | # We log the error, rather than hard failing, because we don't want 222 | # to hard fail if one out of many repositories are bad. 223 | log.error(str(e)) 224 | return True 225 | 226 | 227 | def _is_valid_crontab(crontab): 228 | if CronSlices.is_valid(crontab): 229 | return crontab 230 | 231 | raise argparse.ArgumentTypeError( 232 | 'Invalid crontab.', 233 | ) 234 | -------------------------------------------------------------------------------- /detect_secrets_server/core/usage/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/detect-secrets-server/f2a708fa1a4628a7a93039cae92ef90e8b94d9db/detect_secrets_server/core/usage/common/__init__.py -------------------------------------------------------------------------------- /detect_secrets_server/core/usage/common/hooks.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | 4 | class HookDescriptor(namedtuple( 5 | 'HookDescriptor', 6 | [ 7 | # The value that users can input, to refer to this hook. 8 | # e.g. `--output-hook ` 9 | 'display_name', 10 | 11 | # module name of plugin, used for initialization 12 | 'module_name', 13 | 14 | 'class_name', 15 | 16 | # A HookDescriptor config enum 17 | 'config_setting', 18 | ] 19 | )): 20 | CONFIG_NOT_SUPPORTED = 0 21 | CONFIG_OPTIONAL = 1 22 | CONFIG_REQUIRED = 2 23 | 24 | def __new__(cls, config_setting=None, **kwargs): 25 | if config_setting is None: 26 | config_setting = cls.CONFIG_NOT_SUPPORTED 27 | 28 | return super(HookDescriptor, cls).__new__( 29 | cls, 30 | config_setting=config_setting, 31 | **kwargs 32 | ) 33 | 34 | 35 | ALL_HOOKS = [ 36 | HookDescriptor( 37 | display_name='pysensu', 38 | module_name='detect_secrets_server.hooks.pysensu_yelp', 39 | class_name='PySensuYelpHook', 40 | config_setting=HookDescriptor.CONFIG_REQUIRED, 41 | ), 42 | ] 43 | -------------------------------------------------------------------------------- /detect_secrets_server/core/usage/common/install.py: -------------------------------------------------------------------------------- 1 | try: 2 | from functools import lru_cache 3 | except ImportError: 4 | from functools32 import lru_cache 5 | 6 | 7 | @lru_cache(maxsize=1) 8 | def get_install_options(): 9 | return ['cron'] 10 | -------------------------------------------------------------------------------- /detect_secrets_server/core/usage/common/options.py: -------------------------------------------------------------------------------- 1 | import os 2 | from abc import ABCMeta 3 | 4 | from .. import s3 5 | from .storage import get_storage_options 6 | 7 | 8 | class CommonOptions(object): 9 | """There are some common flags between the various different subparsers, that 10 | we don't want to display in the main help section. 11 | 12 | This contains those flags. 13 | """ 14 | __metaclass__ = ABCMeta 15 | 16 | def __init__(self, subparser, action): 17 | self.parser = subparser.add_parser(action) 18 | self._add_common_arguments() 19 | 20 | def add_arguments(self): 21 | self.add_local_flag() 22 | 23 | return self 24 | 25 | def add_local_flag(self): 26 | self.parser.add_argument( 27 | '-L', 28 | '--local', 29 | action='store_true', 30 | help=( 31 | 'Indicates that the repo argument is a locally stored ' 32 | 'repository (rather than a git URL to be cloned).' 33 | ), 34 | ) 35 | 36 | return self 37 | 38 | def _add_common_arguments(self): 39 | self.parser.add_argument( 40 | '-s', 41 | '--storage', 42 | choices=get_storage_options(), 43 | default='file', 44 | help=( 45 | 'Determines the datastore to use for storing metadata.' 46 | ), 47 | ) 48 | 49 | self.parser.add_argument( 50 | '--root-dir', 51 | type=str, 52 | nargs=1, 53 | default=['~/.detect-secrets-server'], 54 | help=( 55 | 'Specify location to clone git repositories to. This ' 56 | 'folder will also hold any metadata tracking files, if ' 57 | 'no other persistent storage option is selected. ' 58 | 'Default: ~/.detect-secrets-server' 59 | ), 60 | ) 61 | 62 | s3.S3Options(self.parser).add_arguments() 63 | 64 | @staticmethod 65 | def consolidate_args(args): 66 | args.root_dir = os.path.abspath( 67 | os.path.expanduser(args.root_dir[0]) 68 | ) 69 | 70 | s3.S3Options.consolidate_args(args) 71 | -------------------------------------------------------------------------------- /detect_secrets_server/core/usage/common/output.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import textwrap 3 | from importlib import import_module 4 | 5 | from .hooks import ALL_HOOKS 6 | from .hooks import HookDescriptor 7 | from .validators import is_valid_file 8 | from detect_secrets_server.hooks.external import ExternalHook 9 | from detect_secrets_server.hooks.stdout import StdoutHook 10 | 11 | 12 | class OutputOptions(object): 13 | 14 | def __init__(self, parser): 15 | self.parser = parser.add_argument_group( 16 | title='output settings', 17 | description=( 18 | 'Configure output method, for alerting upon secrets found ' 19 | 'in the tracked repositories.' 20 | ), 21 | ) 22 | 23 | def add_arguments(self): 24 | self.parser.add_argument( 25 | '--output-hook', 26 | type=_is_valid_output_hook, 27 | help=( 28 | 'Either one of the pre-registered hooks ({}) ' 29 | 'or a path to a valid executable file.'.format( 30 | ', '.join( 31 | map( 32 | lambda x: x.display_name, 33 | ALL_HOOKS, 34 | ) 35 | ) 36 | ) 37 | ), 38 | metavar='HOOK', 39 | ) 40 | 41 | self.parser.add_argument( 42 | '--output-config', 43 | type=is_valid_file, 44 | help=( 45 | 'Configuration file to initialize output hook, if required.' 46 | ), 47 | metavar='CONFIG_FILENAME', 48 | ) 49 | 50 | return self 51 | 52 | @staticmethod 53 | def consolidate_args(args): 54 | """ 55 | Based on the --output-hook specified, performs validation and 56 | initializes args.output_hook, args.output_hook_command. 57 | """ 58 | args.output_hook, args.output_hook_command = \ 59 | _initialize_output_hook_and_raw_command( 60 | args.output_hook, 61 | args.output_config, 62 | ) 63 | 64 | 65 | def _is_valid_output_hook(hook): 66 | """ 67 | A valid hook is either one of the pre-defined hooks, or a filename 68 | to an arbitrary executable script (for further customization). 69 | """ 70 | for registered_hook in ALL_HOOKS: 71 | if hook == registered_hook.display_name: 72 | return hook 73 | 74 | is_valid_file( 75 | hook, 76 | textwrap.dedent(""" 77 | output-hook must be one of the following values: 78 | {} 79 | or a valid executable filename. 80 | 81 | "{}" does not qualify. 82 | """)[:-1].format( 83 | '\n'.join( 84 | map( 85 | lambda x: ' - {}'.format(x.display_name), 86 | ALL_HOOKS, 87 | ) 88 | ), 89 | hook, 90 | ), 91 | ) 92 | 93 | return hook 94 | 95 | 96 | def _initialize_output_hook_and_raw_command(hook_name, config_filename): 97 | """ 98 | :rtype: tuple(BaseHook, str) 99 | :returns: 100 | BaseHook is the the hook instance to interact with 101 | output_hook_command is how to call the output-hook as CLI args. 102 | """ 103 | hook_found = None 104 | command = '--output-hook {}'.format(hook_name) 105 | 106 | for hook in ALL_HOOKS: 107 | if hook_name == hook.display_name: 108 | hook_found = hook 109 | break 110 | else: 111 | if hook_name: 112 | return ExternalHook(hook_name), command 113 | 114 | return StdoutHook(), '' 115 | 116 | if hook_found.config_setting == HookDescriptor.CONFIG_REQUIRED and \ 117 | not config_filename: 118 | # We want to display this error, as if it was during argument validation. 119 | raise argparse.ArgumentTypeError( 120 | '{} hook requires a config file. ' 121 | 'Pass one in through --output-config.'.format( 122 | hook_found.display_name, 123 | ) 124 | ) 125 | 126 | # These values are not user injectable, so it should be ok. 127 | try: 128 | module = import_module(hook_found.module_name) 129 | hook_class = getattr(module, hook_found.class_name) 130 | except ImportError as e: 131 | raise argparse.ArgumentTypeError(str(e)) 132 | 133 | if hook_found.config_setting == HookDescriptor.CONFIG_NOT_SUPPORTED: 134 | return hook_class(), command 135 | 136 | command += ' --output-config {}'.format(config_filename) 137 | with open(config_filename) as f: 138 | return hook_class(f.read()), command 139 | -------------------------------------------------------------------------------- /detect_secrets_server/core/usage/common/storage.py: -------------------------------------------------------------------------------- 1 | try: 2 | from functools import lru_cache 3 | except ImportError: # pragma: no cover 4 | from functools32 import lru_cache 5 | 6 | 7 | @lru_cache(maxsize=1) 8 | def get_storage_options(): 9 | output = ['file'] 10 | 11 | if should_enable_s3_options(): 12 | output.append('s3') 13 | 14 | return output 15 | 16 | 17 | @lru_cache(maxsize=1) 18 | def should_enable_s3_options(): # pragma: no cover 19 | try: 20 | import boto3 # noqa: F401 21 | return True 22 | except ImportError: 23 | return False 24 | -------------------------------------------------------------------------------- /detect_secrets_server/core/usage/common/validators.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import os 4 | 5 | import yaml 6 | 7 | 8 | def is_valid_file(path, error_msg=None): 9 | if not os.path.exists(path): 10 | if not error_msg: 11 | error_msg = 'File does not exist: %s' % path 12 | 13 | raise argparse.ArgumentTypeError(error_msg) 14 | 15 | return path 16 | 17 | 18 | def is_git_url(url): 19 | if not url.startswith('git@') and not url.startswith('https://'): 20 | raise argparse.ArgumentTypeError( 21 | '"{}" is not a cloneable git URL.'.format(url) 22 | ) 23 | 24 | 25 | def config_file(path): 26 | """ 27 | Custom type to enforce input is valid filepath, and if valid, 28 | extract file contents. 29 | """ 30 | is_valid_file(path) 31 | 32 | with open(path) as f: 33 | return yaml.safe_load(f.read()) 34 | 35 | 36 | def json_file(path): 37 | is_valid_file(path) 38 | 39 | with open(path) as f: 40 | return json.load(f) 41 | -------------------------------------------------------------------------------- /detect_secrets_server/core/usage/install.py: -------------------------------------------------------------------------------- 1 | from .common.install import get_install_options 2 | from .common.options import CommonOptions 3 | from .common.output import OutputOptions 4 | 5 | 6 | class InstallOptions(CommonOptions): 7 | """Describes how to use this tool to install via various means.""" 8 | 9 | def __init__(self, subparser): 10 | super(InstallOptions, self).__init__(subparser, 'install') 11 | 12 | def add_arguments(self): 13 | self.parser.add_argument( 14 | 'method', 15 | choices=get_install_options(), 16 | default='cron', 17 | help='Method of installation.', 18 | ) 19 | 20 | OutputOptions(self.parser).add_arguments() 21 | 22 | return self 23 | 24 | @staticmethod 25 | def consolidate_args(args): 26 | for option in [CommonOptions, OutputOptions]: 27 | option.consolidate_args(args) 28 | -------------------------------------------------------------------------------- /detect_secrets_server/core/usage/list.py: -------------------------------------------------------------------------------- 1 | from .common.options import CommonOptions 2 | 3 | 4 | class ListOptions(CommonOptions): 5 | 6 | def __init__(self, subparser): 7 | super(ListOptions, self).__init__(subparser, 'list') 8 | -------------------------------------------------------------------------------- /detect_secrets_server/core/usage/parser.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from detect_secrets.core.usage import ParserBuilder 4 | 5 | import detect_secrets_server 6 | from .add import AddOptions 7 | from .install import InstallOptions 8 | from .list import ListOptions 9 | from .scan import ScanOptions 10 | 11 | 12 | class ServerParserBuilder(ParserBuilder): 13 | """Arguments, for the server component""" 14 | 15 | def __init__(self): 16 | super(ServerParserBuilder, self).__init__() 17 | self._add_server_use_arguments() 18 | 19 | def _add_version_argument(self): 20 | """Overridden, because we don't want to be showing the version 21 | of detect-secrets plugin that we depend on. 22 | """ 23 | self.parser.add_argument( 24 | '--version', 25 | action='version', 26 | version=detect_secrets_server.__version__, 27 | help='Display version information.', 28 | ) 29 | 30 | return self 31 | 32 | def _add_server_use_arguments(self): 33 | subparser = self.parser.add_subparsers( 34 | dest='action', 35 | ) 36 | 37 | for option in (AddOptions, ListOptions, InstallOptions, ScanOptions): 38 | option(subparser).add_arguments() 39 | 40 | return self 41 | 42 | def parse_args(self, argv): 43 | # NOTE: We can't just call `super`, because we need to parse the PluginOptions 44 | # after we parse the config file, since we need to be able to distinguish 45 | # between default values, and values that are set. 46 | output = self.parser.parse_args(argv) 47 | 48 | try: 49 | if output.action == 'add': 50 | AddOptions.consolidate_args(output) 51 | if getattr(output, 'config', False): 52 | apply_default_plugin_options_to_repos(output) 53 | 54 | elif output.action == 'scan': 55 | ScanOptions.consolidate_args(output) 56 | 57 | elif output.action == 'install': 58 | InstallOptions.consolidate_args(output) 59 | 60 | elif output.action == 'list': 61 | ListOptions.consolidate_args(output) 62 | 63 | except argparse.ArgumentTypeError as e: 64 | self.parser.error(e) 65 | 66 | return output 67 | 68 | 69 | def apply_default_plugin_options_to_repos(args): 70 | """ 71 | There are three ways to configure options (in order of priority): 72 | 1. command line 73 | 2. config file 74 | 3. default values 75 | 76 | This applies default values to the config file, if appropriate. 77 | """ 78 | for tracked_repo in args.repo: 79 | # TODO Issue 17: Not touching exclude_regex in repo metadata 80 | # Just ignoring it for now and using the exclusion CLI args given when calling `scan` 81 | # (This can be ignored because this function is only called by `add`) 82 | for key in ( 83 | 'baseline', 84 | 'crontab', 85 | 'exclude_regex', 86 | 'storage', 87 | ): 88 | if key not in tracked_repo: 89 | tracked_repo[key] = getattr(args, key) 90 | 91 | if 'plugins' not in tracked_repo: 92 | tracked_repo['plugins'] = {} 93 | 94 | for key, value in args.plugins.items(): 95 | if key not in tracked_repo['plugins']: 96 | tracked_repo['plugins'][key] = value 97 | 98 | disabled_plugins = [ 99 | plugin_name 100 | for plugin_name, value in tracked_repo['plugins'].items() 101 | if value is False 102 | ] 103 | for plugin_name in disabled_plugins: 104 | del tracked_repo['plugins'][plugin_name] 105 | 106 | if 'sha' not in tracked_repo: 107 | tracked_repo['sha'] = '' 108 | -------------------------------------------------------------------------------- /detect_secrets_server/core/usage/s3.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from .common.storage import should_enable_s3_options 4 | from .common.validators import config_file 5 | from .common.validators import json_file 6 | 7 | 8 | class S3Options(object): 9 | 10 | def __init__(self, parser): 11 | self.parser = None 12 | if not should_enable_s3_options(): 13 | return 14 | 15 | self.parser = parser.add_argument_group( 16 | title='s3 storage settings', 17 | description=( 18 | 'Configure options for using Amazon S3 as a storage option.' 19 | ), 20 | ) 21 | 22 | def add_arguments(self): 23 | if not self.parser: 24 | return self 25 | 26 | self.parser.add_argument( 27 | '--s3-credentials-file', 28 | nargs=1, 29 | type=str, 30 | help='Specify keys for storing files on S3.', 31 | metavar='FILENAME', 32 | ) 33 | self.parser.add_argument( 34 | '--s3-bucket', 35 | nargs=1, 36 | type=str, 37 | help='Specify which bucket to perform S3 operations on.', 38 | metavar='BUCKET_NAME', 39 | ) 40 | self.parser.add_argument( 41 | '--s3-prefix', 42 | nargs=1, 43 | type=str, 44 | default=[''], 45 | help='Specify the path prefix within the S3 bucket.', 46 | metavar='PREFIX', 47 | ) 48 | self.parser.add_argument( 49 | '--s3-config', 50 | nargs=1, 51 | type=config_file, 52 | default=None, 53 | help='Specify config file for all S3 config options.', 54 | metavar='CONFIG_FILE', 55 | ) 56 | 57 | return self 58 | 59 | @staticmethod 60 | def consolidate_args(args): 61 | if not _needs_s3_config(args): 62 | return 63 | 64 | try: 65 | args.s3_config = args.s3_config[0] 66 | except AttributeError: 67 | raise argparse.ArgumentTypeError( 68 | 'Please pip install the `boto3` library.' 69 | ) 70 | except TypeError: 71 | # If nothing is specified, then args.s3_config == None. 72 | # This is sufficient for conditional logic to determine whether 73 | # user supplied a config file. 74 | pass 75 | 76 | if args.s3_config and any([ 77 | args.s3_bucket, 78 | args.s3_credentials_file, 79 | args.s3_prefix[0], 80 | ]): 81 | raise argparse.ArgumentTypeError( 82 | 'Can\'t specify --s3-config with other s3 command line arguments.', 83 | ) 84 | elif not args.s3_config and not all([ 85 | args.s3_credentials_file, 86 | args.s3_bucket, 87 | ]): 88 | raise argparse.ArgumentTypeError( 89 | 'the following arguments are required: --s3-credentials-file, --s3-bucket', 90 | ) 91 | 92 | if args.s3_config: 93 | bucket_name = args.s3_config['bucket_name'] 94 | prefix = args.s3_config['prefix'] 95 | creds_filename = args.s3_config['credentials_filename'] 96 | else: 97 | bucket_name = args.s3_bucket[0] 98 | prefix = args.s3_prefix[0] 99 | creds_filename = args.s3_credentials_file[0] 100 | 101 | creds = json_file(creds_filename) 102 | 103 | # We do not need these anymore. 104 | del args.s3_bucket 105 | del args.s3_prefix 106 | del args.s3_credentials_file 107 | 108 | args.s3_config = { 109 | 'prefix': prefix, 110 | 'bucket': bucket_name, 111 | 'creds_filename': creds_filename, 112 | 'access_key': creds['accessKeyId'], 113 | 'secret_access_key': creds['secretAccessKey'], 114 | } 115 | 116 | 117 | def _needs_s3_config(args): 118 | if args.storage == 's3': 119 | return True 120 | 121 | if args.action == 'add' and args.config: 122 | for repo in args.repo: 123 | if repo.get('storage') == 's3': 124 | return True 125 | 126 | return False 127 | -------------------------------------------------------------------------------- /detect_secrets_server/core/usage/scan.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | from detect_secrets.core.usage import PluginOptions 5 | 6 | from .common.options import CommonOptions 7 | from .common.output import OutputOptions 8 | from .common.validators import is_valid_file 9 | 10 | 11 | class ScanOptions(CommonOptions): 12 | """Describes how to use this tool for scanning purposes.""" 13 | 14 | def __init__(self, subparser): 15 | super(ScanOptions, self).__init__(subparser, 'scan') 16 | 17 | def add_arguments(self): 18 | self.parser.add_argument( 19 | 'repo', 20 | nargs=1, 21 | help=( 22 | 'Scans an already tracked repository by specifying a git URL ' 23 | '(that you would `git clone`).' 24 | ), 25 | ) 26 | self.parser.add_argument( 27 | '--dry-run', 28 | action='store_true', 29 | help=( 30 | 'Scan the repository with specified plugin configuration, without ' 31 | 'saving state.' 32 | ), 33 | ) 34 | self.parser.add_argument( 35 | '--scan-head', 36 | action='store_true', 37 | help=( 38 | 'Scan the entire repo as opposed to diffs' 39 | ), 40 | 41 | ) 42 | self.parser.add_argument( 43 | '--always-update-state', 44 | action='store_true', 45 | help=( 46 | 'Always update the internal tracking state (latest commit scanned) ' 47 | 'after a successful scan, despite finding secrets.' 48 | ), 49 | ) 50 | 51 | self.parser.add_argument( 52 | '--exclude-files', 53 | type=str, 54 | help=( 55 | 'Filenames that match this regex will be ignored when ' 56 | 'scanning for secrets.' 57 | ), 58 | metavar='REGEX', 59 | ) 60 | 61 | self.parser.add_argument( 62 | '--exclude-lines', 63 | type=str, 64 | help=( 65 | 'Lines that match this regex will be ignored when ' 66 | 'scanning for secrets.' 67 | ), 68 | metavar='REGEX', 69 | ) 70 | 71 | self.parser.add_argument( 72 | '--always-run-output-hook', 73 | action='store_true', 74 | help=( 75 | 'Always run the output hook, even if no issues have been found, ' 76 | 'must be run with the --output-hook option.' 77 | ), 78 | ) 79 | 80 | self.add_local_flag() 81 | for option in [PluginOptions, OutputOptions]: 82 | option(self.parser).add_arguments() 83 | 84 | return self 85 | 86 | @staticmethod 87 | def consolidate_args(args): 88 | """Validation and appropriate formatting of args.repo""" 89 | if args.dry_run and args.always_update_state: 90 | raise argparse.ArgumentTypeError( 91 | 'Can\'t use --dry-run with --always-update-state.', 92 | ) 93 | if (args.always_run_output_hook and (None is args.output_hook)): 94 | raise argparse.ArgumentTypeError( 95 | '--always-run-output-hook must be run with --output-hook', 96 | ) 97 | 98 | for option in [CommonOptions, OutputOptions]: 99 | option.consolidate_args(args) 100 | 101 | args.repo = args.repo[0] 102 | if args.local: 103 | is_valid_file(args.repo) 104 | args.repo = os.path.abspath(args.repo) 105 | 106 | PluginOptions.consolidate_args(args) 107 | -------------------------------------------------------------------------------- /detect_secrets_server/hooks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/detect-secrets-server/f2a708fa1a4628a7a93039cae92ef90e8b94d9db/detect_secrets_server/hooks/__init__.py -------------------------------------------------------------------------------- /detect_secrets_server/hooks/base.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta 2 | from abc import abstractmethod 3 | 4 | 5 | class BaseHook(object): # pragma: no cover 6 | """This is an abstract class to define Hooks API. A hook is an alerting system 7 | that allows you connect your server scanning results to your larger ecosystem 8 | (e.g. email alerts, IRC pings...) 9 | """ 10 | 11 | __metaclass__ = ABCMeta 12 | 13 | @abstractmethod 14 | def alert(self, repo_name, secrets): 15 | """ 16 | :type repo_name: str 17 | :param repo_name: the repository where secrets were found. 18 | 19 | :type secrets: dict 20 | :param secrets: dictionary; where keys are filenames 21 | """ 22 | pass 23 | -------------------------------------------------------------------------------- /detect_secrets_server/hooks/external.py: -------------------------------------------------------------------------------- 1 | """ 2 | This provides a common interface to run arbitrary scripts, provided 3 | by the `--output-hook` option. 4 | 5 | The script you provide must be executable, and accept two command line 6 | inputs: 7 | 1. the name of the repository where secrets are found, and 8 | 2. the json output of secrets found. 9 | """ 10 | import json 11 | import os 12 | import subprocess 13 | 14 | from .base import BaseHook 15 | 16 | 17 | class ExternalHook(BaseHook): 18 | 19 | def __init__(self, filename): 20 | if filename.startswith('/'): 21 | self.filename = filename 22 | else: 23 | self.filename = os.path.join( 24 | os.getcwd(), 25 | filename, 26 | ) 27 | 28 | def alert(self, repo_name, secrets): 29 | subprocess.call([ 30 | self.filename, 31 | repo_name, 32 | json.dumps(secrets), 33 | ]) 34 | -------------------------------------------------------------------------------- /detect_secrets_server/hooks/pysensu_yelp.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example config file (yaml) format: 3 | 4 | # Name needs to be one word 5 | name: SecretFound 6 | alert_after: 0 7 | 8 | # -1 denotes exponential backoff 9 | realert_every: -1 10 | runbook: no-runbook-available 11 | dependencies: [] 12 | team: team-security 13 | irc_channels: [] 14 | notification_email: to-whom-it-may-concern@example.com 15 | ticket: False 16 | project: False 17 | page: False 18 | tip: detect_secrets found a secret 19 | 20 | # status needs to be 1 (warning) or higher to send the email 21 | status: 1 22 | 23 | # null turns into None 24 | ttl: null 25 | 26 | This will send an alert to Sensu, with the above configurations. 27 | See https://github.com/Yelp/pysensu-yelp for more details. 28 | """ 29 | import pysensu_yelp 30 | import yaml 31 | 32 | from .base import BaseHook 33 | 34 | 35 | class PySensuYelpHook(BaseHook): 36 | 37 | def __init__(self, config): 38 | """ 39 | :type config: str, yaml formatted 40 | """ 41 | self.config_data = yaml.safe_load(config) 42 | 43 | def alert(self, repo_name, secrets): 44 | self.config_data['output'] = "In repo " + repo_name + "\n" + str(secrets) 45 | pysensu_yelp.send_event(**self.config_data) 46 | -------------------------------------------------------------------------------- /detect_secrets_server/hooks/stdout.py: -------------------------------------------------------------------------------- 1 | """ 2 | Print to stdout. 3 | """ 4 | import json 5 | 6 | from .base import BaseHook 7 | 8 | 9 | class StdoutHook(BaseHook): 10 | 11 | def alert(self, repo_name, secrets): 12 | print(json.dumps({ 13 | 'repo': repo_name, 14 | 'output': secrets, 15 | })) 16 | -------------------------------------------------------------------------------- /detect_secrets_server/repos/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/detect-secrets-server/f2a708fa1a4628a7a93039cae92ef90e8b94d9db/detect_secrets_server/repos/__init__.py -------------------------------------------------------------------------------- /detect_secrets_server/repos/base_tracked_repo.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import sys 4 | from enum import Enum 5 | 6 | from detect_secrets.core.baseline import get_secrets_not_in_baseline 7 | from detect_secrets.core.secrets_collection import SecretsCollection 8 | from detect_secrets.plugins.common import initialize as initialize_plugins 9 | 10 | from detect_secrets_server.storage.core import git 11 | from detect_secrets_server.storage.file import FileStorage 12 | 13 | 14 | class OverrideLevel(Enum): 15 | NEVER = 0 16 | ASK_USER = 1 17 | ALWAYS = 2 18 | 19 | 20 | class BaseTrackedRepo(object): 21 | 22 | # This should be overriden in subclasses. 23 | STORAGE_CLASS = FileStorage 24 | 25 | @classmethod 26 | def initialize_storage(cls, base_directory): 27 | return cls.STORAGE_CLASS(base_directory) 28 | 29 | def __init__( 30 | self, 31 | repo, 32 | sha, 33 | plugins, 34 | baseline_filename, 35 | exclude_regex, 36 | crontab='', 37 | rootdir=None, 38 | **kwargs 39 | ): 40 | """ 41 | :type repo: str 42 | :param repo: git URL or local path of repo 43 | 44 | :type sha: str 45 | :param sha: last commit hash scanned 46 | 47 | :type plugins: dict 48 | :param plugins: values to configure various plugins, formatted as 49 | described in detect_secrets.core.usage 50 | 51 | :type rootdir: str 52 | :param rootdir: the directory to clone git repositories to. 53 | 54 | :type exclude_regex: str 55 | :param exclude_regex: used for repository scanning; if a filename 56 | matches this exclude_regex, it is not scanned. 57 | 58 | :type crontab: str 59 | :param crontab: crontab syntax, for periodic scanning. 60 | 61 | :type baseline_filename: str 62 | :param baseline_filename: each repository may have a different 63 | baseline filename. This allows us to customize these filenames 64 | per repository. 65 | """ 66 | self.last_commit_hash = sha 67 | self.repo = repo 68 | self.crontab = crontab 69 | self.plugin_config = plugins 70 | self.baseline_filename = baseline_filename 71 | self.exclude_regex = exclude_regex 72 | 73 | if rootdir: 74 | self.storage = self.initialize_storage(rootdir).setup(repo) 75 | 76 | @classmethod 77 | def load_from_file( 78 | cls, 79 | repo_name, 80 | base_directory, 81 | *args, 82 | **kwargs 83 | ): 84 | """This will load a TrackedRepo to memory, from a given meta tracked 85 | file. For automated management without a database. 86 | 87 | The meta tracked file is in the format of self.__dict__ 88 | 89 | :type repo_name: str 90 | :param repo_name: If the git URL is `git@github.com:yelp/detect-secrets` 91 | this value will be `yelp/detect-secrets` 92 | 93 | :rtype: TrackedRepo 94 | :raises: FileNotFoundError 95 | """ 96 | storage = cls.initialize_storage(base_directory) 97 | 98 | data = cls.get_tracked_repo_data(storage, repo_name) 99 | 100 | output = cls(**data) 101 | output.storage = storage.setup(output.repo) 102 | 103 | return output 104 | 105 | @classmethod 106 | def get_tracked_repo_data(cls, storage, repo_name): 107 | if repo_name.startswith('git@') or repo_name.startswith('http'): 108 | repo_name = storage.get_repo_name(repo_name) 109 | 110 | return storage.get(storage.hash_filename(repo_name)) 111 | 112 | @property 113 | def name(self): 114 | return self.storage.repository_name 115 | 116 | def scan(self, exclude_files_regex=None, exclude_lines_regex=None, scan_head=False): 117 | """Fetches latest changes, and scans the git diff between last_commit_hash 118 | and HEAD. 119 | 120 | :raises: subprocess.CalledProcessError 121 | 122 | :type exclude_files_regex: str|None 123 | :param exclude_files_regex: A regex matching filenames to skip over. 124 | 125 | :type exclude_lines: str|None 126 | :param exclude_lines: A regex matching lines to skip over. 127 | 128 | :rtype: SecretsCollection 129 | :returns: secrets found. 130 | """ 131 | self.storage.fetch_new_changes() 132 | 133 | default_plugins = initialize_plugins.from_parser_builder( 134 | self.plugin_config, 135 | exclude_lines_regex=exclude_lines_regex, 136 | ) 137 | # TODO Issue 17: Ignoring self.exclude_regex, using the server scan CLI arg 138 | secrets = SecretsCollection( 139 | plugins=default_plugins, 140 | exclude_files=exclude_files_regex, 141 | exclude_lines=exclude_lines_regex, 142 | ) 143 | 144 | scan_from_this_commit = git.get_empty_tree_commit_hash() if scan_head else self.last_commit_hash 145 | try: 146 | diff_name_only = self.storage.get_diff_name_only(scan_from_this_commit) 147 | 148 | # do a per-file diff + scan so we don't get a OOM if the the commit-diff is too large 149 | for filename in diff_name_only: 150 | file_diff = self.storage.get_diff(scan_from_this_commit, filename) 151 | 152 | secrets.scan_diff( 153 | file_diff, 154 | baseline_filename=self.baseline_filename, 155 | last_commit_hash=scan_from_this_commit, 156 | repo_name=self.name, 157 | ) 158 | except subprocess.CalledProcessError: 159 | self.update() 160 | return secrets 161 | 162 | if self.baseline_filename: 163 | baseline = self.storage.get_baseline_file(self.baseline_filename) 164 | if baseline: 165 | baseline_collection = SecretsCollection.load_baseline_from_string(baseline) 166 | secrets = get_secrets_not_in_baseline(secrets, baseline_collection) 167 | 168 | return secrets 169 | 170 | def update(self): 171 | self.last_commit_hash = self.storage.get_last_commit_hash() 172 | 173 | def save(self, override_level=OverrideLevel.ASK_USER): 174 | """Saves tracked repo config to file. Returns True if successful. 175 | 176 | :type override_level: OverrideLevel 177 | :param override_level: determines if we overwrite the JSON file, if exists. 178 | 179 | :rtype: bool 180 | :returns: True if repository is saved. 181 | """ 182 | name = self.name 183 | if os.path.isfile( 184 | self.storage.get_tracked_file_location( 185 | self.storage.hash_filename(name), 186 | ) 187 | ): 188 | if override_level == OverrideLevel.NEVER: 189 | return False 190 | 191 | elif override_level == OverrideLevel.ASK_USER: 192 | if not self._prompt_user_override(): 193 | return False 194 | 195 | self.storage.put( 196 | self.storage.hash_filename(name), 197 | self.__dict__, 198 | ) 199 | 200 | return True 201 | 202 | @property 203 | def __dict__(self): 204 | """This is written to the filesystem, and used in load_from_file. 205 | Should contain all variables needed to initialize TrackedRepo.""" 206 | output = { 207 | 'repo': self.repo, 208 | 'sha': self.last_commit_hash, 209 | 'crontab': self.crontab, 210 | 211 | 'baseline_filename': self.baseline_filename, 212 | 'exclude_regex': self.exclude_regex, 213 | 214 | 'plugins': self.plugin_config, 215 | } 216 | 217 | return output 218 | 219 | def _prompt_user_override(self): # pragma: no cover 220 | """Prompts for user input to check if should override file. 221 | 222 | :rtype: bool 223 | """ 224 | # Make sure to write to stderr, because crontab output is going to be to stdout 225 | sys.stdout = sys.stderr 226 | 227 | override = None 228 | while override not in ['y', 'n']: 229 | override = str( 230 | input( 231 | '"{}" repo already tracked! Do you want to override this (y|n)? '.format( 232 | self.name, 233 | ) 234 | ) 235 | ).lower() 236 | 237 | sys.stdout = sys.__stdout__ 238 | 239 | if override == 'n': 240 | return False 241 | 242 | return True 243 | -------------------------------------------------------------------------------- /detect_secrets_server/repos/factory.py: -------------------------------------------------------------------------------- 1 | from .base_tracked_repo import BaseTrackedRepo 2 | from .local_tracked_repo import LocalTrackedRepo 3 | from .s3_tracked_repo import S3LocalTrackedRepo 4 | from .s3_tracked_repo import S3TrackedRepo 5 | 6 | 7 | def tracked_repo_factory(is_local=False, is_s3=False): 8 | if is_s3: 9 | if is_local: 10 | return S3LocalTrackedRepo 11 | else: 12 | return S3TrackedRepo 13 | else: 14 | if is_local: 15 | return LocalTrackedRepo 16 | else: 17 | return BaseTrackedRepo 18 | -------------------------------------------------------------------------------- /detect_secrets_server/repos/local_tracked_repo.py: -------------------------------------------------------------------------------- 1 | from .base_tracked_repo import BaseTrackedRepo 2 | from detect_secrets_server.storage.file import FileStorageWithLocalGit 3 | 4 | 5 | class LocalTrackedRepo(BaseTrackedRepo): 6 | 7 | STORAGE_CLASS = FileStorageWithLocalGit 8 | 9 | @property 10 | def name(self): 11 | return self.repo 12 | -------------------------------------------------------------------------------- /detect_secrets_server/repos/s3_tracked_repo.py: -------------------------------------------------------------------------------- 1 | from .base_tracked_repo import BaseTrackedRepo 2 | from .base_tracked_repo import OverrideLevel 3 | from .local_tracked_repo import LocalTrackedRepo 4 | from detect_secrets_server.storage.s3 import S3Storage 5 | from detect_secrets_server.storage.s3 import S3StorageWithLocalGit 6 | 7 | 8 | class S3TrackedRepo(BaseTrackedRepo): 9 | 10 | STORAGE_CLASS = S3Storage 11 | 12 | @classmethod 13 | def initialize_storage(cls, base_directory): 14 | return cls.STORAGE_CLASS( 15 | base_directory, 16 | **cls.init_vars 17 | ) 18 | 19 | def __init__( 20 | self, 21 | repo, 22 | sha, 23 | plugins, 24 | baseline_filename, 25 | exclude_regex, 26 | s3_config, 27 | crontab='', 28 | rootdir=None, 29 | *args, 30 | **kwargs 31 | ): 32 | """ 33 | :type s3_config: dict 34 | :param s3_config: initialized in usage.S3Options. Contains the 35 | following keys: 36 | 37 | prefix: str 38 | an optional prefix to append to the start of the path, 39 | so it can be placed in the s3 bucket appropriately. 40 | 41 | bucket_name: str 42 | the bucket name to upload the meta files to 43 | 44 | credentials_filename: str 45 | filepath to s3 credentials file. This is needed for cron 46 | output. 47 | 48 | access_key: str 49 | s3 access key 50 | 51 | secret_access_key: str 52 | secret s3 access key 53 | """ 54 | self.s3_config = s3_config 55 | 56 | # Store it in the class and the instance, because we need to 57 | # initialize_storage with the class variables. 58 | self._store_variables_in_class( 59 | s3_config=s3_config, 60 | ) 61 | 62 | super(S3TrackedRepo, self).__init__( 63 | repo, 64 | sha, 65 | plugins, 66 | baseline_filename, 67 | exclude_regex, 68 | crontab, 69 | rootdir, 70 | **kwargs 71 | ) 72 | 73 | @classmethod 74 | def load_from_file( 75 | cls, 76 | repo_name, 77 | base_directory, 78 | s3_config, 79 | *args, 80 | **kwargs 81 | ): 82 | cls._store_variables_in_class( 83 | s3_config=s3_config, 84 | ) 85 | 86 | return super(S3TrackedRepo, cls).load_from_file( 87 | repo_name, 88 | base_directory, 89 | ) 90 | 91 | @classmethod 92 | def get_tracked_repo_data(cls, storage, repo_name): 93 | output = super(S3TrackedRepo, cls).get_tracked_repo_data(storage, repo_name) 94 | output['s3_config'] = cls.init_vars['s3_config'] 95 | 96 | return output 97 | 98 | def cron(self): # pragma: no cover 99 | # TODO: deprecate this 100 | output = super(S3TrackedRepo, self).cron() 101 | return '{} --s3-credentials-file {} --s3-bucket {} --s3-prefix {}'.format( 102 | output, 103 | self.s3_config['credentials_filename'], 104 | self.s3_config['bucket'], 105 | self.s3_config['prefix'], 106 | ) 107 | 108 | def save(self, override_level=OverrideLevel.ASK_USER): 109 | success = super(S3TrackedRepo, self).save(override_level) 110 | name = self.name 111 | 112 | is_file_uploaded = self.storage.is_file_uploaded( 113 | self.storage.hash_filename(name), 114 | ) 115 | 116 | # Even if it does not succeed, we may still want to upload something, 117 | # if the file does not already exist in the s3 bucket. 118 | # NOTE: We leverage short-circuiting here. 119 | if not is_file_uploaded or (success and override_level != OverrideLevel.NEVER): 120 | self.storage.upload( 121 | self.storage.hash_filename(name), 122 | self.__dict__, 123 | ) 124 | 125 | return success 126 | 127 | @classmethod 128 | def _store_variables_in_class(cls, **kwargs): 129 | """We tag these variables onto the class, so that we can take 130 | advantage of inheritance. 131 | """ 132 | cls.init_vars = kwargs 133 | 134 | 135 | class S3LocalTrackedRepo(S3TrackedRepo, LocalTrackedRepo): 136 | 137 | STORAGE_CLASS = S3StorageWithLocalGit 138 | -------------------------------------------------------------------------------- /detect_secrets_server/storage/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/detect-secrets-server/f2a708fa1a4628a7a93039cae92ef90e8b94d9db/detect_secrets_server/storage/__init__.py -------------------------------------------------------------------------------- /detect_secrets_server/storage/base.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import os 3 | import subprocess 4 | from abc import ABCMeta 5 | from abc import abstractmethod 6 | 7 | from detect_secrets.core.log import log 8 | 9 | from .core import git 10 | from detect_secrets_server.util.version import is_python_2 11 | 12 | if is_python_2(): # pragma: no cover 13 | FileNotFoundError = IOError 14 | import urlparse 15 | else: 16 | import urllib.parse as urlparse 17 | 18 | 19 | class BaseStorage(object): 20 | """The base class handles git interactions with the local copy 21 | of the git repositories. 22 | 23 | Structure: 24 | root 25 | |- repos # This is where git repos are cloned to 26 | """ 27 | __metaclass__ = ABCMeta 28 | 29 | def __init__(self, base_directory): 30 | self.root = base_directory 31 | 32 | @abstractmethod 33 | def get(self, key): 34 | """Retrieve from storage.""" 35 | pass 36 | 37 | @abstractmethod 38 | def put(self, key, value): 39 | """Store in storage.""" 40 | pass 41 | 42 | @abstractmethod 43 | def get_tracked_repositories(self): 44 | """Return iterator over tracked repositories. 45 | 46 | :rtype: (dict, bool) 47 | dict: metadata for tracked repo 48 | bool: True if local git repo 49 | """ 50 | pass 51 | 52 | def setup(self, repo_url): 53 | """ 54 | :param repo_url: this is placed in setup, rather than __init__, 55 | because we want to use this class without pinning it down 56 | to a repository. 57 | 58 | e.g. We should be able to retrieve information about the 59 | repo_url from a file, with delayed setup. 60 | 61 | :returns: self, for better chaining 62 | """ 63 | self.repo_url = repo_url 64 | 65 | if not os.path.isdir(self.root): 66 | os.makedirs(self.root) 67 | 68 | self._initialize_git_repos_directory() 69 | 70 | return self 71 | 72 | @property 73 | def repository_name(self): 74 | """Human friendly name of git repository tracked.""" 75 | return self.get_repo_name(self.repo_url) 76 | 77 | def clone(self): 78 | git.clone_repo_to_location( 79 | self.repo_url, 80 | self._repo_location, 81 | ) 82 | 83 | def fetch_new_changes(self): 84 | git.fetch_new_changes(self._repo_location) 85 | 86 | def get_diff(self, from_sha, filename=None): 87 | try: 88 | return git.get_diff(self._repo_location, from_sha, files=[filename]) 89 | except subprocess.CalledProcessError: 90 | # This sometimes complains, if the hash does not exist. 91 | # There could be a variety of reasons for this, including: 92 | # - some sort of rewrite of git history 93 | # - this scanner being run on an out-of-date repo 94 | # 95 | # To prevent from any further alerting on this, we are going to 96 | # update the last_commit_hash, to prevent re-alerting on old 97 | # secrets. 98 | # 99 | # TODO: Fix this to be more robust. 100 | log.error( 101 | self._construct_debugging_output(from_sha), 102 | ) 103 | 104 | raise 105 | 106 | def get_diff_name_only(self, from_sha): 107 | return git.get_diff_name_only(self._repo_location, from_sha) 108 | 109 | def _construct_debugging_output(self, sha): # pragma: no cover 110 | alert = { 111 | 'alert': 'Hash not found during git diff', 112 | 'hash': sha, 113 | 'repo_location': self._repo_location, 114 | 'repo_name': self.repository_name, 115 | } 116 | 117 | if not os.path.exists(self._repo_location): 118 | alert['info'] = 'repo_location does not exist' 119 | return alert 120 | 121 | path_to_HEAD = os.path.join(self._repo_location, '/logs/HEAD') 122 | if not os.path.exists(path_to_HEAD): 123 | alert['info'] = 'logs/HEAD does not exist' 124 | return alert 125 | 126 | try: 127 | with open(path_to_HEAD) as f: 128 | first_line = f.readline().strip() 129 | except FileNotFoundError: 130 | first_line = '' 131 | 132 | alert['info'] = 'first_line of logs/HEAD is {}'.format( 133 | str(first_line), 134 | ) 135 | return alert 136 | 137 | def get_last_commit_hash(self): 138 | return git.get_last_commit_hash(self._repo_location) 139 | 140 | def get_baseline_file(self, baseline_filename): 141 | return git.get_baseline_file( 142 | self._repo_location, 143 | baseline_filename, 144 | ) 145 | 146 | def get_blame(self, filename, line_number): 147 | return git.get_blame( 148 | self._repo_location, 149 | filename, 150 | line_number, 151 | ) 152 | 153 | @staticmethod 154 | def hash_filename(name): 155 | """Function broken out, so it can be referenced in test cases""" 156 | return hashlib.sha512(name.encode('utf-8')).hexdigest() 157 | 158 | def get_repo_name(self, url): 159 | """Function broken out, so can be extended in subclass. 160 | 161 | Example: 'git@github.com:yelp/detect-secrets' => yelp/detect-secrets 162 | """ 163 | if url.startswith('git@'): 164 | name = url.split(':')[1] 165 | else: 166 | components = urlparse.urlparse(url) 167 | name = components.path.lstrip('/') 168 | 169 | if name.endswith('.git'): 170 | return name[:-4] 171 | 172 | return name 173 | 174 | def _initialize_git_repos_directory(self): 175 | git_repos_root = os.path.join(self.root, 'repos') 176 | if not os.path.isdir(git_repos_root): 177 | os.makedirs(git_repos_root) 178 | 179 | @property 180 | def _repo_location(self): 181 | return get_filepath_safe( 182 | os.path.join(self.root, 'repos'), 183 | self.hash_filename(self.repository_name), 184 | ) 185 | 186 | 187 | class LocalGitRepository(BaseStorage): 188 | """This mixin surpresses some automated management for git repositories, 189 | for cases when you already have the git repository on your system, and 190 | want to scan that (instead of having this system track it for you). 191 | 192 | Since this surpresses parent functionality, the declaration order for 193 | the mixin is important. 194 | 195 | Example: 196 | >>> class Example(LocalGitRepository, FileStorage): 197 | ... pass 198 | 199 | Structure: 200 | root 201 | |- tracked 202 | |- local # meta files for local tracked repositories 203 | """ 204 | 205 | @property 206 | def repository_name(self): 207 | """Human friendly name of git repository tracked. 208 | 209 | Example: 'git@github.com:yelp/detect-secrets' => yelp/detect-secrets 210 | """ 211 | path = self.repo_url 212 | if not path.endswith('/.git'): 213 | path = os.path.join(path, '.git') 214 | 215 | return super(LocalGitRepository, self).get_repo_name( 216 | git.get_remote_url(path), 217 | ) 218 | 219 | def clone(self): 220 | """If it is locally on disk, no need to clone it.""" 221 | return 222 | 223 | def fetch_new_changes(self): 224 | """The assumption is, if you are scanning a local git repository, 225 | then you are "actively" working on it. Therefore, this module will 226 | not bear the responsibility of auto-updating the repo with `git fetch`. 227 | """ 228 | return 229 | 230 | def _initialize_git_repos_directory(self): 231 | """Don't need to create a place for tracking git repos""" 232 | return 233 | 234 | @property 235 | def _repo_location(self): 236 | """When we're performing git commands on a local repository, we need 237 | to reference the `/.git` folder within the cloned git repo. 238 | 239 | Unless it is a local bare repo. 240 | """ 241 | inner_git_dir = os.path.join(self.repo_url, '.git') 242 | if os.path.exists(inner_git_dir): 243 | return inner_git_dir 244 | # Bare repo 245 | return self.repo_url 246 | 247 | 248 | def get_filepath_safe(prefix, file): 249 | """Attempts to prevent file traversal when trying to get `prefix/file`""" 250 | prefix_realpath = os.path.realpath(prefix) 251 | filepath = os.path.realpath( 252 | '%(prefix_realpath)s/%(file)s' % { 253 | 'prefix_realpath': prefix_realpath, 254 | 'file': file, 255 | } 256 | ) 257 | if not filepath.startswith(prefix_realpath): 258 | raise ValueError 259 | 260 | return filepath 261 | -------------------------------------------------------------------------------- /detect_secrets_server/storage/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/detect-secrets-server/f2a708fa1a4628a7a93039cae92ef90e8b94d9db/detect_secrets_server/storage/core/__init__.py -------------------------------------------------------------------------------- /detect_secrets_server/storage/core/git.py: -------------------------------------------------------------------------------- 1 | """Collection of all git command interactions""" 2 | import os 3 | import re 4 | import subprocess 5 | import sys 6 | 7 | from detect_secrets.core.log import log 8 | 9 | from detect_secrets_server.constants import IGNORED_FILE_EXTENSIONS 10 | 11 | GIT_EMPTY_TREE_HASH = '4b825dc642cb6eb9a060e54bf8d69288fbee4904' 12 | 13 | 14 | def get_last_commit_hash(directory): 15 | return _git( 16 | directory, 17 | 'rev-parse', 18 | 'HEAD', 19 | ) 20 | 21 | 22 | def get_empty_tree_commit_hash(): 23 | return GIT_EMPTY_TREE_HASH 24 | 25 | 26 | def clone_repo_to_location(repo, directory): 27 | """ 28 | :type repo: str 29 | :param repo: git url to clone 30 | 31 | :type directory: str 32 | :param directory: local directory path 33 | """ 34 | try: 35 | # We need to run it through check_output, because we want to trigger 36 | # a subprocess.CalledProcessError upon failure. 37 | subprocess.check_output([ 38 | 'git', 'clone', 39 | repo, 40 | directory, 41 | 42 | # We clone a bare repo, because we're not interested in the 43 | # files themselves. This will be more space efficient. 44 | '--bare', 45 | ], stderr=subprocess.STDOUT) 46 | except subprocess.CalledProcessError as e: 47 | error_message = e.output.decode('utf-8') 48 | 49 | # Ignore this message, because it's expected if the repo 50 | # has already been tracked. 51 | if not re.match( 52 | r"fatal: destination path '[^']+' already exists", 53 | error_message 54 | ): 55 | raise 56 | 57 | 58 | def fetch_new_changes(directory): 59 | main_branch = _get_main_branch(directory) 60 | _git( 61 | directory, 62 | 'fetch', 63 | '--quiet', 64 | 'origin', 65 | '{}:{}'.format( 66 | main_branch, 67 | main_branch, 68 | ), 69 | '--force', 70 | ) 71 | 72 | 73 | def get_baseline_file(directory, filename): 74 | """Take the most updated baseline, because want to get the most updated 75 | baseline. Note that this means it's still "user-dependent", but at the 76 | same time, we want to ignore new explicit whitelists. 77 | Also, this would mean that we **always** get a whitelist, if exists 78 | (rather than worrying about fixing on a commit that has a whitelist) 79 | 80 | :returns: file contents of baseline_file 81 | """ 82 | try: 83 | return _git( 84 | directory, 85 | 'show', 'HEAD:{}'.format(filename), 86 | ) 87 | 88 | except subprocess.CalledProcessError as e: 89 | error_message = e.output.decode('utf-8') 90 | 91 | # Some repositories may not have baselines. 92 | # If so, this is a non-breaking error. 93 | if not ( 94 | re.match( 95 | r"fatal: Path '[^']+' does not exist", 96 | error_message, 97 | ) 98 | or 99 | # It is possible for the directory you are running detect-secret-server from 100 | # to also contain a file with the name , but 101 | # is not part of the repo being scanned. 102 | # This will create a different error message, which we also want to catch. 103 | re.match( 104 | r"fatal: Path '[^']+' exists on disk, but not in 'HEAD'", 105 | error_message, 106 | ) 107 | ): 108 | raise 109 | 110 | 111 | def get_diff(directory, last_commit_hash, files=None): 112 | """Returns the git diff between last commit hash, and HEAD.""" 113 | kwargs = {'should_strip_output': False} 114 | git_args = [ 115 | directory, 116 | 'diff', 117 | last_commit_hash, 118 | 'HEAD', 119 | ] 120 | 121 | if not files: 122 | filenames_to_include_in_diff = _filter_filenames_from_diff(directory, last_commit_hash) 123 | else: 124 | filenames_to_include_in_diff = files 125 | 126 | if ( 127 | filenames_to_include_in_diff 128 | and 129 | # This happens when there are too many files in the diff 130 | # e.g. 'warning: inexact rename detection was skipped due to too many files.' 131 | # We choose to not include specific files out in this case 132 | # Because if we did, we would later get a 133 | # `OSError: [Errno 7] Argument list too long` 134 | # when including the list of files. 135 | not filenames_to_include_in_diff[-1].startswith('warning:') 136 | ): 137 | git_args.extend( 138 | ['--'] + filenames_to_include_in_diff, 139 | ) 140 | 141 | return _git( 142 | *git_args, 143 | # Python 2 made me do this 144 | **kwargs 145 | ) 146 | 147 | 148 | def get_diff_name_only(directory, last_commit_hash): 149 | return _filter_filenames_from_diff(directory, last_commit_hash) 150 | 151 | 152 | def get_remote_url(directory): 153 | return _git( 154 | directory, 155 | 'remote', 156 | 'get-url', 157 | 'origin', 158 | ) 159 | 160 | 161 | def get_blame(directory, filename, line_number): 162 | """Returns the author who last made the change, to a given file, 163 | on a given line. 164 | """ 165 | return _git( 166 | directory, 167 | 'blame', 168 | _get_main_branch(directory), 169 | '-L', '{},{}'.format(line_number, line_number), 170 | '--show-email', 171 | '--line-porcelain', 172 | '--', 173 | filename, 174 | ) 175 | 176 | 177 | def _get_main_branch(directory): 178 | """While this is `master` most of the time, there are some exceptions""" 179 | return _git( 180 | directory, 181 | 'rev-parse', 182 | '--abbrev-ref', 183 | 'HEAD', 184 | ) 185 | 186 | 187 | def _filter_filenames_from_diff(directory, last_commit_hash): 188 | filenames = _git( 189 | directory, 190 | 'diff', 191 | last_commit_hash, 192 | 'HEAD', 193 | '--name-only', 194 | '--diff-filter', 'ACM', 195 | ).splitlines() 196 | 197 | return [ 198 | filename 199 | for filename in filenames 200 | if os.path.splitext(filename)[1] not in IGNORED_FILE_EXTENSIONS 201 | ] 202 | 203 | 204 | def _git(directory, *args, **kwargs): 205 | try: 206 | output = subprocess.check_output( 207 | [ 208 | 'git', 209 | '--git-dir', directory, 210 | ] + list(args), 211 | stderr=subprocess.STDOUT 212 | ).decode('utf-8', errors='ignore') 213 | 214 | # This is to fix https://github.com/matiasb/python-unidiff/issues/54 215 | if not kwargs.get('should_strip_output', True): 216 | return output 217 | return output.strip() 218 | except subprocess.CalledProcessError as e: 219 | error_message = e.output.decode('utf-8') 220 | 221 | # Catch this error, this happens during scanning and means it's an empty repo. This bails out 222 | # of the scan process and logs error. 223 | if re.match( 224 | r"fatal: couldn't find remote ref (None|HEAD)", 225 | error_message 226 | ): 227 | # directory is the best/only output without drastic rewrites, hashed path correlates to repo 228 | log.error("Empty repository cannot be scanned: %s", directory) 229 | sys.exit(1) 230 | # TODO: This won't work if scan loops through repos, but works since it's a single scan currently 231 | 232 | # Catch this error, this happens during initialization and means it's an empty repo. This allows 233 | # the repo metadata to be written to /tracked 234 | elif re.match( 235 | r"fatal: ambiguous argument 'HEAD': unknown revision or path not in the working tree.", 236 | error_message 237 | ): 238 | return None 239 | else: 240 | raise 241 | -------------------------------------------------------------------------------- /detect_secrets_server/storage/file.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | from .base import BaseStorage 5 | from .base import get_filepath_safe 6 | from .base import LocalGitRepository 7 | 8 | 9 | class FileStorage(BaseStorage): 10 | """For state management without a database. 11 | 12 | Structure: 13 | root 14 | |- repos # This is where git repos are cloned to 15 | |- tracked # This is where meta files containing state reside 16 | """ 17 | 18 | def setup(self, repo_url): 19 | super(FileStorage, self).setup(repo_url) 20 | 21 | storage_root = os.path.join(self.root, 'tracked') 22 | if not os.path.isdir(storage_root): 23 | os.makedirs(storage_root) 24 | 25 | return self 26 | 27 | def get(self, key): 28 | """ 29 | :raises: FileNotFoundError 30 | :raises: ValueError 31 | """ 32 | filename = self.get_tracked_file_location(key) 33 | with open(filename) as f: 34 | return json.load(f) 35 | 36 | def put(self, key, value): 37 | """ 38 | :raises: ValueError 39 | """ 40 | filename = self.get_tracked_file_location(key) 41 | with open(filename, 'w') as f: 42 | f.write(json.dumps(value, indent=2, sort_keys=True)) 43 | 44 | def get_tracked_file_location(self, key): 45 | return get_filepath_safe( 46 | os.path.join(self.root, 'tracked'), 47 | '{}.json'.format(key), 48 | ) 49 | 50 | def get_tracked_repositories(self): 51 | filepath = get_filepath_safe( 52 | self.root, 53 | 'tracked', 54 | ) 55 | 56 | for root, _, files in os.walk(filepath): 57 | for filename in files: 58 | with open(os.path.join(root, filename)) as f: 59 | yield json.loads(f.read()), False 60 | 61 | break 62 | 63 | 64 | class FileStorageWithLocalGit(LocalGitRepository, FileStorage): 65 | 66 | def setup(self, repo_url): 67 | super(FileStorage, self).setup(repo_url) 68 | 69 | storage_root = os.path.join(self.root, 'tracked') 70 | if not os.path.isdir(storage_root): 71 | os.makedirs(storage_root) 72 | 73 | local_storage_root = os.path.join(self.root, 'tracked', 'local') 74 | if not os.path.isdir(local_storage_root): 75 | os.makedirs(local_storage_root) 76 | 77 | return self 78 | 79 | def get_tracked_file_location(self, key): 80 | return get_filepath_safe( 81 | os.path.join(self.root, 'tracked', 'local'), 82 | '{}.json'.format(key), 83 | ) 84 | 85 | def get_tracked_repositories(self): 86 | for tup in super(FileStorageWithLocalGit, self).get_tracked_repositories(): 87 | yield tup 88 | 89 | filepath = get_filepath_safe( 90 | os.path.join(self.root, 'tracked'), 91 | 'local', 92 | ) 93 | 94 | for root, _, files in os.walk(filepath): 95 | for filename in files: 96 | with open(os.path.join(root, filename)) as f: 97 | yield json.loads(f.read()), True 98 | 99 | break 100 | -------------------------------------------------------------------------------- /detect_secrets_server/storage/s3.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from .file import FileStorage 4 | from .file import FileStorageWithLocalGit 5 | from detect_secrets_server.core.usage.s3 import should_enable_s3_options 6 | 7 | 8 | class S3Storage(FileStorage): 9 | """For file state management, backed to Amazon S3. 10 | 11 | See detect_secrets_server.storage.file.FileStorage for the expected 12 | file layout in the S3 bucket. 13 | """ 14 | 15 | def __init__( 16 | self, 17 | base_directory, 18 | s3_config 19 | ): 20 | super(S3Storage, self).__init__(base_directory) 21 | 22 | self.access_key = s3_config['access_key'] 23 | self.secret_access_key = s3_config['secret_access_key'] 24 | self.bucket_name = s3_config['bucket'] 25 | self.prefix = s3_config['prefix'] 26 | 27 | self._initialize_client() 28 | 29 | def get(self, key, force_download=True): 30 | """Downloads file from S3 into local storage.""" 31 | file_on_disk = self.get_tracked_file_location(key) 32 | if force_download or not os.path.exists(file_on_disk): 33 | self.client.download_file( 34 | Bucket=self.bucket_name, 35 | Key=self.get_s3_tracked_file_location(key), 36 | Filename=file_on_disk, 37 | ) 38 | 39 | return super(S3Storage, self).get(key) 40 | 41 | # NOTE: There's no `put` functionality, because S3TrackedRepo handles uploads 42 | # separately. That is, there are cases when you want to store a local 43 | # copy, but not upload it. 44 | 45 | def get_tracked_repositories(self): 46 | # Source: https://adamj.eu/tech/2018/01/09/using-boto3-think-pagination/ 47 | pages = self.client.get_paginator('list_objects').paginate( 48 | Bucket=self.bucket_name, 49 | Prefix=self.prefix, 50 | ) 51 | for page in pages: 52 | for obj in page['Contents']: 53 | filename = os.path.splitext(obj['Key'][len(self.prefix):])[0] 54 | if filename.startswith('/'): 55 | filename = filename[1:] 56 | 57 | yield ( 58 | self.get(filename, force_download=False), 59 | 60 | # TODO: In it's current state, you can't distinguish the 61 | # difference between S3StorageWithLocalGit and S3Storage, 62 | # because there's no separate paths in S3. 63 | # 64 | # Therefore, return None so that the results will be 65 | # displayed irregardless of the user's `--local` flag. 66 | None, 67 | ) 68 | 69 | def upload(self, key, value): 70 | """This is different than `put`, to support situations where you 71 | may want to upload locally, but not to be sync'ed with the cloud. 72 | """ 73 | self.client.upload_file( 74 | Filename=self.get_tracked_file_location(key), 75 | Bucket=self.bucket_name, 76 | Key=self.get_s3_tracked_file_location(key), 77 | ) 78 | 79 | def is_file_uploaded(self, key): 80 | """Note: that we are using the filename as a prefix, so we will 81 | never run into the 1000 object limit of `list_objects_v2`. 82 | 83 | :rtype: bool 84 | """ 85 | filename = self.get_s3_tracked_file_location(key) 86 | response = self.client.list_objects_v2( 87 | Bucket=self.bucket_name, 88 | Prefix=filename, 89 | ) 90 | 91 | for obj in response.get('Contents', []): 92 | if obj['Key'] == filename: 93 | return bool(obj['Size']) 94 | 95 | return False 96 | 97 | def _initialize_client(self): 98 | boto3 = self._get_boto3() 99 | if not boto3: 100 | return 101 | 102 | self.client = boto3.client( 103 | 's3', 104 | aws_access_key_id=self.access_key, 105 | aws_secret_access_key=self.secret_access_key, 106 | ) 107 | 108 | def _get_boto3(self): 109 | """Used for mocking purposes.""" 110 | if not should_enable_s3_options(): 111 | return 112 | 113 | import boto3 114 | return boto3 115 | 116 | def get_s3_tracked_file_location(self, key): 117 | return os.path.join( 118 | self.prefix, 119 | key + '.json' 120 | ) 121 | 122 | 123 | class S3StorageWithLocalGit(S3Storage, FileStorageWithLocalGit): 124 | pass 125 | -------------------------------------------------------------------------------- /detect_secrets_server/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/detect-secrets-server/f2a708fa1a4628a7a93039cae92ef90e8b94d9db/detect_secrets_server/util/__init__.py -------------------------------------------------------------------------------- /detect_secrets_server/util/version.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | def is_python_2(): 5 | return sys.version_info[0] < 3 6 | -------------------------------------------------------------------------------- /examples/aws_credentials.json: -------------------------------------------------------------------------------- 1 | { 2 | "accessKeyId": "access_key", 3 | "secretAccessKey": "secret_key" 4 | } 5 | -------------------------------------------------------------------------------- /examples/config.yaml: -------------------------------------------------------------------------------- 1 | default: 2 | plugins: 3 | HexHighEntropyString: 3 4 | Base64HighEntropyString: 4.5 5 | PrivateKeyDetector: true 6 | baseline: .secrets.baseline 7 | base_temp_dir: /tmp/detect_secrets_tracked_repos 8 | exclude_regex: ^(\.git|build|logs|node_modules|virtualenv_run)|.*tests/.* 9 | -------------------------------------------------------------------------------- /examples/pysensu.config.yaml: -------------------------------------------------------------------------------- 1 | name: SecretFound # name needs to be one word 2 | alert_after: 0 3 | realert_every: -1 # -1 means exponential backoff 4 | runbook: no-runbook-available 5 | dependencies: [] 6 | team: team-security 7 | irc_channels: [] 8 | notification_email: to-whom-it-may-concern@example.com 9 | ticket: False 10 | project: False 11 | page: False 12 | tip: detect_secrets found a secret 13 | status: 1 # status needs to be 1 (warning) or higher to send the email 14 | ttl: null # null gets constructed into None 15 | -------------------------------------------------------------------------------- /examples/repos.yaml: -------------------------------------------------------------------------------- 1 | tracked: 2 | - repo: git@github.com:yelp/detect-secrets.git 3 | crontab: "* * 4 * *" 4 | sha: ce9232e49873ff11cf3f377d755aa033253328ed 5 | is_local_repo: False 6 | plugins: 7 | Base64HighEntropyString: 8 | base64_limit: 4 9 | PrivateKeyDetector: False 10 | -------------------------------------------------------------------------------- /examples/s3.yaml: -------------------------------------------------------------------------------- 1 | credentials_filename: examples/aws_credentials.json 2 | bucket_name: my-bucket-in-us-east-1 3 | prefix: secret_detector/tracked_repos 4 | -------------------------------------------------------------------------------- /examples/standalone_hook.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.6 2 | """ 3 | This is an example bare-bone script that you can use as an argument for 4 | setting up an output hook. 5 | 6 | >>> $ detect-secrets-server scan yelp/detect-secrets 7 | ... --output-hook examples/standalone_hook.py 8 | """ 9 | import json 10 | import sys 11 | 12 | 13 | def main(): 14 | print('repo:', sys.argv[1]) 15 | print(json.dumps(json.loads(sys.argv[2]), indent=2, sort_keys=True)) 16 | 17 | 18 | if __name__ == '__main__': 19 | main() 20 | -------------------------------------------------------------------------------- /requirements-dev-minimal.txt: -------------------------------------------------------------------------------- 1 | coverage 2 | detect-secrets>=0.13.0 3 | PyYAML 4 | pre-commit 5 | pytest 6 | python-crontab 7 | tox 8 | unidiff 9 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | aspy.yaml==1.3.0 2 | attrs==19.3.0 3 | certifi==2020.4.5.1 4 | cfgv==2.0.1 5 | chardet==3.0.4 6 | coverage==5.1 7 | detect-secrets==0.13.1 8 | filelock==3.0.12 9 | identify==1.4.25 10 | idna==2.9 11 | importlib-metadata==1.6.0 12 | importlib-resources==1.5.0 13 | more-itertools==8.2.0 14 | nodeenv==1.3.5 15 | packaging==20.3 16 | pathlib2==2.3.5 17 | pluggy==0.13.1 18 | pre-commit==1.21.0 19 | py==1.8.1 20 | pyparsing==2.4.7 21 | pytest==5.4.3 22 | python-crontab==2.4.1 23 | python-dateutil==2.8.1 24 | PyYAML==5.3.1 25 | requests==2.24.0 26 | six==1.15.0 27 | toml==0.10.0 28 | tox==3.16.0 29 | unidiff==0.5.5 30 | urllib3==1.25.9 31 | virtualenv==16.7.7 32 | wcwidth==0.1.9 33 | zipp==1.2.0 34 | -------------------------------------------------------------------------------- /scripts/uploader.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script pretty much just follows the instructions from 3 | # https://packaging.python.org/tutorials/packaging-projects/ 4 | # and uploads this package to pypi. 5 | 6 | function usage() { 7 | echo "Usage: uploader.sh [test|main]" 8 | echo "Specify the pypi instance you want to upload to." 9 | echo " - test: uploads to test.pypi.org" 10 | echo " - main: uploads to pypi.org" 11 | } 12 | 13 | function main() { 14 | local mode="$1" 15 | if [[ -z "$mode" ]]; then 16 | usage 17 | return 0 18 | fi 19 | if [[ "$mode" != "main" ]] && [[ "$mode" != "test" ]]; then 20 | usage 21 | return 1 22 | fi 23 | 24 | gitTagVersion "$mode" 25 | if [[ "$?" != 0 ]]; then 26 | return 1 27 | fi 28 | 29 | # Install dependencies 30 | pip install setuptools wheel twine 31 | 32 | # Create distribution files 33 | python setup.py sdist bdist_wheel 34 | 35 | uploadToPyPI "$mode" 36 | testUpload "$mode" 37 | if [[ $? == 0 ]]; then 38 | echo "Success!" 39 | rm -r build/ dist/ 40 | fi 41 | } 42 | 43 | function gitTagVersion() { 44 | # Usage: gitTagVersion 45 | # This tags the latest upload with the latest version. 46 | local mode="$1" 47 | 48 | local version 49 | version=`python -m detect_secrets_server --version` 50 | if [[ "$?" != 0 ]]; then 51 | echo "Unable to get version information." 52 | return 1 53 | fi 54 | 55 | local extraArgs="" 56 | if [[ "$mode" == "test" ]]; then 57 | extraArgs="--index-url https://test.pypi.org/simple/" 58 | fi 59 | 60 | # Check pip for existing version 61 | local buffer 62 | buffer=$((pip install $extraArgs detect_secrets_server==no_version_found) 2>&1) 63 | buffer=`echo "$buffer" | grep "$version"` 64 | if [[ "$?" == 0 ]]; then 65 | echo "error: Version already exists in PyPI." 66 | return 1 67 | fi 68 | 69 | # Ignore output 70 | buffer=`git tag --list | grep "^v$version$"` 71 | if [[ "$?" != 0 ]]; then 72 | git tag "v$version" && git push origin --tags 73 | fi 74 | 75 | } 76 | 77 | function uploadToPyPI() { 78 | # Usage: uploadToPyPI 79 | local mode="$1" 80 | if [[ "$mode" == "main" ]]; then 81 | twine upload dist/* 82 | else 83 | twine upload --repository-url https://test.pypi.org/legacy/ dist/* 84 | fi 85 | } 86 | 87 | function testUpload() { 88 | # Usage: testUpload 89 | local mode="$1" 90 | 91 | installFromPyPI "$mode" 92 | 93 | detect-secrets-server --version 94 | if [[ $? != 0 ]]; then 95 | echo "Failed installation!" 96 | return 1 97 | fi 98 | } 99 | 100 | function installFromPyPI() { 101 | # Usage: installFromPyPI 102 | local mode="$1" 103 | if [[ "$mode" == "main" ]]; then 104 | pip install detect-secrets-server 105 | else 106 | pip install --index-url https://test.pypi.org/simple/ detect-secrets-server 107 | fi 108 | } 109 | 110 | 111 | main "$@" 112 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | 4 | [wheel] 5 | universal = True 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages 2 | from setuptools import setup 3 | 4 | import detect_secrets_server 5 | 6 | 7 | setup( 8 | name='detect_secrets_server', 9 | packages=find_packages(exclude=(['test*', 'tmp*'])), 10 | version=detect_secrets_server.__version__, 11 | description='Tool for setting up a detect-secrets server', 12 | long_description=( 13 | 'Check out detect-secrets-server on ' 14 | '`GitHub `_!' 15 | ), 16 | license="Copyright Yelp, Inc. 2018", 17 | author='Aaron Loo', 18 | author_email='aaronloo@yelp.com', 19 | url='https://github.com/Yelp/detect-secrets-server', 20 | download_url='https://github.com/Yelp/detect-secrets-server/archive/{}.tar.gz'.format(detect_secrets_server.__version__), 21 | keywords=[ 22 | 'secret-management', 23 | 'pre-commit', 24 | 'security', 25 | 'entropy-checks' 26 | ], 27 | install_requires=[ 28 | 'detect-secrets==0.13.1', 29 | 'pyyaml', 30 | 'unidiff', 31 | ], 32 | extras_require={ 33 | 'cron': [ 34 | 'python-crontab', 35 | ], 36 | }, 37 | entry_points={ 38 | 'console_scripts': [ 39 | 'detect-secrets-server = detect_secrets_server.__main__:main', 40 | ], 41 | }, 42 | classifiers=[ 43 | "Programming Language :: Python :: 3", 44 | "License :: OSI Approved :: Apache Software License", 45 | "Intended Audience :: Developers", 46 | "Topic :: Software Development", 47 | "Topic :: Utilities", 48 | "Environment :: Console", 49 | "Operating System :: OS Independent", 50 | "Development Status :: 5 - Production/Stable", 51 | ] 52 | ) 53 | -------------------------------------------------------------------------------- /test_data/sample.diff: -------------------------------------------------------------------------------- 1 | diff --git a/examples/aws_credentials.json b/examples/aws_credentials.json 2 | index 05c305b..4764bb2 100644 3 | --- a/examples/aws_credentials.json 4 | +++ b/examples/aws_credentials.json 5 | @@ -1,4 +1,4 @@ 6 | { 7 | - "accessKeyId": "access_key", 8 | - "secretAccessKey": "secret_key" 9 | + "accessKeyId": "AKIAIOSFODNN7EXAMPLE", 10 | + "secretAccessKey": "81442db33ceee1092586feb635d7b3c5d72257c775ee323414bac8e673d41af3" 11 | } 12 | -------------------------------------------------------------------------------- /testing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/detect-secrets-server/f2a708fa1a4628a7a93039cae92ef90e8b94d9db/testing/__init__.py -------------------------------------------------------------------------------- /testing/base_usage_test.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta 2 | from unittest import mock 3 | 4 | from .util import cache_buster 5 | from detect_secrets_server.core.usage.parser import ServerParserBuilder 6 | 7 | 8 | class UsageTest(object): 9 | 10 | __metaclass__ = ABCMeta 11 | 12 | def parse_args(self, argument_string='', has_boto=False): 13 | with mock.patch( 14 | 'detect_secrets_server.core.usage.common.storage.should_enable_s3_options', 15 | return_value=has_boto, 16 | ), mock.patch( 17 | 'detect_secrets_server.core.usage.s3.should_enable_s3_options', 18 | return_value=has_boto, 19 | ): 20 | return ServerParserBuilder().parse_args(argument_string.split()) 21 | 22 | def teardown(self): 23 | cache_buster() 24 | -------------------------------------------------------------------------------- /testing/factories.py: -------------------------------------------------------------------------------- 1 | import json as json_module 2 | 3 | from detect_secrets.core.potential_secret import PotentialSecret 4 | from detect_secrets.core.secrets_collection import SecretsCollection 5 | 6 | 7 | def metadata_factory(repo, json=False, **kwargs): 8 | """ 9 | This generates a layout you would expect for metadata storage with files. 10 | 11 | :type json: bool 12 | :param json: if True, will return string instead. 13 | """ 14 | output = { 15 | "baseline_filename": None, 16 | "crontab": "0 0 * * *", 17 | "exclude_regex": None, 18 | "plugins": { 19 | "AWSKeyDetector": {}, 20 | "ArtifactoryDetector": {}, 21 | "Base64HighEntropyString": { 22 | "base64_limit": 4.5, 23 | }, 24 | "BasicAuthDetector": {}, 25 | "CloudantDetector": {}, 26 | "HexHighEntropyString": { 27 | "hex_limit": 3, 28 | }, 29 | "IbmCloudIamDetector": {}, 30 | "IbmCosHmacDetector": {}, 31 | "JwtTokenDetector": {}, 32 | "KeywordDetector": { 33 | 'keyword_exclude': None 34 | }, 35 | "MailchimpDetector": {}, 36 | "PrivateKeyDetector": {}, 37 | "SlackDetector": {}, 38 | "SoftlayerDetector": {}, 39 | "StripeDetector": {}, 40 | "TwilioKeyDetector": {}, 41 | }, 42 | "repo": repo, 43 | "sha": 'sha256-hash', 44 | } 45 | 46 | output.update(kwargs) 47 | 48 | if json: 49 | return json_module.dumps(output, indent=2, sort_keys=True) 50 | return output 51 | 52 | 53 | def single_repo_config_factory(repo, **kwargs): 54 | """ 55 | This generates a layout used in passing config files when initializing repos. 56 | """ 57 | output = { 58 | 'repo': repo, 59 | } 60 | output.update(kwargs) 61 | 62 | return output 63 | 64 | 65 | def potential_secret_factory(type_='type', filename='filename', lineno=1, secret='secret'): 66 | """This is only marginally better than creating PotentialSecret objects directly, 67 | because of default values. 68 | """ 69 | return PotentialSecret( 70 | typ=type_, 71 | filename=filename, 72 | lineno=lineno, 73 | secret=secret, 74 | ) 75 | 76 | 77 | def secrets_collection_factory(secrets=None, plugins=(), exclude_regex=''): # pragma: no cover 78 | """ 79 | :type secrets: list(dict) 80 | :param secrets: list of params to pass to add_secret. 81 | Eg. [ {'secret': 'blah'}, ] 82 | 83 | :type plugins: tuple 84 | :type exclude_regex: str 85 | 86 | :rtype: SecretsCollection 87 | """ 88 | collection = SecretsCollection(plugins, exclude_regex) 89 | 90 | if plugins: 91 | collection.plugins = plugins 92 | 93 | # Handle secrets 94 | if secrets is None: 95 | return collection 96 | 97 | for kwargs in secrets: 98 | _add_secret(collection, **kwargs) 99 | 100 | return collection 101 | 102 | 103 | def _add_secret(collection, type_='type', secret='secret', filename='filename', lineno=1): 104 | """Utility function to add individual secrets to a SecretCollection. 105 | 106 | :param collection: SecretCollection; will be modified by this function. 107 | :param filename: string 108 | :param secret: string; secret to add 109 | :param lineno: integer; line number of occurring secret 110 | """ 111 | if filename not in collection.data: # pragma: no cover 112 | collection[filename] = {} 113 | 114 | tmp_secret = potential_secret_factory( 115 | type_=type_, 116 | filename=filename, 117 | lineno=lineno, 118 | secret=secret, 119 | ) 120 | collection.data[filename][tmp_secret] = tmp_secret 121 | -------------------------------------------------------------------------------- /testing/github/commit_comment.json: -------------------------------------------------------------------------------- 1 | { 2 | "action": "created", 3 | "comment": { 4 | "url": "https://api.github.com/repos/Codertocat/Hello-World/comments/33548674", 5 | "html_url": "https://github.com/Codertocat/Hello-World/commit/6113728f27ae82c7b1a177c8d03f9e96e0adf246#commitcomment-33548674", 6 | "id": 33548674, 7 | "node_id": "MDEzOkNvbW1pdENvbW1lbnQzMzU0ODY3NA==", 8 | "user": { 9 | "login": "Codertocat", 10 | "id": 21031067, 11 | "node_id": "MDQ6VXNlcjIxMDMxMDY3", 12 | "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", 13 | "gravatar_id": "", 14 | "url": "https://api.github.com/users/Codertocat", 15 | "html_url": "https://github.com/Codertocat", 16 | "followers_url": "https://api.github.com/users/Codertocat/followers", 17 | "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", 18 | "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", 19 | "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", 20 | "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", 21 | "organizations_url": "https://api.github.com/users/Codertocat/orgs", 22 | "repos_url": "https://api.github.com/users/Codertocat/repos", 23 | "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", 24 | "received_events_url": "https://api.github.com/users/Codertocat/received_events", 25 | "type": "User", 26 | "site_admin": false 27 | }, 28 | "position": null, 29 | "line": null, 30 | "path": null, 31 | "commit_id": "6113728f27ae82c7b1a177c8d03f9e96e0adf246", 32 | "created_at": "2019-05-15T15:20:39Z", 33 | "updated_at": "2019-05-15T15:20:39Z", 34 | "author_association": "OWNER", 35 | "body": "This is a really good change! :+1:" 36 | }, 37 | "repository": { 38 | "id": 186853002, 39 | "node_id": "MDEwOlJlcG9zaXRvcnkxODY4NTMwMDI=", 40 | "name": "Hello-World", 41 | "full_name": "Codertocat/Hello-World", 42 | "private": false, 43 | "owner": { 44 | "login": "Codertocat", 45 | "id": 21031067, 46 | "node_id": "MDQ6VXNlcjIxMDMxMDY3", 47 | "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", 48 | "gravatar_id": "", 49 | "url": "https://api.github.com/users/Codertocat", 50 | "html_url": "https://github.com/Codertocat", 51 | "followers_url": "https://api.github.com/users/Codertocat/followers", 52 | "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", 53 | "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", 54 | "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", 55 | "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", 56 | "organizations_url": "https://api.github.com/users/Codertocat/orgs", 57 | "repos_url": "https://api.github.com/users/Codertocat/repos", 58 | "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", 59 | "received_events_url": "https://api.github.com/users/Codertocat/received_events", 60 | "type": "User", 61 | "site_admin": false 62 | }, 63 | "html_url": "https://github.com/Codertocat/Hello-World", 64 | "description": null, 65 | "fork": false, 66 | "url": "https://api.github.com/repos/Codertocat/Hello-World", 67 | "forks_url": "https://api.github.com/repos/Codertocat/Hello-World/forks", 68 | "keys_url": "https://api.github.com/repos/Codertocat/Hello-World/keys{/key_id}", 69 | "collaborators_url": "https://api.github.com/repos/Codertocat/Hello-World/collaborators{/collaborator}", 70 | "teams_url": "https://api.github.com/repos/Codertocat/Hello-World/teams", 71 | "hooks_url": "https://api.github.com/repos/Codertocat/Hello-World/hooks", 72 | "issue_events_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/events{/number}", 73 | "events_url": "https://api.github.com/repos/Codertocat/Hello-World/events", 74 | "assignees_url": "https://api.github.com/repos/Codertocat/Hello-World/assignees{/user}", 75 | "branches_url": "https://api.github.com/repos/Codertocat/Hello-World/branches{/branch}", 76 | "tags_url": "https://api.github.com/repos/Codertocat/Hello-World/tags", 77 | "blobs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/blobs{/sha}", 78 | "git_tags_url": "https://api.github.com/repos/Codertocat/Hello-World/git/tags{/sha}", 79 | "git_refs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/refs{/sha}", 80 | "trees_url": "https://api.github.com/repos/Codertocat/Hello-World/git/trees{/sha}", 81 | "statuses_url": "https://api.github.com/repos/Codertocat/Hello-World/statuses/{sha}", 82 | "languages_url": "https://api.github.com/repos/Codertocat/Hello-World/languages", 83 | "stargazers_url": "https://api.github.com/repos/Codertocat/Hello-World/stargazers", 84 | "contributors_url": "https://api.github.com/repos/Codertocat/Hello-World/contributors", 85 | "subscribers_url": "https://api.github.com/repos/Codertocat/Hello-World/subscribers", 86 | "subscription_url": "https://api.github.com/repos/Codertocat/Hello-World/subscription", 87 | "commits_url": "https://api.github.com/repos/Codertocat/Hello-World/commits{/sha}", 88 | "git_commits_url": "https://api.github.com/repos/Codertocat/Hello-World/git/commits{/sha}", 89 | "comments_url": "https://api.github.com/repos/Codertocat/Hello-World/comments{/number}", 90 | "issue_comment_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/comments{/number}", 91 | "contents_url": "https://api.github.com/repos/Codertocat/Hello-World/contents/{+path}", 92 | "compare_url": "https://api.github.com/repos/Codertocat/Hello-World/compare/{base}...{head}", 93 | "merges_url": "https://api.github.com/repos/Codertocat/Hello-World/merges", 94 | "archive_url": "https://api.github.com/repos/Codertocat/Hello-World/{archive_format}{/ref}", 95 | "downloads_url": "https://api.github.com/repos/Codertocat/Hello-World/downloads", 96 | "issues_url": "https://api.github.com/repos/Codertocat/Hello-World/issues{/number}", 97 | "pulls_url": "https://api.github.com/repos/Codertocat/Hello-World/pulls{/number}", 98 | "milestones_url": "https://api.github.com/repos/Codertocat/Hello-World/milestones{/number}", 99 | "notifications_url": "https://api.github.com/repos/Codertocat/Hello-World/notifications{?since,all,participating}", 100 | "labels_url": "https://api.github.com/repos/Codertocat/Hello-World/labels{/name}", 101 | "releases_url": "https://api.github.com/repos/Codertocat/Hello-World/releases{/id}", 102 | "deployments_url": "https://api.github.com/repos/Codertocat/Hello-World/deployments", 103 | "created_at": "2019-05-15T15:19:25Z", 104 | "updated_at": "2019-05-15T15:20:34Z", 105 | "pushed_at": "2019-05-15T15:20:33Z", 106 | "git_url": "git://github.com/Codertocat/Hello-World.git", 107 | "ssh_url": "git@github.com:Codertocat/Hello-World.git", 108 | "clone_url": "https://github.com/Codertocat/Hello-World.git", 109 | "svn_url": "https://github.com/Codertocat/Hello-World", 110 | "homepage": null, 111 | "size": 0, 112 | "stargazers_count": 0, 113 | "watchers_count": 0, 114 | "language": "Ruby", 115 | "has_issues": true, 116 | "has_projects": true, 117 | "has_downloads": true, 118 | "has_wiki": true, 119 | "has_pages": true, 120 | "forks_count": 0, 121 | "mirror_url": null, 122 | "archived": false, 123 | "disabled": false, 124 | "open_issues_count": 2, 125 | "license": null, 126 | "forks": 0, 127 | "open_issues": 2, 128 | "watchers": 0, 129 | "default_branch": "master" 130 | }, 131 | "sender": { 132 | "login": "Codertocat", 133 | "id": 21031067, 134 | "node_id": "MDQ6VXNlcjIxMDMxMDY3", 135 | "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", 136 | "gravatar_id": "", 137 | "url": "https://api.github.com/users/Codertocat", 138 | "html_url": "https://github.com/Codertocat", 139 | "followers_url": "https://api.github.com/users/Codertocat/followers", 140 | "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", 141 | "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", 142 | "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", 143 | "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", 144 | "organizations_url": "https://api.github.com/users/Codertocat/orgs", 145 | "repos_url": "https://api.github.com/users/Codertocat/repos", 146 | "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", 147 | "received_events_url": "https://api.github.com/users/Codertocat/received_events", 148 | "type": "User", 149 | "site_admin": false 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /testing/mocks.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is a collection of utility functions for easier, DRY testing. 3 | """ 4 | import json 5 | from collections import namedtuple 6 | from contextlib import contextmanager 7 | from subprocess import CalledProcessError 8 | from unittest import mock 9 | 10 | from detect_secrets_server.util.version import is_python_2 11 | 12 | 13 | @contextmanager 14 | def mock_open(data=None): 15 | if not data: 16 | data = {} 17 | 18 | namespace = '__builtin__.open' if is_python_2() else 'builtins.open' 19 | 20 | mock_open = mock.mock_open(read_data=json.dumps(data)) 21 | with mock.patch(namespace, mock_open): 22 | yield mock_open 23 | 24 | 25 | @contextmanager 26 | def mock_git_calls(*cases): 27 | """We perform several subprocess.check_output calls for git commands, 28 | but we only want to mock one at a time. This function helps us do that. 29 | 30 | However, the idea is that we *never* want to call out to git in tests, 31 | so we should mock out everything that does that. 32 | 33 | :type cases: iterable(SubprocessMock) 34 | """ 35 | # We need to use a dictionary, because python2.7 does not support 36 | # the `nonlocal` keyword (and needs to share scope with 37 | # _mock_single_git_call function) 38 | current_case = {'index': 0} 39 | 40 | def _mock_subprocess_git_call(cmds, **kwargs): 41 | command = ' '.join(cmds) 42 | 43 | try: 44 | case = cases[current_case['index']] 45 | except IndexError: 46 | raise AssertionError( 47 | '\nExpected: ""\n' 48 | 'Actual: "{}"'.format( 49 | command 50 | ) 51 | ) 52 | current_case['index'] += 1 53 | 54 | if command != case.expected_input: 55 | # Pretty it up a little, for display 56 | if not case.expected_input.startswith('git'): 57 | case.expected_input = 'git ' + case.expected_input 58 | 59 | raise AssertionError( 60 | '\nExpected: "{}"\n' 61 | 'Actual: "{}"'.format( 62 | case.expected_input, 63 | command, 64 | ) 65 | ) 66 | 67 | if case.should_throw_exception: 68 | raise CalledProcessError(1, '', case.mocked_output) 69 | 70 | return case.mocked_output 71 | 72 | def _mock_single_git_call(directory, *args, **kwargs): 73 | return _mock_subprocess_git_call(['git'] + list(args)) 74 | 75 | # mock_subprocess is needed for `clone_repo_to_location`. 76 | with mock.patch( 77 | 'detect_secrets_server.storage.core.git._git' 78 | ) as mock_git, mock.patch( 79 | 'detect_secrets_server.storage.core.git.subprocess.check_output' 80 | ) as mock_subprocess: 81 | mock_git.side_effect = _mock_single_git_call 82 | mock_subprocess.side_effect = _mock_subprocess_git_call 83 | 84 | yield 85 | 86 | if current_case['index'] != len(cases): 87 | raise AssertionError( 88 | '\nExpected: "{}"\n' 89 | 'Actual: ""'.format(cases[current_case['index']].expected_input) 90 | ) 91 | 92 | 93 | class SubprocessMock(namedtuple( 94 | 'SubprocessMock', 95 | [ 96 | 'expected_input', 97 | 'mocked_output', 98 | 'should_throw_exception', 99 | ] 100 | )): 101 | """For use with mock_subprocess. 102 | 103 | :type expected_input: string 104 | :param expected_input: only return mocked_output if input matches this 105 | 106 | :type mocked_output: mixed 107 | :param mocked_output: value you want to return, when expected_input matches. 108 | 109 | :type should_throw_exception: bool 110 | :param should_throw_exception: if True, will throw subprocess.CalledProcessError 111 | with mocked output as error message 112 | """ 113 | def __new__(cls, expected_input, mocked_output='', should_throw_exception=False): 114 | return super(SubprocessMock, cls).__new__( 115 | cls, 116 | expected_input, 117 | mocked_output, 118 | should_throw_exception 119 | ) 120 | -------------------------------------------------------------------------------- /testing/util.py: -------------------------------------------------------------------------------- 1 | from detect_secrets_server.core.usage.common import storage 2 | 3 | 4 | EICAR = 'aHR0cHM6Ly93d3cueW91dHViZS5jb20vd2F0Y2g/dj1vSGc1U0pZUkhBMA==' 5 | JWT = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.1YeuHyvmlKMBvPDchxf71EkHMSRmYPD0Vb8Hza1ypbM' 6 | 7 | 8 | def cache_buster(): 9 | storage.get_storage_options.cache_clear() 10 | storage.should_enable_s3_options.cache_clear() 11 | -------------------------------------------------------------------------------- /tests/actions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/detect-secrets-server/f2a708fa1a4628a7a93039cae92ef90e8b94d9db/tests/actions/__init__.py -------------------------------------------------------------------------------- /tests/actions/conftest.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | import pytest 4 | 5 | 6 | @pytest.fixture 7 | def mock_file_operations(): 8 | """Mocks out certain calls in BaseTrackedRepo that attempts to 9 | write to disk. 10 | """ 11 | mock_open = mock.mock_open() 12 | with mock.patch( 13 | 'detect_secrets_server.storage.base.os.makedirs', 14 | ), mock.patch( 15 | 'detect_secrets_server.storage.file.open', 16 | mock_open, 17 | ): 18 | yield mock_open() 19 | -------------------------------------------------------------------------------- /tests/actions/initialize_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | from contextlib import contextmanager 3 | from unittest import mock 4 | 5 | import pytest 6 | 7 | from detect_secrets_server.actions import add_repo 8 | from detect_secrets_server.actions import initialize 9 | from detect_secrets_server.core.usage.parser import ServerParserBuilder 10 | from detect_secrets_server.storage.base import BaseStorage 11 | from testing.factories import metadata_factory 12 | from testing.factories import single_repo_config_factory 13 | from testing.mocks import mock_git_calls 14 | from testing.mocks import SubprocessMock 15 | from testing.util import cache_buster 16 | 17 | 18 | class TestInitialize: 19 | def teardown(self): 20 | cache_buster() 21 | 22 | @staticmethod 23 | def parse_args(argument_string='', has_s3=False): 24 | base_argument = ( 25 | 'add will_be_mocked --config ' 26 | ) 27 | if has_s3: 28 | base_argument += '--s3-config examples/s3.yaml ' 29 | 30 | with mock.patch( 31 | 'detect_secrets_server.core.usage.s3.should_enable_s3_options', 32 | return_value=has_s3, 33 | ): 34 | return ServerParserBuilder().parse_args( 35 | (base_argument + argument_string).split() 36 | ) 37 | 38 | def test_no_tracked_repos(self): 39 | with mock_repos_config({ 40 | 'tracked': [], 41 | }): 42 | args = self.parse_args() 43 | 44 | assert not initialize(args) 45 | 46 | def test_simple_success(self, mock_rootdir): 47 | with mock_repos_config({ 48 | 'tracked': [ 49 | single_repo_config_factory( 50 | 'git@github.com:yelp/detect-secrets', 51 | ), 52 | ] 53 | }), mock_repo_class( 54 | 'BaseTrackedRepo' 55 | ) as repo_class: 56 | args = self.parse_args( 57 | '--root-dir {}'.format(mock_rootdir) 58 | ) 59 | initialize(args) 60 | 61 | kwargs = repo_class.call_args[1] 62 | assert kwargs['repo'] == 'git@github.com:yelp/detect-secrets' 63 | assert kwargs['sha'] == '' 64 | assert kwargs['crontab'] == '0 0 * * *' 65 | assert kwargs['rootdir'] == mock_rootdir 66 | 67 | @pytest.mark.parametrize( 68 | 'data,expected_repo_class', 69 | [ 70 | ( 71 | { 72 | 'is_local_repo': True, 73 | 'repo': 'examples', 74 | }, 75 | 'LocalTrackedRepo', 76 | ), 77 | ( 78 | { 79 | 'storage': 's3', 80 | 'repo': 'git@github.com:yelp/detect-secrets', 81 | }, 82 | 'S3TrackedRepo', 83 | ), 84 | ( 85 | { 86 | 'is_local_repo': True, 87 | 'repo': 'examples', 88 | 'storage': 's3', 89 | }, 90 | 'S3LocalTrackedRepo', 91 | ), 92 | ] 93 | ) 94 | def test_flags_set_tracked_repo_classes(self, data, expected_repo_class): 95 | with mock_repos_config({ 96 | 'tracked': [ 97 | single_repo_config_factory( 98 | **data 99 | ), 100 | ] 101 | }): 102 | args = self.parse_args(has_s3=data.get('storage') == 's3') 103 | 104 | with mock_repo_class(expected_repo_class) as repo_class: 105 | initialize(args) 106 | assert repo_class.called 107 | 108 | def test_repo_config_overrides_defaults(self, mock_rootdir): 109 | with mock_repos_config({ 110 | 'tracked': [ 111 | single_repo_config_factory( 112 | 'git@github.com:yelp/detect-secrets', 113 | plugins={ 114 | # This checks that CLI overrides config file 115 | 'HexHighEntropyString': { 116 | 'hex_limit': 5, 117 | }, 118 | 119 | # This checks it overrides default values 120 | 'Base64HighEntropyString': { 121 | 'base64_limit': 2, 122 | }, 123 | 124 | # This checks for disabling functionality 125 | 'PrivateKeyDetector': False, 126 | }, 127 | 128 | # This checks it overrides CLI (non-plugin) 129 | baseline_filename='will_be_overriden', 130 | 131 | # This checks it overrides default value (non-plugin) 132 | exclude_regex='something_here', 133 | crontab='* * 4 * *', 134 | ) 135 | ], 136 | }): 137 | args = self.parse_args( 138 | '--hex-limit 4 ' 139 | '--baseline baseline.file ' 140 | '--root-dir {}'.format(mock_rootdir) 141 | ) 142 | 143 | with mock_repo_class('BaseTrackedRepo') as repo_class: 144 | initialize(args) 145 | 146 | kwargs = repo_class.call_args[1] 147 | assert kwargs['repo'] == 'git@github.com:yelp/detect-secrets' 148 | assert kwargs['sha'] == '' 149 | assert kwargs['crontab'] == '* * 4 * *' 150 | # NOTE: This is disabled, since it's `False` above. 151 | assert 'PrivateKeyDetector' not in kwargs['plugins'] 152 | assert kwargs['plugins']['Base64HighEntropyString']['base64_limit'] == 2.0 153 | assert kwargs['plugins']['HexHighEntropyString']['hex_limit'] == 4.0 154 | assert kwargs['rootdir'] == mock_rootdir 155 | assert kwargs['baseline_filename'] == 'baseline.file' 156 | assert kwargs['exclude_regex'] == 'something_here' 157 | 158 | 159 | class TestAddRepo: 160 | @staticmethod 161 | def parse_args(argument_string='', has_s3=False): 162 | with mock.patch( 163 | 'detect_secrets_server.core.usage.s3.should_enable_s3_options', 164 | return_value=has_s3, 165 | ): 166 | return ServerParserBuilder().parse_args( 167 | argument_string.split() 168 | ) 169 | 170 | def teardown(self): 171 | cache_buster() 172 | 173 | def test_add_non_local_repo(self, mock_file_operations, mock_rootdir): 174 | self.add_non_local_repo(mock_rootdir) 175 | mock_file_operations.write.assert_called_with( 176 | metadata_factory( 177 | repo='git@github.com:yelp/detect-secrets', 178 | sha='mocked_sha', 179 | json=True, 180 | ), 181 | ) 182 | 183 | def test_override_meta_tracking_if_already_exists( 184 | self, 185 | mock_file_operations, 186 | mock_rootdir, 187 | ): 188 | with mock.patch( 189 | 'detect_secrets_server.storage.file.FileStorage.get_tracked_file_location', 190 | 191 | # This doesn't matter what it is, just that it exists. 192 | return_value='examples/config.yaml', 193 | ): 194 | self.add_non_local_repo(mock_rootdir) 195 | 196 | assert mock_file_operations.write.called 197 | 198 | def add_non_local_repo(self, mock_rootdir): 199 | repo = 'git@github.com:yelp/detect-secrets' 200 | directory = '{}/repos/{}'.format( 201 | mock_rootdir, 202 | BaseStorage.hash_filename('yelp/detect-secrets'), 203 | ) 204 | 205 | git_calls = [ 206 | SubprocessMock( 207 | expected_input='git clone {} {} --bare'.format(repo, directory), 208 | ), 209 | SubprocessMock( 210 | expected_input='git rev-parse HEAD', 211 | mocked_output='mocked_sha', 212 | ), 213 | ] 214 | 215 | with mock_git_calls(*git_calls): 216 | args = self.parse_args('add {} --root-dir {}'.format(repo, mock_rootdir)) 217 | add_repo(args) 218 | 219 | def test_add_local_repo(self, mock_file_operations, mock_rootdir): 220 | # This just needs to exist; no actual operations will be done to this. 221 | repo = 'examples' 222 | 223 | git_calls = [ 224 | # repo.update 225 | SubprocessMock( 226 | expected_input='git rev-parse HEAD', 227 | mocked_output='mocked_sha', 228 | ), 229 | ] 230 | 231 | with mock_git_calls(*git_calls): 232 | args = self.parse_args( 233 | 'add {} --baseline .secrets.baseline --local --root-dir {}'.format( 234 | repo, 235 | mock_rootdir, 236 | ) 237 | ) 238 | 239 | add_repo(args) 240 | 241 | mock_file_operations.write.assert_called_with( 242 | metadata_factory( 243 | sha='mocked_sha', 244 | repo=os.path.abspath( 245 | os.path.join( 246 | os.path.dirname(__file__), 247 | '../../examples', 248 | ), 249 | ), 250 | baseline_filename='.secrets.baseline', 251 | json=True, 252 | ), 253 | ) 254 | 255 | def test_add_s3_backend_repo(self, mock_file_operations, mocked_boto): 256 | args = self.parse_args( 257 | 'add {} ' 258 | '--local ' 259 | '--storage s3 ' 260 | '--s3-credentials-file examples/aws_credentials.json ' 261 | '--s3-bucket pail'.format('examples'), 262 | has_s3=True, 263 | ) 264 | 265 | git_calls = [ 266 | # repo.update 267 | SubprocessMock( 268 | expected_input='git rev-parse HEAD', 269 | mocked_output='mocked_sha', 270 | ), 271 | ] 272 | 273 | with mock_git_calls( 274 | *git_calls 275 | ): 276 | mocked_boto.list_objects_v2.return_value = {} 277 | add_repo(args) 278 | 279 | 280 | @contextmanager 281 | def mock_repos_config(data): 282 | """Unfortunately, mocking this means that we can't test more than 283 | one config file at a time. However, all consolidation tests with 284 | --config-file should have been done in usage_test, so we should 285 | be OK. 286 | """ 287 | with mock.patch( 288 | 'detect_secrets_server.core.usage.add.config_file', 289 | return_value=data, 290 | ): 291 | yield 292 | 293 | 294 | @contextmanager 295 | def mock_repo_class(classname): 296 | """ 297 | :type classname: str 298 | """ 299 | with mock.patch( 300 | 'detect_secrets_server.repos.factory.{}'.format(classname), 301 | ) as repo_class: 302 | yield repo_class 303 | -------------------------------------------------------------------------------- /tests/actions/install_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | import textwrap 4 | from contextlib import contextmanager 5 | from unittest import mock 6 | 7 | import pytest 8 | 9 | from detect_secrets_server.actions.install import install_mapper 10 | from detect_secrets_server.core.usage.parser import ServerParserBuilder 11 | from testing.factories import metadata_factory 12 | 13 | 14 | class TestInstallCron(object): 15 | 16 | @staticmethod 17 | def parse_args(rootdir, argument_string=''): 18 | with mock.patch( 19 | 'detect_secrets_server.core.usage.s3.should_enable_s3_options', 20 | return_value=False, 21 | ): 22 | return ServerParserBuilder().parse_args( 23 | 'install cron --root-dir {} {}'.format( 24 | rootdir, 25 | argument_string, 26 | ).split() 27 | ) 28 | 29 | def test_writes_crontab(self, mock_crontab, mock_rootdir, mock_metadata): 30 | args = self.parse_args(mock_rootdir) 31 | with mock_metadata( 32 | remote_files=( 33 | metadata_factory( 34 | repo='git@github.com:yelp/detect-secrets', 35 | json=True, 36 | ), 37 | ), 38 | local_files=( 39 | metadata_factory( 40 | repo='examples', 41 | json=True, 42 | ), 43 | ) 44 | ): 45 | install_mapper(args) 46 | 47 | assert mock_crontab.content == textwrap.dedent(""" 48 | 0 0 * * * detect-secrets-server scan git@github.com:yelp/detect-secrets --root-dir {} 49 | 0 0 * * * detect-secrets-server scan examples --local --root-dir {} 50 | """).format(mock_rootdir, mock_rootdir)[1:-1] 51 | mock_crontab.write_to_user.assert_called_with(user=True) 52 | 53 | def test_crontab_writes_with_output_hook( 54 | self, 55 | mock_crontab, 56 | mock_rootdir, 57 | mock_metadata, 58 | ): 59 | args = self.parse_args( 60 | mock_rootdir, 61 | '--output-hook examples/standalone_hook.py' 62 | ) 63 | 64 | with mock_metadata( 65 | remote_files=( 66 | metadata_factory( 67 | repo='git@github.com:yelp/detect-secrets', 68 | crontab='1 2 3 4 5', 69 | json=True, 70 | ), 71 | ), 72 | ): 73 | install_mapper(args) 74 | 75 | assert mock_crontab.content == ( 76 | '1 2 3 4 5 detect-secrets-server scan git@github.com:yelp/detect-secrets' 77 | ' --root-dir {}' 78 | ' --output-hook examples/standalone_hook.py'.format(mock_rootdir) 79 | ) 80 | mock_crontab.write_to_user.assert_called_with(user=True) 81 | 82 | def test_does_not_override_existing_crontab( 83 | self, 84 | mock_crontab, 85 | mock_rootdir, 86 | mock_metadata, 87 | ): 88 | mock_crontab.old_content = textwrap.dedent(""" 89 | * * * * * detect-secrets-server scan old_config_will_be_deleted --local 90 | * * * * * some_content_here 91 | """)[1:] 92 | 93 | args = self.parse_args(mock_rootdir) 94 | with mock_metadata( 95 | local_files=( 96 | metadata_factory( 97 | repo='examples', 98 | crontab='1 2 3 4 5', 99 | json=True, 100 | ), 101 | ), 102 | ): 103 | install_mapper(args) 104 | 105 | assert mock_crontab.content == textwrap.dedent(""" 106 | * * * * * some_content_here 107 | 108 | 1 2 3 4 5 detect-secrets-server scan examples --local --root-dir {} 109 | """).format(mock_rootdir)[1:-1] 110 | 111 | 112 | @pytest.fixture 113 | def mock_crontab(): 114 | output = mock.Mock() 115 | 116 | def mock_constructor(user, tab='', *args, **kwargs): 117 | output.content = tab 118 | 119 | return output 120 | 121 | def writer(filename): 122 | with open(filename, 'w') as f: 123 | f.write(output.old_content) 124 | output.write = writer 125 | output.old_content = '' 126 | 127 | with mock.patch( 128 | 'detect_secrets_server.actions.install.CronTab', 129 | mock_constructor, 130 | ): 131 | yield output 132 | 133 | 134 | @pytest.fixture 135 | def mock_metadata(mock_rootdir): 136 | def _write_content(content, is_local=False): 137 | directory = os.path.join(mock_rootdir, 'tracked') 138 | if is_local: 139 | directory = os.path.join(directory, 'local') 140 | 141 | file = tempfile.NamedTemporaryFile( 142 | dir=directory, 143 | suffix='.json', 144 | ) 145 | file.write(content.encode()) 146 | 147 | # This makes it so that we can immediately read from it. 148 | file.seek(0) 149 | 150 | return file 151 | 152 | @contextmanager 153 | def wrapped(remote_files=None, local_files=None): 154 | """ 155 | :type remote_files: list(str) 156 | :param remote_files: list of JSON encoded metadata_factory, for remote repos 157 | :type local_files: list(str) 158 | :param local_files: list of JSON encoded metadata_factory, for local repos 159 | """ 160 | if not remote_files: 161 | remote_files = [] 162 | if not local_files: 163 | local_files = [] 164 | 165 | for path in ( 166 | os.path.join(mock_rootdir, 'tracked'), 167 | os.path.join(mock_rootdir, 'tracked/local'), 168 | ): 169 | os.mkdir(path) 170 | 171 | temp_files = [] 172 | for content in remote_files: 173 | temp_files.append(_write_content(content)) 174 | 175 | for content in local_files: 176 | temp_files.append(_write_content(content, is_local=True)) 177 | 178 | try: 179 | yield 180 | finally: 181 | for file in temp_files: 182 | file.close() 183 | 184 | return wrapped 185 | -------------------------------------------------------------------------------- /tests/actions/scan_test.py: -------------------------------------------------------------------------------- 1 | import json 2 | import textwrap 3 | from contextlib import contextmanager 4 | from unittest import mock 5 | 6 | import pytest 7 | from detect_secrets.core.secrets_collection import SecretsCollection 8 | 9 | from detect_secrets_server.actions import scan_repo 10 | from detect_secrets_server.core.usage.parser import ServerParserBuilder 11 | from detect_secrets_server.hooks.stdout import StdoutHook 12 | from testing.factories import secrets_collection_factory 13 | from testing.mocks import mock_git_calls 14 | from testing.mocks import SubprocessMock 15 | 16 | 17 | class TestScanRepo(object): 18 | 19 | @staticmethod 20 | def parse_args(argument_string=''): 21 | base_argument = ( 22 | 'scan will_be_mocked ' 23 | '--output-hook examples/standalone_hook.py ' 24 | ) 25 | 26 | with mock.patch( 27 | 'detect_secrets_server.core.usage.s3.should_enable_s3_options', 28 | return_value=False, 29 | ): 30 | return ServerParserBuilder().parse_args( 31 | (base_argument + argument_string).split() 32 | ) 33 | 34 | def test_quits_early_if_cannot_load_meta_tracking_file(self): 35 | args = self.parse_args() 36 | 37 | assert scan_repo(args) == 1 38 | 39 | def test_updates_tracked_repo_when_no_secrets_are_found( 40 | self, 41 | mock_file_operations, 42 | mock_logger 43 | ): 44 | with self.setup_env( 45 | SecretsCollection(), 46 | updates_repo=True, 47 | ) as args: 48 | assert scan_repo(args) == 0 49 | 50 | mock_logger.info.assert_called_with( 51 | 'No secrets found for %s', 52 | 'yelp/detect-secrets', 53 | ) 54 | 55 | mock_file_operations.write.assert_called_with( 56 | json.dumps(mock_tracked_file('new_sha'), indent=2, sort_keys=True) 57 | ) 58 | 59 | def test_alerts_on_secrets_found( 60 | self, 61 | mock_file_operations, 62 | mock_logger, 63 | ): 64 | secrets = secrets_collection_factory([ 65 | { 66 | 'filename': 'file_with_secrets', 67 | 'lineno': 5, 68 | }, 69 | ]) 70 | 71 | with self.setup_env(secrets) as args: 72 | secret_hash = list( 73 | secrets.data['file_with_secrets'].values() 74 | )[0].secret_hash 75 | 76 | args.output_hook = mock_external_hook( 77 | 'yelp/detect-secrets', 78 | { 79 | 'file_with_secrets': [{ 80 | 'type': 'type', 81 | 'hashed_secret': secret_hash, 82 | 'is_verified': False, 83 | 'line_number': 5, 84 | 'author': 'khock', 85 | 'commit': 'new_sha', 86 | }], 87 | }, 88 | ) 89 | 90 | assert scan_repo(args) == 0 91 | 92 | mock_logger.error.assert_called_with( 93 | 'Secrets found in %s', 94 | 'yelp/detect-secrets', 95 | ) 96 | assert not mock_file_operations.write.called 97 | 98 | def test_does_not_write_state_when_dry_run(self, mock_file_operations): 99 | with self.setup_env( 100 | SecretsCollection(), 101 | '--dry-run', 102 | ) as args: 103 | assert scan_repo(args) == 0 104 | 105 | assert not mock_file_operations.write.called 106 | 107 | def test_scan_head_and_does_not_write_state_when_scan_head( 108 | self, 109 | mock_file_operations, 110 | mock_logger, 111 | ): 112 | secrets = secrets_collection_factory([ 113 | { 114 | 'filename': 'file_with_secrets', 115 | 'lineno': 5, 116 | }, 117 | ]) 118 | 119 | with self.setup_env( 120 | secrets, 121 | '--scan-head', 122 | ) as args: 123 | 124 | secret_hash = list( 125 | secrets.data['file_with_secrets'].values() 126 | )[0].secret_hash 127 | 128 | args.output_hook = mock_external_hook( 129 | 'yelp/detect-secrets', 130 | { 131 | 'file_with_secrets': [{ 132 | 'type': 'type', 133 | 'hashed_secret': secret_hash, 134 | 'is_verified': False, 135 | 'line_number': 5, 136 | 'author': 'khock', 137 | 'commit': 'new_sha', 138 | }], 139 | }, 140 | ) 141 | 142 | assert scan_repo(args) == 0 143 | 144 | mock_logger.error.assert_called_with( 145 | 'Secrets found in %s', 146 | 'yelp/detect-secrets', 147 | ) 148 | 149 | assert not mock_file_operations.write.called 150 | 151 | def test_always_writes_state_with_always_update_state_flag( 152 | self, 153 | mock_file_operations, 154 | ): 155 | secrets = secrets_collection_factory([ 156 | { 157 | 'filename': 'file_with_secrets', 158 | 'lineno': 5, 159 | }, 160 | ]) 161 | 162 | with self.setup_env( 163 | secrets, 164 | '--always-update-state', 165 | updates_repo=True, 166 | ) as args: 167 | assert scan_repo(args) == 0 168 | 169 | assert mock_file_operations.write.called 170 | 171 | @contextmanager 172 | def setup_env(self, scan_results, argument_string='', updates_repo=False): 173 | """This sets up the relevant mocks, so that we can conduct testing. 174 | 175 | :type scan_results: SecretsCollection 176 | 177 | :type argument_string: str 178 | :param argument_string: additional arguments to parse_args 179 | 180 | :type updates_repo: bool 181 | :param updates_repo: True if scan should update its internal state 182 | """ 183 | @contextmanager 184 | def wrapped_mock_git_calls(git_calls): 185 | if not git_calls: 186 | # Need to yield **something** 187 | yield 188 | return 189 | 190 | with mock_git_calls(*git_calls): 191 | yield 192 | 193 | args = self.parse_args(argument_string) 194 | 195 | with mock.patch( 196 | 'detect_secrets_server.repos.base_tracked_repo.BaseTrackedRepo.scan', 197 | return_value=scan_results, 198 | ), mock.patch( 199 | # We mock this, so that we can successfully load_from_file 200 | 'detect_secrets_server.storage.file.FileStorage.get', 201 | return_value=mock_tracked_file('old_sha'), 202 | ), wrapped_mock_git_calls( 203 | get_subprocess_mocks(scan_results, updates_repo), 204 | ): 205 | yield args 206 | 207 | 208 | def get_subprocess_mocks(secrets, updates_repo): 209 | """ 210 | :type secrets: SecretsCollection 211 | :type updates_repo: bool 212 | """ 213 | subprocess_mocks = [] 214 | if secrets.data: 215 | # TODO: If we need to, we should modify this for more filenames 216 | secrets_dict = secrets.json() 217 | filenames = list(secrets_dict.keys()) 218 | 219 | subprocess_mocks.append( 220 | # First, we get the main branch 221 | SubprocessMock( 222 | expected_input='git rev-parse --abbrev-ref HEAD', 223 | mocked_output='master', 224 | ), 225 | ) 226 | 227 | subprocess_mocks.append( 228 | # then, we get the blame info for that branch 229 | SubprocessMock( 230 | expected_input=( 231 | 'git blame master -L {},{} --show-email ' 232 | '--line-porcelain -- {}'.format( 233 | secrets_dict[filenames[0]][0]['line_number'], 234 | secrets_dict[filenames[0]][0]['line_number'], 235 | filenames[0], 236 | ) 237 | ), 238 | mocked_output=mock_blame_info(), 239 | ), 240 | ) 241 | 242 | subprocess_mocks.append( 243 | # and get the current HEAD. 244 | SubprocessMock( 245 | expected_input='git rev-parse HEAD', 246 | mocked_output='new_sha', 247 | ), 248 | ) 249 | 250 | if updates_repo: 251 | # We also get the current HEAD when updating. 252 | subprocess_mocks.append( 253 | SubprocessMock( 254 | expected_input='git rev-parse HEAD', 255 | mocked_output='new_sha', 256 | ), 257 | ) 258 | 259 | return subprocess_mocks 260 | 261 | 262 | def mock_tracked_file(sha): 263 | return { 264 | 'sha': sha, 265 | 'repo': 'git@github.com:yelp/detect-secrets', 266 | 'plugins': { 267 | 'HexHighEntropyString': { 268 | 'hex_limit': 3, 269 | }, 270 | 'Base64HighEntropyString': { 271 | 'base64_limit': 4.5, 272 | }, 273 | 'PrivateKeyDetector': {}, 274 | 'BasicAuthDetector': {}, 275 | }, 276 | 'crontab': '* * 4 * *', 277 | 'baseline_filename': None, 278 | 'exclude_regex': None, 279 | } 280 | 281 | 282 | def mock_blame_info(): 283 | return textwrap.dedent(""" 284 | d39c008353447bbc1845812fcaf0a03b50af439f 177 174 1 285 | author Kevin Hock 286 | author-mail 287 | author-time 1513196047 288 | author-tz -0800 289 | committer Foo 290 | committer-mail 291 | committer-time 1513196047 292 | committer-tz -0800 293 | summary mock 294 | previous 23c630620c23843559485fd2ada02e9e7bc5a07e4 mock_output.java 295 | filename some_file.java 296 | "super:secret f8616fefbo41fdc31960ehef078f85527"))); 297 | """)[1:] 298 | 299 | 300 | def mock_external_hook(expected_repo_name, expected_secrets): 301 | def wrapped(repo_name, secrets): 302 | assert repo_name == expected_repo_name 303 | assert secrets == expected_secrets 304 | 305 | mock_hook = StdoutHook() 306 | mock_hook.alert = wrapped 307 | 308 | return mock_hook 309 | 310 | 311 | @pytest.fixture 312 | def mock_logger(): 313 | with mock.patch( 314 | 'detect_secrets_server.actions.scan.log' 315 | ) as log: 316 | yield log 317 | -------------------------------------------------------------------------------- /tests/adhoc/github/github_webhook_test.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | import pytest 5 | 6 | from detect_secrets_server.adhoc.github.webhook import scan_for_secrets 7 | from testing.util import EICAR 8 | from testing.util import JWT 9 | 10 | 11 | @pytest.mark.parametrize( 12 | 'action', 13 | { 14 | 'created', 15 | 'edited', 16 | }, 17 | ) 18 | @pytest.mark.parametrize( 19 | 'event', 20 | { 21 | 'commit_comment', 22 | 'issue_comment', 23 | 'pull_request_review_comment', 24 | }, 25 | ) 26 | def test_comment_with_secret(event, action): 27 | payload = get_payload(event) 28 | 29 | # We make sure to add "multiple words" here, since we want to make 30 | # sure that it supports multi-word bodies (as we would expect in 31 | # regular usage). 32 | payload['comment']['body'] = 'multiple words {}'.format(EICAR) 33 | payload['action'] = action 34 | 35 | assert scan_for_secrets(event, payload) 36 | 37 | 38 | @pytest.mark.parametrize( 39 | 'action', 40 | { 41 | 'created', 42 | 'edited', 43 | }, 44 | ) 45 | @pytest.mark.parametrize( 46 | 'event', 47 | { 48 | 'commit_comment', 49 | 'issue_comment', 50 | 'pull_request_review_comment', 51 | }, 52 | ) 53 | def test_comment_no_secret(event, action): 54 | payload = get_payload(event) 55 | payload['action'] = action 56 | 57 | assert not scan_for_secrets(event, payload) 58 | 59 | 60 | @pytest.mark.parametrize( 61 | 'event', 62 | { 63 | 'commit_comment', 64 | 'issue_comment', 65 | 'pull_request_review_comment', 66 | }, 67 | ) 68 | def test_comment_deleted(event): 69 | payload = get_payload(event) 70 | payload['comment']['body'] = 'multiple words {}'.format(EICAR) 71 | payload['action'] = 'deleted' 72 | 73 | assert not scan_for_secrets(event, payload) 74 | 75 | 76 | @pytest.mark.parametrize( 77 | 'event_key', 78 | { 79 | 'issues,issue', 80 | 'pull_request,pull_request', 81 | }, 82 | ) 83 | @pytest.mark.parametrize( 84 | 'action', 85 | { 86 | 'opened', 87 | 'edited', 88 | }, 89 | ) 90 | def test_issue_success(event_key, action): 91 | event, key = event_key.split(',') 92 | payload = get_payload(event) 93 | payload['action'] = action 94 | payload[key]['body'] = 'multiple words {}'.format(EICAR) 95 | 96 | assert scan_for_secrets(event, payload) 97 | 98 | 99 | @pytest.mark.parametrize( 100 | 'event_key', 101 | { 102 | 'issues,issue', 103 | 'pull_request,pull_request', 104 | }, 105 | ) 106 | @pytest.mark.parametrize( 107 | 'action', 108 | { 109 | 'opened', 110 | 'edited', 111 | }, 112 | ) 113 | def test_issue_no_secret(event_key, action): 114 | event, key = event_key.split(',') 115 | payload = get_payload(event) 116 | payload['action'] = action 117 | 118 | assert not scan_for_secrets(event, payload) 119 | 120 | 121 | @pytest.mark.parametrize( 122 | 'event_key', 123 | { 124 | 'issues,issue', 125 | 'pull_request,pull_request', 126 | }, 127 | ) 128 | def test_issue_not_applicable(event_key): 129 | event, key = event_key.split(',') 130 | payload = get_payload(event) 131 | payload['action'] = 'deleted' 132 | payload[key]['body'] = 'multiple words {}'.format(EICAR) 133 | 134 | assert not scan_for_secrets(event, payload) 135 | 136 | 137 | def test_parameter_passing(): 138 | event = 'commit_comment' 139 | payload = get_payload(event) 140 | payload['comment']['body'] = JWT 141 | 142 | assert not scan_for_secrets( 143 | event, 144 | payload, 145 | '--no-jwt-scan', 146 | # kind of annoying that this is necessary but each part the 147 | # JWT is random-looking base64 too 148 | '--no-base64-string-scan', 149 | ) 150 | 151 | 152 | def get_payload(name): 153 | filepath = os.path.join( 154 | os.path.dirname(__file__), 155 | '../../../testing/github/', 156 | '{}.json'.format(name), 157 | ) 158 | 159 | with open(filepath) as f: 160 | return json.loads(f.read()) 161 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import tempfile 4 | from unittest import mock 5 | 6 | import pytest 7 | 8 | 9 | @pytest.fixture 10 | def mock_rootdir(): 11 | # We create a temp directory in the current repo, because it will be 12 | # platform-independent. 13 | tempdir = os.path.abspath( 14 | os.path.join( 15 | os.path.dirname(__file__), 16 | '../tmp' 17 | ) 18 | ) 19 | if not os.path.isdir(tempdir): # pragma: no cover 20 | os.mkdir(tempdir) 21 | 22 | pathname = tempfile.mkdtemp(dir=tempdir) 23 | try: 24 | yield pathname 25 | finally: 26 | shutil.rmtree(pathname) 27 | 28 | 29 | @pytest.fixture 30 | def mocked_boto(): 31 | mock_client = mock.Mock() 32 | with mock.patch( 33 | 'detect_secrets_server.storage.s3.S3Storage._get_boto3', 34 | return_value=mock_client, 35 | ), mock.patch( 36 | 'detect_secrets_server.core.usage.common.storage.should_enable_s3_options', 37 | return_value=True, 38 | ), mock.patch( 39 | 'detect_secrets_server.core.usage.s3.should_enable_s3_options', 40 | return_value=True, 41 | ): 42 | yield mock_client.client() 43 | -------------------------------------------------------------------------------- /tests/core/usage/add_test.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | import pytest 4 | 5 | from testing.base_usage_test import UsageTest 6 | 7 | 8 | class TestAddOptions(UsageTest): 9 | 10 | @pytest.mark.parametrize( 11 | 'command, will_raise_error', 12 | [ 13 | ( 14 | 'add git@github.com:yelp/detect-secrets', 15 | False, 16 | ), 17 | ( 18 | 'add https://github.com/Yelp/detect-secrets.git', 19 | False, 20 | ), 21 | 22 | # If local, the git url will fail because it's not a folder 23 | ( 24 | 'add git@github.com:yelp/detect-secrets -L', 25 | True, 26 | ), 27 | 28 | # Doesn't matter where the repo is: the path just needs to exist 29 | ( 30 | 'add examples -L ', 31 | False, 32 | ), 33 | ], 34 | ) 35 | def test_ensure_parameter_is_git_url(self, command, will_raise_error): 36 | if will_raise_error: 37 | with pytest.raises(SystemExit): 38 | self.parse_args(command) 39 | else: 40 | self.parse_args(command) 41 | 42 | def test_adhoc_settings(self): 43 | args = self.parse_args( 44 | 'add examples -L ' 45 | '--baseline .baseline ' 46 | '--exclude-regex regex ' 47 | '--root-dir /tmp' 48 | ) 49 | 50 | assert args.baseline == '.baseline' 51 | assert args.exclude_regex == 'regex' 52 | assert args.root_dir == '/tmp' 53 | 54 | def test_local_config_does_not_make_sense(self): 55 | with pytest.raises(SystemExit): 56 | self.parse_args( 57 | 'add examples/repos.yaml --config --local' 58 | ) 59 | 60 | def test_invalid_config_file(self): 61 | with pytest.raises(SystemExit), mock.patch( 62 | 'detect_secrets_server.core.usage.add.config_file', 63 | return_value={'foo': 'bar'}, 64 | ): 65 | self.parse_args('add will_be_mocked') 66 | 67 | def test_config_file(self): 68 | args = self.parse_args( 69 | 'add examples/repos.yaml --config ' 70 | ) 71 | 72 | assert args.repo[0]['repo'] == \ 73 | 'git@github.com:yelp/detect-secrets.git' 74 | 75 | def test_config_file_does_not_override_command_line_args(self): 76 | args = self.parse_args( 77 | 'add examples/repos.yaml --config' 78 | ) 79 | 80 | # Config file overrides default values 81 | assert args.repo[0]['plugins']['Base64HighEntropyString']['base64_limit'] == 4 82 | 83 | # Default values are used 84 | assert args.plugins['PrivateKeyDetector'] == {} 85 | assert args.repo[0]['plugins']['HexHighEntropyString']['hex_limit'] == 3 86 | 87 | args = self.parse_args( 88 | 'add examples/repos.yaml --config ' 89 | '--no-private-key-scan ' 90 | '--base64-limit 5' 91 | ) 92 | 93 | # CLI options overrides config file 94 | assert args.repo[0]['plugins']['Base64HighEntropyString']['base64_limit'] == 5 95 | assert 'PrivateKeyDetector' not in args.plugins 96 | 97 | # Default values still used 98 | assert args.repo[0]['plugins']['HexHighEntropyString']['hex_limit'] == 3 99 | 100 | def test_config_file_unknown_plugin_does_nothing(self): 101 | mock_config_file = { 102 | 'tracked': [ 103 | { 104 | 'repo': 'git@github.com:yelp/detect-secrets', 105 | 'plugins': { 106 | 'blah': { 107 | 'arg_name': 1, 108 | }, 109 | }, 110 | }, 111 | ], 112 | } 113 | with mock.patch( 114 | 'detect_secrets_server.core.usage.add.config_file', 115 | return_value=mock_config_file, 116 | ): 117 | args = self.parse_args('add will_be_mocked --config') 118 | 119 | assert 'blah' not in args.repo[0]['plugins'] 120 | 121 | @pytest.mark.parametrize( 122 | 'repo', 123 | ( 124 | { 125 | 'repo': 'non_existent_file', 126 | 'crontab': '* * 4 * *', 127 | 'is_local_repo': True, 128 | }, 129 | { 130 | 'repo': 'examples', 131 | 'crontab': '* * 4 * *', 132 | }, 133 | ), 134 | ) 135 | def test_config_file_unknown_repos_are_discarded(self, repo): 136 | with mock.patch( 137 | 'detect_secrets_server.core.usage.add.config_file', 138 | return_value={ 139 | 'tracked': [repo], 140 | }, 141 | ): 142 | args = self.parse_args('add will_be_mocked --config') 143 | 144 | assert not args.repo 145 | -------------------------------------------------------------------------------- /tests/core/usage/output_test.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | from unittest import mock 3 | 4 | import pytest 5 | 6 | from detect_secrets_server.core.usage.common.hooks import HookDescriptor 7 | from detect_secrets_server.hooks.external import ExternalHook 8 | from detect_secrets_server.hooks.stdout import StdoutHook 9 | from testing.base_usage_test import UsageTest 10 | 11 | 12 | class TestOutputOptions(UsageTest): 13 | 14 | def parse_args(self, argument_string=''): 15 | return super(TestOutputOptions, self).parse_args( 16 | argument_string, 17 | ) 18 | 19 | @pytest.mark.parametrize( 20 | 'hook_input', 21 | [ 22 | # No such hook 23 | 'asdf', 24 | 25 | # config file required 26 | 'pysensu', 27 | 28 | # no such file 29 | 'test_data/invalid_file', 30 | ] 31 | ) 32 | def test_invalid_output_hook(self, hook_input): 33 | with pytest.raises(SystemExit): 34 | self.parse_args('scan --output-hook {} examples -L'.format(hook_input)) 35 | 36 | def test_valid_external_hook(self): 37 | args = self.parse_args( 38 | 'scan --output-hook examples/standalone_hook.py examples -L', 39 | ) 40 | assert isinstance(args.output_hook, ExternalHook) 41 | 42 | def test_valid_hook_with_config_file(self): 43 | """ 44 | We don't want test cases to require extra dependencies, only to test 45 | whether they are compatible. Therefore, we mock ALL_HOOKS with a 46 | stand-in replacement for a hook that requires a config file. 47 | """ 48 | with mock.patch( 49 | 'detect_secrets_server.core.usage.common.output.ALL_HOOKS', 50 | [ 51 | HookDescriptor( 52 | display_name='config_needed', 53 | module_name='will_be_mocked', 54 | class_name='ConfigFileRequiredHook', 55 | config_setting=HookDescriptor.CONFIG_REQUIRED, 56 | ), 57 | ], 58 | ), mock.patch( 59 | 'detect_secrets_server.core.usage.common.output.import_module', 60 | return_value=Module( 61 | ConfigFileRequiredHook=ConfigFileRequiredMockClass, 62 | ), 63 | ): 64 | args = self.parse_args( 65 | 'scan ' 66 | '--output-hook config_needed ' 67 | '--output-config examples/pysensu.config.yaml ' 68 | 'examples ' 69 | ) 70 | 71 | with open('examples/pysensu.config.yaml') as f: 72 | content = f.read() 73 | 74 | assert args.output_hook.config == content 75 | 76 | def test_no_hook_provided(self): 77 | args = self.parse_args('scan git@git.github.com:Yelp/detect-secrets') 78 | assert isinstance(args.output_hook, StdoutHook) 79 | assert args.output_hook_command == '' 80 | 81 | 82 | Module = namedtuple( 83 | 'Module', 84 | [ 85 | 'ConfigFileRequiredHook', 86 | ] 87 | ) 88 | 89 | 90 | class ConfigFileRequiredMockClass(object): 91 | def __init__(self, config): 92 | self.config = config 93 | -------------------------------------------------------------------------------- /tests/core/usage/parser_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import detect_secrets_server 4 | from detect_secrets_server.core.usage.parser import ServerParserBuilder 5 | from detect_secrets_server.util.version import is_python_2 6 | 7 | 8 | def test_version(capsys): 9 | with pytest.raises(SystemExit) as e: 10 | ServerParserBuilder().parse_args(['--version']) 11 | 12 | assert str(e.value) == '0' 13 | 14 | # Oh, the joys of writing compatible code 15 | if is_python_2(): # pragma: no cover 16 | assert capsys.readouterr().err.strip() == detect_secrets_server.__version__ 17 | else: # pragma: no cover 18 | assert capsys.readouterr().out.strip() == detect_secrets_server.__version__ 19 | -------------------------------------------------------------------------------- /tests/core/usage/s3_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from testing.base_usage_test import UsageTest 4 | 5 | 6 | class TestS3Options(UsageTest): 7 | 8 | def parse_args(self, argument_string='', has_boto=True): 9 | # This test suite uses `scan` to test, because the API is a lot simpler. 10 | argument_string = '{} {} {}'.format( 11 | 'scan --output-hook examples/standalone_hook.py --storage s3', 12 | argument_string, 13 | 'examples -L', 14 | ) 15 | return super(TestS3Options, self).parse_args( 16 | argument_string, 17 | has_boto, 18 | ) 19 | 20 | def test_should_fail_to_find_s3_arguments(self): 21 | with pytest.raises(SystemExit): 22 | self.parse_args( 23 | '--s3-credentials-file examples/aws_credentials.json --s3-bucket BUCKET', 24 | has_boto=False, 25 | ) 26 | 27 | def test_success(self): 28 | args = self.parse_args( 29 | ( 30 | '--s3-credentials-file examples/aws_credentials.json ' 31 | '--s3-bucket BUCKET ' 32 | '--s3-prefix p' 33 | ) 34 | ) 35 | 36 | assert not any([ 37 | getattr(args, 's3_bucket', None), 38 | getattr(args, 's3_prefix', None), 39 | getattr(args, 's3_credentials_file', None), 40 | ]) 41 | assert args.s3_config == { 42 | 'prefix': 'p', 43 | 'bucket': 'BUCKET', 44 | 'creds_filename': 'examples/aws_credentials.json', 45 | 'access_key': 'access_key', 46 | 'secret_access_key': 'secret_key', 47 | } 48 | -------------------------------------------------------------------------------- /tests/core/usage/scan_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from detect_secrets_server.hooks.external import ExternalHook 4 | from testing.base_usage_test import UsageTest 5 | 6 | 7 | class TestScanOptions(UsageTest): 8 | 9 | def test_invalid_local_file(self): 10 | with pytest.raises(SystemExit): 11 | self.parse_args('scan --output-hook examples/standalone_hook.py fake_dir -L') 12 | 13 | def test_valid_local_file(self): 14 | args = self.parse_args('scan examples -L --output-hook examples/standalone_hook.py') 15 | 16 | assert args.action == 'scan' 17 | assert args.local 18 | assert isinstance(args.output_hook, ExternalHook) 19 | 20 | def test_conflicting_args(self): 21 | with pytest.raises(SystemExit): 22 | self.parse_args( 23 | 'scan' 24 | ' --dry-run --always-update-state' 25 | ' -L examples' 26 | ' --output-hook examples/standalone_hook.py' 27 | ) 28 | -------------------------------------------------------------------------------- /tests/main_test.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | import pytest 4 | 5 | from detect_secrets_server.__main__ import main 6 | from detect_secrets_server.storage.base import BaseStorage 7 | from testing.mocks import mock_git_calls 8 | from testing.mocks import SubprocessMock 9 | from testing.util import cache_buster 10 | 11 | 12 | class TestMain(object): 13 | 14 | def teardown(self): 15 | cache_buster() 16 | 17 | def test_no_args(self): 18 | with pytest.raises(SystemExit): 19 | main([]) 20 | 21 | @pytest.mark.parametrize( 22 | 'argument_string,action_executed', 23 | [ 24 | ( 25 | 'add examples/repos.yaml --config ' 26 | '--storage s3 ' 27 | '--s3-config examples/s3.yaml', 28 | 'initialize', 29 | ), 30 | ( 31 | 'add git@github.com:yelp/detect-secrets ' 32 | '--s3-credentials-file examples/aws_credentials.json ' 33 | '--s3-bucket pail ' 34 | '--storage s3', 35 | 'add_repo', 36 | ), 37 | ( 38 | 'scan yelp/detect-secrets', 39 | 'scan_repo', 40 | ), 41 | ] 42 | ) 43 | def test_actions(self, argument_string, action_executed): 44 | """All detailed actions tests are covered in their individual 45 | test cases. This just makes sure they run, for coverage. 46 | """ 47 | with mock.patch( 48 | 'detect_secrets_server.__main__.actions', 49 | autospec=True, 50 | ) as mock_actions, mock.patch( 51 | 'detect_secrets_server.core.usage.s3.should_enable_s3_options', 52 | return_value=True, 53 | ), mock.patch( 54 | 'detect_secrets_server.core.usage.common.storage.should_enable_s3_options', 55 | return_value=True, 56 | ): 57 | mock_actions.initialize.return_value = '' 58 | mock_actions.scan_repo.return_value = 0 59 | 60 | assert main(argument_string.split()) == 0 61 | assert getattr(mock_actions, action_executed).called 62 | 63 | @pytest.mark.parametrize( 64 | 'repo_to_scan', 65 | ( 66 | 'Yelp/detect-secrets', 67 | 'https://github.com/Yelp/detect-secrets', 68 | 'git@github.com:Yelp/detect-secrets', 69 | ), 70 | ) 71 | def test_repositories_added_can_be_scanned(self, mock_rootdir, repo_to_scan): 72 | directory = '{}/repos/{}'.format( 73 | mock_rootdir, 74 | BaseStorage.hash_filename('Yelp/detect-secrets'), 75 | ) 76 | mocked_sha = 'aabbcc' 77 | 78 | # We don't **actually** want to clone the repo per test run. 79 | with mock_git_calls( 80 | SubprocessMock( 81 | expected_input=( 82 | 'git clone https://github.com/Yelp/detect-secrets {} --bare' 83 | ).format( 84 | directory, 85 | ), 86 | ), 87 | # Since there is no prior sha to retrieve 88 | SubprocessMock( 89 | expected_input='git rev-parse HEAD', 90 | mocked_output=mocked_sha, 91 | ) 92 | ): 93 | assert main([ 94 | 'add', 'https://github.com/Yelp/detect-secrets', 95 | '--root-dir', mock_rootdir, 96 | ]) == 0 97 | 98 | with mock_git_calls( 99 | # Getting latest changes 100 | SubprocessMock( 101 | expected_input='git rev-parse --abbrev-ref HEAD', 102 | mocked_output='master', 103 | ), 104 | SubprocessMock( 105 | expected_input='git fetch --quiet origin master:master --force', 106 | ), 107 | # Getting relevant diff 108 | SubprocessMock( 109 | expected_input='git diff {} HEAD --name-only --diff-filter ACM'.format(mocked_sha), 110 | mocked_output='filenameA', 111 | ), 112 | SubprocessMock( 113 | expected_input='git diff {} HEAD -- filenameA'.format(mocked_sha), 114 | mocked_output='', 115 | ), 116 | # Storing latest sha 117 | SubprocessMock( 118 | expected_input='git rev-parse HEAD', 119 | ), 120 | ): 121 | assert main([ 122 | 'scan', repo_to_scan, 123 | '--root-dir', mock_rootdir, 124 | ]) == 0 125 | -------------------------------------------------------------------------------- /tests/repos/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/detect-secrets-server/f2a708fa1a4628a7a93039cae92ef90e8b94d9db/tests/repos/__init__.py -------------------------------------------------------------------------------- /tests/repos/s3_tracked_repo_test.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | from unittest import mock 3 | 4 | import pytest 5 | 6 | from detect_secrets_server.repos.base_tracked_repo import OverrideLevel 7 | from detect_secrets_server.repos.s3_tracked_repo import S3LocalTrackedRepo 8 | from detect_secrets_server.repos.s3_tracked_repo import S3TrackedRepo 9 | from testing.factories import metadata_factory 10 | 11 | 12 | class TestS3TrackedRepo(object): 13 | 14 | def test_load_from_file(self, mock_logic, mock_rootdir): 15 | with mock_logic() as (client, repo): 16 | assert repo.s3_config == mock_s3_config() 17 | 18 | filename = '{}.json'.format( 19 | repo.storage.hash_filename('mocked_repository_name'), 20 | ) 21 | client.download_file.assert_called_with( 22 | Bucket='pail', 23 | Key='prefix/{}'.format(filename), 24 | Filename='{}/tracked/{}'.format( 25 | mock_rootdir, 26 | filename, 27 | ), 28 | ) 29 | 30 | @pytest.mark.parametrize( 31 | 'is_file_uploaded,override_level,should_upload', 32 | [ 33 | # If not uploaded, always upload despite OverrideLevel. 34 | (False, OverrideLevel.NEVER, True,), 35 | (False, OverrideLevel.ASK_USER, True,), 36 | (False, OverrideLevel.ALWAYS, True,), 37 | 38 | # Upload if OverrideLevel != NEVER 39 | (True, OverrideLevel.ALWAYS, True,), 40 | (True, OverrideLevel.NEVER, False,), 41 | ] 42 | ) 43 | def test_save( 44 | self, 45 | mock_logic, 46 | is_file_uploaded, 47 | override_level, 48 | should_upload, 49 | ): 50 | with mock_logic() as (client, repo): 51 | filename = 'prefix/{}.json'.format( 52 | repo.storage.hash_filename('yelp/detect-secrets') 53 | ) 54 | 55 | mock_list_objects_return_value = {} 56 | if is_file_uploaded: 57 | mock_list_objects_return_value = { 58 | 'Contents': [ 59 | { 60 | 'Key': filename, 61 | 'Size': 1, 62 | }, 63 | ], 64 | } 65 | 66 | client.list_objects_v2.return_value = \ 67 | mock_list_objects_return_value 68 | 69 | repo.save(override_level) 70 | 71 | client.list_objects_v2.assert_called_with( 72 | Bucket='pail', 73 | Prefix=filename, 74 | ) 75 | assert client.upload_file.called is should_upload 76 | 77 | 78 | def mock_s3_config(): 79 | return { 80 | 'prefix': 'prefix', 81 | 'bucket': 'pail', 82 | 'credentials_filename': 'examples/aws_credentials.json', 83 | 'access_key': 'access_key', 84 | 'secret_access_key': 'secret_access_key', 85 | } 86 | 87 | 88 | @pytest.fixture 89 | def mock_logic(mocked_boto, mock_rootdir): 90 | @contextmanager 91 | def wrapped(is_local=False): 92 | klass = S3LocalTrackedRepo if is_local else S3TrackedRepo 93 | 94 | with mock.patch( 95 | 'detect_secrets_server.storage.file.open', 96 | mock.mock_open( 97 | read_data=metadata_factory( 98 | 'git@github.com:yelp/detect-secrets', 99 | json=True, 100 | ), 101 | ) 102 | ), mock.patch( 103 | 'detect_secrets_server.storage.file.os.path.isdir', 104 | return_value=True, 105 | ): 106 | yield ( 107 | mocked_boto, 108 | klass.load_from_file( 109 | 'mocked_repository_name', 110 | mock_rootdir, 111 | mock_s3_config(), 112 | ) 113 | ) 114 | 115 | return wrapped 116 | -------------------------------------------------------------------------------- /tests/storage/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/detect-secrets-server/f2a708fa1a4628a7a93039cae92ef90e8b94d9db/tests/storage/__init__.py -------------------------------------------------------------------------------- /tests/storage/base_test.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | from contextlib import contextmanager 3 | from unittest import mock 4 | 5 | import pytest 6 | 7 | from detect_secrets_server.storage.base import BaseStorage 8 | from detect_secrets_server.storage.base import get_filepath_safe 9 | from detect_secrets_server.storage.base import LocalGitRepository 10 | from testing.mocks import mock_git_calls 11 | from testing.mocks import SubprocessMock 12 | 13 | 14 | class TestBaseStorage(object): 15 | 16 | def test_setup_creates_directories(self, mock_rootdir, base_storage): 17 | with assert_directories_created([ 18 | mock_rootdir, 19 | mock_rootdir + '/repos' 20 | ]): 21 | base_storage.setup('git@github.com:yelp/detect-secrets') 22 | 23 | @pytest.mark.parametrize( 24 | 'repo,name', 25 | [ 26 | ( 27 | 'git@github.com:yelp/detect-secrets', 28 | 'yelp/detect-secrets', 29 | ), 30 | 31 | # Ends with .git 32 | ( 33 | 'git@github.com:yelp/detect-secrets.git', 34 | 'yelp/detect-secrets', 35 | ), 36 | 37 | # Not git@ prefixed 38 | ( 39 | 'https://github.com/yelp/detect-secrets', 40 | 'yelp/detect-secrets', 41 | ), 42 | ( 43 | 'https://example.com/yelp/detect-secrets', 44 | 'yelp/detect-secrets', 45 | ), 46 | ( 47 | 'https://example.com/yelp/detect-secrets.git', 48 | 'yelp/detect-secrets', 49 | ), 50 | 51 | # Throw a PORT number in for good measure 52 | ( 53 | 'https://example.com:23456/yelp/detect-secrets', 54 | 'yelp/detect-secrets', 55 | ), 56 | ], 57 | ) 58 | def test_repository_name(self, repo, name, base_storage): 59 | with assert_directories_created(): 60 | assert base_storage.setup(repo).repository_name == name 61 | 62 | def test_baseline_file_does_not_exist(self, base_storage): 63 | """This also conveniently tests our _git function""" 64 | with assert_directories_created(): 65 | repo = base_storage.setup('git@github.com:yelp/detect-secrets') 66 | 67 | with pytest.raises(subprocess.CalledProcessError): 68 | repo.get_baseline_file('does_not_exist') 69 | 70 | def test_clone_repo_if_exists(self, base_storage, mock_rootdir): 71 | with assert_directories_created(): 72 | repo = base_storage.setup('git@github.com:yelp/detect-secrets') 73 | 74 | with mock_git_calls( 75 | self.construct_subprocess_mock_git_clone( 76 | repo, 77 | b'fatal: destination path \'blah\' already exists', 78 | mock_rootdir, 79 | ), 80 | ): 81 | repo.clone() 82 | 83 | def test_clone_repo_something_else_went_wrong(self, mock_rootdir, base_storage): 84 | with assert_directories_created(): 85 | repo = base_storage.setup('git@github.com:yelp/detect-secrets') 86 | 87 | with mock_git_calls( 88 | self.construct_subprocess_mock_git_clone( 89 | repo, 90 | b'Some other error message, not expected', 91 | mock_rootdir, 92 | ) 93 | ), pytest.raises( 94 | subprocess.CalledProcessError 95 | ): 96 | repo.clone() 97 | 98 | @staticmethod 99 | def construct_subprocess_mock_git_clone(repo, mocked_output, mock_rootdir): 100 | return SubprocessMock( 101 | expected_input=( 102 | 'git clone git@github.com:yelp/detect-secrets {} --bare'.format( 103 | '{}/repos/{}'.format( 104 | mock_rootdir, 105 | repo.hash_filename('yelp/detect-secrets'), 106 | ), 107 | ) 108 | ), 109 | mocked_output=mocked_output, 110 | should_throw_exception=True, 111 | ) 112 | 113 | 114 | class TestLocalGitRepository(object): 115 | 116 | @pytest.mark.parametrize( 117 | 'repo,name', 118 | [ 119 | ( 120 | '/file/to/yelp/detect-secrets', 121 | 'yelp/detect-secrets', 122 | ), 123 | ( 124 | '/file/to/yelp/detect-secrets/.git', 125 | 'yelp/detect-secrets', 126 | ), 127 | ] 128 | ) 129 | def test_name(self, repo, name, local_storage): 130 | """OK, I admit this is kinda a lame test case, because technically 131 | everything is mocked out. However, it's needed for coverage, and 132 | it *does* test things (kinda). 133 | """ 134 | with mock_git_calls( 135 | SubprocessMock( 136 | expected_input='git remote get-url origin', 137 | mocked_output='git@github.com:yelp/detect-secrets', 138 | ), 139 | ), assert_directories_created(): 140 | assert local_storage.setup(repo).repository_name == name 141 | 142 | def test_clone(self, local_storage): 143 | # We're asserting that nothing is run in this case. 144 | with mock_git_calls(), assert_directories_created(): 145 | local_storage.setup('git@github.com:yelp/detect-secrets')\ 146 | .clone() 147 | 148 | 149 | class TestGetFilepathSafe(object): 150 | 151 | @pytest.mark.parametrize( 152 | 'prefix,filename,expected', 153 | [ 154 | ('/path/to', 'file', '/path/to/file',), 155 | ('/path/to', '../to/file', '/path/to/file',), 156 | ('/path/to/../to', 'file', '/path/to/file',), 157 | ] 158 | ) 159 | def test_success(self, prefix, filename, expected): 160 | assert get_filepath_safe(prefix, filename) == expected 161 | 162 | def test_failure(self): 163 | with pytest.raises(ValueError): 164 | get_filepath_safe('/path/to', '../../etc/pwd') 165 | 166 | 167 | @contextmanager 168 | def assert_directories_created(directories_created=None): 169 | """ 170 | :type directories_created: list 171 | """ 172 | with mock.patch( 173 | 'detect_secrets_server.storage.base.os.makedirs' 174 | ) as makedirs, mock.patch( 175 | 'detect_secrets_server.storage.base.os.path.isdir', 176 | return_value=False, 177 | ): 178 | yield 179 | 180 | if directories_created: 181 | makedirs.assert_has_calls(map( 182 | lambda x: mock.call(x), 183 | directories_created, 184 | )) 185 | else: 186 | assert makedirs.called 187 | 188 | 189 | @pytest.fixture 190 | def base_storage(mock_rootdir): 191 | return get_mocked_class(BaseStorage)(mock_rootdir) 192 | 193 | 194 | @pytest.fixture 195 | def local_storage(mock_rootdir): 196 | return get_mocked_class(LocalGitRepository)(mock_rootdir) 197 | 198 | 199 | def get_mocked_class(class_object): 200 | class MockStorage(class_object): # pragma: no cover 201 | def get(self, key): 202 | pass 203 | 204 | def put(self, key, value): 205 | pass 206 | 207 | def get_tracked_repositories(self): 208 | return () 209 | 210 | return MockStorage 211 | -------------------------------------------------------------------------------- /tests/storage/file_test.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import pytest 4 | 5 | from .base_test import assert_directories_created 6 | from detect_secrets_server.storage.file import FileStorage 7 | from detect_secrets_server.storage.file import FileStorageWithLocalGit 8 | from testing.mocks import mock_open 9 | 10 | 11 | @pytest.fixture 12 | def file_storage(mock_rootdir): 13 | return FileStorage(mock_rootdir) 14 | 15 | 16 | @pytest.fixture 17 | def local_file_storage(mock_rootdir): 18 | return FileStorageWithLocalGit(mock_rootdir) 19 | 20 | 21 | class TestFileStorage(object): 22 | 23 | def test_setup_creates_directories(self, file_storage, mock_rootdir): 24 | with assert_directories_created([ 25 | mock_rootdir, 26 | mock_rootdir + '/repos', 27 | mock_rootdir + '/tracked', 28 | ]): 29 | file_storage.setup('git@github.com:yelp/detect-secrets') 30 | 31 | def test_get_success(self, file_storage): 32 | with mock_open({'key': 'value'}): 33 | data = file_storage.get('does_not_matter') 34 | 35 | assert data == {'key': 'value'} 36 | 37 | def test_get_failure(self, file_storage): 38 | with pytest.raises(FileNotFoundError): 39 | file_storage.get('file_does_not_exist') 40 | 41 | def test_put_success(self, file_storage): 42 | with mock_open() as m: 43 | file_storage.put('filename', { 44 | 'key': 'value', 45 | }) 46 | 47 | m().write.assert_called_with( 48 | json.dumps( 49 | { 50 | 'key': 'value', 51 | }, 52 | indent=2, 53 | sort_keys=True, 54 | ) 55 | ) 56 | 57 | 58 | class TestFileStorageWithLocalGit(object): 59 | 60 | def test_setup_creates_directories(self, local_file_storage, mock_rootdir): 61 | with assert_directories_created([ 62 | mock_rootdir, 63 | mock_rootdir + '/tracked', 64 | mock_rootdir + '/tracked/local', 65 | ]): 66 | local_file_storage.setup('git@github.com:yelp/detect-secrets') 67 | 68 | def test_get_success(self, local_file_storage, mock_rootdir): 69 | with mock_open() as m: 70 | local_file_storage.get('mock_filename') 71 | 72 | m.assert_called_with( 73 | '{}/tracked/local/mock_filename.json'.format( 74 | mock_rootdir, 75 | ), 76 | ) 77 | -------------------------------------------------------------------------------- /tests/storage/s3_test.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from unittest import mock 3 | 4 | import pytest 5 | 6 | from detect_secrets_server.storage.s3 import S3Storage 7 | from testing.mocks import mock_open 8 | 9 | 10 | def test_get_does_not_download_if_exists(mock_logic): 11 | with mock.patch( 12 | 'detect_secrets_server.storage.s3.os.path.exists', 13 | return_value=True, 14 | ), mock_open(): 15 | mock_logic.get('filename', force_download=False) 16 | 17 | assert not mock_logic.client.download_file.called 18 | 19 | 20 | def test_get_tracked_repositories(mock_logic): 21 | # Honestly, this is just a smoke test, because to get proper testing, 22 | # you would need to hook this up to an s3 bucket. 23 | with mock.patch.object( 24 | mock_logic.client, 25 | 'get_paginator', 26 | ) as mock_paginator, mock.patch.object( 27 | mock_logic, 28 | 'get', 29 | ) as mock_get: 30 | mock_paginator().paginate.return_value = ( 31 | { 32 | 'Contents': [ 33 | { 34 | 'Key': 'prefix/filenameA.json', 35 | 'LastModified': datetime.now(), 36 | 'Size': 500, 37 | }, 38 | ], 39 | }, 40 | { 41 | 'Contents': [ 42 | { 43 | 'Key': 'prefix/filenameB.json', 44 | 'LastModified': datetime.now(), 45 | 'Size': 200, 46 | }, 47 | ], 48 | }, 49 | 50 | ) 51 | list(mock_logic.get_tracked_repositories()) 52 | 53 | mock_get.assert_has_calls([ 54 | mock.call('filenameA', force_download=False), 55 | mock.call('filenameB', force_download=False), 56 | ]) 57 | 58 | 59 | @pytest.fixture 60 | def mock_logic(mocked_boto, mock_rootdir): 61 | yield S3Storage( 62 | mock_rootdir, 63 | { 64 | 'access_key': 'will_be_mocked', 65 | 'secret_access_key': 'will_be_mocked', 66 | 'bucket': 'pail', 67 | 'prefix': 'prefix', 68 | }, 69 | ) 70 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | project = detect_secrets_server 3 | # These should match the travis env list 4 | envlist = py{35,36,37,py3} 5 | skip_missing_interpreters = true 6 | tox_pip_extensions_ext_pip_custom_platform = true 7 | 8 | [testenv] 9 | passenv = SSH_AUTH_SOCK 10 | deps = 11 | -rrequirements-dev.txt 12 | --no-cache-dir 13 | whitelist_externals = coverage 14 | commands = 15 | coverage erase 16 | coverage run -m pytest {posargs:tests} 17 | coverage report --show-missing --include=tests/* --fail-under 100 18 | coverage report --show-missing --include=detect_secrets_server/* --fail-under 93 19 | pre-commit run --all-files 20 | 21 | [testenv:py35] 22 | # this overrides the default testenv and removes pre-commit, because 23 | # pre-commit and its hooks installs non-py35-friendly packages 24 | commands = 25 | coverage erase 26 | coverage run -m pytest {posargs:tests} 27 | coverage report --show-missing --include=tests/* --fail-under 100 28 | coverage report --show-missing --include=detect_secrets_server/* --fail-under 93 29 | 30 | [testenv:pypy3] 31 | # this overrides the default testenv and removes pre-commit, because 32 | # pre-commit and its hooks installs non-py35-friendly packages 33 | commands = 34 | coverage erase 35 | coverage run -m pytest {posargs:tests} 36 | coverage report --show-missing --include=tests/* --fail-under 100 37 | coverage report --show-missing --include=detect_secrets_server/* --fail-under 93 38 | 39 | [testenv:venv] 40 | basepython = /usr/bin/python3.6 41 | envdir = venv 42 | commands = 43 | pre-commit install -f --install-hooks 44 | 45 | [testenv:pre-commit] 46 | deps = pre-commit>=0.16.3 47 | commands = pre-commit {posargs} 48 | 49 | [pep8] 50 | ignore = E501 51 | --------------------------------------------------------------------------------