├── .github └── workflows │ ├── build-and-test.yml │ ├── codeql-analysis.yml │ └── release.yml ├── .gitignore ├── .pre-commit-config.yaml ├── AUTHORS ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── OSSMETADATA ├── README.md ├── docker-compose.yml ├── docs └── images │ └── Repokid.png ├── repokid ├── __init__.py ├── cli │ ├── __init__.py │ ├── dispatcher_cli.py │ └── repokid_cli.py ├── commands │ ├── __init__.py │ ├── repo.py │ ├── role.py │ ├── role_cache.py │ └── schedule.py ├── datasource │ ├── __init__.py │ ├── access_advisor.py │ ├── iam.py │ └── plugin.py ├── dispatcher │ ├── __init__.py │ └── types.py ├── exceptions.py ├── filters │ ├── __init__.py │ ├── age │ │ └── __init__.py │ ├── blocklist │ │ └── __init__.py │ ├── exclusive │ │ └── __init__.py │ ├── lambda │ │ └── __init__.py │ ├── optout │ │ └── __init__.py │ └── utils.py ├── hooks │ ├── __init__.py │ └── loggers │ │ └── __init__.py ├── lib │ └── __init__.py ├── plugin.py ├── py.typed ├── role.py ├── types.py └── utils │ ├── __init__.py │ ├── dynamo.py │ ├── iam.py │ ├── logging.py │ ├── permissions.py │ └── roledata.py ├── requirements-test.in ├── requirements-test.txt ├── requirements.in ├── requirements.txt ├── setup.cfg ├── setup.py ├── test-requirements.txt └── tests ├── __init__.py ├── artifacts ├── __init__.py └── hook │ └── __init__.py ├── conftest.py ├── datasource ├── __init__.py ├── conftest.py ├── test_access_advisor.py └── test_iam.py ├── filters ├── __init__.py └── test_age.py ├── test_commands.py ├── test_dispatcher_cli.py ├── test_hooks.py ├── test_permissions.py.bak ├── test_role.py ├── test_roledata.py └── vars.py /.github/workflows/build-and-test.yml: -------------------------------------------------------------------------------- 1 | name: Build and Test 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - master 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | python-version: [3.7, 3.8, 3.9] 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v2 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install --upgrade pip 26 | pip install -r requirements.txt -r requirements-test.txt 27 | - name: Run pre-commit 28 | run: | 29 | pre-commit run -a 30 | coverage run --source repokid -m py.test 31 | bandit -r . -ll -ii -x repokid/tests/ 32 | - name: Test with pytest 33 | run: | 34 | coverage run --source repokid -m py.test 35 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | name: "CodeQL" 7 | 8 | on: 9 | push: 10 | branches: [master] 11 | pull_request: 12 | # The branches below must be a subset of the branches above 13 | branches: [master] 14 | schedule: 15 | - cron: '0 10 * * 2' 16 | 17 | jobs: 18 | analyze: 19 | name: Analyze 20 | runs-on: ubuntu-latest 21 | 22 | strategy: 23 | fail-fast: false 24 | matrix: 25 | # Override automatic language detection by changing the below list 26 | # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] 27 | language: ['python'] 28 | # Learn more... 29 | # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection 30 | 31 | steps: 32 | - name: Checkout repository 33 | uses: actions/checkout@v2 34 | with: 35 | # We must fetch at least the immediate parents so that if this is 36 | # a pull request then we can checkout the head. 37 | fetch-depth: 2 38 | 39 | # If this run was triggered by a pull request event, then checkout 40 | # the head of the pull request instead of the merge commit. 41 | - run: git checkout HEAD^2 42 | if: ${{ github.event_name == 'pull_request' }} 43 | 44 | # Initializes the CodeQL tools for scanning. 45 | - name: Initialize CodeQL 46 | uses: github/codeql-action/init@v1 47 | with: 48 | languages: ${{ matrix.language }} 49 | # If you wish to specify custom queries, you can do so here or in a config file. 50 | # By default, queries listed here will override any specified in a config file. 51 | # Prefix the list here with "+" to use these queries and those in the config file. 52 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 53 | 54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 55 | # If this step fails, then you should remove it and run the build manually (see below) 56 | - name: Autobuild 57 | uses: github/codeql-action/autobuild@v1 58 | 59 | # ℹ️ Command-line programs to run using the OS shell. 60 | # 📚 https://git.io/JvXDl 61 | 62 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 63 | # and modify them (or add more) to build your code if your project 64 | # uses a compiled language 65 | 66 | #- run: | 67 | # make bootstrap 68 | # make release 69 | 70 | - name: Perform CodeQL Analysis 71 | uses: github/codeql-action/analyze@v1 72 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | # Sequence of patterns matched against refs/tags 4 | tags: 5 | - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 6 | 7 | name: Release & Publish 8 | 9 | jobs: 10 | release: 11 | name: Create Release 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout code 15 | uses: actions/checkout@v2 16 | - name: Create Release 17 | id: create_release 18 | uses: actions/create-release@v1 19 | env: 20 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 21 | with: 22 | tag_name: ${{ github.ref }} 23 | release_name: ${{ github.ref }} 24 | draft: false 25 | prerelease: false 26 | publish: 27 | runs-on: ubuntu-latest 28 | steps: 29 | - uses: actions/checkout@v2 30 | - name: Set up Python 31 | uses: actions/setup-python@v2 32 | with: 33 | python-version: '3.x' 34 | - name: Install dependencies 35 | run: | 36 | python -m pip install --upgrade pip 37 | pip install setuptools wheel twine 38 | - name: Build and publish 39 | env: 40 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 41 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 42 | run: | 43 | python setup.py sdist bdist_wheel 44 | twine upload dist/* 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .run/ 3 | venv/ 4 | *.pyc 5 | *.json 6 | *.log 7 | *.csv 8 | build 9 | dist 10 | .coverage 11 | .cache 12 | .DS_Store 13 | .DS_Store? 14 | *.egg-info/ 15 | .newt.yml 16 | tox.ini 17 | test-reports/* 18 | config.json 19 | .eggs/ 20 | dynamodb-data/ 21 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_language_version: 2 | python: python3 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v3.4.0 # Use the ref you want to point at 6 | hooks: 7 | - id: trailing-whitespace 8 | - id: check-ast 9 | - id: check-case-conflict 10 | - id: check-yaml 11 | - id: pretty-format-json 12 | args: ["--autofix"] 13 | 14 | - repo: https://github.com/pre-commit/mirrors-autopep8 15 | rev: v1.5.4 16 | hooks: 17 | - id: autopep8 18 | 19 | - repo: https://gitlab.com/pycqa/flake8 20 | rev: 3.8.4 21 | hooks: 22 | - id: flake8 23 | args: ["--exclude", "venv/,.tox/,.eggs/"] 24 | 25 | - repo: https://github.com/pre-commit/mirrors-isort 26 | rev: 'v5.7.0' 27 | hooks: 28 | - id: isort 29 | args: ["--sl", "--profile", "black"] 30 | 31 | - repo: https://github.com/ambv/black 32 | rev: 20.8b1 33 | hooks: 34 | - id: black 35 | 36 | - repo: https://github.com/pre-commit/mirrors-mypy 37 | rev: v0.790 38 | hooks: 39 | - id: mypy 40 | args: ["--strict"] 41 | additional_dependencies: 42 | - pydantic 43 | 44 | - repo: local 45 | hooks: 46 | - id: python-bandit-vulnerability-check 47 | name: bandit 48 | entry: bandit 49 | args: ['--ini', 'setup.cfg', '-x', 'repokid/tests/', '-r', 'repokid'] 50 | language: system 51 | pass_filenames: false 52 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | - Repokid Maintainers 2 | - Patrick Kelley 3 | - Travis McPeak 4 | - Patrick Sanders 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | Thank you for considering contributing to the Repokid project! Get started by reading `development setup` then look for something in our [open issues](https://github.com/Netflix/repokid/issues) that matches your interest and skillset (or file an issue for something if it doesn't exist yet). Please make sure to follow the guidelines outlined in `submitting changes` and consider joining our [Gitter](https://gitter.im/netflix-repokid/Lobby). Happy hacking! 4 | 5 | Development Setup 6 | ----------------- 7 | Please follow the instructions in Repokid's [getting started](https://github.com/Netflix/repokid/#getting-started) section. If you encounter any problems with the instructions please file an issue. 8 | 9 | Issue Tags 10 | ---------- 11 | First time contributors should consider tackling an issue marked with `difficulty:newcomer` first. These issues are relatively easy and a good way to get started. 12 | 13 | We also have some issues tagged with `help wanted`. These issues are something we'd love help with and are a great way to get significantly involved with the project. 14 | 15 | 16 | Submitting Changes 17 | ------------------ 18 | If you're adding something to one of our modules that are already covered by unit tests please add tests that exercise your change. We're working on adding unit tests for other modules and looking for help to get to 100% coverage. 19 | 20 | Please also verify that the changes you've made work as expected locally. 21 | 22 | Pushing New Versions to Pypi 23 | ---------------------------- 24 | For the majority of contributors, this will not be a huge factor, however, for those deploying new versions, the instructions are as follows: 25 | 1. Ensure you have `twine` installed, this is part of the requirements file, so if you've installed via the recommended paths, you should be fine. 26 | 2. Edit the version number present in the `repokid/init.py`: 27 | 28 | $ vim repokid/__init__.py 29 | 30 | 3. Create a package for your newest version: 31 | 32 | $ python setup.py sdist 33 | 34 | 4. Upload to Pypi using Twine: 35 | 36 | $ twine upload dist/* 37 | 38 | #### Problems that can be found when creating new versions. 39 | Pypi has a "known problem" with versions of the same title, for example, reuploading a failed attempt, or trying to patch over the top of a dist that has a file missing. This will result in a failure, and will require you to bump numbers again. As they are immutable, effectively. So version numbers can be used fairly liberally. 40 | 41 | #### Context around using Twine: 42 | Please refer to the "Why Should I Use This" section: 43 | > https://github.com/pypa/twine 44 | 45 | Chat with our Developers 46 | ------------------------ 47 | Our developers are in [Gitter](https://gitter.im/netflix-repokid/Lobby). Whether you're seeing a bug, thinking about a new feature, or wondering about a design decision drop by and ask! 48 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7 2 | 3 | WORKDIR /usr/src/app 4 | COPY . . 5 | 6 | RUN pip install bandit coveralls && \ 7 | pip install . && \ 8 | pip install -r requirements-test.txt && \ 9 | python setup.py develop && \ 10 | repokid config config.json # Generate example config 11 | 12 | ENTRYPOINT ["repokid"] 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2020 Netflix, Inc. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | include requirements.in 3 | include repokid/py.typed 4 | 5 | include LICENSE README.md 6 | 7 | global-exclude *.py[cod] -------------------------------------------------------------------------------- /OSSMETADATA: -------------------------------------------------------------------------------- 1 | osslifecycle=active 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Repokid 2 | ======= 3 | [![NetflixOSS Lifecycle](https://img.shields.io/osslifecycle/Netflix/osstracker.svg)]() 4 | [![Build Status](https://travis-ci.com/Netflix/repokid.svg?branch=master)](https://travis-ci.com/Netflix/repokid) 5 | [![PyPI version](https://badge.fury.io/py/repokid.svg)](https://badge.fury.io/py/repokid) 6 | [![Coverage Status](https://coveralls.io/repos/github/Netflix/repokid/badge.svg?branch=master)](https://coveralls.io/github/Netflix/repokid?branch=master) 7 | [![Discord chat](https://img.shields.io/discord/754080763070382130?logo=discord)](https://discord.gg/9kwMWa6) 8 | 9 | Repokid Logo 10 | 11 | Repokid uses Access Advisor provided by [Aardvark](https://github.com/Netflix-Skunkworks/aardvark) 12 | to remove permissions granting access to unused services from the inline policies of IAM roles in 13 | an AWS account. 14 | 15 | ## Getting Started 16 | 17 | ### Install 18 | 19 | ```bash 20 | mkvirtualenv repokid 21 | git clone git@github.com:Netflix/repokid.git 22 | cd repokid 23 | pip install -e . 24 | repokid config config.json 25 | ``` 26 | 27 | #### DynamoDB 28 | 29 | You will need a [DynamoDB](https://aws.amazon.com/dynamodb/) table called `repokid_roles` (specify account and endpoint in `dynamo_db` in config file). 30 | 31 | The table should have the following properties: 32 | - `RoleId` (string) as a primary partition key, no primary sort key 33 | - A global secondary index named `Account` with a primary partition key of `Account` and `RoleId` and `Account` as projected attributes 34 | - A global secondary index named `RoleName` with a primary partition key of `RoleName` and `RoleId` and `RoleName` as projected attributes 35 | 36 | For development, you can run dynamo [locally](http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.html). 37 | 38 | To run locally: 39 | 40 | ```bash 41 | docker-compose up 42 | ``` 43 | 44 | The endpoint for DynamoDB will be `http://localhost:8000`. A DynamoDB admin panel can be found at `http://localhost:8001`. 45 | 46 | If you run the development version the table and index will be created for you automatically. 47 | 48 | #### IAM Permissions 49 | 50 | Repokid needs an IAM Role in each account that will be queried. Additionally, Repokid needs to be launched with a role or user which can `sts:AssumeRole` into the different account roles. 51 | 52 | RepokidInstanceProfile: 53 | - Only create one. 54 | - Needs the ability to call `sts:AssumeRole` into all of the RepokidRoles. 55 | - DynamoDB permissions for the `repokid_roles` table and all indexes (specified in `assume_role` subsection of `dynamo_db` in config) and the ability to run `dynamodb:ListTables` 56 | 57 | RepokidRole: 58 | - Must exist in every account to be managed by repokid. 59 | - Must have a trust policy allowing `RepokidInstanceProfile`. 60 | - Name must be specified in `connection_iam` in config file. 61 | - Has these permissions: 62 | ```json 63 | { 64 | "Version": "2012-10-17", 65 | "Statement": [ 66 | { 67 | "Action": [ 68 | "iam:DeleteInstanceProfile", 69 | "iam:DeleteRole", 70 | "iam:DeleteRolePolicy", 71 | "iam:GetAccountAuthorizationDetails", 72 | "iam:GetInstanceProfile", 73 | "iam:GetRole", 74 | "iam:GetRolePolicy", 75 | "iam:ListInstanceProfiles", 76 | "iam:ListInstanceProfilesForRole", 77 | "iam:ListRolePolicies", 78 | "iam:PutRolePolicy", 79 | "iam:UpdateRoleDescription" 80 | ], 81 | "Effect": "Allow", 82 | "Resource": "*" 83 | } 84 | ] 85 | } 86 | ``` 87 | 88 | So if you are monitoring `n` accounts, you will always need `n+1` roles. (`n` RepokidRoles and `1` RepokidInstanceProfile). 89 | 90 | #### Editing config.json 91 | 92 | Running `repokid config config.json` creates a file that you will need to edit. Find and update these fields: 93 | - `dynamodb`: If using dynamo locally, set the endpoint to `http://localhost:8010`. If using AWS hosted dynamo, set the `region`, `assume_role`, and `account_number`. 94 | - `aardvark_api_location`: The location to your Aardvark REST API. Something like `https://aardvark.yourcompany.net/api/1/advisors` 95 | - `connection_iam`: Set `assume_role` to `RepokidRole`, or whatever you have called it. 96 | 97 | ## Optional Config 98 | Repokid uses filters to decide which roles are candidates to be repoed. Filters may be configured to suit your 99 | environment as described below. 100 | 101 | ### Blocklist Filter 102 | Roles may be excluded by adding them to the Blocklist filter. One common reason to exclude a role is if 103 | the corresponding workload performs occasional actions that may not have been observed but are known to be 104 | required. There are two ways to exclude a role: 105 | 106 | - Exclude role name for all accounts: add it to a list in the config `filter_config.BlocklistFilter.all` 107 | - Exclude role name for specific account: add it to a list in the config `filter_config.BlocklistFilter.` 108 | 109 | Blocklists can also be maintained in an S3 blocklist file. They should be in the following form: 110 | ```json 111 | { 112 | "arns": ["arn1", "arn2"], 113 | "names": {"role_name_1": ["all", "account_number_1"], "role_name_2": ["account_number_2", "account_number_3"]} 114 | } 115 | ``` 116 | 117 | ### Exclusive Filter 118 | If you prefer to repo only certain roles you can use the Exclusive Filter. Maybe you want to consider only roles used in production or by certain teams. 119 | To select roles for repo-ing you may list their names in the configuration files. Shell style glob patterns are also supported. 120 | Role selection can be specified per individual account or globally. 121 | To activate this filter put `"repokid.filters.exclusive:ExclusiveFilter"`in the section `active_filters` of the config file. 122 | To configure it you can start with the autogenerated config file, which has an example config in the `"filter_config"` section: 123 | 124 | ``` 125 | "ExclusiveFilter": { 126 | "all": [ 127 | "" 128 | ], 129 | "": [ 130 | "" 131 | ] 132 | } 133 | ``` 134 | 135 | ### Age Filter 136 | By default the age filter excludes roles that are younger than 90 days. To change this edit the config setting: 137 | `filter_config.AgeFilter.minimum_age`. 138 | 139 | ### Active Filters 140 | 141 | New filters can be created to support internal logic. At Netflix we have several that are specific to our 142 | use cases. To make them active make sure they are in the Python path and add them in the config to the list in 143 | the section `active_filters`. 144 | 145 | ## Extending Repokid 146 | 147 | ### Hooks 148 | 149 | Repokid is extensible via hooks that are called before, during, and after various operations as listed below. 150 | 151 | | Hook name | Context | 152 | |-----------|---------| 153 | | `AFTER_REPO` | role, errors | 154 | | `AFTER_REPO_ROLES` | roles, errors | 155 | | `BEFORE_REPO_ROLES` | account_number, roles | 156 | | `AFTER_SCHEDULE_REPO` | roles | 157 | | `DURING_REPOABLE_CALCULATION` | role_id, arn, account_number, role_name, potentially_repoable_permissions, minimum_age | 158 | | `DURING_REPOABLE_CALCULATION_BATCH` | role_batch, potentially_repoable_permissions, minimum_age | 159 | 160 | Hooks must adhere to the following interface: 161 | 162 | ```python 163 | from repokid.hooks import implements_hook 164 | from repokid.types import RepokidHookInput, RepokidHookOutput 165 | 166 | @implements_hook("TARGET_HOOK_NAME", 1) 167 | def custom_hook(input_dict: RepokidHookInput) -> RepokidHookOutput: 168 | """Hook functions are called with a dict containing the keys listed above based on the target hook. 169 | Any mutations made to the input and returned in the output will be passed on to subsequent hook funtions. 170 | """ 171 | ... 172 | ``` 173 | 174 | Examples of hook implementations can be found in [`repokid.hooks.loggers`](repokid/hooks/loggers/__init__.py). 175 | 176 | ### Filters 177 | 178 | Custom filters can be written to exclude roles from being repoed. Filters must adhere to the following interface: 179 | 180 | ```python 181 | from repokid.filters import Filter 182 | from repokid.types import RepokidFilterConfig 183 | from repokid.role import RoleList 184 | 185 | 186 | class CustomFilterName(Filter): 187 | def __init__(self, config: RepokidFilterConfig = None) -> None: 188 | """Filters are initialized with a dict containing the contents of `filter_config.FilterName` 189 | from the config file. This example would be initialized with `filter_config.CustomFilterName`. 190 | The configuration can be accessed via `self.config` 191 | 192 | If you don't need any custom initialization logic, you can leave this function out of your 193 | filter class. 194 | """ 195 | super().__init__(config=config) 196 | # custom initialization logic goes here 197 | ... 198 | 199 | def apply(self, input_list: RoleList) -> RoleList: 200 | """Determine roles to be excluded and return them as a RoleList""" 201 | ... 202 | ``` 203 | 204 | A simple filter implementation can be found in [`repokid.filters.age`](repokid/filters/age/__init__.py). A more complex example is in [`repokid.blocklist.age`](repokid/filters/blocklist/__init__.py). 205 | 206 | ## How to Use 207 | 208 | Once Repokid is configured, use it as follows: 209 | 210 | ### Standard flow 211 | - Update role cache: `repokid update_role_cache ` 212 | - Display role cache: `repokid display_role_cache ` 213 | - Display information about a specific role: `repokid display_role ` 214 | - Repo a specific role: `repokid repo_role ` 215 | - Repo all roles in an account: `repokid repo_all_roles -c` 216 | 217 | ### Scheduling 218 | Rather than running a repo right now you can schedule one (`schedule_repo` command). The duration between scheduling and eligibility is configurable, but by default roles can be repoed 7 days after scheduling. You can then run a command `repo_scheduled_roles` to only repo roles which have already been scheduled. 219 | 220 | ### Targeting a specific permission 221 | 222 | Say that you find a given permission especially dangerous in your environment. Here I'll use `s3:PutObjectACL` as an example. You can use Repokid to find all roles that have this permission (even those hidden in a wildcard), and then remove just that single permission. 223 | 224 | Find & Remove: 225 | - Ensure the role cache is updated before beginning. 226 | - Find roles with a given permission: `repokid find_roles_with_permissions ... [--output=ROLE_FILE]` 227 | - Remove permission from roles: `repokid remove_permissions_from_roles --role-file=ROLE_FILE ... [-c]` 228 | 229 | Example: 230 | ``` 231 | $ repokid find_roles_with_permissions "s3:putobjectacl" "sts:assumerole" --output=myroles.json 232 | ... 233 | $ repokid remove_permissions_from_roles --role-file=myroles.json "s3:putobjectacl" "sts:assumerole" -c 234 | ``` 235 | 236 | ### Rolling back 237 | Repokid stores a copy of each version of inline policies it knows about. These are added when 238 | a different version of a policy is found during `update_role_cache` and any time a repo action 239 | occurs. To restore a previous version run: 240 | 241 | See all versions of roles: `repokid rollback_role ` 242 | Restore a specific version: `repokid rollback_role --selection= -c` 243 | 244 | ### Stats 245 | Repokid keeps counts of the total permissions for each role. Stats are added any time an `update_role_cache` or 246 | `repo_role` action occur. To output all stats to a CSV file run: `repokid repo_stats `. An optional account number can be specified to output stats for a specific account only. 247 | 248 | ### Library 249 | 250 | > New in `v0.14.2` 251 | 252 | Repokid can be called as a library using the `repokid.lib` module: 253 | 254 | ```python 255 | from repokid.lib import display_role, repo_role, update_role_cache 256 | 257 | account_number = "123456789012" 258 | 259 | display_role(account_number, "superCoolRoleName") 260 | update_role_cache(account_number) 261 | repo_role(account_number, "superCoolRoleName", commit=True) 262 | ``` 263 | 264 | ## Dispatcher ## 265 | Repokid Dispatcher is designed to listen for messages on a queue and perform actions. So far the actions are: 266 | - List repoable services from a role 267 | - Set or remove an opt-out 268 | - List and perform rollbacks for a role 269 | 270 | Repokid will respond on a configurable SNS topic with information about any success or failures. The Dispatcher 271 | component exists to help with operationalization of the repo lifecycle across your organization. You may choose 272 | to expose the queue directly to developers, but more likely this should be guarded because rolling back can be 273 | a destructive action if not done carefully. 274 | 275 | ## Development 276 | 277 | ### Releasing 278 | 279 | Versioning is handled by [setupmeta](https://github.com/zsimic/setupmeta). To create a new release: 280 | 281 | ```bash 282 | python setup.py version --bump patch --push 283 | 284 | # Inspect output and make sure it's what you expect 285 | # If all is well, commit and push the new tag: 286 | python setup.py version --bump patch --push --commit 287 | ``` 288 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | dynamodb: 4 | container_name: dynamodb 5 | image: amazon/dynamodb-local:latest 6 | entrypoint: java 7 | networks: 8 | - dynamo 9 | command: "-jar DynamoDBLocal.jar -sharedDb -dbPath /data" 10 | restart: always 11 | volumes: 12 | - ./dynamodb-data:/data 13 | ports: 14 | - "8000:8000" 15 | dynamodb_admin: 16 | container_name: dynamodb-admin 17 | image: aaronshaf/dynamodb-admin:latest 18 | networks: 19 | - dynamo 20 | environment: 21 | - DYNAMO_ENDPOINT=http://dynamodb:8000 22 | ports: 23 | - "8001:8001" 24 | 25 | networks: 26 | dynamo: 27 | -------------------------------------------------------------------------------- /docs/images/Repokid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/repokid/376aa82ed31fe66ac4b1aecc3c22b0fc4fcfc0ea/docs/images/Repokid.png -------------------------------------------------------------------------------- /repokid/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import collections 15 | import inspect 16 | import json 17 | import logging 18 | import logging.config 19 | import os 20 | from typing import DefaultDict 21 | from typing import List 22 | from typing import Tuple 23 | 24 | import import_string 25 | 26 | from repokid.types import RepokidConfig 27 | from repokid.types import RepokidHook 28 | from repokid.types import RepokidHooks 29 | 30 | __version__ = "0.19.5" 31 | 32 | 33 | def init_config() -> RepokidConfig: 34 | """ 35 | Try to find config by searching for it in a few paths, load it, and store it in the global CONFIG 36 | 37 | Args: 38 | account_number (string): The current account number Repokid is being run against. This is needed to provide 39 | the right config to the blocklist filter. 40 | 41 | Returns: 42 | None 43 | """ 44 | load_config_paths = [ 45 | os.environ.get("REPOKID_CONFIG_LOCATION"), 46 | os.path.join(os.getcwd(), "config.json"), 47 | "/etc/repokid/config.json", 48 | "/apps/repokid/config.json", 49 | ] 50 | config: RepokidConfig = {} 51 | for path in load_config_paths: 52 | if not path: 53 | continue 54 | try: 55 | with open(path, "r") as f: 56 | print("Loaded config from {}".format(path)) 57 | config = json.load(f) 58 | return config 59 | 60 | except IOError: 61 | print("Unable to load config from {}, trying next location".format(path)) 62 | 63 | print("Config not found in any path, using defaults") 64 | return config 65 | 66 | 67 | def init_logging() -> logging.Logger: 68 | """ 69 | Initialize global LOGGER object with config defined in the global CONFIG object 70 | 71 | Args: 72 | None 73 | 74 | Returns: 75 | None 76 | """ 77 | 78 | if CONFIG: 79 | logging.config.dictConfig(CONFIG["logging"]) 80 | 81 | # these loggers are very noisy 82 | suppressed_loggers = [ 83 | "botocore.vendored.requests.packages.urllib3.connectionpool", 84 | "urllib3", 85 | "botocore.credentials", 86 | ] 87 | 88 | for logger in suppressed_loggers: 89 | logging.getLogger(logger).setLevel(logging.ERROR) 90 | 91 | log = logging.getLogger(__name__) 92 | log.propagate = False 93 | return log 94 | 95 | 96 | def get_hooks(hooks_list: List[str]) -> RepokidHooks: 97 | """ 98 | Output should be a dictionary with keys as the names of hooks and values as a list of functions (in order) to call 99 | 100 | Args: 101 | hooks_list: A list of paths to load hooks from 102 | 103 | Returns: 104 | dict: Keys are hooks by name (AFTER_SCHEDULE_REPO) and values are a list of functions to execute 105 | """ 106 | # hooks is a temporary dictionary of priority/RepokidHook tuples 107 | hooks: DefaultDict[str, List[Tuple[int, RepokidHook]]] = collections.defaultdict( 108 | list 109 | ) 110 | 111 | for hook in hooks_list: 112 | module = import_string(hook) 113 | # get members retrieves all the functions from a given module 114 | all_funcs = inspect.getmembers(module, inspect.isfunction) 115 | # first argument is the function name (which we don't need) 116 | for (_, func) in all_funcs: 117 | # we only look at functions that have been decorated with _implements_hook 118 | if hasattr(func, "_implements_hook"): 119 | h: Tuple[int, RepokidHook] = (func._implements_hook["priority"], func) 120 | # append to the dictionary in whatever order we see them, we'll sort later. Dictionary value should be 121 | # a list of tuples (priority, function) 122 | hooks[func._implements_hook["hook_name"]].append(h) 123 | 124 | # sort by priority 125 | for k in hooks.keys(): 126 | hooks[k] = sorted(hooks[k], key=lambda priority: int(priority[0])) 127 | # get rid of the priority - we don't need it anymore 128 | # save to a new dict that conforms to the RepokidHooks spec 129 | final_hooks: RepokidHooks = RepokidHooks() 130 | for k in hooks.keys(): 131 | final_hooks[k] = [func_tuple[1] for func_tuple in hooks[k]] 132 | 133 | return final_hooks 134 | 135 | 136 | CONFIG: RepokidConfig = init_config() 137 | LOGGER: logging.Logger = init_logging() 138 | -------------------------------------------------------------------------------- /repokid/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/repokid/376aa82ed31fe66ac4b1aecc3c22b0fc4fcfc0ea/repokid/cli/__init__.py -------------------------------------------------------------------------------- /repokid/cli/dispatcher_cli.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import inspect 3 | import json 4 | from typing import Any 5 | from typing import Dict 6 | from typing import Generator 7 | from typing import Optional 8 | 9 | from cloudaux.aws.sts import boto3_cached_conn 10 | from mypy_boto3_sns.client import SNSClient 11 | from mypy_boto3_sqs.client import SQSClient 12 | from mypy_boto3_sqs.type_defs import ReceiveMessageResultTypeDef 13 | 14 | import repokid.dispatcher 15 | from repokid import CONFIG 16 | from repokid.dispatcher.types import Message 17 | 18 | 19 | def get_failure_message(channel: str, message: str) -> Dict[str, Any]: 20 | return {"channel": channel, "message": message, "title": "Repokid Failure"} 21 | 22 | 23 | def delete_message(receipt_handle: str, conn_details: Dict[str, Any]) -> None: 24 | client: SQSClient = boto3_cached_conn("sqs", **conn_details) 25 | client.delete_message( 26 | QueueUrl=CONFIG["dispatcher"]["to_rr_queue"], ReceiptHandle=receipt_handle 27 | ) 28 | 29 | 30 | def receive_message(conn_details: Dict[str, Any]) -> ReceiveMessageResultTypeDef: 31 | client: SQSClient = boto3_cached_conn("sqs", **conn_details) 32 | return client.receive_message( 33 | QueueUrl=CONFIG["dispatcher"]["to_rr_queue"], 34 | MaxNumberOfMessages=1, 35 | WaitTimeSeconds=10, 36 | ) 37 | 38 | 39 | def send_message(message_dict: Dict[str, Any], conn_details: Dict[str, Any]) -> None: 40 | client: SNSClient = boto3_cached_conn("sns", **conn_details) 41 | client.publish( 42 | TopicArn=CONFIG["dispatcher"]["from_rr_sns"], Message=json.dumps(message_dict) 43 | ) 44 | 45 | 46 | @contextlib.contextmanager 47 | def message_context( 48 | message_object: ReceiveMessageResultTypeDef, connection: Dict[str, Any] 49 | ) -> Generator[Optional[str], Dict[str, Any], None]: 50 | try: 51 | receipt_handle = message_object["Messages"][0]["ReceiptHandle"] 52 | yield json.loads(message_object["Messages"][0]["Body"]) 53 | except KeyError: 54 | # we might not actually have a message 55 | yield None 56 | else: 57 | if receipt_handle: 58 | delete_message(receipt_handle, connection) 59 | 60 | 61 | all_funcs = inspect.getmembers(repokid.dispatcher, inspect.isfunction) 62 | RESPONDER_FUNCTIONS = { 63 | func[1]._implements_command: func[1] 64 | for func in all_funcs 65 | if hasattr(func[1], "_implements_command") 66 | } 67 | 68 | 69 | def main() -> None: 70 | conn_details = { 71 | "assume_role": CONFIG["dispatcher"].get("assume_role", None), 72 | "session_name": CONFIG["dispatcher"].get("session_name", "Repokid"), 73 | "region": CONFIG["dispatcher"].get("region", "us-west-2"), 74 | } 75 | 76 | while True: 77 | message = receive_message(conn_details) 78 | if not message or "Messages" not in message: 79 | continue 80 | 81 | with message_context(message, conn_details) as msg: 82 | if not msg: 83 | continue 84 | 85 | parsed_msg = Message.parse_obj(msg) 86 | 87 | if parsed_msg.errors: 88 | failure_message = get_failure_message( 89 | channel=parsed_msg.respond_channel, 90 | message="Malformed message: {}".format(parsed_msg.errors), 91 | ) 92 | send_message(failure_message, conn_details) 93 | continue 94 | 95 | try: 96 | return_val = RESPONDER_FUNCTIONS[parsed_msg.command](parsed_msg) 97 | except KeyError: 98 | failure_message = get_failure_message( 99 | channel=parsed_msg.respond_channel, 100 | message="Unknown function {}".format(parsed_msg.command), 101 | ) 102 | send_message(failure_message, conn_details) 103 | continue 104 | 105 | send_message( 106 | { 107 | "message": "@{} {}".format( 108 | parsed_msg.respond_user, return_val.return_message 109 | ), 110 | "channel": parsed_msg.respond_channel, 111 | "title": "Repokid Success" 112 | if return_val.successful 113 | else "Repokid Failure", 114 | }, 115 | conn_details, 116 | ) 117 | 118 | 119 | if __name__ == "__main__": 120 | main() 121 | -------------------------------------------------------------------------------- /repokid/cli/repokid_cli.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import json 16 | import logging 17 | from typing import List 18 | from typing import Optional 19 | 20 | from click import Command 21 | from click import Context 22 | from click import Group 23 | from click import argument 24 | from click import group 25 | from click import option 26 | from click import pass_context 27 | 28 | from repokid import CONFIG 29 | from repokid import get_hooks 30 | from repokid.commands.repo import _repo_all_roles 31 | from repokid.commands.repo import _repo_role 32 | from repokid.commands.repo import _repo_stats 33 | from repokid.commands.repo import _rollback_role 34 | from repokid.commands.role import _display_role 35 | from repokid.commands.role import _display_roles 36 | from repokid.commands.role import _find_roles_with_permissions 37 | from repokid.commands.role import _remove_permissions_from_roles 38 | from repokid.commands.role_cache import _update_role_cache 39 | from repokid.commands.schedule import _cancel_scheduled_repo 40 | from repokid.commands.schedule import _schedule_repo 41 | from repokid.commands.schedule import _show_scheduled_roles 42 | from repokid.types import RepokidConfig 43 | 44 | logger = logging.getLogger("repokid") 45 | 46 | 47 | def _generate_default_config(filename: str = "") -> RepokidConfig: 48 | """ 49 | Generate and return a config dict; will write the config to a file if a filename is provided 50 | 51 | Args: 52 | filename (string): Name of file to write the generated config (represented in JSON) 53 | 54 | Returns: 55 | dict: Template for Repokid config as a dictionary 56 | """ 57 | config_dict = { 58 | "query_role_data_in_batch": False, 59 | "batch_processing_size": 100, 60 | "filter_config": { 61 | "AgeFilter": {"minimum_age": 90}, 62 | "BlocklistFilter": { 63 | "all": [], 64 | "blocklist_bucket": { 65 | "bucket": "", 66 | "key": "", 67 | "account_number": "", 68 | "region": "", 70 | }, 71 | }, 72 | "ExclusiveFilter": { 73 | "all": [""], 74 | "": [""], 75 | }, 76 | }, 77 | "active_filters": [ 78 | "repokid.filters.age:AgeFilter", 79 | "repokid.filters.lambda:LambdaFilter", 80 | "repokid.filters.blocklist:BlocklistFilter", 81 | "repokid.filters.optout:OptOutFilter", 82 | ], 83 | "aardvark_api_location": "", 84 | "connection_iam": { 85 | "assume_role": "RepokidRole", 86 | "session_name": "repokid", 87 | "region": "us-east-1", 88 | }, 89 | "dynamo_db": { 90 | "assume_role": "RepokidRole", 91 | "account_number": "", 92 | "endpoint": "", 93 | "region": "", 94 | "session_name": "repokid", 95 | }, 96 | "hooks": ["repokid.hooks.loggers"], 97 | "logging": { 98 | "version": 1, 99 | "disable_existing_loggers": "False", 100 | "formatters": { 101 | "standard": { 102 | "format": "%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]" 103 | }, 104 | "json": {"class": "json_log_formatter.JSONFormatter"}, 105 | }, 106 | "handlers": { 107 | "file": { 108 | "class": "logging.handlers.RotatingFileHandler", 109 | "level": "INFO", 110 | "formatter": "standard", 111 | "filename": "repokid.log", 112 | "maxBytes": 10485760, 113 | "backupCount": 100, 114 | "encoding": "utf8", 115 | }, 116 | "json_file": { 117 | "class": "logging.handlers.RotatingFileHandler", 118 | "level": "INFO", 119 | "formatter": "json", 120 | "filename": "repokid.json", 121 | "maxBytes": 10485760, 122 | "backupCount": 100, 123 | "encoding": "utf8", 124 | }, 125 | "console": { 126 | "class": "logging.StreamHandler", 127 | "level": "INFO", 128 | "formatter": "standard", 129 | "stream": "ext://sys.stdout", 130 | }, 131 | }, 132 | "loggers": { 133 | "repokid": { 134 | "handlers": ["file", "json_file", "console"], 135 | "level": "INFO", 136 | } 137 | }, 138 | }, 139 | "opt_out_period_days": 90, 140 | "dispatcher": { 141 | "session_name": "repokid", 142 | "region": "us-west-2", 143 | "to_rr_queue": "COMMAND_QUEUE_TO_REPOKID_URL", 144 | "from_rr_sns": "RESPONSES_FROM_REPOKID_SNS_ARN", 145 | }, 146 | "repo_requirements": { 147 | "oldest_aa_data_days": 5, 148 | "exclude_new_permissions_for_days": 14, 149 | }, 150 | "repo_schedule_period_days": 7, 151 | "warnings": {"unknown_permissions": False}, 152 | } 153 | if filename: 154 | try: 155 | with open(filename, "w") as f: 156 | json.dump(config_dict, f, indent=4, sort_keys=True) 157 | except OSError as e: 158 | print(f"Unable to open {filename} for writing: {e}") 159 | else: 160 | print(f"Successfully wrote sample config to {filename}") 161 | return config_dict 162 | 163 | 164 | class AliasedGroup(Group): 165 | """AliasedGroup provides backward compatibility with the previous Repokid CLI commands""" 166 | 167 | def get_command(self, ctx: Context, cmd_name: str) -> Optional[Command]: 168 | rv = Group.get_command(self, ctx, cmd_name) 169 | if rv: 170 | return rv 171 | dashed = cmd_name.replace("_", "-") 172 | for cmd in self.list_commands(ctx): 173 | if cmd == dashed: 174 | return Group.get_command(self, ctx, cmd) 175 | return None 176 | 177 | 178 | @group(cls=AliasedGroup) 179 | @pass_context 180 | def cli(ctx: Context) -> None: 181 | ctx.ensure_object(dict) 182 | 183 | if not CONFIG: 184 | config = _generate_default_config() 185 | else: 186 | config = CONFIG 187 | 188 | ctx.obj["config"] = config 189 | ctx.obj["hooks"] = get_hooks(config.get("hooks", ["repokid.hooks.loggers"])) 190 | 191 | 192 | @cli.command() 193 | @argument("filename") 194 | @pass_context 195 | def config(ctx: Context, filename: str) -> None: 196 | _generate_default_config(filename=filename) 197 | 198 | 199 | @cli.command() 200 | @argument("account_number") 201 | @pass_context 202 | def update_role_cache(ctx: Context, account_number: str) -> None: 203 | config = ctx.obj["config"] 204 | hooks = ctx.obj["hooks"] 205 | _update_role_cache(account_number, config, hooks) 206 | 207 | 208 | @cli.command() 209 | @argument("account_number") 210 | @option("--inactive", is_flag=True, default=False, help="Include inactive roles") 211 | @pass_context 212 | def display_role_cache(ctx: Context, account_number: str, inactive: bool) -> None: 213 | _display_roles(account_number, inactive=inactive) 214 | 215 | 216 | @cli.command() 217 | @argument("permissions", nargs=-1) 218 | @option("--output", "-o", required=False, help="File to write results to") 219 | @pass_context 220 | def find_roles_with_permissions( 221 | ctx: Context, permissions: List[str], output: str 222 | ) -> None: 223 | _find_roles_with_permissions(permissions, output) 224 | 225 | 226 | @cli.command() 227 | @argument("permissions", nargs=-1) 228 | @option("--role-file", "-f", required=True, help="File to read roles from") 229 | @option("--commit", "-c", is_flag=True, default=False, help="Commit changes") 230 | @pass_context 231 | def remove_permissions_from_roles( 232 | ctx: Context, permissions: List[str], role_file: str, commit: bool 233 | ) -> None: 234 | config = ctx.obj["config"] 235 | hooks = ctx.obj["hooks"] 236 | _remove_permissions_from_roles(permissions, role_file, config, hooks, commit=commit) 237 | 238 | 239 | @cli.command() 240 | @argument("account_number") 241 | @argument("role_name") 242 | @pass_context 243 | def display_role(ctx: Context, account_number: str, role_name: str) -> None: 244 | config = ctx.obj["config"] 245 | _display_role(account_number, role_name, config) 246 | 247 | 248 | @cli.command() 249 | @argument("account_number") 250 | @argument("role_name") 251 | @option("--commit", "-c", is_flag=True, default=False, help="Commit changes") 252 | @pass_context 253 | def repo_role(ctx: Context, account_number: str, role_name: str, commit: bool) -> None: 254 | config = ctx.obj["config"] 255 | hooks = ctx.obj["hooks"] 256 | _repo_role(account_number, role_name, config, hooks, commit=commit) 257 | 258 | 259 | @cli.command() 260 | @argument("account_number") 261 | @argument("role_name") 262 | @option("--selection", "-s", required=True, type=int) 263 | @option("--commit", "-c", is_flag=True, default=False, help="Commit changes") 264 | @pass_context 265 | def rollback_role( 266 | ctx: Context, 267 | account_number: str, 268 | role_name: str, 269 | selection: int, 270 | commit: bool, 271 | ) -> None: 272 | config = ctx.obj["config"] 273 | hooks = ctx.obj["hooks"] 274 | _rollback_role( 275 | account_number, role_name, config, hooks, selection=selection, commit=commit 276 | ) 277 | 278 | 279 | @cli.command() 280 | @argument("account_number") 281 | @option("--commit", "-c", is_flag=True, default=False, help="Commit changes") 282 | @pass_context 283 | def repo_all_roles(ctx: Context, account_number: str, commit: bool) -> None: 284 | config = ctx.obj["config"] 285 | hooks = ctx.obj["hooks"] 286 | logger.info("Updating role data") 287 | _update_role_cache(account_number, config, hooks) 288 | _repo_all_roles(account_number, config, hooks, commit=commit, scheduled=False) 289 | 290 | 291 | @cli.command() 292 | @argument("account_number") 293 | @pass_context 294 | def schedule_repo(ctx: Context, account_number: str) -> None: 295 | config = ctx.obj["config"] 296 | hooks = ctx.obj["hooks"] 297 | logger.info("Updating role data") 298 | _update_role_cache(account_number, config, hooks) 299 | _schedule_repo(account_number, config, hooks) 300 | 301 | 302 | @cli.command() 303 | @argument("account_number") 304 | @pass_context 305 | def show_scheduled_roles(ctx: Context, account_number: str) -> None: 306 | _show_scheduled_roles(account_number) 307 | 308 | 309 | @cli.command() 310 | @argument("account_number") 311 | @option("--role", "-r", required=False, type=str) 312 | @option("--all", "-a", is_flag=True, default=False, help="cancel for all roles") 313 | @pass_context 314 | def cancel_scheduled_repo( 315 | ctx: Context, account_number: str, role: str, all: bool 316 | ) -> None: 317 | _cancel_scheduled_repo(account_number, role_name=role, is_all=all) 318 | 319 | 320 | @cli.command() 321 | @argument("account_number") 322 | @option("--commit", "-c", is_flag=True, default=False, help="Commit changes") 323 | @pass_context 324 | def repo_scheduled_roles(ctx: Context, account_number: str, commit: bool) -> None: 325 | config = ctx.obj["config"] 326 | hooks = ctx.obj["hooks"] 327 | _update_role_cache(account_number, config, hooks) 328 | _repo_all_roles(account_number, config, hooks, commit=commit, scheduled=True) 329 | 330 | 331 | @cli.command() 332 | @argument("account_number") 333 | @option("--output", "-o", required=True, help="File to write results to") 334 | @pass_context 335 | def repo_stats(ctx: Context, account_number: str, output: str) -> None: 336 | _repo_stats(output, account_number=account_number) 337 | 338 | 339 | if __name__ == "__main__": 340 | cli() 341 | -------------------------------------------------------------------------------- /repokid/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/repokid/376aa82ed31fe66ac4b1aecc3c22b0fc4fcfc0ea/repokid/commands/__init__.py -------------------------------------------------------------------------------- /repokid/commands/repo.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import csv 15 | import json 16 | import logging 17 | import pprint 18 | from typing import Iterable 19 | from typing import List 20 | 21 | from botocore.exceptions import ClientError 22 | from cloudaux.aws.iam import delete_role_policy 23 | from cloudaux.aws.iam import get_role_inline_policies 24 | from cloudaux.aws.iam import put_role_policy 25 | from tabulate import tabulate 26 | 27 | import repokid.hooks 28 | from repokid.datasource.access_advisor import AccessAdvisorDatasource 29 | from repokid.datasource.iam import IAMDatasource 30 | from repokid.exceptions import RoleStoreError 31 | from repokid.role import Role 32 | from repokid.role import RoleList 33 | from repokid.types import RepokidConfig 34 | from repokid.types import RepokidHooks 35 | from repokid.utils.dynamo import find_role_in_cache 36 | from repokid.utils.dynamo import role_arns_for_all_accounts 37 | from repokid.utils.permissions import get_services_in_permissions 38 | 39 | LOGGER = logging.getLogger("repokid") 40 | 41 | 42 | def _repo_role( 43 | account_number: str, 44 | role_name: str, 45 | config: RepokidConfig, 46 | hooks: RepokidHooks, 47 | commit: bool = False, 48 | scheduled: bool = False, 49 | ) -> List[str]: 50 | """ 51 | Calculate what repoing can be done for a role and then actually do it if commit is set 52 | 1) Check that a role exists, it isn't being disqualified by a filter, and that is has fresh AA data 53 | 2) Get the role's current permissions, repoable permissions, and the new policy if it will change 54 | 3) Make the changes if commit is set 55 | Args: 56 | account_number (string) 57 | role_name (string) 58 | commit (bool) 59 | 60 | Returns: 61 | None 62 | """ 63 | role_id = find_role_in_cache(role_name, account_number) 64 | # only load partial data that we need to determine if we should keep going 65 | role = Role(role_id=role_id, config=config) 66 | role.fetch() 67 | return role.repo(hooks, commit=commit, scheduled=scheduled) 68 | 69 | 70 | def _rollback_role( 71 | account_number: str, 72 | role_name: str, 73 | config: RepokidConfig, 74 | hooks: RepokidHooks, 75 | selection: int = -1, 76 | commit: bool = False, 77 | ) -> List[str]: 78 | """ 79 | Display the historical policy versions for a role as a numbered list. Restore to a specific version if selected. 80 | Indicate changes that will be made and then actually make them if commit is selected. 81 | 82 | Args: 83 | account_number (string) 84 | role_name (string) 85 | selection (int): which policy version in the list to rollback to 86 | commit (bool): actually make the change 87 | 88 | Returns: 89 | errors (list): if any 90 | """ 91 | errors = [] 92 | 93 | role_id = find_role_in_cache(role_name, account_number) 94 | if not role_id: 95 | message = "Could not find role with name {} in account {}".format( 96 | role_name, account_number 97 | ) 98 | errors.append(message) 99 | LOGGER.warning(message) 100 | return errors 101 | else: 102 | role = Role(role_id=role_id) 103 | role.fetch() 104 | 105 | # no option selected, display a table of options 106 | if selection < 0: 107 | headers = ["Number", "Source", "Discovered", "Permissions", "Services"] 108 | rows = [] 109 | for index, policies_version in enumerate(role.policies): 110 | policy_permissions, _ = repokid.utils.permissions.get_permissions_in_policy( 111 | policies_version["Policy"] 112 | ) 113 | rows.append( 114 | [ 115 | index, 116 | policies_version["Source"], 117 | policies_version["Discovered"], 118 | len(policy_permissions), 119 | get_services_in_permissions(policy_permissions), 120 | ] 121 | ) 122 | print(tabulate(rows, headers=headers)) 123 | return errors 124 | 125 | conn = config["connection_iam"] 126 | conn["account_number"] = account_number 127 | 128 | current_policies = get_role_inline_policies(role.dict(by_alias=True), **conn) 129 | 130 | pp = pprint.PrettyPrinter() 131 | 132 | print("Will restore the following policies:") 133 | pp.pprint(role.policies[int(selection)]["Policy"]) 134 | 135 | print("Current policies:") 136 | pp.pprint(current_policies) 137 | 138 | current_permissions, _ = role.get_permissions_for_policy_version() 139 | selected_permissions, _ = role.get_permissions_for_policy_version( 140 | selection=selection 141 | ) 142 | restored_permissions = selected_permissions - current_permissions 143 | 144 | print("\nResore will return these permissions:") 145 | print("\n".join([perm for perm in sorted(restored_permissions)])) 146 | 147 | if not commit: 148 | return errors 149 | 150 | # if we're restoring from a version with fewer policies than we have now, we need to remove them to 151 | # complete the restore. To do so we'll store all the policy names we currently have and remove them 152 | # from the list as we update. Any policy names left need to be manually removed 153 | policies_to_remove = current_policies.keys() 154 | 155 | for policy_name, policy in role.policies[int(selection)]["Policy"].items(): 156 | try: 157 | LOGGER.info( 158 | f"Pushing cached policy: {policy_name} (role: {role.role_name} account {account_number})" 159 | ) 160 | 161 | put_role_policy( 162 | RoleName=role.role_name, 163 | PolicyName=policy_name, 164 | PolicyDocument=json.dumps(policy, indent=2, sort_keys=True), 165 | **conn, 166 | ) 167 | 168 | except ClientError: 169 | message = f"Unable to push policy {policy_name}. (role: {role.role_name} account {account_number})" 170 | LOGGER.error(message, exc_info=True) 171 | errors.append(message) 172 | 173 | else: 174 | # remove the policy name if it's in the list 175 | try: 176 | policies_to_remove.remove(policy_name) 177 | except Exception: # nosec 178 | pass 179 | 180 | if policies_to_remove: 181 | for policy_name in policies_to_remove: 182 | try: 183 | LOGGER.info( 184 | f"Deleting policy {policy_name} for rollback (role: {role.role_name} account {account_number})" 185 | ) 186 | delete_role_policy( 187 | RoleName=role.role_name, PolicyName=policy_name, **conn 188 | ) 189 | 190 | except ClientError: 191 | message = f"Unable to delete policy {policy_name}. (role: {role.role_name} account {account_number})" 192 | LOGGER.error(message, exc_info=True) 193 | errors.append(message) 194 | 195 | try: 196 | role.store() 197 | except RoleStoreError: 198 | message = ( 199 | f"failed to store role data for {role.role_name} in account {role.account}" 200 | ) 201 | errors.append(message) 202 | LOGGER.exception(message, exc_info=True) 203 | 204 | if not errors: 205 | LOGGER.info( 206 | f"Successfully restored selected version {selection} of role policies (role: {role.role_name} " 207 | f"account: {account_number}" 208 | ) 209 | return errors 210 | 211 | 212 | def _repo_all_roles( 213 | account_number: str, 214 | config: RepokidConfig, 215 | hooks: RepokidHooks, 216 | commit: bool = False, 217 | scheduled: bool = True, 218 | limit: int = -1, 219 | ) -> None: 220 | """ 221 | Repo all scheduled or eligible roles in an account. Collect any errors and display them at the end. 222 | 223 | Args: 224 | account_number (string) 225 | dynamo_table 226 | config 227 | commit (bool): actually make the changes 228 | scheduled (bool): if True only repo the scheduled roles, if False repo all the (eligible) roles 229 | limit (int): limit number of roles to be repoed per run (< 0 is unlimited) 230 | 231 | Returns: 232 | None 233 | """ 234 | access_advisor_datasource = AccessAdvisorDatasource() 235 | access_advisor_datasource.seed(account_number) 236 | iam_datasource = IAMDatasource() 237 | role_arns = iam_datasource.seed(account_number) 238 | errors = [] 239 | 240 | roles = RoleList.from_arns(role_arns, config=config) 241 | roles = roles.get_active() 242 | if scheduled: 243 | roles = roles.get_scheduled() 244 | if not roles: 245 | LOGGER.info(f"No roles to repo in account {account_number}") 246 | return 247 | 248 | LOGGER.info( 249 | "Repoing these {}roles from account {}:\n\t{}".format( 250 | "scheduled " if scheduled else "", 251 | account_number, 252 | ", ".join([role.role_name for role in roles]), 253 | ) 254 | ) 255 | 256 | repokid.hooks.call_hooks( 257 | hooks, "BEFORE_REPO_ROLES", {"account_number": account_number, "roles": roles} 258 | ) 259 | 260 | count = 0 261 | repoed = RoleList([]) 262 | for role in roles: 263 | if limit >= 0 and count == limit: 264 | break 265 | role_errors = role.repo(hooks, commit=commit, scheduled=scheduled) 266 | if role_errors: 267 | errors.extend(role_errors) 268 | repoed.append(role) 269 | count += 1 270 | 271 | if errors: 272 | LOGGER.error(f"Error(s) during repo in account: {account_number}: {errors}") 273 | LOGGER.info(f"Successfully repoed {count} roles in account {account_number}") 274 | 275 | repokid.hooks.call_hooks( 276 | hooks, 277 | "AFTER_REPO_ROLES", 278 | {"account_number": account_number, "roles": repoed, "errors": errors}, 279 | ) 280 | 281 | 282 | def _repo_stats(output_file: str, account_number: str = "") -> None: 283 | """ 284 | Create a csv file with stats about roles, total permissions, and applicable filters over time 285 | 286 | Args: 287 | output_file (string): the name of the csv file to write 288 | account_number (string): if specified only display roles from selected account, otherwise display all 289 | 290 | Returns: 291 | None 292 | """ 293 | role_ids: Iterable[str] 294 | if account_number: 295 | access_advisor_datasource = AccessAdvisorDatasource() 296 | access_advisor_datasource.seed(account_number) 297 | iam_datasource = IAMDatasource() 298 | role_arns = iam_datasource.seed(account_number) 299 | else: 300 | role_arns = role_arns_for_all_accounts() 301 | 302 | headers = [ 303 | "RoleId", 304 | "Role Name", 305 | "Account", 306 | "Active", 307 | "Date", 308 | "Source", 309 | "Permissions Count", 310 | "Repoable Permissions Count", 311 | "Disqualified By", 312 | ] 313 | rows = [] 314 | roles = RoleList.from_arns( 315 | role_arns, fields=["RoleId", "RoleName", "Account", "Active", "Stats"] 316 | ) 317 | 318 | for role in roles: 319 | for stats_entry in role.stats: 320 | rows.append( 321 | [ 322 | role.role_id, 323 | role.role_name, 324 | role.account, 325 | role.active, 326 | stats_entry["Date"], 327 | stats_entry["Source"], 328 | stats_entry["PermissionsCount"], 329 | stats_entry.get("RepoablePermissionsCount", 0), 330 | stats_entry.get("DisqualifiedBy", []), 331 | ] 332 | ) 333 | 334 | try: 335 | with open(output_file, "w") as csvfile: 336 | csv_writer = csv.writer(csvfile) 337 | csv_writer.writerow(headers) 338 | for row in rows: 339 | csv_writer.writerow(row) 340 | except IOError as e: 341 | LOGGER.error( 342 | "Unable to write file {}: {}".format(output_file, e), exc_info=True 343 | ) 344 | else: 345 | LOGGER.info("Successfully wrote stats to {}".format(output_file)) 346 | -------------------------------------------------------------------------------- /repokid/commands/role.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import csv 15 | import json 16 | import logging 17 | from typing import Any 18 | from typing import List 19 | from typing import Optional 20 | 21 | import tabview as t 22 | from policyuniverse.arn import ARN 23 | from tabulate import tabulate 24 | from tqdm import tqdm 25 | 26 | import repokid.hooks 27 | from repokid.exceptions import MissingRepoableServices 28 | from repokid.role import Role 29 | from repokid.role import RoleList 30 | from repokid.types import RepokidConfig 31 | from repokid.types import RepokidHooks 32 | from repokid.utils.dynamo import find_role_in_cache 33 | from repokid.utils.dynamo import get_all_role_ids_for_account 34 | from repokid.utils.dynamo import role_arns_for_all_accounts 35 | from repokid.utils.iam import inline_policies_size_exceeds_maximum 36 | from repokid.utils.permissions import get_permissions_in_policy 37 | from repokid.utils.permissions import get_services_in_permissions 38 | 39 | LOGGER = logging.getLogger("repokid") 40 | 41 | 42 | def _display_roles(account_number: str, inactive: bool = False) -> None: 43 | """ 44 | Display a table with data about all roles in an account and write a csv file with the data. 45 | 46 | Args: 47 | account_number (string) 48 | inactive (bool): show roles that have historically (but not currently) existed in the account if True 49 | 50 | Returns: 51 | None 52 | """ 53 | headers = [ 54 | "Name", 55 | "Refreshed", 56 | "Disqualified By", 57 | "Can be repoed", 58 | "Permissions", 59 | "Repoable", 60 | "Repoed", 61 | "Services", 62 | ] 63 | 64 | rows: List[List[Any]] = [] 65 | 66 | role_ids = get_all_role_ids_for_account(account_number) 67 | roles = RoleList.from_ids(role_ids) 68 | 69 | if not inactive: 70 | roles = roles.get_active() 71 | 72 | for role in roles: 73 | rows.append( 74 | [ 75 | role.role_name, 76 | role.refreshed, 77 | role.disqualified_by, 78 | len(role.disqualified_by) == 0, 79 | role.total_permissions, 80 | role.repoable_permissions, 81 | role.repoed, 82 | role.repoable_services, 83 | ] 84 | ) 85 | 86 | rows = sorted(rows, key=lambda x: (x[5], x[0], x[4])) 87 | rows.insert(0, headers) 88 | # print tabulate(rows, headers=headers) 89 | t.view(rows) 90 | with open("table.csv", "w") as csvfile: 91 | csv_writer = csv.writer(csvfile) 92 | csv_writer.writerow(headers) 93 | for row in rows: 94 | csv_writer.writerow(row) 95 | 96 | 97 | def _find_roles_with_permissions(permissions: List[str], output_file: str) -> None: 98 | """ 99 | Search roles in all accounts for a policy with any of the provided permissions, log the ARN of each role. 100 | 101 | Args: 102 | permissions (list[string]): The name of the permissions to find 103 | output_file (string): filename to write the output 104 | 105 | Returns: 106 | None 107 | """ 108 | arns: List[str] = list() 109 | role_ids = role_arns_for_all_accounts() 110 | roles = RoleList.from_ids( 111 | role_ids, fields=["Policies", "RoleName", "Arn", "Active"] 112 | ) 113 | for role in roles: 114 | role_permissions, _ = role.get_permissions_for_policy_version() 115 | 116 | permissions_set = set([p.lower() for p in permissions]) 117 | found_permissions = permissions_set.intersection(role_permissions) 118 | 119 | if found_permissions and role.active: 120 | arns.append(role.arn) 121 | LOGGER.info( 122 | "ARN {arn} has {permissions}".format( 123 | arn=role.arn, permissions=list(found_permissions) 124 | ) 125 | ) 126 | 127 | if not output_file: 128 | return 129 | 130 | with open(output_file, "w") as fd: 131 | json.dump(arns, fd) 132 | 133 | LOGGER.info(f"Output written to file {output_file}") 134 | 135 | 136 | def _display_role( 137 | account_number: str, 138 | role_name: str, 139 | config: RepokidConfig, 140 | ) -> None: 141 | """ 142 | Displays data about a role in a given account: 143 | 1) Name, which filters are disqualifying it from repo, if it's repoable, total/repoable permissions, 144 | when it was last repoed, which services can be repoed 145 | 2) The policy history: how discovered (repo, scan, etc), the length of the policy, and start of the contents 146 | 3) Captured stats entry for the role 147 | 4) A list of all services/actions currently allowed and whether they are repoable 148 | 5) What the new policy would look like after repoing (if it is repoable) 149 | 150 | Args: 151 | account_number (string) 152 | role_name (string) 153 | 154 | Returns: 155 | None 156 | """ 157 | role_id = find_role_in_cache(role_name, account_number) 158 | if not role_id: 159 | LOGGER.warning("Could not find role with name {}".format(role_name)) 160 | return 161 | 162 | role = Role(role_id=role_id) 163 | role.fetch() 164 | 165 | print("\n\nRole repo data:") 166 | headers = [ 167 | "Name", 168 | "Refreshed", 169 | "Disqualified By", 170 | "Can be repoed", 171 | "Permissions", 172 | "Repoable", 173 | "Repoed", 174 | "Services", 175 | ] 176 | rows = [ 177 | [ 178 | role.role_name, 179 | role.refreshed, 180 | role.disqualified_by, 181 | len(role.disqualified_by) == 0, 182 | role.total_permissions, 183 | role.repoable_permissions, 184 | role.repoed, 185 | role.repoable_services, 186 | ] 187 | ] 188 | print(tabulate(rows, headers=headers) + "\n\n") 189 | 190 | print("Policy history:") 191 | headers = ["Number", "Source", "Discovered", "Permissions", "Services"] 192 | rows = [] 193 | for index, policies_version in enumerate(role.policies): 194 | policy_permissions, _ = get_permissions_in_policy(policies_version["Policy"]) 195 | rows.append( 196 | [ 197 | index, 198 | policies_version["Source"], 199 | policies_version["Discovered"], 200 | len(policy_permissions), 201 | get_services_in_permissions(policy_permissions), 202 | ] 203 | ) 204 | print(tabulate(rows, headers=headers) + "\n\n") 205 | 206 | print("Stats:") 207 | headers = ["Date", "Event Type", "Permissions Count", "Disqualified By"] 208 | rows = [] 209 | for stats_entry in role.stats: 210 | rows.append( 211 | [ 212 | stats_entry["Date"], 213 | stats_entry["Source"], 214 | stats_entry["PermissionsCount"], 215 | stats_entry.get("DisqualifiedBy", []), 216 | ] 217 | ) 218 | print(tabulate(rows, headers=headers) + "\n\n") 219 | 220 | # can't do anymore if we don't have AA data 221 | if not role.aa_data: 222 | LOGGER.warning("ARN not found in Access Advisor: {}".format(role.arn)) 223 | return 224 | 225 | warn_unknown_permissions = config.get("warnings", {}).get( 226 | "unknown_permissions", False 227 | ) 228 | 229 | permissions, eligible_permissions = role.get_permissions_for_policy_version( 230 | warn_unknown_perms=warn_unknown_permissions 231 | ) 232 | 233 | print("Repoable services and permissions") 234 | headers = ["Service", "Action", "Repoable"] 235 | rows = [] 236 | for permission in permissions: 237 | service = permission.split(":")[0] 238 | action = permission.split(":")[1] 239 | is_repoable_permission = permission in role.repoable_services 240 | is_repoable_service = permission.split(":")[0] in role.repoable_services 241 | # repoable is is True if the action (`permission`) is in the list of repoable 242 | # services OR if the service (`permission.split(":")[0]`) is in the list 243 | repoable = is_repoable_permission or is_repoable_service 244 | rows.append([service, action, repoable]) 245 | 246 | rows = sorted(rows, key=lambda x: (x[2], x[0], x[1])) 247 | print(tabulate(rows, headers=headers) + "\n\n") 248 | 249 | try: 250 | repoed_policies, _ = role.get_repoed_policy() 251 | print( 252 | "Repo'd Policies: \n{}".format( 253 | json.dumps(repoed_policies, indent=2, sort_keys=True) 254 | ) 255 | ) 256 | except MissingRepoableServices: 257 | print("All Policies Removed") 258 | repoed_policies = {} 259 | 260 | # need to check if all policies would be too large 261 | if inline_policies_size_exceeds_maximum(repoed_policies): 262 | LOGGER.warning( 263 | "Policies would exceed the AWS size limit after repo for role: {}. " 264 | "Please manually minify.".format(role_name) 265 | ) 266 | 267 | 268 | def _remove_permissions_from_roles( 269 | permissions: List[str], 270 | role_filename: str, 271 | config: Optional[RepokidConfig], 272 | hooks: RepokidHooks, 273 | commit: bool = False, 274 | ) -> None: 275 | """Loads roles specified in file and calls _remove_permissions_from_role() for each one. 276 | 277 | Args: 278 | permissions (list) 279 | role_filename (string) 280 | commit (bool) 281 | 282 | Returns: 283 | None 284 | """ 285 | with open(role_filename, "r") as fd: 286 | roles = json.load(fd) 287 | 288 | for role_arn in tqdm(roles): 289 | arn = ARN(role_arn) 290 | if arn.error: 291 | LOGGER.error("INVALID ARN: {arn}".format(arn=role_arn)) 292 | return 293 | 294 | account_number = arn.account_number 295 | role_name = arn.name.split("/")[-1] 296 | 297 | role_id = find_role_in_cache(role_name, account_number) 298 | role = Role(role_id=role_id, config=config) 299 | role.fetch() 300 | 301 | role.remove_permissions(permissions, hooks, commit=commit) 302 | 303 | repokid.hooks.call_hooks(hooks, "AFTER_REPO", {"role": role}) 304 | -------------------------------------------------------------------------------- /repokid/commands/role_cache.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import logging 15 | 16 | from tqdm import tqdm 17 | 18 | from repokid.datasource.access_advisor import AccessAdvisorDatasource 19 | from repokid.datasource.iam import IAMDatasource 20 | from repokid.filters.utils import get_filter_plugins 21 | from repokid.role import RoleList 22 | from repokid.types import RepokidConfig 23 | from repokid.types import RepokidHooks 24 | from repokid.utils.roledata import find_and_mark_inactive 25 | 26 | LOGGER = logging.getLogger("repokid") 27 | 28 | 29 | def _update_role_cache( 30 | account_number: str, 31 | config: RepokidConfig, 32 | hooks: RepokidHooks, 33 | ) -> None: 34 | """ 35 | Update data about all roles in a given account: 36 | 1) list all the roles and initiate a role object with basic data including name and roleID 37 | 2) get inline policies for each of the roles 38 | 3) build a list of active roles - we'll want to keep data about roles that may have been deleted in case we 39 | need to restore them, so if we used to have a role and now we don't see it we'll mark it inactive 40 | 4) update data about the roles in Dynamo 41 | 5) mark inactive roles in Dynamo 42 | 6) load and instantiate filter plugins 43 | 7) for each filter determine the list of roles that it filters 44 | 8) update data in Dynamo about filters 45 | 9) get Aardvark data for each role 46 | 10) update Dynamo with Aardvark data 47 | 11) calculate repoable permissions/policies for all the roles 48 | 12) update Dynamo with information about how many total and repoable permissions and which services are repoable 49 | 13) update stats in Dynamo with basic information like total permissions and which filters are applicable 50 | 51 | Args: 52 | account_number (string): The current account number Repokid is being run against 53 | 54 | Returns: 55 | None 56 | """ 57 | access_advisor_datasource = AccessAdvisorDatasource() 58 | access_advisor_datasource.seed(account_number) 59 | iam_datasource = IAMDatasource() 60 | role_arns = iam_datasource.seed(account_number) 61 | 62 | # We only iterate over the newly-seeded data (`role_arns`) so we don't duplicate work for runs on multiple accounts 63 | roles = RoleList.from_arns(role_arns) 64 | 65 | LOGGER.info("Updating role data for account {}".format(account_number)) 66 | for role in tqdm(roles): 67 | role.account = account_number 68 | role.gather_role_data(hooks, config=config, source="Scan", store=False) 69 | # Reseting previous filters 70 | role.disqualified_by = list() 71 | 72 | LOGGER.info("Finding inactive roles in account {}".format(account_number)) 73 | find_and_mark_inactive(account_number, roles) 74 | 75 | LOGGER.info("Filtering roles") 76 | plugins = get_filter_plugins(account_number, config=config) 77 | for plugin in plugins.filter_plugins: 78 | filtered_list = plugin.apply(roles) 79 | class_name = plugin.__class__.__name__ 80 | for filtered_role in filtered_list: 81 | LOGGER.debug( 82 | "Role {} filtered by {}".format(filtered_role.role_name, class_name) 83 | ) 84 | # There may be existing duplicate records, so we do a dance here to dedupe them. 85 | disqualified_by = set(filtered_role.disqualified_by) 86 | disqualified_by.add(class_name) 87 | filtered_role.disqualified_by = list(disqualified_by) 88 | 89 | for role in roles: 90 | role.calculate_repo_scores( 91 | config["filter_config"]["AgeFilter"]["minimum_age"], hooks 92 | ) 93 | LOGGER.debug( 94 | "Role {} in account {} has\nrepoable permissions: {}\nrepoable services: {}".format( 95 | role.role_name, 96 | account_number, 97 | role.repoable_permissions, 98 | role.repoable_services, 99 | ) 100 | ) 101 | 102 | LOGGER.info("Storing updated role data in account {}".format(account_number)) 103 | roles.store() 104 | -------------------------------------------------------------------------------- /repokid/commands/schedule.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import logging 15 | import time 16 | from datetime import datetime as dt 17 | 18 | from tabulate import tabulate 19 | 20 | import repokid.hooks 21 | from repokid.exceptions import RoleStoreError 22 | from repokid.role import Role 23 | from repokid.role import RoleList 24 | from repokid.types import RepokidConfig 25 | from repokid.types import RepokidHooks 26 | from repokid.utils.dynamo import find_role_in_cache 27 | from repokid.utils.dynamo import get_all_role_ids_for_account 28 | 29 | LOGGER = logging.getLogger("repokid") 30 | 31 | 32 | def _schedule_repo( 33 | account_number: str, 34 | config: RepokidConfig, 35 | hooks: RepokidHooks, 36 | ) -> None: 37 | """ 38 | Schedule a repo for a given account. Schedule repo for a time in the future (default 7 days) for any roles in 39 | the account with repoable permissions. 40 | """ 41 | scheduled_roles = [] 42 | role_ids = get_all_role_ids_for_account(account_number) 43 | roles = RoleList.from_ids(role_ids) 44 | roles.fetch_all(fetch_aa_data=True) 45 | 46 | scheduled_time = int(time.time()) + ( 47 | 86400 * config.get("repo_schedule_period_days", 7) 48 | ) 49 | for role in roles: 50 | if not role.aa_data: 51 | LOGGER.warning("Not scheduling %s; missing Access Advisor data", role.arn) 52 | continue 53 | if not role.repoable_permissions > 0: 54 | LOGGER.debug("Not scheduling %s; no repoable permissions", role.arn) 55 | continue 56 | if role.repo_scheduled: 57 | LOGGER.debug( 58 | "Not scheduling %s; already scheduled for %s", 59 | role.arn, 60 | role.repo_scheduled, 61 | ) 62 | continue 63 | 64 | role.repo_scheduled = scheduled_time 65 | # freeze the scheduled perms to whatever is repoable right now 66 | role.repo_scheduled = scheduled_time 67 | role.scheduled_perms = role.repoable_services 68 | try: 69 | role.store(["repo_scheduled", "scheduled_perms"]) 70 | except RoleStoreError: 71 | logging.exception("failed to store role", exc_info=True) 72 | 73 | scheduled_roles.append(role) 74 | 75 | LOGGER.info( 76 | "Scheduled repo for {} days from now for account {} and these roles:\n\t{}".format( 77 | config.get("repo_schedule_period_days", 7), 78 | account_number, 79 | ", ".join([r.role_name for r in scheduled_roles]), 80 | ) 81 | ) 82 | 83 | repokid.hooks.call_hooks(hooks, "AFTER_SCHEDULE_REPO", {"roles": scheduled_roles}) 84 | 85 | 86 | def _show_scheduled_roles(account_number: str) -> None: 87 | """ 88 | Show scheduled repos for a given account. For each scheduled show whether scheduled time is elapsed or not. 89 | """ 90 | role_ids = get_all_role_ids_for_account(account_number) 91 | roles = RoleList.from_ids(role_ids) 92 | 93 | # filter to show only roles that are scheduled 94 | roles = roles.get_active().get_scheduled() 95 | 96 | header = ["Role name", "Scheduled", "Scheduled Time Elapsed?"] 97 | rows = [] 98 | 99 | curtime = int(time.time()) 100 | 101 | for role in roles: 102 | rows.append( 103 | [ 104 | role.role_name, 105 | dt.fromtimestamp(role.repo_scheduled).strftime("%Y-%m-%d %H:%M"), 106 | role.repo_scheduled < curtime, 107 | ] 108 | ) 109 | 110 | print(tabulate(rows, headers=header)) 111 | 112 | 113 | def _cancel_scheduled_repo( 114 | account_number: str, role_name: str = "", is_all: bool = False 115 | ) -> None: 116 | """ 117 | Cancel scheduled repo for a role in an account 118 | """ 119 | if not is_all and not role_name: 120 | LOGGER.error("Either a specific role to cancel or all must be provided") 121 | return 122 | 123 | if is_all: 124 | role_ids = get_all_role_ids_for_account(account_number) 125 | roles = RoleList.from_ids(role_ids) 126 | 127 | # filter to show only roles that are scheduled 128 | roles = roles.get_scheduled() 129 | 130 | for role in roles: 131 | role.repo_scheduled = 0 132 | role.scheduled_perms = [] 133 | try: 134 | role.store(["repo_scheduled", "scheduled_perms"]) 135 | except RoleStoreError: 136 | LOGGER.exception("failed to store role", exc_info=True) 137 | 138 | LOGGER.info( 139 | "Canceled scheduled repo for roles: {}".format( 140 | ", ".join([role.role_name for role in roles]) 141 | ) 142 | ) 143 | return 144 | 145 | role_id = find_role_in_cache(role_name, account_number) 146 | if not role_id: 147 | LOGGER.warning( 148 | f"Could not find role with name {role_name} in account {account_number}" 149 | ) 150 | return 151 | 152 | role = Role(role_id=role_id) 153 | role.fetch() 154 | 155 | if not role.repo_scheduled: 156 | LOGGER.warning( 157 | "Repo was not scheduled for role {} in account {}".format( 158 | role.role_name, account_number 159 | ) 160 | ) 161 | return 162 | 163 | role.repo_scheduled = 0 164 | role.scheduled_perms = [] 165 | try: 166 | role.store(["repo_scheduled", "scheduled_perms"]) 167 | except RoleStoreError: 168 | LOGGER.exception("failed to store role", exc_info=True) 169 | raise 170 | 171 | LOGGER.info( 172 | "Successfully cancelled scheduled repo for role {} in account {}".format( 173 | role.role_name, role.account 174 | ) 175 | ) 176 | -------------------------------------------------------------------------------- /repokid/datasource/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /repokid/datasource/access_advisor.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | from typing import Any 17 | from typing import Dict 18 | from typing import Iterable 19 | from typing import Optional 20 | 21 | import requests 22 | 23 | from repokid.datasource.plugin import DatasourcePlugin 24 | from repokid.exceptions import AardvarkError 25 | from repokid.exceptions import NotFoundError 26 | from repokid.plugin import Singleton 27 | from repokid.types import AardvarkResponse 28 | from repokid.types import AccessAdvisorEntry 29 | from repokid.types import RepokidConfig 30 | 31 | logger = logging.getLogger("repokid") 32 | 33 | 34 | class AccessAdvisorDatasource( 35 | DatasourcePlugin[str, AccessAdvisorEntry], metaclass=Singleton 36 | ): 37 | def __init__(self, config: Optional[RepokidConfig] = None): 38 | super().__init__(config=config) 39 | 40 | def _fetch( 41 | self, account_number: str = "", arn: str = "" 42 | ) -> Dict[str, AccessAdvisorEntry]: 43 | """ 44 | Make a request to the Aardvark server to get all data about a given account or ARN. 45 | We'll request in groups of PAGE_SIZE and check the current count to see if we're done. Keep requesting as long 46 | as the total count (reported by the API) is greater than the number of pages we've received times the page size. 47 | As we go, keeping building the dict and return it when done. 48 | 49 | Args: 50 | account_number (string): Used to form the phrase query for Aardvark 51 | arn (string) 52 | 53 | Returns: 54 | dict: Aardvark data is a dict with the role ARN as the key and a list of services as value 55 | """ 56 | api_location = self.config.get("aardvark_api_location") 57 | if not api_location: 58 | raise AardvarkError("aardvark not configured") 59 | 60 | response_data: AardvarkResponse = {} 61 | 62 | PAGE_SIZE = 1000 63 | page_num = 1 64 | 65 | payload: Dict[str, Any] 66 | if account_number: 67 | payload = {"phrase": account_number} 68 | elif arn: 69 | payload = {"arn": [arn]} 70 | else: 71 | return {} 72 | while True: 73 | params = {"count": PAGE_SIZE, "page": page_num} 74 | try: 75 | r_aardvark = requests.post(api_location, params=params, json=payload) 76 | except requests.exceptions.RequestException as e: 77 | logger.exception("unable to get Aardvark data: {}".format(e)) 78 | raise AardvarkError("unable to get aardvark data") 79 | else: 80 | if r_aardvark.status_code != 200: 81 | logger.exception("unable to get Aardvark data") 82 | raise AardvarkError("unable to get aardvark data") 83 | 84 | response_data.update(r_aardvark.json()) 85 | # don't want these in our Aardvark data 86 | response_data.pop("count") 87 | response_data.pop("page") 88 | response_data.pop("total") 89 | if PAGE_SIZE * page_num < r_aardvark.json().get("total"): 90 | page_num += 1 91 | else: 92 | break 93 | return response_data 94 | 95 | def get(self, arn: str) -> AccessAdvisorEntry: 96 | result = self._data.get(arn) 97 | if result: 98 | return result 99 | 100 | # Try to get data from Aardvark 101 | result = self._fetch(arn=arn).get(arn) 102 | if result: 103 | self._data[arn] = result 104 | return result 105 | raise NotFoundError 106 | 107 | def _get_arns_for_account(self, account_number: str) -> Iterable[str]: 108 | return filter(lambda x: x.split(":")[4] == account_number, self.keys()) 109 | 110 | def seed(self, account_number: str) -> Iterable[str]: 111 | if account_number not in self._seeded: 112 | aa_data = self._fetch(account_number=account_number) 113 | self._data.update(aa_data) 114 | self._seeded.append(account_number) 115 | return aa_data.keys() 116 | else: 117 | return self._get_arns_for_account(account_number) 118 | -------------------------------------------------------------------------------- /repokid/datasource/iam.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import copy 16 | import logging 17 | from typing import Dict 18 | from typing import Iterable 19 | from typing import Optional 20 | 21 | from cloudaux.aws.iam import get_account_authorization_details 22 | from cloudaux.orchestration.aws.iam.role import FLAGS 23 | from cloudaux.orchestration.aws.iam.role import get_role 24 | 25 | from repokid.datasource.plugin import DatasourcePlugin 26 | from repokid.exceptions import NotFoundError 27 | from repokid.plugin import Singleton 28 | from repokid.types import IAMEntry 29 | from repokid.types import RepokidConfig 30 | 31 | logger = logging.getLogger("repokid") 32 | 33 | 34 | class IAMDatasource(DatasourcePlugin[str, IAMEntry], metaclass=Singleton): 35 | _arn_to_id: Dict[str, str] = {} 36 | 37 | def __init__(self, config: Optional[RepokidConfig] = None): 38 | super().__init__(config=config) 39 | 40 | def _fetch_account(self, account_number: str) -> Dict[str, IAMEntry]: 41 | logger.info("getting role data for account %s", account_number) 42 | conn = copy.deepcopy(self.config.get("connection_iam", {})) 43 | conn["account_number"] = account_number 44 | auth_details = get_account_authorization_details(filter="Role", **conn) 45 | auth_details_by_id = {item["Arn"]: item for item in auth_details} 46 | self._arn_to_id.update({item["Arn"]: item["RoleId"] for item in auth_details}) 47 | # convert policies list to dictionary to maintain consistency with old call which returned a dict 48 | for _, data in auth_details_by_id.items(): 49 | data["RolePolicyList"] = { 50 | item["PolicyName"]: item["PolicyDocument"] 51 | for item in data["RolePolicyList"] 52 | } 53 | return auth_details_by_id 54 | 55 | def _fetch(self, arn: str) -> IAMEntry: 56 | logger.info("getting role data for role %s", arn) 57 | conn = copy.deepcopy(self.config["connection_iam"]) 58 | conn["account_number"] = arn.split(":")[4] 59 | role = {"RoleName": arn.split("/")[-1]} 60 | role_info: IAMEntry = get_role(role, flags=FLAGS.INLINE_POLICIES, **conn) 61 | self._arn_to_id[arn] = role_info["RoleId"] 62 | if not role_info: 63 | raise NotFoundError 64 | self._data[arn] = role_info 65 | return role_info 66 | 67 | def get(self, arn: str) -> IAMEntry: 68 | result = self._data.get(arn) 69 | if not result: 70 | return self._fetch(arn) 71 | return result 72 | 73 | def get_id_for_arn(self, arn: str) -> Optional[str]: 74 | return self._arn_to_id.get(arn) 75 | 76 | def _get_ids_for_account(self, account_number: str) -> Iterable[str]: 77 | ids_for_account = [ 78 | k for k, v in self.items() if v["Arn"].split(":")[4] == account_number 79 | ] 80 | return ids_for_account 81 | 82 | def seed(self, account_number: str) -> Iterable[str]: 83 | if account_number in self._seeded: 84 | return self._get_ids_for_account(account_number) 85 | fetched_data = self._fetch_account(account_number) 86 | new_keys = fetched_data.keys() 87 | self._data.update(fetched_data) 88 | self._seeded.append(account_number) 89 | return new_keys 90 | 91 | 92 | # TODO: Implement retrieval of IAM data from AWS Config 93 | class ConfigDatasource(DatasourcePlugin[str, IAMEntry], metaclass=Singleton): 94 | def __init__(self, config: Optional[RepokidConfig] = None): 95 | super().__init__(config=config) 96 | 97 | def get(self, identifier: str) -> IAMEntry: 98 | pass 99 | 100 | def seed(self, identifier: str) -> Iterable[str]: 101 | pass 102 | -------------------------------------------------------------------------------- /repokid/datasource/plugin.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | from typing import Dict 17 | from typing import Generic 18 | from typing import ItemsView 19 | from typing import Iterable 20 | from typing import Iterator 21 | from typing import List 22 | from typing import Optional 23 | from typing import ValuesView 24 | from typing import cast 25 | 26 | from repokid.plugin import RepokidPlugin 27 | from repokid.types import KT 28 | from repokid.types import VT 29 | from repokid.types import RepokidConfig 30 | 31 | logger = logging.getLogger("repokid") 32 | 33 | 34 | class DatasourcePlugin(RepokidPlugin, Generic[KT, VT]): 35 | """A dict-like container that can be used to retrieve and store data""" 36 | 37 | _data: Dict[KT, VT] 38 | _seeded: List[str] 39 | 40 | def __init__(self, config: Optional[RepokidConfig] = None): 41 | super().__init__(config=config) 42 | self._data = {} 43 | self._seeded = [] 44 | 45 | def __getitem__(self, name: KT) -> VT: 46 | return self._data[name] 47 | 48 | def __iter__(self) -> Iterator[VT]: 49 | return iter(cast(Iterable[VT], self._data)) 50 | 51 | def keys(self) -> Iterable[KT]: 52 | return self._data.keys() 53 | 54 | def items(self) -> ItemsView[KT, VT]: 55 | return self._data.items() 56 | 57 | def values(self) -> ValuesView[VT]: 58 | return self._data.values() 59 | 60 | def get(self, identifier: KT) -> VT: 61 | raise NotImplementedError 62 | 63 | def seed(self, identifier: KT) -> Iterable[KT]: 64 | raise NotImplementedError 65 | 66 | def reset(self) -> None: 67 | logger.debug("resetting %s", type(self).__name__) 68 | self._data = {} 69 | self._seeded = [] 70 | -------------------------------------------------------------------------------- /repokid/dispatcher/__init__.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import time 3 | from collections import namedtuple 4 | from typing import Callable 5 | 6 | from repokid import CONFIG 7 | from repokid import get_hooks 8 | from repokid.commands.repo import _rollback_role 9 | from repokid.dispatcher.types import Message 10 | from repokid.exceptions import RoleStoreError 11 | from repokid.role import Role 12 | from repokid.utils.dynamo import find_role_in_cache 13 | from repokid.utils.permissions import get_permissions_in_policy 14 | from repokid.utils.permissions import get_services_and_permissions_from_repoable 15 | 16 | ResponderReturn = namedtuple("ResponderReturn", "successful, return_message") 17 | 18 | if CONFIG: 19 | hooks_list = CONFIG.get("hooks", ["repokid.hooks.loggers"]) 20 | else: 21 | hooks_list = ["repokid.hooks.loggers"] 22 | 23 | hooks = get_hooks(hooks_list) 24 | DispatcherCommand = Callable[[Message], ResponderReturn] 25 | 26 | 27 | def implements_command( 28 | command: str, 29 | ) -> Callable[[DispatcherCommand], DispatcherCommand]: 30 | def _implements_command(func: DispatcherCommand) -> DispatcherCommand: 31 | if not hasattr(func, "_implements_command"): 32 | setattr(func, "_implements_command", command) 33 | return func 34 | 35 | return _implements_command 36 | 37 | 38 | @implements_command("list_repoable_services") 39 | def list_repoable_services(message: Message) -> ResponderReturn: 40 | role_id = find_role_in_cache(message.role_name, message.account) 41 | 42 | if not role_id: 43 | return ResponderReturn( 44 | successful=False, 45 | return_message="Unable to find role {} in account {}".format( 46 | message.role_name, message.account 47 | ), 48 | ) 49 | else: 50 | role = Role(role_id=role_id) 51 | role.fetch(fields=["RepoableServices"]) 52 | 53 | ( 54 | repoable_permissions, 55 | repoable_services, 56 | ) = get_services_and_permissions_from_repoable(role.repoable_services) 57 | 58 | return ResponderReturn( 59 | successful=True, 60 | return_message=( 61 | "Role {} in account {} has:\n Repoable Services: \n{}\n\n Repoable Permissions: \n{}".format( 62 | message.role_name, 63 | message.account, 64 | "\n".join([service for service in repoable_services]), 65 | "\n".join([perm for perm in repoable_permissions]), 66 | ) 67 | ), 68 | ) 69 | 70 | 71 | @implements_command("list_role_rollbacks") 72 | def list_role_rollbacks(message: Message) -> ResponderReturn: 73 | role_id = find_role_in_cache(message.role_name, message.account) 74 | 75 | if not role_id: 76 | return ResponderReturn( 77 | successful=False, 78 | return_message="Unable to find role {} in account {}".format( 79 | message.role_name, message.account 80 | ), 81 | ) 82 | 83 | role = Role(role_id=role_id) 84 | role.fetch(fields=["Policies"]) 85 | return_val = "Restorable versions for role {} in account {}\n".format( 86 | message.role_name, message.account 87 | ) 88 | for index, policy_version in enumerate(role.policies): 89 | total_permissions, _ = get_permissions_in_policy(policy_version["Policy"]) 90 | return_val += "({:>3}): {:<5} {:<15} {}\n".format( 91 | index, 92 | len(total_permissions), 93 | policy_version["Discovered"], 94 | policy_version["Source"], 95 | ) 96 | return ResponderReturn(successful=True, return_message=return_val) 97 | 98 | 99 | @implements_command("opt_out") 100 | def opt_out(message: Message) -> ResponderReturn: 101 | if CONFIG: 102 | opt_out_period = CONFIG.get("opt_out_period_days", 90) 103 | else: 104 | opt_out_period = 90 105 | 106 | if not message.reason or not message.requestor: 107 | return ResponderReturn( 108 | successful=False, return_message="Reason and requestor must be specified" 109 | ) 110 | 111 | role_id = find_role_in_cache(message.role_name, message.account) 112 | 113 | if not role_id: 114 | return ResponderReturn( 115 | successful=False, 116 | return_message="Unable to find role {} in account {}".format( 117 | message.role_name, message.account 118 | ), 119 | ) 120 | 121 | role = Role(role_id=role_id) 122 | role.fetch(fields=["OptOut"]) 123 | if role.opt_out: 124 | timestr = time.strftime("%m/%d/%y", time.localtime(role.opt_out["expire"])) 125 | return ResponderReturn( 126 | successful=False, 127 | return_message=( 128 | "Role {} in account {} is already opted out by {} for reason {} " 129 | "until {}".format( 130 | message.role_name, 131 | message.account, 132 | role.opt_out["owner"], 133 | role.opt_out["reason"], 134 | timestr, 135 | ) 136 | ), 137 | ) 138 | else: 139 | current_dt = datetime.datetime.fromtimestamp(time.time()) 140 | expire_dt = current_dt + datetime.timedelta(opt_out_period) 141 | expire_epoch = int((expire_dt - datetime.datetime(1970, 1, 1)).total_seconds()) 142 | new_opt_out = { 143 | "owner": message.requestor, 144 | "reason": message.reason, 145 | "expire": expire_epoch, 146 | } 147 | role.opt_out = new_opt_out 148 | try: 149 | role.store(fields=["opt_out"]) 150 | except RoleStoreError: 151 | return ResponderReturn( 152 | successful=False, 153 | return_message=f"Failed to opt out role {message.role_name} in account {message.account}", 154 | ) 155 | return ResponderReturn( 156 | successful=True, 157 | return_message="Role {} in account {} opted-out until {}".format( 158 | message.role_name, message.account, expire_dt.strftime("%m/%d/%y") 159 | ), 160 | ) 161 | 162 | 163 | @implements_command("remove_opt_out") 164 | def remove_opt_out(message: Message) -> ResponderReturn: 165 | role_id = find_role_in_cache(message.role_name, message.account) 166 | 167 | if not role_id: 168 | return ResponderReturn( 169 | successful=False, 170 | return_message="Unable to find role {} in account {}".format( 171 | message.role_name, message.account 172 | ), 173 | ) 174 | 175 | role = Role(role_id=role_id) 176 | role.fetch(fields=["OptOut"]) 177 | 178 | if not role.opt_out: 179 | return ResponderReturn( 180 | successful=False, 181 | return_message="Role {} in account {} wasn't opted out".format( 182 | message.role_name, message.account 183 | ), 184 | ) 185 | else: 186 | role.opt_out = {} 187 | try: 188 | role.store(fields=["opt_out"]) 189 | except RoleStoreError: 190 | return ResponderReturn( 191 | successful=False, 192 | return_message=f"Failed to cancel opt out for role {message.role_name} in account {message.account}", 193 | ) 194 | return ResponderReturn( 195 | successful=True, 196 | return_message="Cancelled opt-out for role {} in account {}".format( 197 | message.role_name, message.account 198 | ), 199 | ) 200 | 201 | 202 | @implements_command("rollback_role") 203 | def rollback_role(message: Message) -> ResponderReturn: 204 | if not message.selection: 205 | return ResponderReturn( 206 | successful=False, return_message="Rollback must contain a selection number" 207 | ) 208 | 209 | errors = _rollback_role( 210 | message.account, 211 | message.role_name, 212 | CONFIG, 213 | hooks, 214 | selection=int(message.selection), 215 | commit=True, 216 | ) 217 | if errors: 218 | return ResponderReturn( 219 | successful=False, return_message="Errors during rollback: {}".format(errors) 220 | ) 221 | else: 222 | return ResponderReturn( 223 | successful=True, 224 | return_message="Successfully rolled back role {} in account {}".format( 225 | message.role_name, message.account 226 | ), 227 | ) 228 | -------------------------------------------------------------------------------- /repokid/dispatcher/types.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from pydantic import Field 4 | from pydantic.main import BaseModel 5 | 6 | 7 | class Message(BaseModel): 8 | account: str 9 | command: str 10 | role_name: str 11 | respond_channel: str 12 | errors: List[str] = Field(default=[]) 13 | respond_user: str = Field(default="") 14 | requestor: str = Field(default="") 15 | reason: str = Field(default="") 16 | selection: str = Field(default="") 17 | -------------------------------------------------------------------------------- /repokid/exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | class RoleError(Exception): 15 | pass 16 | 17 | 18 | class UnexpectedDynamoUpdateValue(Exception): 19 | pass 20 | 21 | 22 | class BlocklistError(Exception): 23 | pass 24 | 25 | 26 | class AardvarkError(Exception): 27 | pass 28 | 29 | 30 | class RoleNotFoundError(RoleError): 31 | pass 32 | 33 | 34 | class MissingRepoableServices(RoleError): 35 | pass 36 | 37 | 38 | class RoleStoreError(RoleError): 39 | pass 40 | 41 | 42 | class IAMError(Exception): 43 | pass 44 | 45 | 46 | class ModelError(AttributeError): 47 | pass 48 | 49 | 50 | class DynamoDBError(Exception): 51 | pass 52 | 53 | 54 | class IntegrityError(Exception): 55 | pass 56 | 57 | 58 | class NotFoundError(Exception): 59 | pass 60 | 61 | 62 | class DynamoDBMaxItemSizeError(Exception): 63 | pass 64 | 65 | 66 | class IAMActionError(Exception): 67 | pass 68 | -------------------------------------------------------------------------------- /repokid/filters/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | from typing import List 5 | 6 | import import_string 7 | 8 | from repokid.role import RoleList 9 | from repokid.types import RepokidFilterConfig 10 | 11 | LOGGER = logging.getLogger("repokid") 12 | 13 | 14 | # inspiration from https://github.com/slackhq/python-rtmbot/blob/master/rtmbot/core.py 15 | class FilterPlugins: 16 | """ 17 | FilterPlugins is used to hold a list of instantiated plugins. The internal object filter_plugins contains a list 18 | of active plugins that can be iterated. 19 | """ 20 | 21 | def __init__(self) -> None: 22 | """Initialize empty list""" 23 | self.filter_plugins: List[Filter] = [] 24 | 25 | def load_plugin(self, module: str, config: RepokidFilterConfig = None) -> None: 26 | """Import a module by path, instantiate it with plugin specific config and add to the list of active plugins""" 27 | cls = None 28 | try: 29 | cls = import_string(module) 30 | except ImportError as e: 31 | LOGGER.warn("Unable to find plugin {}, exception: {}".format(module, e)) 32 | else: 33 | try: 34 | plugin = cls(config=config) 35 | except KeyError: 36 | plugin = cls() 37 | LOGGER.info("Loaded plugin {}".format(module)) 38 | self.filter_plugins.append(plugin) 39 | 40 | 41 | class Filter: 42 | """Base class for filter plugins to inherit. Passes config if supplied and requires the apply method be defined""" 43 | 44 | def __init__(self, config: RepokidFilterConfig = None) -> None: 45 | self.config = config 46 | 47 | def apply(self, input_list: RoleList) -> RoleList: 48 | raise NotImplementedError 49 | -------------------------------------------------------------------------------- /repokid/filters/age/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import datetime 16 | import logging 17 | 18 | from repokid.filters import Filter 19 | from repokid.role import RoleList 20 | 21 | log = logging.getLogger("repokid") 22 | 23 | 24 | class AgeFilter(Filter): 25 | def apply(self, input_list: RoleList) -> RoleList: 26 | now = datetime.datetime.now() 27 | if self.config: 28 | days_delta = self.config.get("minimum_age", 90) 29 | else: 30 | log.info("Minimum age not set in config, using default 90 days") 31 | days_delta = 90 32 | 33 | ago = datetime.timedelta(days=days_delta) 34 | 35 | too_young = RoleList([]) 36 | for role in input_list: 37 | if not role.create_date: 38 | log.warning(f"Role {role.role_name} is missing create_date") 39 | too_young.append(role) 40 | continue 41 | 42 | # Ensure create_date is an offset-naive datetime 43 | create_date = datetime.datetime.fromtimestamp(role.create_date.timestamp()) 44 | 45 | if create_date > now - ago: 46 | log.info( 47 | f"Role {role.role_name} created too recently to cleanup. ({create_date})" 48 | ) 49 | too_young.append(role) 50 | return too_young 51 | -------------------------------------------------------------------------------- /repokid/filters/blocklist/__init__.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | from typing import Any 4 | from typing import Dict 5 | from typing import Set 6 | 7 | import botocore 8 | from cloudaux.aws.sts import boto3_cached_conn 9 | 10 | from repokid.exceptions import BlocklistError 11 | from repokid.filters import Filter 12 | from repokid.role import RoleList 13 | from repokid.types import RepokidFilterConfig 14 | 15 | LOGGER = logging.getLogger("repokid") 16 | 17 | 18 | def get_blocklist_from_bucket(bucket_config: Dict[str, Any]) -> Dict[str, Any]: 19 | blocklist_json: Dict[str, Any] 20 | try: 21 | s3_resource = boto3_cached_conn( 22 | "s3", 23 | service_type="resource", 24 | account_number=bucket_config.get("account_number"), 25 | assume_role=bucket_config.get("assume_role", None), 26 | session_name="repokid", 27 | region=bucket_config.get("region", "us-west-2"), 28 | ) 29 | 30 | s3_obj = s3_resource.Object( 31 | bucket_name=bucket_config["bucket_name"], key=bucket_config["key"] 32 | ) 33 | blocklist = s3_obj.get()["Body"].read().decode("utf-8") 34 | blocklist_json = json.loads(blocklist) 35 | # Blocklist problems are really bad and we should quit rather than silently continue 36 | except (botocore.exceptions.ClientError, AttributeError): 37 | LOGGER.critical( 38 | "S3 blocklist config was set but unable to connect retrieve object, quitting" 39 | ) 40 | raise BlocklistError("Could not retrieve blocklist") 41 | except ValueError: 42 | LOGGER.critical( 43 | "S3 blocklist config was set but the returned file is bad, quitting" 44 | ) 45 | raise BlocklistError("Could not parse blocklist") 46 | if set(blocklist_json.keys()) != {"arns", "names"}: 47 | LOGGER.critical("S3 blocklist file is malformed, quitting") 48 | raise BlocklistError("Could not parse blocklist") 49 | return blocklist_json 50 | 51 | 52 | class BlocklistFilter(Filter): 53 | blocklist_json: Dict[str, Any] = {} 54 | 55 | def __init__(self, config: RepokidFilterConfig = None) -> None: 56 | super().__init__(config=config) 57 | if not config: 58 | LOGGER.error( 59 | "No configuration provided, cannot initialize Blocklist Filter" 60 | ) 61 | return 62 | current_account = config.get("current_account") or "" 63 | if not current_account: 64 | LOGGER.error("Unable to get current account for Blocklist Filter") 65 | 66 | blocklisted_role_names = set() 67 | blocklisted_role_names.update( 68 | [rolename.lower() for rolename in config.get(current_account, [])] 69 | ) 70 | blocklisted_role_names.update( 71 | [rolename.lower() for rolename in config.get("all", [])] 72 | ) 73 | 74 | if BlocklistFilter.blocklist_json: 75 | blocklisted_role_names.update( 76 | [ 77 | name.lower() 78 | for name, accounts in BlocklistFilter.blocklist_json[ 79 | "names" 80 | ].items() 81 | if ("all" in accounts or config.get("current_account") in accounts) 82 | ] 83 | ) 84 | 85 | self.blocklisted_arns: Set[str] = ( 86 | set() 87 | if not BlocklistFilter.blocklist_json 88 | else set(BlocklistFilter.blocklist_json.get("arns", [])) 89 | ) 90 | self.blocklisted_role_names = blocklisted_role_names 91 | 92 | @classmethod 93 | def init_blocklist(cls, config: RepokidFilterConfig) -> None: 94 | if not config: 95 | LOGGER.error("No config provided for blocklist filter") 96 | raise BlocklistError("No config provided for blocklist filter") 97 | if not cls.blocklist_json: 98 | bucket_config = config.get( 99 | "blocklist_bucket", config.get("blacklist_bucket", {}) 100 | ) 101 | if bucket_config: 102 | cls.blocklist_json = get_blocklist_from_bucket(bucket_config) 103 | 104 | def apply(self, input_list: RoleList) -> RoleList: 105 | blocklisted_roles = RoleList([]) 106 | 107 | for role in input_list: 108 | if ( 109 | role.role_name.lower() in self.blocklisted_role_names 110 | or role.arn in self.blocklisted_arns 111 | ): 112 | blocklisted_roles.append(role) 113 | return blocklisted_roles 114 | -------------------------------------------------------------------------------- /repokid/filters/exclusive/__init__.py: -------------------------------------------------------------------------------- 1 | import fnmatch 2 | import logging 3 | 4 | from repokid.filters import Filter 5 | from repokid.role import RoleList 6 | from repokid.types import RepokidFilterConfig 7 | 8 | LOGGER = logging.getLogger("repokid") 9 | 10 | 11 | class ExclusiveFilter(Filter): 12 | def __init__(self, config: RepokidFilterConfig = None): 13 | super().__init__(config=config) 14 | if not config: 15 | LOGGER.error( 16 | "No configuration provided, cannot initialize Exclusive Filter" 17 | ) 18 | return 19 | current_account = config.get("current_account", "") 20 | if not current_account: 21 | LOGGER.error("Unable to get current account for Exclusive Filter") 22 | 23 | exclusive_role_globs = set() 24 | exclusive_role_globs.update( 25 | [role_glob.lower() for role_glob in config.get(current_account, [])] 26 | ) 27 | exclusive_role_globs.update( 28 | [role_glob.lower() for role_glob in config.get("all", [])] 29 | ) 30 | 31 | self.exclusive_role_globs = exclusive_role_globs 32 | 33 | def apply(self, input_list: RoleList) -> RoleList: 34 | exclusive_roles = [] 35 | 36 | for role_glob in self.exclusive_role_globs: 37 | exclusive_roles += [ 38 | role 39 | for role in input_list 40 | if fnmatch.fnmatch(role.role_name.lower(), role_glob) 41 | ] 42 | filtered_roles = list(set(input_list) - set(exclusive_roles)) 43 | return RoleList(filtered_roles) 44 | -------------------------------------------------------------------------------- /repokid/filters/lambda/__init__.py: -------------------------------------------------------------------------------- 1 | from repokid.filters import Filter 2 | from repokid.role import RoleList 3 | 4 | 5 | class LambdaFilter(Filter): 6 | def apply(self, input_list: RoleList) -> RoleList: 7 | lambda_roles: RoleList = RoleList([]) 8 | 9 | for role in input_list: 10 | if "lambda" in str(role.assume_role_policy_document).lower(): 11 | lambda_roles.append(role) 12 | return lambda_roles 13 | -------------------------------------------------------------------------------- /repokid/filters/optout/__init__.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from repokid.filters import Filter 4 | from repokid.role import RoleList 5 | from repokid.types import RepokidFilterConfig 6 | 7 | 8 | class OptOutFilter(Filter): 9 | def __init__(self, config: RepokidFilterConfig = None) -> None: 10 | super().__init__(config=config) 11 | self.current_time_epoch = int(time.time()) 12 | 13 | def apply(self, input_list: RoleList) -> RoleList: 14 | opt_out_roles: RoleList = RoleList([]) 15 | 16 | for role in input_list: 17 | if role.opt_out and role.opt_out["expire"] > self.current_time_epoch: 18 | opt_out_roles.append(role) 19 | return opt_out_roles 20 | -------------------------------------------------------------------------------- /repokid/filters/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from repokid import CONFIG 4 | from repokid.filters import FilterPlugins 5 | from repokid.types import RepokidConfig 6 | 7 | 8 | def get_filter_plugins( 9 | account_number: str, config: Optional[RepokidConfig] = None 10 | ) -> FilterPlugins: 11 | config = config or CONFIG 12 | 13 | plugins = FilterPlugins() 14 | # Blocklist needs to know the current account 15 | filter_config = config["filter_config"] 16 | blocklist_filter_config = filter_config.get( 17 | "BlocklistFilter", filter_config.get("BlacklistFilter") 18 | ) 19 | blocklist_filter_config["current_account"] = account_number 20 | 21 | for plugin_path in config.get("active_filters", []): 22 | plugin_name = plugin_path.split(":")[1] 23 | if plugin_name == "ExclusiveFilter": 24 | # ExclusiveFilter plugin active; try loading its config. Also, it requires the current account, so add it. 25 | exclusive_filter_config = filter_config.get("ExclusiveFilter", {}) 26 | exclusive_filter_config["current_account"] = account_number 27 | plugins.load_plugin( 28 | plugin_path, config=config["filter_config"].get(plugin_name, None) 29 | ) 30 | 31 | return plugins 32 | -------------------------------------------------------------------------------- /repokid/hooks/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | 3 | from repokid.types import RepokidHook 4 | from repokid.types import RepokidHookInput 5 | from repokid.types import RepokidHookOutput 6 | from repokid.types import RepokidHooks 7 | 8 | 9 | def call_hooks( 10 | hooks_dict: RepokidHooks, hook_name: str, inputs_dict: RepokidHookInput 11 | ) -> RepokidHookOutput: 12 | """ 13 | Call all hooks of a given name in order. The output of one function is the input to the next. Return the final 14 | output. 15 | 16 | Args: 17 | hooks_dict: Dict with all functions to run for each hook name 18 | hook_name: The selected hook name to run 19 | inputs_dict: All required inputs 20 | 21 | Returns: 22 | dict: Outputs of the final function in the chain 23 | """ 24 | if hook_name not in hooks_dict: 25 | return inputs_dict 26 | 27 | for func in hooks_dict[hook_name]: 28 | inputs_dict = func(inputs_dict) 29 | if not inputs_dict: 30 | raise MissingOutputInHook("Function {} didn't return output".format(func)) 31 | return inputs_dict 32 | 33 | 34 | def implements_hook( 35 | hook_name: str, priority: int 36 | ) -> Callable[[RepokidHook], RepokidHook]: 37 | def _implements_hook(func: RepokidHook) -> RepokidHook: 38 | if not hasattr(func, "_implements_hook"): 39 | setattr( 40 | func, "_implements_hook", {"hook_name": hook_name, "priority": priority} 41 | ) 42 | return func 43 | 44 | return _implements_hook 45 | 46 | 47 | class MissingHookParameter(Exception): 48 | pass 49 | 50 | 51 | class MissingOutputInHook(Exception): 52 | pass 53 | -------------------------------------------------------------------------------- /repokid/hooks/loggers/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import repokid.hooks as hooks 4 | from repokid.role import Role 5 | from repokid.types import RepokidHookInput 6 | from repokid.types import RepokidHookOutput 7 | 8 | LOGGER = logging.getLogger("repokid") 9 | 10 | 11 | @hooks.implements_hook("BEFORE_REPO_ROLES", 1) 12 | def log_before_repo_roles(input_dict: RepokidHookInput) -> RepokidHookOutput: 13 | LOGGER.debug("Calling DURING_REPOABLE_CALCULATION hooks") 14 | if not all(required in input_dict for required in ["account_number", "roles"]): 15 | raise hooks.MissingHookParameter( 16 | "Did not get all required parameters for BEFORE_REPO_ROLES hook" 17 | ) 18 | return input_dict 19 | 20 | 21 | @hooks.implements_hook("DURING_REPOABLE_CALCULATION", 1) 22 | def log_during_repoable_calculation_hooks( 23 | input_dict: RepokidHookInput, 24 | ) -> RepokidHookOutput: 25 | LOGGER.debug("Calling DURING_REPOABLE_CALCULATION hooks") 26 | if not all( 27 | required in input_dict 28 | for required in [ 29 | "account_number", 30 | "role_name", 31 | "potentially_repoable_permissions", 32 | "minimum_age", 33 | ] 34 | ): 35 | raise hooks.MissingHookParameter( 36 | "Did not get all required parameters for DURING_REPOABLE_CALCULATION hook" 37 | ) 38 | return input_dict 39 | 40 | 41 | @hooks.implements_hook("DURING_REPOABLE_CALCULATION_BATCH", 1) 42 | def log_during_repoable_calculation_batch_hooks( 43 | input_dict: RepokidHookInput, 44 | ) -> RepokidHookOutput: 45 | LOGGER.debug("Calling DURING_REPOABLE_CALCULATION_BATCH hooks") 46 | 47 | if not all( 48 | required in input_dict 49 | for required in [ 50 | "role_batch", 51 | "potentially_repoable_permissions", 52 | "minimum_age", 53 | ] 54 | ): 55 | raise hooks.MissingHookParameter( 56 | "Did not get all required parameters for DURING_REPOABLE_CALCULATION_BATCH hook" 57 | ) 58 | for role in input_dict["role_batch"]: 59 | if not isinstance(role, Role): 60 | raise hooks.MissingHookParameter( 61 | "Role_batch needs to be a series of Role objects in DURING_REPOABLE_CALCULATION_BATCH hook" 62 | ) 63 | return input_dict 64 | 65 | 66 | @hooks.implements_hook("AFTER_SCHEDULE_REPO", 1) 67 | def log_after_schedule_repo_hooks(input_dict: RepokidHookInput) -> RepokidHookOutput: 68 | LOGGER.debug("Calling AFTER_SCHEDULE_REPO hooks") 69 | if "roles" not in input_dict: 70 | raise hooks.MissingHookParameter( 71 | "Required key 'roles' not passed to AFTER_SCHEDULE_REPO" 72 | ) 73 | return input_dict 74 | 75 | 76 | @hooks.implements_hook("AFTER_REPO", 1) 77 | def log_after_repo_hooks(input_dict: RepokidHookInput) -> RepokidHookOutput: 78 | LOGGER.debug("Calling AFTER_REPO hooks") 79 | if "role" not in input_dict: 80 | raise hooks.MissingHookParameter("Required key 'role' not passed to AFTER_REPO") 81 | return input_dict 82 | -------------------------------------------------------------------------------- /repokid/lib/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | This module contains wrapper functions for the functions contained in the child 16 | modules so developers don't have to worry about passing configs, hooks, and dynamo 17 | clients. 18 | """ 19 | from typing import List 20 | 21 | from repokid import CONFIG 22 | from repokid import get_hooks 23 | from repokid.commands.repo import _repo_all_roles 24 | from repokid.commands.repo import _repo_role 25 | from repokid.commands.repo import _repo_stats 26 | from repokid.commands.repo import _rollback_role 27 | from repokid.commands.role import _display_role 28 | from repokid.commands.role import _display_roles 29 | from repokid.commands.role import _find_roles_with_permissions 30 | from repokid.commands.role import _remove_permissions_from_roles 31 | from repokid.commands.role_cache import _update_role_cache 32 | from repokid.commands.schedule import _cancel_scheduled_repo 33 | from repokid.commands.schedule import _schedule_repo 34 | from repokid.commands.schedule import _show_scheduled_roles 35 | 36 | hooks = get_hooks(CONFIG.get("hooks", ["repokid.hooks.loggers"])) 37 | 38 | 39 | def update_role_cache(account_number: str) -> None: 40 | """ 41 | Library wrapper to update data about all roles in a given account. 42 | 43 | Ref: :func:`~repokid.commands.role_cache._update_role_cache` 44 | 45 | Args: 46 | account_number (string): The current account number Repokid is being run against 47 | 48 | Returns: 49 | None 50 | """ 51 | return _update_role_cache(account_number, CONFIG, hooks) 52 | 53 | 54 | def display_role_cache(account_number: str, inactive: bool = False) -> None: 55 | """ 56 | Library wrapper to display a table with data about all roles in an account and write a csv file with the data. 57 | 58 | Ref: :func:`~repokid.commands.role_cache._display_roles` 59 | 60 | Args: 61 | account_number (string): The current account number Repokid is being run against 62 | inactive (bool): show roles that have historically (but not currently) existed in the account if True 63 | 64 | Returns: 65 | None 66 | """ 67 | return _display_roles(account_number, inactive=inactive) 68 | 69 | 70 | def find_roles_with_permissions(permissions: List[str], output_file: str = "") -> None: 71 | """ 72 | Library wrapper to search roles in all accounts for a policy with any of the provided permissions, log the ARN of 73 | each role. 74 | 75 | Ref: :func:`~repokid.commands.role._find_roles_with_permissions` 76 | 77 | Args: 78 | permissions (list[string]): The name of the permissions to find 79 | output_file (string): filename to write the output 80 | 81 | Returns: 82 | None 83 | """ 84 | return _find_roles_with_permissions(permissions, output_file) 85 | 86 | 87 | def remove_permissions_from_roles( 88 | permissions: List[str], role_filename: str, commit: bool = False 89 | ) -> None: 90 | """ 91 | Library wrapper to loads role specified in file and call _remove_permissions_from_role() for each one. 92 | 93 | Ref: :func:`~repokid.commands.role._remove_permissions_from_roles` 94 | 95 | Args: 96 | permissions (list) 97 | role_filename (string) 98 | commit (bool) 99 | 100 | Returns: 101 | None 102 | """ 103 | return _remove_permissions_from_roles( 104 | permissions, role_filename, CONFIG, hooks, commit=commit 105 | ) 106 | 107 | 108 | def display_role(account_number: str, role_name: str) -> None: 109 | """ 110 | Library wrapper to display data about a role in a given account 111 | 112 | Ref: :func:`~repokid.commands.role._display_role` 113 | 114 | Args: 115 | account_number (string): The current account number Repokid is being run against 116 | role_name (string) 117 | 118 | Returns: 119 | None 120 | """ 121 | return _display_role(account_number, role_name, CONFIG) 122 | 123 | 124 | def repo_role( 125 | account_number: str, role_name: str, commit: bool = False, update: bool = True 126 | ) -> List[str]: 127 | """ 128 | Library wrapper to calculate what repoing can be done for a role and then actually do it if commit is set. 129 | 130 | Ref: :func:`~repokid.commands.repo._repo_role` 131 | 132 | Args: 133 | account_number (string): The current account number Repokid is being run against 134 | role_name (string) 135 | commit (bool) 136 | update (bool) 137 | 138 | Returns: 139 | errors (list): if any 140 | """ 141 | return _repo_role(account_number, role_name, CONFIG, hooks, commit=commit) 142 | 143 | 144 | def rollback_role( 145 | account_number: str, role_name: str, selection: int = 0, commit: bool = False 146 | ) -> List[str]: 147 | """ 148 | Library wrapper to display the historical policy versions for a roll as a numbered list. Restore to a specific 149 | version if selected. Indicate changes that will be made and then actually make them if commit is selected. 150 | 151 | Ref: :func:`~repokid.commands.repo._rollback_role` 152 | 153 | Args: 154 | account_number (string): The current account number Repokid is being run against 155 | role_name (string) 156 | selection (int): which policy version in the list to rollback to 157 | commit (bool): actually make the change 158 | 159 | Returns: 160 | errors (list): if any 161 | """ 162 | return _rollback_role( 163 | account_number, role_name, CONFIG, hooks, selection=selection, commit=commit 164 | ) 165 | 166 | 167 | def schedule_repo(account_number: str) -> None: 168 | """ 169 | Library wrapper to schedule a repo for a given account. Schedule repo for a time in the future (default 7 days) for 170 | any roles in the account with repoable permissions. 171 | 172 | Ref: :func:`~repokid.commands.repo._repo_all_roles` 173 | 174 | Args: 175 | account_number (string): The current account number Repokid is being run against 176 | 177 | Returns: 178 | None 179 | """ 180 | _update_role_cache(account_number, CONFIG, hooks) 181 | return _schedule_repo(account_number, CONFIG, hooks) 182 | 183 | 184 | def repo_all_roles( 185 | account_number: str, commit: bool = False, update: bool = True, limit: int = -1 186 | ) -> None: 187 | """ 188 | Convenience wrapper for repo_roles() with scheduled=False. 189 | 190 | Ref: :func:`~repokid.commands.repo_roles` 191 | 192 | Args: 193 | account_number (string): The current account number Repokid is being run against 194 | commit (bool): actually make the changes 195 | update (bool): if True run update_role_cache before repoing 196 | limit (int): limit number of roles to be repoed per run (< 0 is unlimited) 197 | 198 | Returns: 199 | None 200 | """ 201 | return repo_roles( 202 | account_number, commit=commit, scheduled=False, update=update, limit=limit 203 | ) 204 | 205 | 206 | def repo_scheduled_roles( 207 | account_number: str, commit: bool = False, update: bool = True, limit: int = -1 208 | ) -> None: 209 | """ 210 | Convenience wrapper for repo_roles() with scheduled=True. 211 | 212 | Ref: :func:`~repokid.commands.repo_roles` 213 | 214 | Args: 215 | account_number (string): The current account number Repokid is being run against 216 | commit (bool): actually make the changes 217 | update (bool): if True run update_role_cache before repoing 218 | limit (int): limit number of roles to be repoed per run (< 0 is unlimited) 219 | 220 | Returns: 221 | None 222 | """ 223 | return repo_roles( 224 | account_number, commit=commit, scheduled=True, update=update, limit=limit 225 | ) 226 | 227 | 228 | def repo_roles( 229 | account_number: str, 230 | commit: bool = False, 231 | scheduled: bool = False, 232 | update: bool = True, 233 | limit: int = -1, 234 | ) -> None: 235 | """ 236 | Library wrapper to repo all scheduled or eligible roles in an account. Collect any errors and display them at the 237 | end. 238 | 239 | Ref: :func:`~repokid.commands.repo._repo_all_roles` 240 | 241 | Args: 242 | account_number (string): The current account number Repokid is being run against 243 | commit (bool): actually make the changes 244 | scheduled (bool): if True only repo the scheduled roles, if False repo all the (eligible) roles 245 | update (bool): if True run update_role_cache before repoing 246 | limit (int): limit number of roles to be repoed per run (< 0 is unlimited) 247 | 248 | Returns: 249 | None 250 | """ 251 | if update: 252 | _update_role_cache(account_number, CONFIG, hooks) 253 | return _repo_all_roles( 254 | account_number, CONFIG, hooks, commit=commit, scheduled=scheduled, limit=limit 255 | ) 256 | 257 | 258 | def show_scheduled_roles(account_number: str) -> None: 259 | """ 260 | Library wrapper to show scheduled repos for a given account. For each scheduled show whether scheduled time is 261 | elapsed or not. 262 | 263 | Ref: :func:`~repokid.commands.schedule._show_scheduled_roles` 264 | 265 | Args: 266 | account_number (string): The current account number Repokid is being run against 267 | 268 | Returns: 269 | None 270 | """ 271 | return _show_scheduled_roles(account_number) 272 | 273 | 274 | def cancel_scheduled_repo( 275 | account_number: str, role_name: str = "", is_all: bool = False 276 | ) -> None: 277 | """ 278 | Library wrapper to cancel scheduled repo for a role in an account. 279 | 280 | Ref: :func:`~repokid.commands.schedule._cancel_scheduled_repo` 281 | 282 | Args: 283 | account_number (string): The current account number Repokid is being run against 284 | role_name (string): Role name to cancel scheduled repo for 285 | is_all (bool): Cancel schedule repos on all roles if True 286 | 287 | Returns: 288 | None 289 | """ 290 | return _cancel_scheduled_repo(account_number, role_name=role_name, is_all=is_all) 291 | 292 | 293 | def repo_stats(output_filename: str = "", account_number: str = "") -> None: 294 | """ 295 | Library wrapper to create a csv file with stats about roles, total permissions, and applicable filters over time. 296 | 297 | Ref: :func:`~repokid.commands.repo._repo_stats` 298 | 299 | Args: 300 | output_filename (string): the name of the csv file to write 301 | account_number (string): if specified only display roles from selected account, otherwise display all 302 | 303 | Returns: 304 | None 305 | """ 306 | return _repo_stats(output_filename, account_number=account_number) 307 | -------------------------------------------------------------------------------- /repokid/plugin.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from __future__ import annotations 15 | 16 | import logging 17 | from typing import Any 18 | from typing import Dict 19 | from typing import Optional 20 | 21 | from repokid import CONFIG 22 | from repokid.types import RepokidConfig 23 | 24 | logger = logging.getLogger("repokid") 25 | 26 | 27 | class RepokidPlugin: 28 | def __init__(self, config: Optional[RepokidConfig] = None): 29 | if config: 30 | self.config = config 31 | else: 32 | self.config = CONFIG 33 | 34 | 35 | class M_A(type): 36 | pass 37 | 38 | 39 | class Singleton(M_A): 40 | _instances: Dict[str, Singleton] = {} 41 | 42 | def __call__(cls, *args: Any, **kwargs: Any) -> Singleton: 43 | if cls.__name__ not in cls._instances: 44 | cls._instances[cls.__name__] = super(Singleton, cls).__call__( 45 | *args, **kwargs 46 | ) 47 | return cls._instances[cls.__name__] 48 | -------------------------------------------------------------------------------- /repokid/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/repokid/376aa82ed31fe66ac4b1aecc3c22b0fc4fcfc0ea/repokid/py.typed -------------------------------------------------------------------------------- /repokid/types.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from typing import Any 15 | from typing import Callable 16 | from typing import DefaultDict 17 | from typing import Dict 18 | from typing import List 19 | from typing import Optional 20 | from typing import TypeVar 21 | 22 | RepokidConfig = Dict[str, Any] 23 | RepokidFilterConfig = Optional[Dict[str, Any]] 24 | RepokidHook = Callable[[Dict[str, Any]], Dict[str, Any]] 25 | RepokidHooks = DefaultDict[str, List[RepokidHook]] 26 | RepokidHookInput = Dict[str, Any] 27 | RepokidHookOutput = RepokidHookInput 28 | AccessAdvisorEntry = List[Dict[str, Any]] 29 | AardvarkResponse = Dict[str, AccessAdvisorEntry] 30 | IAMEntry = Dict[str, Any] 31 | 32 | # Reusable typevars for generics 33 | KT = TypeVar("KT") 34 | VT = TypeVar("VT") 35 | -------------------------------------------------------------------------------- /repokid/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/repokid/376aa82ed31fe66ac4b1aecc3c22b0fc4fcfc0ea/repokid/utils/__init__.py -------------------------------------------------------------------------------- /repokid/utils/iam.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import datetime 15 | import json 16 | import logging 17 | import re 18 | from typing import Any 19 | from typing import Dict 20 | 21 | import botocore 22 | from cloudaux.aws.iam import delete_role_policy 23 | from cloudaux.aws.iam import put_role_policy 24 | from cloudaux.aws.sts import boto3_cached_conn 25 | from mypy_boto3_iam.client import IAMClient 26 | 27 | from repokid.exceptions import IAMError 28 | 29 | LOGGER = logging.getLogger("repokid") 30 | MAX_AWS_POLICY_SIZE = 10240 31 | 32 | 33 | def update_repoed_description(role_name: str, conn_details: Dict[str, Any]) -> None: 34 | client: IAMClient = boto3_cached_conn("iam", **conn_details) 35 | try: 36 | description = client.get_role(RoleName=role_name)["Role"].get("Description", "") 37 | except KeyError: 38 | return 39 | date_string = datetime.datetime.now(tz=datetime.timezone.utc).strftime("%m/%d/%y") 40 | if "; Repokid repoed" in description: 41 | new_description = re.sub( 42 | r"; Repokid repoed [0-9]{2}\/[0-9]{2}\/[0-9]{2}", 43 | f"; Repokid repoed {date_string}", 44 | description, 45 | ) 46 | else: 47 | new_description = description + " ; Repokid repoed {}".format(date_string) 48 | # IAM role descriptions have a max length of 1000, if our new length would be longer, skip this 49 | if len(new_description) < 1000: 50 | client.update_role_description(RoleName=role_name, Description=new_description) 51 | else: 52 | LOGGER.error( 53 | "Unable to set repo description ({}) for role {}, length would be too long".format( 54 | new_description, role_name 55 | ) 56 | ) 57 | 58 | 59 | def inline_policies_size_exceeds_maximum(policies: Dict[str, Any]) -> bool: 60 | """Validate the policies, when converted to JSON without whitespace, remain under the size limit. 61 | 62 | Args: 63 | policies (list) 64 | Returns: 65 | bool 66 | """ 67 | exported_no_whitespace = json.dumps(policies, separators=(",", ":")) 68 | if len(exported_no_whitespace) > MAX_AWS_POLICY_SIZE: 69 | return True 70 | return False 71 | 72 | 73 | def delete_policy( 74 | name: str, role_name: str, account_number: str, conn: Dict[str, Any] 75 | ) -> None: 76 | """Deletes the specified IAM Role inline policy. 77 | 78 | Args: 79 | name (string) 80 | role (Role object) 81 | account_number (string) 82 | conn (dict) 83 | 84 | Returns: 85 | error (string) or None 86 | """ 87 | LOGGER.info( 88 | "Deleting policy with name {} from {} in account {}".format( 89 | name, role_name, account_number 90 | ) 91 | ) 92 | try: 93 | delete_role_policy(RoleName=role_name, PolicyName=name, **conn) 94 | except botocore.exceptions.ClientError as e: 95 | raise IAMError( 96 | f"Error deleting policy: {name} from role: {role_name} in account {account_number}" 97 | ) from e 98 | 99 | 100 | def replace_policies( 101 | repoed_policies: Dict[str, Any], 102 | role_name: str, 103 | account_number: str, 104 | conn: Dict[str, Any], 105 | ) -> None: 106 | """Overwrite IAM Role inline policies with those supplied. 107 | 108 | Args: 109 | repoed_policies (dict) 110 | role (Role object) 111 | account_number (string) 112 | conn (dict) 113 | 114 | Returns: 115 | error (string) or None 116 | """ 117 | LOGGER.info( 118 | "Replacing Policies With: \n{} (role: {} account: {})".format( 119 | json.dumps(repoed_policies, indent=2, sort_keys=True), 120 | role_name, 121 | account_number, 122 | ) 123 | ) 124 | 125 | for policy_name, policy in repoed_policies.items(): 126 | try: 127 | put_role_policy( 128 | RoleName=role_name, 129 | PolicyName=policy_name, 130 | PolicyDocument=json.dumps(policy, indent=2, sort_keys=True), 131 | **conn, 132 | ) 133 | 134 | except botocore.exceptions.ClientError as e: 135 | error = "Exception calling PutRolePolicy on {role}/{policy} in account {account}".format( 136 | role=role_name, 137 | policy=policy_name, 138 | account=account_number, 139 | ) 140 | raise IAMError(error) from e 141 | -------------------------------------------------------------------------------- /repokid/utils/logging.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import json 15 | import logging 16 | from typing import Any 17 | from typing import Dict 18 | from typing import List 19 | 20 | LOGGER = logging.getLogger("repokid") 21 | 22 | 23 | def log_deleted_and_repoed_policies( 24 | deleted_policy_names: List[str], 25 | repoed_policies: Dict[str, Any], 26 | role_name: str, 27 | account_number: str, 28 | ) -> None: 29 | """Logs data on policies that would otherwise be modified or deleted if the commit flag were set. 30 | 31 | Args: 32 | deleted_policy_names (list) 33 | repoed_policies (list) 34 | role_name (string) 35 | account_number (string) 36 | 37 | Returns: 38 | None 39 | """ 40 | for name in deleted_policy_names: 41 | LOGGER.info( 42 | "Would delete policy from {} with name {} in account {}".format( 43 | role_name, name, account_number 44 | ) 45 | ) 46 | 47 | if repoed_policies: 48 | LOGGER.info( 49 | "Would replace policies for role {} with: \n{} in account {}".format( 50 | role_name, 51 | json.dumps(repoed_policies, indent=2, sort_keys=True), 52 | account_number, 53 | ) 54 | ) 55 | -------------------------------------------------------------------------------- /repokid/utils/roledata.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import copy 15 | import logging 16 | from typing import Any 17 | from typing import Dict 18 | from typing import List 19 | from typing import Set 20 | from typing import Tuple 21 | 22 | import repokid.hooks 23 | from repokid.role import Role 24 | from repokid.role import RoleList 25 | from repokid.types import RepokidHooks 26 | from repokid.utils.dynamo import get_all_role_ids_for_account 27 | from repokid.utils.permissions import _get_potentially_repoable_permissions 28 | 29 | LOGGER = logging.getLogger("repokid") 30 | 31 | 32 | def find_and_mark_inactive(account_number: str, active_roles: RoleList) -> None: 33 | """ 34 | Mark roles in the account that aren't currently active inactive. Do this by getting all roles in the account and 35 | subtracting the active roles, any that are left are inactive and should be marked thusly. 36 | 37 | Args: 38 | account_number (string) 39 | active_roles (set): the currently active roles discovered in the most recent scan 40 | 41 | Returns: 42 | None 43 | """ 44 | known_roles = set(get_all_role_ids_for_account(account_number)) 45 | inactive_roles: Set[Role] = { 46 | role for role in active_roles if role.role_id not in known_roles 47 | } 48 | 49 | for role in inactive_roles: 50 | if role.active: 51 | role.mark_inactive() 52 | 53 | 54 | def _convert_repoed_service_to_sorted_perms_and_services( 55 | repoed_services: Set[str], 56 | ) -> Tuple[List[str], List[str]]: 57 | """ 58 | Repokid stores a field RepoableServices that historically only stored services (when Access Advisor was only data). 59 | Now this field is repurposed to store both services and permissions. We can tell the difference because permissions 60 | always have the form :. This function splits the contents of the field to sorted sets of 61 | repoable services and permissions. 62 | 63 | Args: 64 | repoed_services (list): List from Dynamo of repoable services and permissions 65 | 66 | Returns: 67 | list: Sorted list of repoable permissions (where there are other permissions that aren't repoed) 68 | list: Sorted list of repoable services (where the entire service is removed) 69 | """ 70 | repoable_permissions = set() 71 | repoable_services = set() 72 | 73 | for entry in repoed_services: 74 | if len(entry.split(":")) == 2: 75 | repoable_permissions.add(entry) 76 | else: 77 | repoable_services.add(entry) 78 | 79 | return sorted(repoable_permissions), sorted(repoable_services) 80 | 81 | 82 | def _get_repoable_permissions_batch( 83 | repo_able_roles: RoleList, 84 | permissions_dict: Dict[str, Any], 85 | minimum_age: int, 86 | hooks: RepokidHooks, 87 | batch_size: int, 88 | ) -> Dict[str, Any]: 89 | """ 90 | Generate a dictionary mapping of role arns to their repoable permissions based on the list of all permissions the 91 | role's policies currently allow and Access Advisor data for the services included in the role's policies. 92 | 93 | The first step is to come up with a list of services that were used within the time threshold (the same defined) 94 | in the age filter config. Permissions are repoable if they aren't in the used list, aren't in the constant list 95 | of unsupported services/actions (IAM_ACCESS_ADVISOR_UNSUPPORTED_SERVICES, IAM_ACCESS_ADVISOR_UNSUPPORTED_ACTIONS), 96 | and aren't being temporarily ignored because they're on the no_repo_permissions list (newly added). 97 | 98 | Args: 99 | repo_able_roles: (list): List of the roles that can be checked for repoing 100 | permissions_dict (dict): Mapping role arns to their full list of permissions that the role's permissions allow 101 | minimum_age: Minimum age of a role (in days) for it to be repoable 102 | hooks: Dict containing hook names and functions to run 103 | 104 | Returns: 105 | dict: Mapping role arns to set of permissions that are 'repoable' (not used within the time threshold) 106 | """ 107 | 108 | if len(repo_able_roles) == 0: 109 | return {} 110 | 111 | repo_able_roles_batches = copy.deepcopy(repo_able_roles) 112 | potentially_repoable_permissions_dict = {} 113 | repoable_dict = {} 114 | repoable_log_dict = {} 115 | 116 | for role in repo_able_roles: 117 | potentially_repoable_permissions_dict[ 118 | role.arn 119 | ] = _get_potentially_repoable_permissions( 120 | role.role_name, 121 | role.account, 122 | role.aa_data or [], 123 | permissions_dict[role.arn], 124 | role.no_repo_permissions, 125 | minimum_age, 126 | ) 127 | 128 | while len(repo_able_roles_batches) > 0: 129 | role_batch = repo_able_roles_batches[:batch_size] 130 | repo_able_roles_batches = repo_able_roles_batches[batch_size:] 131 | 132 | hooks_output = repokid.hooks.call_hooks( 133 | hooks, 134 | "DURING_REPOABLE_CALCULATION_BATCH", 135 | { 136 | "role_batch": role_batch, 137 | "potentially_repoable_permissions": potentially_repoable_permissions_dict, 138 | "minimum_age": minimum_age, 139 | }, 140 | ) 141 | for role_arn, output in list(hooks_output.items()): 142 | repoable = { 143 | permission_name 144 | for permission_name, permission_value in list( 145 | output["potentially_repoable_permissions"].items() 146 | ) 147 | if permission_value.repoable 148 | } 149 | repoable_dict[role_arn] = repoable 150 | repoable_log_dict[role_arn] = "".join( 151 | "{}: {}\n".format(perm, decision.decider) 152 | for perm, decision in list( 153 | output["potentially_repoable_permissions"].items() 154 | ) 155 | ) 156 | 157 | for role in repo_able_roles: 158 | LOGGER.debug( 159 | "Repoable permissions for role {role_name} in {account_number}:\n{repoable}".format( 160 | role_name=role.role_name, 161 | account_number=role.account, 162 | repoable=repoable_log_dict[role.arn], 163 | ) 164 | ) 165 | return repoable_dict 166 | -------------------------------------------------------------------------------- /requirements-test.in: -------------------------------------------------------------------------------- 1 | bandit 2 | black 3 | coveralls 4 | flake8 5 | flake8-import-order 6 | mypy 7 | pre-commit 8 | python-dateutil 9 | mock 10 | pytest 11 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --no-emit-index-url --output-file=requirements-test.txt requirements-test.in 6 | # 7 | appdirs==1.4.4 8 | # via black 9 | attrs==21.2.0 10 | # via pytest 11 | backports.entry-points-selectable==1.1.0 12 | # via virtualenv 13 | bandit==1.7.0 14 | # via -r requirements-test.in 15 | black==21.7b0 16 | # via -r requirements-test.in 17 | certifi==2021.5.30 18 | # via requests 19 | cfgv==3.3.1 20 | # via pre-commit 21 | charset-normalizer==2.0.4 22 | # via requests 23 | click==8.0.1 24 | # via black 25 | coverage==5.5 26 | # via coveralls 27 | coveralls==3.2.0 28 | # via -r requirements-test.in 29 | distlib==0.3.2 30 | # via virtualenv 31 | docopt==0.6.2 32 | # via coveralls 33 | filelock==3.0.12 34 | # via virtualenv 35 | flake8-import-order==0.18.1 36 | # via -r requirements-test.in 37 | flake8==3.9.2 38 | # via -r requirements-test.in 39 | gitdb==4.0.7 40 | # via gitpython 41 | gitpython==3.1.18 42 | # via bandit 43 | identify==2.2.13 44 | # via pre-commit 45 | idna==3.2 46 | # via requests 47 | importlib-metadata==4.6.4 48 | # via 49 | # backports.entry-points-selectable 50 | # click 51 | # flake8 52 | # pluggy 53 | # pre-commit 54 | # pytest 55 | # stevedore 56 | # virtualenv 57 | iniconfig==1.1.1 58 | # via pytest 59 | mccabe==0.6.1 60 | # via flake8 61 | mock==4.0.3 62 | # via -r requirements-test.in 63 | mypy-extensions==0.4.3 64 | # via 65 | # black 66 | # mypy 67 | mypy==0.910 68 | # via -r requirements-test.in 69 | nodeenv==1.6.0 70 | # via pre-commit 71 | packaging==21.0 72 | # via pytest 73 | pathspec==0.9.0 74 | # via black 75 | pbr==5.6.0 76 | # via stevedore 77 | platformdirs==2.2.0 78 | # via virtualenv 79 | pluggy==0.13.1 80 | # via pytest 81 | pre-commit==2.14.0 82 | # via -r requirements-test.in 83 | py==1.10.0 84 | # via pytest 85 | pycodestyle==2.7.0 86 | # via 87 | # flake8 88 | # flake8-import-order 89 | pyflakes==2.3.1 90 | # via flake8 91 | pyparsing==2.4.7 92 | # via packaging 93 | pytest==6.2.4 94 | # via -r requirements-test.in 95 | python-dateutil==2.8.2 96 | # via -r requirements-test.in 97 | pyyaml==5.4.1 98 | # via 99 | # bandit 100 | # pre-commit 101 | regex==2021.8.21 102 | # via black 103 | requests==2.26.0 104 | # via coveralls 105 | six==1.16.0 106 | # via 107 | # bandit 108 | # python-dateutil 109 | # virtualenv 110 | smmap==4.0.0 111 | # via gitdb 112 | stevedore==3.4.0 113 | # via bandit 114 | toml==0.10.2 115 | # via 116 | # mypy 117 | # pre-commit 118 | # pytest 119 | tomli==1.2.1 120 | # via black 121 | typed-ast==1.4.3 122 | # via 123 | # black 124 | # mypy 125 | typing-extensions==3.10.0.0 126 | # via 127 | # black 128 | # gitpython 129 | # importlib-metadata 130 | # mypy 131 | urllib3==1.26.6 132 | # via requests 133 | virtualenv==20.7.2 134 | # via pre-commit 135 | zipp==3.5.0 136 | # via importlib-metadata 137 | 138 | # The following packages are considered to be unsafe in a requirements file: 139 | # setuptools 140 | -------------------------------------------------------------------------------- /requirements.in: -------------------------------------------------------------------------------- 1 | boto3<=1.17.71 2 | boto3-stubs[dynamodb,iam,sns,sqs]<=1.17.71 3 | cloudaux 4 | click 5 | import_string 6 | json_log_formatter 7 | policyuniverse 8 | requests 9 | tabulate 10 | tabview 11 | tqdm 12 | pip-tools 13 | pydantic 14 | python-dateutil 15 | pytz 16 | raven 17 | twine 18 | types-click 19 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --no-emit-index-url --output-file=requirements.txt requirements.in 6 | # 7 | bleach==4.1.0 8 | # via readme-renderer 9 | boto3-stubs[dynamodb,iam,sns,sqs]==1.17.71 10 | # via -r requirements.in 11 | boto3==1.17.71 12 | # via 13 | # -r requirements.in 14 | # cloudaux 15 | boto==2.49.0 16 | # via cloudaux 17 | botocore==1.20.112 18 | # via 19 | # boto3 20 | # cloudaux 21 | # s3transfer 22 | certifi==2021.5.30 23 | # via requests 24 | charset-normalizer==2.0.4 25 | # via requests 26 | click==8.0.1 27 | # via 28 | # -r requirements.in 29 | # pip-tools 30 | cloudaux==1.9.6 31 | # via -r requirements.in 32 | colorama==0.4.4 33 | # via twine 34 | defusedxml==0.7.1 35 | # via cloudaux 36 | docutils==0.17.1 37 | # via readme-renderer 38 | flagpole==1.1.1 39 | # via cloudaux 40 | idna==3.2 41 | # via requests 42 | import-string==0.1.0 43 | # via -r requirements.in 44 | importlib-metadata==4.6.4 45 | # via 46 | # click 47 | # keyring 48 | # pep517 49 | # twine 50 | inflection==0.5.1 51 | # via cloudaux 52 | jmespath==0.10.0 53 | # via 54 | # boto3 55 | # botocore 56 | joblib==1.0.1 57 | # via cloudaux 58 | json-log-formatter==0.4.0 59 | # via -r requirements.in 60 | keyring==23.1.0 61 | # via twine 62 | mypy-boto3-dynamodb==1.17.71 63 | # via boto3-stubs 64 | mypy-boto3-iam==1.17.71 65 | # via boto3-stubs 66 | mypy-boto3-sns==1.17.71 67 | # via boto3-stubs 68 | mypy-boto3-sqs==1.17.71 69 | # via boto3-stubs 70 | packaging==21.0 71 | # via bleach 72 | pep517==0.11.0 73 | # via pip-tools 74 | pip-tools==6.2.0 75 | # via -r requirements.in 76 | pkginfo==1.7.1 77 | # via twine 78 | policyuniverse==1.4.0.20210816 79 | # via -r requirements.in 80 | pydantic==1.8.2 81 | # via -r requirements.in 82 | pygments==2.10.0 83 | # via readme-renderer 84 | pyparsing==2.4.7 85 | # via packaging 86 | python-dateutil==2.8.2 87 | # via 88 | # -r requirements.in 89 | # botocore 90 | pytz==2021.1 91 | # via -r requirements.in 92 | raven==6.10.0 93 | # via -r requirements.in 94 | readme-renderer==29.0 95 | # via twine 96 | requests-toolbelt==0.9.1 97 | # via twine 98 | requests==2.26.0 99 | # via 100 | # -r requirements.in 101 | # requests-toolbelt 102 | # twine 103 | rfc3986==1.5.0 104 | # via twine 105 | s3transfer==0.4.2 106 | # via boto3 107 | six==1.16.0 108 | # via 109 | # bleach 110 | # cloudaux 111 | # import-string 112 | # python-dateutil 113 | # readme-renderer 114 | tabulate==0.8.9 115 | # via -r requirements.in 116 | tabview==1.4.4 117 | # via -r requirements.in 118 | tomli==1.2.1 119 | # via pep517 120 | tqdm==4.62.2 121 | # via 122 | # -r requirements.in 123 | # twine 124 | twine==3.4.2 125 | # via -r requirements.in 126 | types-click==7.1.5 127 | # via -r requirements.in 128 | typing-extensions==3.10.0.0 129 | # via 130 | # boto3-stubs 131 | # importlib-metadata 132 | # mypy-boto3-dynamodb 133 | # mypy-boto3-iam 134 | # mypy-boto3-sns 135 | # mypy-boto3-sqs 136 | # pydantic 137 | urllib3==1.26.6 138 | # via 139 | # botocore 140 | # requests 141 | webencodings==0.5.1 142 | # via bleach 143 | wheel==0.37.0 144 | # via pip-tools 145 | zipp==3.5.0 146 | # via 147 | # importlib-metadata 148 | # pep517 149 | 150 | # The following packages are considered to be unsafe in a requirements file: 151 | # pip 152 | # setuptools 153 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | import-order-style = google 3 | application-import-names = repokid 4 | max-line-length = 120 5 | 6 | [mypy] 7 | plugins = pydantic.mypy 8 | ignore_missing_imports = True 9 | show_error_codes = True 10 | 11 | [mypy-tests.*] 12 | ignore_errors = True 13 | 14 | [pydantic-mypy] 15 | init_forbid_extra = True 16 | init_typed = True 17 | warn_required_dynamic_aliases = True 18 | warn_untyped_fields = True 19 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from setuptools import find_packages 16 | from setuptools import setup 17 | 18 | setup( 19 | name="repokid", 20 | description="AWS Least Privilege for Distributed, High-Velocity Deployment", 21 | url="https://github.com/Netflix/repokid", 22 | packages=find_packages(), 23 | package_data={"repokid": ["py.typed"]}, 24 | versioning="dev", 25 | setup_requires=["setupmeta"], 26 | python_requires=">=3.7", 27 | keywords=["aws", "iam", "access_advisor"], 28 | entry_points={ 29 | "console_scripts": [ 30 | "repokid = repokid.cli.repokid_cli:cli", 31 | "dispatcher = repokid.cli.dispatcher_cli:main", 32 | ] 33 | }, 34 | classifiers=[ 35 | "Development Status :: 5 - Production/Stable", 36 | "Intended Audience :: Developers", 37 | "Intended Audience :: System Administrators", 38 | "License :: OSI Approved :: Apache Software License", 39 | "Natural Language :: English", 40 | "Operating System :: OS Independent", 41 | "Programming Language :: Python :: 3.7", 42 | "Topic :: Security", 43 | "Topic :: System", 44 | "Topic :: System :: Systems Administration", 45 | ], 46 | zip_safe=False, 47 | ) 48 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | flake8==3.5.0 2 | flake8-import-order==0.18.1 3 | python-dateutil==2.6.0 4 | mock==2.0.0 5 | pytest==3.2.3 6 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/repokid/376aa82ed31fe66ac4b1aecc3c22b0fc4fcfc0ea/tests/__init__.py -------------------------------------------------------------------------------- /tests/artifacts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/repokid/376aa82ed31fe66ac4b1aecc3c22b0fc4fcfc0ea/tests/artifacts/__init__.py -------------------------------------------------------------------------------- /tests/artifacts/hook/__init__.py: -------------------------------------------------------------------------------- 1 | import repokid.hooks as hooks 2 | from repokid.types import RepokidHookInput 3 | from repokid.types import RepokidHookOutput 4 | 5 | 6 | @hooks.implements_hook("TEST_HOOK", 2) 7 | def function_2(input_dict: RepokidHookInput) -> RepokidHookOutput: 8 | return input_dict 9 | 10 | 11 | @hooks.implements_hook("TEST_HOOK", 1) 12 | def function_1(input_dict: RepokidHookInput) -> RepokidHookOutput: 13 | return input_dict 14 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | 17 | from repokid.role import Role 18 | from tests import vars 19 | 20 | 21 | @pytest.fixture(scope="function") 22 | def mock_role(role_dict): 23 | return Role(**role_dict) 24 | 25 | 26 | @pytest.fixture(scope="session") 27 | def role_dict(): 28 | return { 29 | "aa_data": vars.aa_data, 30 | "account": vars.account, 31 | "active": vars.active, 32 | "arn": vars.arn, 33 | "assume_role_policy_document": vars.assume_role_policy_document, 34 | "create_date": vars.create_date, 35 | "disqualified_by": vars.disqualified_by, 36 | "last_updated": vars.last_updated, 37 | "no_repo_permissions": vars.no_repo_permissions, 38 | "opt_out": vars.opt_out, 39 | "policies": vars.policies, 40 | "refreshed": vars.refreshed, 41 | "repoable_permissions": vars.repoable_permissions, 42 | "repoable_services": vars.repoable_services, 43 | "repoed": vars.repoed, 44 | "repo_scheduled": vars.repo_scheduled, 45 | "role_id": vars.role_id, 46 | "role_name": vars.role_name, 47 | "scheduled_perms": vars.scheduled_perms, 48 | "stats": vars.stats, 49 | "tags": vars.tags, 50 | "total_permissions": vars.total_permissions, 51 | } 52 | 53 | 54 | @pytest.fixture(scope="session") 55 | def role_dict_with_aliases(): 56 | return { 57 | "AAData": vars.aa_data, 58 | "Account": vars.account, 59 | "Active": vars.active, 60 | "Arn": vars.arn, 61 | "AssumeRolePolicyDocument": vars.assume_role_policy_document, 62 | "CreateDate": vars.create_date, 63 | "DisqualifiedBy": vars.disqualified_by, 64 | "LastUpdated": vars.last_updated, 65 | "NoRepoPermissions": vars.no_repo_permissions, 66 | "OptOut": vars.opt_out, 67 | "Policies": vars.policies, 68 | "Refreshed": vars.refreshed, 69 | "RepoablePermissions": vars.repoable_permissions, 70 | "RepoableServices": vars.repoable_services, 71 | "Repoed": vars.repoed, 72 | "RepoScheduled": vars.repo_scheduled, 73 | "RoleId": vars.role_id, 74 | "RoleName": vars.role_name, 75 | "ScheduledPerms": vars.scheduled_perms, 76 | "Stats": vars.stats, 77 | "Tags": vars.tags, 78 | "TotalPermissions": vars.total_permissions, 79 | } 80 | -------------------------------------------------------------------------------- /tests/datasource/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/repokid/376aa82ed31fe66ac4b1aecc3c22b0fc4fcfc0ea/tests/datasource/__init__.py -------------------------------------------------------------------------------- /tests/datasource/conftest.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import pytest 4 | 5 | from repokid.datasource.access_advisor import AccessAdvisorDatasource 6 | from repokid.datasource.iam import ConfigDatasource 7 | from repokid.datasource.iam import IAMDatasource 8 | from repokid.datasource.plugin import DatasourcePlugin 9 | 10 | 11 | @pytest.fixture(autouse=True) 12 | def purge_datasources(): 13 | datasources: List[DatasourcePlugin] = [ 14 | AccessAdvisorDatasource(), 15 | IAMDatasource(), 16 | ConfigDatasource(), 17 | ] 18 | 19 | for ds in datasources: 20 | ds.reset() 21 | yield 22 | for ds in datasources: 23 | ds.reset() 24 | -------------------------------------------------------------------------------- /tests/datasource/test_access_advisor.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | import pytest 4 | 5 | from repokid.datasource.access_advisor import AccessAdvisorDatasource 6 | from repokid.exceptions import NotFoundError 7 | 8 | 9 | def test_access_advisor_get(): 10 | ds = AccessAdvisorDatasource() 11 | arn = "pretend_arn" 12 | expected = [{"a": "b"}] 13 | ds._data = {arn: expected} 14 | result = ds.get(arn) 15 | assert result == expected 16 | 17 | 18 | @patch("repokid.datasource.access_advisor.AccessAdvisorDatasource._fetch") 19 | def test_access_advisor_get_fallback(mock_fetch): 20 | ds = AccessAdvisorDatasource() 21 | arn = "pretend_arn" 22 | expected = [{"a": "b"}] 23 | mock_fetch.return_value = {arn: expected} 24 | result = ds.get(arn) 25 | mock_fetch.assert_called_once() 26 | assert mock_fetch.call_args[1]["arn"] == arn 27 | assert result == expected 28 | # make sure fetched data gets cached 29 | assert arn in ds._data 30 | assert ds._data[arn] == expected 31 | 32 | 33 | @patch("repokid.datasource.access_advisor.AccessAdvisorDatasource._fetch") 34 | def test_access_advisor_get_fallback_not_found(mock_fetch): 35 | ds = AccessAdvisorDatasource() 36 | arn = "pretend_arn" 37 | mock_fetch.return_value = {} 38 | with pytest.raises(NotFoundError): 39 | _ = ds.get(arn) 40 | mock_fetch.assert_called_once() 41 | assert mock_fetch.call_args[1]["arn"] == arn 42 | 43 | 44 | @patch("repokid.datasource.access_advisor.AccessAdvisorDatasource._fetch") 45 | def test_access_advisor_seed(mock_fetch): 46 | ds = AccessAdvisorDatasource() 47 | arn = "pretend_arn" 48 | account_number = "123456789012" 49 | expected = {arn: [{"a": "b"}]} 50 | mock_fetch.return_value = expected 51 | ds.seed(account_number) 52 | mock_fetch.assert_called_once() 53 | assert mock_fetch.call_args[1]["account_number"] == account_number 54 | assert ds._data == expected 55 | # make sure fetched data gets cached 56 | assert arn in ds._data 57 | assert ds._data[arn] == [{"a": "b"}] 58 | -------------------------------------------------------------------------------- /tests/datasource/test_iam.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | import pytest 4 | 5 | from repokid.datasource.iam import IAMDatasource 6 | from repokid.exceptions import NotFoundError 7 | 8 | 9 | def test_iam_get(): 10 | ds = IAMDatasource() 11 | arn = "pretend_arn" 12 | expected = {"a": "b"} 13 | ds._data = {arn: expected} 14 | result = ds.get(arn) 15 | assert result == expected 16 | 17 | 18 | @patch("repokid.datasource.iam.IAMDatasource._fetch") 19 | def test_iam_get_fallback_not_found(mock_fetch): 20 | mock_fetch.side_effect = NotFoundError 21 | ds = IAMDatasource() 22 | arn = "arn:aws:iam::12345678901:role/test" 23 | with pytest.raises(NotFoundError): 24 | _ = ds.get(arn) 25 | 26 | 27 | @patch("repokid.datasource.iam.IAMDatasource._fetch_account") 28 | def test_iam_seed(mock_fetch_account): 29 | ds = IAMDatasource() 30 | arn = "pretend_arn" 31 | account_number = "123456789012" 32 | expected = {arn: {"a": "b"}} 33 | mock_fetch_account.return_value = expected 34 | ds.seed(account_number) 35 | mock_fetch_account.assert_called_once() 36 | assert mock_fetch_account.call_args[0][0] == account_number 37 | assert ds._data == expected 38 | # make sure fetched data gets cached 39 | assert arn in ds._data 40 | assert ds._data[arn] == {"a": "b"} 41 | -------------------------------------------------------------------------------- /tests/filters/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/filters/test_age.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import datetime 15 | 16 | from dateutil import tz 17 | 18 | from repokid.filters.age import AgeFilter 19 | from repokid.role import Role 20 | from repokid.role import RoleList 21 | 22 | 23 | def test_age_with_tz(mock_role: Role): 24 | age_filter = AgeFilter() 25 | create_date = datetime.datetime.now(tz=tz.tzutc()) - datetime.timedelta(days=100) 26 | assert create_date.tzinfo 27 | mock_role.create_date = create_date 28 | role_list = RoleList([mock_role]) 29 | result = age_filter.apply(role_list) 30 | assert len(result) == 0 31 | 32 | 33 | def test_age_no_tz(mock_role: Role): 34 | age_filter = AgeFilter() 35 | create_date = datetime.datetime.now() - datetime.timedelta(days=100) 36 | assert not create_date.tzinfo 37 | mock_role.create_date = create_date 38 | role_list = RoleList([mock_role]) 39 | result = age_filter.apply(role_list) 40 | assert len(result) == 0 41 | 42 | 43 | def test_age_too_young_with_tz(mock_role: Role): 44 | age_filter = AgeFilter() 45 | create_date = datetime.datetime.now(tz=tz.tzutc()) 46 | assert create_date.tzinfo 47 | mock_role.create_date = create_date 48 | role_list = RoleList([mock_role]) 49 | result = age_filter.apply(role_list) 50 | assert len(result) == 1 51 | 52 | 53 | def test_age_too_young_no_tz(mock_role: Role): 54 | age_filter = AgeFilter() 55 | create_date = datetime.datetime.now() 56 | assert not create_date.tzinfo 57 | mock_role.create_date = create_date 58 | role_list = RoleList([mock_role]) 59 | result = age_filter.apply(role_list) 60 | assert len(result) == 1 61 | -------------------------------------------------------------------------------- /tests/test_dispatcher_cli.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | import pytest 4 | from mock import call 5 | from mock import patch 6 | from pydantic.error_wrappers import ValidationError 7 | 8 | import repokid.dispatcher as dispatcher 9 | from repokid.dispatcher.types import Message 10 | 11 | MESSAGE = Message( 12 | command="command", 13 | account="account", 14 | role_name="role", 15 | respond_channel="respond_channel", 16 | respond_user="some_user", 17 | requestor="a_requestor", 18 | reason="some_reason", 19 | selection="some_selection", 20 | ) 21 | 22 | 23 | class TestDispatcherCLI(object): 24 | def test_message_creation(self): 25 | test_message = MESSAGE 26 | assert test_message.command == "command" 27 | assert test_message.account == "account" 28 | assert test_message.role_name == "role" 29 | assert test_message.respond_channel == "respond_channel" 30 | assert test_message.respond_user == "some_user" 31 | assert test_message.requestor == "a_requestor" 32 | assert test_message.reason == "some_reason" 33 | assert test_message.selection == "some_selection" 34 | 35 | def test_schema(self): 36 | 37 | # happy path 38 | test_message = { 39 | "command": "list_repoable_services", 40 | "account": "123", 41 | "role_name": "abc", 42 | "respond_channel": "channel", 43 | "respond_user": "user", 44 | } 45 | result = Message.parse_obj(test_message) 46 | assert result.command == "list_repoable_services" 47 | 48 | # missing required field command 49 | test_message = { 50 | "account": "123", 51 | "role_name": "abc", 52 | "respond_channel": "channel", 53 | "respond_user": "user", 54 | } 55 | with pytest.raises(ValidationError): 56 | _ = Message.parse_obj(test_message) 57 | 58 | @patch("repokid.dispatcher.get_services_and_permissions_from_repoable") 59 | @patch("repokid.role.Role.fetch") 60 | @patch("repokid.dispatcher.find_role_in_cache") 61 | def test_list_repoable_services( 62 | self, 63 | mock_find_role_in_cache, 64 | mock_role_fetch, 65 | mock_get_services_and_permissions_from_repoable, 66 | ): 67 | mock_find_role_in_cache.side_effect = [None, "ROLE_ID_A"] 68 | mock_get_services_and_permissions_from_repoable.return_value = {"foo", "bar"} 69 | 70 | success, _ = dispatcher.list_repoable_services(MESSAGE) 71 | assert not success 72 | mock_find_role_in_cache.assert_called_once() 73 | mock_role_fetch.assert_not_called() 74 | mock_get_services_and_permissions_from_repoable.assert_not_called() 75 | 76 | mock_role_fetch.reset_mock() 77 | mock_get_services_and_permissions_from_repoable.reset_mock() 78 | mock_find_role_in_cache.reset_mock() 79 | 80 | success, _ = dispatcher.list_repoable_services(MESSAGE) 81 | assert success 82 | mock_find_role_in_cache.assert_called_once() 83 | mock_role_fetch.assert_called_once() 84 | mock_get_services_and_permissions_from_repoable.assert_called_once() 85 | 86 | @patch("repokid.role.Role.fetch") 87 | @patch("repokid.dispatcher.find_role_in_cache") 88 | def test_list_role_rollbacks(self, mock_find_role_in_cache, mock_role_fetch): 89 | mock_find_role_in_cache.side_effect = [None, "ROLE_ID_A"] 90 | 91 | (success, _) = dispatcher.list_role_rollbacks(MESSAGE) 92 | assert not success 93 | mock_find_role_in_cache.assert_called_once() 94 | mock_role_fetch.assert_not_called() 95 | 96 | mock_role_fetch.reset_mock() 97 | mock_find_role_in_cache.reset_mock() 98 | 99 | (success, _) = dispatcher.list_repoable_services(MESSAGE) 100 | assert success 101 | mock_find_role_in_cache.assert_called_once() 102 | mock_role_fetch.assert_called_once() 103 | 104 | @patch("time.time") 105 | @patch("repokid.role.Role.store") 106 | @patch("repokid.role.Role.fetch") 107 | @patch("repokid.dispatcher.find_role_in_cache") 108 | def test_opt_out( 109 | self, mock_find_role_in_cache, mock_role_fetch, mock_role_store, mock_time 110 | ): 111 | mock_find_role_in_cache.side_effect = [None, "ROLE_ID_A"] 112 | 113 | # mock_get_role_data.side_effect = [ 114 | # MockRoleNoOptOut(), # role not found 115 | # MockRoleNoOptOut(), # opt out exists 116 | # MockRoleOptOut(), 117 | # MockRoleNoOptOut(), 118 | # MockRoleEmptyOptOut(), # success 119 | # ] 120 | 121 | mock_time.return_value = 0 122 | 123 | bad_message = copy.deepcopy(MESSAGE) 124 | bad_message.reason = None 125 | # message missing reason 126 | (success, _) = dispatcher.opt_out(bad_message) 127 | assert not success 128 | 129 | # role not found 130 | (success, _) = dispatcher.opt_out(MESSAGE) 131 | assert not success 132 | 133 | (success, msg) = dispatcher.opt_out(MESSAGE) 134 | assert success 135 | assert mock_role_store.mock_calls[0] == call(fields=["opt_out"]) 136 | -------------------------------------------------------------------------------- /tests/test_hooks.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # Licensed under the Apache License, Version 2.0 (the "License"); 16 | # you may not use this file except in compliance with the License. 17 | # You may obtain a copy of the License at 18 | # 19 | # http://www.apache.org/licenses/LICENSE-2.0 20 | # 21 | # Unless required by applicable law or agreed to in writing, software 22 | # distributed under the License is distributed on an "AS IS" BASIS, 23 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 24 | # See the License for the specific language governing permissions and 25 | # limitations under the License. 26 | import pytest 27 | 28 | import repokid.cli.repokid_cli 29 | import repokid.hooks 30 | from repokid.hooks.loggers import log_during_repoable_calculation_batch_hooks 31 | from repokid.role import Role 32 | from tests.artifacts.hook import function_1 33 | from tests.artifacts.hook import function_2 34 | from tests.test_commands import ROLES 35 | 36 | 37 | def func_a(input_dict): 38 | input_dict["value"] += 1 39 | return input_dict 40 | 41 | 42 | def func_b(input_dict): 43 | input_dict["value"] += 1 44 | return input_dict 45 | 46 | 47 | def func_c(input_dict): 48 | input_dict["value"] += 10 49 | return input_dict 50 | 51 | 52 | def func_d(input_value): 53 | required_vals = ["a", "b"] 54 | if not all(val in input_value for val in required_vals): 55 | raise repokid.hooks.MissingHookParameter 56 | 57 | 58 | def func_e(input_value): 59 | pass 60 | 61 | 62 | class TestHooks(object): 63 | def test_call_hooks(self): 64 | hooks = { 65 | "TEST_HOOK": [func_a, func_b], 66 | "NOT_CALLED": [func_c], 67 | "MISSING_PARAMETER": [func_d], 68 | "MISSING_OUTPUT": [func_e], 69 | } 70 | hook_args = {"value": 0} 71 | output_value = repokid.hooks.call_hooks(hooks, "TEST_HOOK", hook_args) 72 | 73 | # func_a and func_b are called to increment 0 --> 2, func_c is not called 74 | assert output_value["value"] == 2 75 | 76 | # missing required parameter b 77 | with pytest.raises(repokid.hooks.MissingHookParameter): 78 | output_value = repokid.hooks.call_hooks( 79 | hooks, "MISSING_PARAMETER", {"a": "1"} 80 | ) 81 | 82 | with pytest.raises(repokid.hooks.MissingOutputInHook): 83 | output_value = repokid.hooks.call_hooks(hooks, "MISSING_OUTPUT", {"a": 1}) 84 | 85 | def test_get_hooks(self): 86 | hooks_config = ["tests.artifacts.hook"] 87 | hooks = repokid.cli.repokid_cli.get_hooks(hooks_config) 88 | 89 | # key is correct, both functions are loaded and in correct priority order 90 | assert hooks == {"TEST_HOOK": [function_1, function_2]} 91 | 92 | def test_implements_hook(self): 93 | def func_a(): 94 | pass 95 | 96 | @repokid.hooks.implements_hook("DECORATOR_TEST", 1) 97 | def func_b(): 98 | pass 99 | 100 | assert not hasattr(func_a, "_implements_hook") 101 | assert hasattr(func_b, "_implements_hook") 102 | assert func_b._implements_hook == {"hook_name": "DECORATOR_TEST", "priority": 1} 103 | 104 | def test_log_during_repoable_calculation_batch_hooks(self): 105 | hooks = { 106 | "DURING_REPOABLE_CALCULATION_BATCH": [ 107 | log_during_repoable_calculation_batch_hooks 108 | ] 109 | } 110 | 111 | input_dict = { 112 | "role_batch": [Role(**ROLES[0]), "def"], 113 | "potentially_repoable_permissions": [], 114 | "minimum_age": 1, 115 | } 116 | 117 | with pytest.raises(repokid.hooks.MissingHookParameter): 118 | # role_batch', 'potentially_repoable_permissions', 'minimum_age' 119 | repokid.hooks.call_hooks( 120 | hooks, "DURING_REPOABLE_CALCULATION_BATCH", input_dict 121 | ) 122 | 123 | input_dict["role_batch"] = [ 124 | Role(**ROLES[0]), 125 | Role(**ROLES[1]), 126 | Role(**ROLES[2]), 127 | ] 128 | assert input_dict == repokid.hooks.call_hooks( 129 | hooks, "DURING_REPOABLE_CALCULATION_BATCH", input_dict 130 | ) 131 | -------------------------------------------------------------------------------- /tests/test_permissions.py.bak: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from repokid.utils.permissions import ( 15 | remove_actions_from_policies, 16 | tidy_policies, 17 | get_repoed_policies_v2, 18 | ) 19 | 20 | 21 | def test_remove_actions_from_policies(): 22 | """ 23 | This tests a few things: 24 | - removing an entire service 25 | - removing an individual action 26 | - removing a subset of an expanded wildcard action 27 | - removing all of an expanded wildcard action 28 | - ensuring NOREPO policy statements are not touched 29 | - ensuring Deny policy statements are not touched 30 | """ 31 | repoable_permissions = { 32 | "ec2", 33 | "sqs:sendmessage", 34 | "sqs:changemessagevisibilitybatch", 35 | } 36 | policies = { 37 | "partial_repo_policy": { 38 | "Statement": [ 39 | { 40 | "Effect": "Allow", 41 | "Sid": "test-sid", 42 | "Action": [ 43 | "ec2:Describe*", 44 | "sqs:SendMessage", 45 | "sqs:GetMessage", 46 | "sqs:Change*", 47 | ], 48 | }, 49 | { 50 | "Effect": "Allow", 51 | "Sid": "NOREPOtest-sid", 52 | "Action": [ 53 | "ec2:Describe*", 54 | "sqs:SendMessage", 55 | "sqs:GetMessage", 56 | ], 57 | }, 58 | { 59 | "Effect": "Deny", 60 | "Sid": "test-deny", 61 | "Action": [ 62 | "iam:Create*", 63 | ], 64 | }, 65 | ], 66 | }, 67 | "full_repo_policy": { 68 | "Statement": [ 69 | { 70 | "Effect": "Allow", 71 | "Sid": "test-sid", 72 | "Action": [ 73 | "ec2:Describe*", 74 | "sqs:SendMessage", 75 | ], 76 | }, 77 | ], 78 | }, 79 | "no_repo_policy": { 80 | "Statement": [ 81 | { 82 | "Effect": "Allow", 83 | "Sid": "test-sid", 84 | "Action": [ 85 | "sqs:GetMessage", 86 | ], 87 | }, 88 | { 89 | "Effect": "Allow", 90 | "Sid": "NOREPOtest-sid", 91 | "Action": [ 92 | "ec2:Describe*", 93 | "sqs:SendMessage", 94 | "sqs:GetMessage", 95 | ], 96 | }, 97 | ], 98 | }, 99 | } 100 | expected_policies = { 101 | "partial_repo_policy": { 102 | "Statement": [ 103 | { 104 | "Effect": "Allow", 105 | "Sid": "test-sid", 106 | "Action": [ 107 | "sqs:changemessagevisibility", 108 | "sqs:getmessage", 109 | ], 110 | }, 111 | { 112 | "Effect": "Allow", 113 | "Sid": "NOREPOtest-sid", 114 | "Action": [ 115 | "ec2:Describe*", 116 | "sqs:SendMessage", 117 | "sqs:GetMessage", 118 | ], 119 | }, 120 | { 121 | "Effect": "Deny", 122 | "Sid": "test-deny", 123 | "Action": [ 124 | "iam:Create*", 125 | ], 126 | }, 127 | ], 128 | }, 129 | "full_repo_policy": { 130 | "Statement": [ 131 | { 132 | "Effect": "Allow", 133 | "Sid": "test-sid", 134 | "Action": [], 135 | }, 136 | ], 137 | }, 138 | "no_repo_policy": { 139 | "Statement": [ 140 | { 141 | "Effect": "Allow", 142 | "Sid": "test-sid", 143 | "Action": [ 144 | "sqs:getmessage", 145 | ], 146 | }, 147 | { 148 | "Effect": "Allow", 149 | "Sid": "NOREPOtest-sid", 150 | "Action": [ 151 | "ec2:Describe*", 152 | "sqs:SendMessage", 153 | "sqs:GetMessage", 154 | ], 155 | }, 156 | ], 157 | }, 158 | } 159 | result = remove_actions_from_policies( 160 | policies, repoable_permissions, exclude_prefix="NOREPO" 161 | ) 162 | assert result == expected_policies 163 | 164 | 165 | def test_tidy_policies(): 166 | policies = { 167 | "partial_repo_policy": { 168 | "Statement": [ 169 | { 170 | "Effect": "Allow", 171 | "Sid": "test-sid", 172 | "Action": [], 173 | }, 174 | { 175 | "Effect": "Allow", 176 | "Sid": "NOREPOtest-sid", 177 | "Action": [ 178 | "ec2:Describe*", 179 | "sqs:SendMessage", 180 | "sqs:GetMessage", 181 | ], 182 | }, 183 | { 184 | "Effect": "Deny", 185 | "Sid": "test-deny", 186 | "Action": [ 187 | "iam:Create*", 188 | ], 189 | }, 190 | ], 191 | }, 192 | "full_repo_policy": { 193 | "Statement": [ 194 | { 195 | "Effect": "Allow", 196 | "Sid": "test-sid", 197 | "Action": [], 198 | }, 199 | ], 200 | }, 201 | "no_repo_policy": { 202 | "Statement": [ 203 | { 204 | "Effect": "Allow", 205 | "Sid": "test-sid", 206 | "Action": [ 207 | "sqs:getmessage", 208 | ], 209 | }, 210 | { 211 | "Effect": "Allow", 212 | "Sid": "NOREPOtest-sid", 213 | "Action": [ 214 | "ec2:Describe*", 215 | "sqs:SendMessage", 216 | "sqs:GetMessage", 217 | ], 218 | }, 219 | ], 220 | }, 221 | } 222 | expected_policies = { 223 | "partial_repo_policy": { 224 | "Statement": [ 225 | { 226 | "Effect": "Allow", 227 | "Sid": "NOREPOtest-sid", 228 | "Action": [ 229 | "ec2:Describe*", 230 | "sqs:SendMessage", 231 | "sqs:GetMessage", 232 | ], 233 | }, 234 | { 235 | "Effect": "Deny", 236 | "Sid": "test-deny", 237 | "Action": [ 238 | "iam:Create*", 239 | ], 240 | }, 241 | ], 242 | }, 243 | "full_repo_policy": { 244 | "Statement": [], 245 | }, 246 | "no_repo_policy": { 247 | "Statement": [ 248 | { 249 | "Effect": "Allow", 250 | "Sid": "test-sid", 251 | "Action": [ 252 | "sqs:getmessage", 253 | ], 254 | }, 255 | { 256 | "Effect": "Allow", 257 | "Sid": "NOREPOtest-sid", 258 | "Action": [ 259 | "ec2:Describe*", 260 | "sqs:SendMessage", 261 | "sqs:GetMessage", 262 | ], 263 | }, 264 | ], 265 | }, 266 | } 267 | expected_empty_policy_names = ["full_repo_policy"] 268 | tidied_policies, empty_policy_names = tidy_policies(policies) 269 | assert tidied_policies == expected_policies 270 | assert empty_policy_names == expected_empty_policy_names 271 | 272 | 273 | def test_get_repoed_policies_v2(): 274 | repoable_permissions = { 275 | "ec2", 276 | "sqs:sendmessage", 277 | "sqs:changemessagevisibilitybatch", 278 | } 279 | policies = { 280 | "partial_repo_policy": { 281 | "Statement": [ 282 | { 283 | "Effect": "Allow", 284 | "Sid": "test-sid", 285 | "Action": [ 286 | "ec2:Describe*", 287 | "sqs:SendMessage", 288 | "sqs:GetMessage", 289 | "sqs:Change*", 290 | ], 291 | }, 292 | { 293 | "Effect": "Allow", 294 | "Sid": "test-sid", 295 | "Action": [ 296 | "sqs:SendMessage", 297 | ], 298 | }, 299 | { 300 | "Effect": "Allow", 301 | "Sid": "NOREPOtest-sid", 302 | "Action": [ 303 | "ec2:Describe*", 304 | "sqs:SendMessage", 305 | "sqs:GetMessage", 306 | ], 307 | }, 308 | { 309 | "Effect": "Deny", 310 | "Sid": "test-deny", 311 | "Action": [ 312 | "iam:Create*", 313 | ], 314 | }, 315 | ], 316 | }, 317 | "full_repo_policy": { 318 | "Statement": [ 319 | { 320 | "Effect": "Allow", 321 | "Sid": "test-sid", 322 | "Action": [ 323 | "ec2:Describe*", 324 | "sqs:SendMessage", 325 | ], 326 | }, 327 | ], 328 | }, 329 | "no_repo_policy": { 330 | "Statement": [ 331 | { 332 | "Effect": "Allow", 333 | "Sid": "test-sid", 334 | "Action": [ 335 | "sqs:GetMessage", 336 | ], 337 | }, 338 | { 339 | "Effect": "Allow", 340 | "Sid": "NOREPOtest-sid", 341 | "Action": [ 342 | "ec2:Describe*", 343 | "sqs:SendMessage", 344 | "sqs:GetMessage", 345 | ], 346 | }, 347 | ], 348 | }, 349 | } 350 | expected_policies = { 351 | "partial_repo_policy": { 352 | "Statement": [ 353 | { 354 | "Action": ["sqs:changemessagevisibility", "sqs:getmessage"], 355 | "Effect": "Allow", 356 | "Sid": "test-sid", 357 | }, 358 | { 359 | "Action": ["ec2:Describe*", "sqs:SendMessage", "sqs:GetMessage"], 360 | "Effect": "Allow", 361 | "Sid": "NOREPOtest-sid", 362 | }, 363 | { 364 | "Effect": "Deny", 365 | "Sid": "test-deny", 366 | "Action": [ 367 | "iam:Create*", 368 | ], 369 | }, 370 | ], 371 | }, 372 | "full_repo_policy": { 373 | "Statement": [], 374 | }, 375 | "no_repo_policy": { 376 | "Statement": [ 377 | { 378 | "Effect": "Allow", 379 | "Sid": "test-sid", 380 | "Action": [ 381 | "sqs:getmessage", 382 | ], 383 | }, 384 | { 385 | "Effect": "Allow", 386 | "Sid": "NOREPOtest-sid", 387 | "Action": [ 388 | "ec2:Describe*", 389 | "sqs:SendMessage", 390 | "sqs:GetMessage", 391 | ], 392 | }, 393 | ], 394 | }, 395 | } 396 | expected_empty_policy_names = ["full_repo_policy"] 397 | repoed_policies, empty_policy_names = get_repoed_policies_v2( 398 | policies, repoable_permissions 399 | ) 400 | assert repoed_policies == expected_policies 401 | assert empty_policy_names == expected_empty_policy_names 402 | -------------------------------------------------------------------------------- /tests/test_roledata.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import repokid.utils.permissions 15 | import repokid.utils.roledata 16 | from tests.test_commands import ROLE_POLICIES 17 | 18 | # AARDVARK_DATA maintained in test_repokid_cli 19 | 20 | 21 | class TestRoledata(object): 22 | def test_get_repoed_policy(self): 23 | policies = ROLE_POLICIES["all_services_used"] 24 | repoable_permissions = { 25 | "iam:addroletoinstanceprofile", 26 | "iam:attachrolepolicy", 27 | "s3:createbucket", 28 | } 29 | 30 | ( 31 | rewritten_policies, 32 | empty_policies, 33 | ) = repokid.utils.permissions.get_repoed_policy(policies, repoable_permissions) 34 | 35 | assert rewritten_policies == { 36 | "s3_perms": { 37 | "Version": "2012-10-17", 38 | "Statement": [ 39 | { 40 | "Action": ["s3:deletebucket"], 41 | "Resource": ["*"], 42 | "Effect": "Allow", 43 | } 44 | ], 45 | } 46 | } 47 | assert empty_policies == ["iam_perms"] 48 | 49 | def test_find_newly_added_permissions(self): 50 | old_policy = ROLE_POLICIES["all_services_used"] 51 | new_policy = ROLE_POLICIES["unused_ec2"] 52 | 53 | new_perms = repokid.utils.permissions.find_newly_added_permissions( 54 | old_policy, new_policy, minimize=False 55 | ) 56 | assert new_perms == {"ec2:allocatehosts", "ec2:associateaddress"} 57 | 58 | def test_find_newly_added_permissions_minimize(self): 59 | old_policy = ROLE_POLICIES["all_services_used"] 60 | new_policy = ROLE_POLICIES["unused_ec2"] 61 | 62 | new_perms = repokid.utils.permissions.find_newly_added_permissions( 63 | old_policy, new_policy, minimize=True 64 | ) 65 | assert new_perms == {"ec2:allocateh*", "ec2:associatea*"} 66 | 67 | def test_convert_repoable_perms_to_perms_and_services(self): 68 | all_perms = {"a:j", "a:k", "b:l", "c:m", "c:n"} 69 | repoable_perms = {"b:l", "c:m"} 70 | expected_repoed_services = {"b"} 71 | expected_repoed_permissions = {"c:m"} 72 | assert repokid.utils.permissions.convert_repoable_perms_to_perms_and_services( 73 | all_perms, repoable_perms 74 | ) == (expected_repoed_permissions, expected_repoed_services) 75 | 76 | def test_convert_repoed_service_to_sorted_perms_and_services(self): 77 | repoed_services = { 78 | "route53", 79 | "ec2", 80 | "s3:abc", 81 | "dynamodb:def", 82 | "ses:ghi", 83 | "ses:jkl", 84 | } 85 | expected_services = ["ec2", "route53"] 86 | expected_permissions = ["dynamodb:def", "s3:abc", "ses:ghi", "ses:jkl"] 87 | assert ( 88 | repokid.utils.roledata._convert_repoed_service_to_sorted_perms_and_services( 89 | repoed_services 90 | ) 91 | == ( 92 | expected_permissions, 93 | expected_services, 94 | ) 95 | ) 96 | 97 | def test_get_epoch_authenticated(self): 98 | assert repokid.utils.permissions._get_epoch_authenticated(1545787620000) == ( 99 | 1545787620, 100 | True, 101 | ) 102 | assert repokid.utils.permissions._get_epoch_authenticated(1545787620) == ( 103 | 1545787620, 104 | True, 105 | ) 106 | assert repokid.utils.permissions._get_epoch_authenticated(154578762) == ( 107 | -1, 108 | False, 109 | ) 110 | -------------------------------------------------------------------------------- /tests/vars.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Netflix, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import datetime 16 | 17 | sample_aa_service = { 18 | "lastUpdated": "Thu, 17 Dec 2020 02:01:36 GMT", 19 | "lastAuthenticatedEntity": None, 20 | "serviceNamespace": "test", 21 | "serviceName": "AWS Test Service", 22 | "totalAuthenticatedEntities": 0, 23 | "lastAuthenticated": 0, 24 | } 25 | 26 | aa_data = [sample_aa_service] 27 | account = "123456789012" 28 | active = True 29 | arn = f"arn:aws:iam:{account}::role/TestRole" 30 | assume_role_policy_document = {} 31 | create_date = datetime.datetime.now() - datetime.timedelta(days=10) 32 | disqualified_by = [] 33 | last_updated = datetime.datetime.now() - datetime.timedelta(hours=2) 34 | no_repo_permissions = {"service3:action4": 0} 35 | opt_out = {} 36 | policies = [{"Policy": {"this_is_fake": "cool"}, "Source": "Fixture"}] 37 | refreshed = (datetime.datetime.now() - datetime.timedelta(hours=1)).isoformat() 38 | repoable_permissions = 5 39 | repoable_services = [ 40 | "service1:action1", 41 | "service1:action2", 42 | "service2:action3", 43 | "service3", 44 | ] 45 | repoed = "" 46 | repo_scheduled = 0.0 47 | role_id = "ARIOABC123BLAHBLAHBLAH" 48 | role_name = "TestRole" 49 | scheduled_perms = [""] 50 | stats = [{}] 51 | tags = [{}] 52 | total_permissions = 5 53 | --------------------------------------------------------------------------------