├── .bumpversion.cfg ├── .editorconfig ├── .github ├── ISSUE_TEMPLATE.md └── workflows │ ├── dev.yml │ ├── preview.yml │ └── release.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .pylintrc ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── docs ├── changelog.md ├── contributing.md ├── datastores │ ├── bigquery.md │ ├── databricks.md │ ├── mongodb.md │ ├── postgresql.md │ ├── redshift.md │ ├── s3.md │ └── snowflake.md ├── index.md └── installation.md ├── makefile ├── mkdocs.yml ├── mypy.ini ├── poetry.lock ├── pyproject.toml ├── pyrightconfig.json ├── setup.cfg ├── tests ├── __init__.py ├── mocks │ ├── __init__.py │ └── mock_writers.py ├── test_authz_analyzer.py ├── test_cli.py ├── test_writers │ ├── __init__.py │ ├── test_csv_writer.py │ └── test_multijson_writer.py └── tests_datastores │ ├── __init__.py │ ├── aws │ ├── __init__.py │ ├── aws_ptrp │ │ ├── __init__.py │ │ ├── resolve_permissions_test_inputs │ │ │ ├── action │ │ │ │ ├── action_resolving_assume_role.json │ │ │ │ ├── action_resolving_federated_user.json │ │ │ │ ├── action_resolving_s3.json │ │ │ │ └── assume_role_actions_for_principal_type.json │ │ │ ├── deny_with_condition │ │ │ │ ├── deny_with_condition_on_assume_role.json │ │ │ │ └── deny_with_condition_on_s3_bucket.json │ │ │ ├── federated_user │ │ │ │ ├── federated_user_cross_account.json │ │ │ │ └── iam_user_to_federated_user.json │ │ │ ├── iam_groups │ │ │ │ ├── iam_user_with_2_groups_includng_allow_and_deny.json │ │ │ │ ├── iam_user_with_group_and_attached_policy.json │ │ │ │ └── iam_user_with_group_simple_policy.json │ │ │ ├── iam_identity_center │ │ │ │ └── iam_identity_center_with_users_and_groups.json │ │ │ ├── iam_roles │ │ │ │ ├── assume_role_cross_account.json │ │ │ │ ├── assume_role_to_role.json │ │ │ │ ├── assume_role_via_resourve_based.json │ │ │ │ ├── simple_assume_role.json │ │ │ │ └── valid_assume_action.json │ │ │ ├── iam_user_to_s3_bucket │ │ │ │ ├── iam_user_with_attached_policy_and_deny.json │ │ │ │ ├── iam_user_with_inline_policy.json │ │ │ │ ├── iam_user_with_inline_policy_and_deny.json │ │ │ │ ├── iam_user_with_inline_policy_and_resource_based.json │ │ │ │ └── iam_user_with_inline_policy_multi_stmts.json │ │ │ ├── multi_iam_users_to_multi_s3_buckets │ │ │ │ ├── iam_users_to_s3_cross_account_via_resource_based.json │ │ │ │ └── multi_iam_users_to_multi_s3_buckets.json │ │ │ ├── no_resource │ │ │ │ ├── assume_role_with_not_resource.json │ │ │ │ ├── federated_users_with_not_resource.json │ │ │ │ ├── iam_user_with_not_resource_combinations_in_both_allow_and_deny.json │ │ │ │ ├── iam_user_with_not_resource_in_all_allow.json │ │ │ │ ├── iam_user_with_not_resource_in_all_deny.json │ │ │ │ ├── iam_user_with_not_resource_including_wildcard_object_regex_allow.json │ │ │ │ ├── iam_user_with_not_resource_including_wildcard_object_regex_deny.json │ │ │ │ ├── iam_user_with_not_resource_multiple_wildcard_regexes.json │ │ │ │ ├── iam_user_with_not_resource_on_no_valid_resource_arn_allow.json │ │ │ │ └── iam_user_with_not_resource_on_no_valid_resource_arn_deny.json │ │ │ ├── not_action │ │ │ │ ├── iam_user_allowed_no_action.json │ │ │ │ ├── iam_user_allowed_no_action_s3_complement.json │ │ │ │ ├── iam_user_with_multiple_actions_in_no_action_resource.json │ │ │ │ ├── iam_user_with_resource_no_action_simple.json │ │ │ │ └── iam_user_with_resource_no_action_wildcard_action_set.json │ │ │ ├── not_principal │ │ │ │ ├── iam_users_roles_and_federated_with_deny_not_principal_single_account.json │ │ │ │ └── iam_users_with_deny_not_principal_cross_accounts.json │ │ │ ├── principal │ │ │ │ ├── irrelevant_principal_types.json │ │ │ │ └── principal_resolving_via_resource_based.json │ │ │ ├── resource │ │ │ │ ├── resouce_resolving_assume_role.json │ │ │ │ ├── resouce_resolving_federated_user.json │ │ │ │ └── resouce_resolving_s3.json │ │ │ └── s3_resource_regex │ │ │ │ ├── deny_and_allow_default.json │ │ │ │ ├── deny_and_allow_on_object_multi_stmts.json │ │ │ │ ├── deny_and_allow_with_one_resource_in_deny_that_is_a_subgroup_of_all_in_allow.json │ │ │ │ ├── deny_on_object_and_allow_bucket.json │ │ │ │ └── valid_actions_objects_and_actions_resources.json │ │ ├── test_create_session_with_assume_role.py │ │ ├── test_resolve_permissions.py │ │ ├── test_satori_dev_account_ptrp.py │ │ └── utils │ │ │ ├── __init__.py │ │ │ └── aws_ptrp_load_from_dict.py │ ├── exporter_test_inputs │ │ ├── ptrp_line_basic_iam_user_and_policy_path.json │ │ ├── ptrp_line_iam_identity_center_user_and_group.json │ │ ├── ptrp_line_multiple_roles_in_path_and_all_principals.json │ │ ├── ptrp_line_with_federated_user.json │ │ ├── ptrp_line_with_iam_group_and_notes.json │ │ └── ptrp_line_with_notes.json │ ├── test_exporter.py │ └── utils │ │ ├── __init__.py │ │ └── test_aws_regex_full_subset.py │ ├── bigquery │ ├── __init__.py │ ├── generate_authz_entry.py │ ├── mocks.py │ └── test_bigquery.py │ ├── databricks │ ├── __init__.py │ ├── generate_authz_entry.py │ ├── mocks.py │ └── test_analyzer.py │ ├── mongodb │ ├── __init__.py │ ├── test_atlas_build_custom_role_from_response.py │ ├── test_atlas_organization_users.py │ └── test_mongodb.py │ ├── postgres │ ├── __init__.py │ ├── mocks │ │ ├── __init__.py │ │ └── postgres_mock_connector.py │ └── test_postgress_analyzer.py │ ├── redshift │ ├── __init__.py │ └── test_redshift.py │ └── snowflake │ ├── __init__.py │ └── test_snowflake_analyzer.py └── universal_data_permissions_scanner ├── __init__.py ├── cli.py ├── datastores ├── __init__.py ├── aws │ ├── __init__.py │ ├── analyzer │ │ ├── __init__.py │ │ ├── analyzer.py │ │ ├── exporter.py │ │ └── redshift │ │ │ ├── __init__.py │ │ │ ├── analyzer.py │ │ │ ├── commands │ │ │ ├── all_databases.sql │ │ │ ├── all_tables.sql │ │ │ ├── datashare_consumers.sql │ │ │ ├── datashare_desc.sql │ │ │ ├── datashares.sql │ │ │ ├── identities_pg_user.sql │ │ │ ├── identities_privileges.sql │ │ │ ├── identities_svv_role_grants.sql │ │ │ └── identities_svv_user_grants.sql │ │ │ ├── exporter.py │ │ │ ├── model.py │ │ │ └── service.py │ └── aws_ptrp_package │ │ └── aws_ptrp │ │ ├── __init__.py │ │ ├── actions │ │ ├── __init__.py │ │ ├── actions_resolver.py │ │ └── aws_actions.py │ │ ├── iam │ │ ├── __init__.py │ │ ├── iam_entities.py │ │ ├── iam_groups.py │ │ ├── iam_policies.py │ │ ├── iam_roles.py │ │ ├── iam_users.py │ │ ├── policy │ │ │ ├── __init__.py │ │ │ ├── effect.py │ │ │ ├── group_policy.py │ │ │ ├── policy.py │ │ │ ├── policy_document.py │ │ │ ├── policy_document_resolver.py │ │ │ ├── policy_document_utils.py │ │ │ └── user_policy.py │ │ ├── public_block_access_config.py │ │ └── role │ │ │ ├── __init__.py │ │ │ └── role_policy.py │ │ ├── iam_identity_center │ │ ├── __init__.py │ │ ├── iam_identity_center_entities.py │ │ ├── iam_identity_center_groups.py │ │ ├── iam_identity_center_users.py │ │ └── permission_sets.py │ │ ├── logger.py │ │ ├── policy_evaluation │ │ ├── __init__.py │ │ └── policy_evaluation.py │ │ ├── principals │ │ ├── __init__.py │ │ ├── aws_principals.py │ │ ├── no_entity_principal.py │ │ ├── principal.py │ │ └── principals_resolver.py │ │ ├── ptrp.py │ │ ├── ptrp_allowed_lines │ │ ├── __init__.py │ │ ├── allowed_line.py │ │ ├── allowed_line_node_notes.py │ │ ├── allowed_line_nodes_base.py │ │ ├── allowed_lines_resolver.py │ │ └── allowed_lines_resolver_result.py │ │ ├── ptrp_models │ │ ├── __init__.py │ │ └── ptrp_model.py │ │ ├── resources │ │ ├── __init__.py │ │ ├── account_resources.py │ │ └── resources_resolver.py │ │ ├── services │ │ ├── __init__.py │ │ ├── assume_role │ │ │ ├── __init__.py │ │ │ ├── assume_role_actions.py │ │ │ ├── assume_role_resources.py │ │ │ └── assume_role_service.py │ │ ├── federated_user │ │ │ ├── __init__.py │ │ │ ├── federated_user_actions.py │ │ │ ├── federated_user_resources.py │ │ │ └── federated_user_service.py │ │ ├── resolved_stmt.py │ │ ├── s3 │ │ │ ├── __init__.py │ │ │ ├── bucket.py │ │ │ ├── bucket_acl.py │ │ │ ├── s3_actions.py │ │ │ ├── s3_resources.py │ │ │ └── s3_service.py │ │ ├── service_action_base.py │ │ ├── service_action_type.py │ │ ├── service_actions_resolver_base.py │ │ ├── service_base.py │ │ ├── service_resource_base.py │ │ ├── service_resource_type.py │ │ └── service_resources_resolver_base.py │ │ └── utils │ │ ├── __init__.py │ │ ├── assume_role.py │ │ ├── create_session.py │ │ ├── pagination.py │ │ ├── regex_subset.py │ │ └── serde.py ├── bigquery │ ├── __init__.py │ ├── analyzer.py │ ├── policy_tree.py │ └── service.py ├── databricks │ ├── __init__.py │ ├── analyzer.py │ ├── exceptions.py │ ├── identities.py │ ├── model.py │ ├── policy_tree.py │ └── service │ │ ├── __init__.py │ │ ├── authentication │ │ ├── __init__.py │ │ ├── authentication.py │ │ ├── basic.py │ │ └── oauth.py │ │ ├── exceptions.py │ │ ├── model.py │ │ └── scim.py ├── mongodb │ ├── __init__.py │ ├── analyzer.py │ ├── atlas │ │ ├── __init__.py │ │ ├── analyzer.py │ │ ├── exceptions.py │ │ ├── model.py │ │ ├── permission_resolvers.py │ │ ├── service.py │ │ └── service_model.py │ ├── model.py │ ├── resolvers.py │ ├── service.py │ └── service_model.py ├── postgres │ ├── __init__.py │ ├── analyzer.py │ ├── commands │ │ ├── all_databases.sql │ │ ├── all_tables.sql │ │ ├── roles.sql │ │ └── roles_grants.sql │ ├── database_query_results.py │ ├── deployment.py │ ├── exporter.py │ └── model.py └── snowflake │ ├── __init__.py │ ├── analyzer.py │ ├── commands │ ├── grants_roles.sql │ ├── grants_to_share.sql │ ├── shares.sql │ └── user_grants.sql │ ├── exporter.py │ ├── model.py │ └── service.py ├── errors ├── __init__.py ├── failed_connection_errors.py └── snowflake.py ├── main.py ├── models ├── __init__.py └── model.py ├── utils ├── __init__.py └── logger.py └── writers ├── __init__.py ├── base_writers.py ├── csv_writer.py ├── get_writers.py └── multi_json_exporter.py /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.1.38 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:pyproject.toml] 7 | search = version = "{current_version}" 8 | replace = version = "{new_version}" 9 | 10 | [bumpversion:file:universal_data_permissions_scanner/__init__.py] 11 | search = __version__ = '{current_version}' 12 | replace = __version__ = '{new_version}' 13 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | [LICENSE] 18 | insert_final_newline = false 19 | 20 | [Makefile] 21 | indent_style = tab 22 | 23 | [*.{yml, yaml}] 24 | indent_size = 2 25 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | * authz-analyzer version: 2 | * Python version: 3 | * Operating System: 4 | 5 | ### Description 6 | 7 | Describe what you were trying to get done. 8 | Tell us what happened, what went wrong, and what you expected to happen. 9 | 10 | ### What I Did 11 | 12 | ``` 13 | Paste the command(s) you ran and the output. 14 | If there was a crash, please include the traceback here. 15 | ``` 16 | -------------------------------------------------------------------------------- /.github/workflows/dev.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions. 2 | 3 | name: dev workflow 4 | 5 | # Controls when the action will run. 6 | on: 7 | # Triggers the workflow on push or pull request events but only for the master branch 8 | push: 9 | branches: [master, main] 10 | pull_request: 11 | branches: [master, main] 12 | 13 | # Allows you to run this workflow manually from the Actions tab 14 | workflow_dispatch: 15 | 16 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 17 | jobs: 18 | # This workflow contains a single job called "test" 19 | test: 20 | # The type of runner that the job will run on 21 | strategy: 22 | matrix: 23 | python-versions: [3.8, 3.9, "3.10", "3.11"] 24 | # os: [ubuntu-22.04, macos-latest, windows-latest] 25 | os: [ubuntu-22.04] 26 | runs-on: ${{ matrix.os }} 27 | 28 | # Steps represent a sequence of tasks that will be executed as part of the job 29 | steps: 30 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 31 | - uses: actions/checkout@v3 32 | - uses: actions/setup-python@v4 33 | with: 34 | python-version: ${{ matrix.python-versions }} 35 | 36 | - name: Install dependencies 37 | run: | 38 | python -m pip install --upgrade pip 39 | pip install poetry tox tox-gh-actions 40 | - name: test with tox 41 | run: tox -v 42 | 43 | - name: list files 44 | run: ls -l . 45 | - name: Publish Test Results 46 | uses: EnricoMi/publish-unit-test-result-action@v2 47 | with: 48 | files: | 49 | .test_results/*.xml 50 | - name: Code Coverage 51 | uses: codecov/codecov-action@v4 52 | env: 53 | CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} 54 | with: 55 | fail_ci_if_error: true 56 | files: coverage.xml 57 | -------------------------------------------------------------------------------- /.github/workflows/preview.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: stage & preview workflow 4 | 5 | # Controls when the action will run. 6 | on: 7 | # Triggers the workflow on push or pull request events but only for the master branch 8 | push: 9 | branches: [master, main] 10 | 11 | # Allows you to run this workflow manually from the Actions tab 12 | workflow_dispatch: 13 | 14 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 15 | jobs: 16 | publish_dev_build: 17 | runs-on: ubuntu-latest 18 | 19 | strategy: 20 | matrix: 21 | python-versions: [3.8] 22 | 23 | steps: 24 | - uses: actions/checkout@v3 25 | - uses: actions/setup-python@v4 26 | with: 27 | python-version: ${{ matrix.python-versions }} 28 | 29 | - name: Install dependencies 30 | run: | 31 | python -m pip install --upgrade pip 32 | pip install poetry 33 | 34 | - name: Build wheels and source tarball 35 | run: | 36 | poetry version $(poetry version --short)-dev.$GITHUB_RUN_NUMBER 37 | poetry version --short 38 | poetry build 39 | 40 | - name: publish to Test PyPI 41 | uses: pypa/gh-action-pypi-publish@release/v1 42 | with: 43 | user: __token__ 44 | password: ${{ secrets.TEST_PYPI_API_TOKEN}} 45 | repository_url: https://test.pypi.org/legacy/ 46 | skip_existing: false 47 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # Publish package on main branch if it's tagged with 'v*' 2 | 3 | name: release & publish workflow 4 | 5 | # Controls when the action will run. 6 | on: 7 | # Triggers the workflow on push events but only for the master branch 8 | push: 9 | tags: 10 | - "v*" 11 | 12 | # Allows you to run this workflow manually from the Actions tab 13 | workflow_dispatch: 14 | 15 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 16 | jobs: 17 | # This workflow contains a single job called "release" 18 | release: 19 | name: Create Release 20 | runs-on: ubuntu-20.04 21 | 22 | strategy: 23 | matrix: 24 | python-versions: [3.8] 25 | 26 | # Steps represent a sequence of tasks that will be executed as part of the job 27 | steps: 28 | - name: Get version from tag 29 | id: tag_name 30 | run: | 31 | echo ::set-output name=current_version::${GITHUB_REF#refs/tags/v} 32 | shell: bash 33 | 34 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 35 | - uses: actions/checkout@v3 36 | 37 | - uses: actions/setup-python@v4 38 | with: 39 | python-version: ${{ matrix.python-versions }} 40 | 41 | - name: Install dependencies 42 | run: | 43 | python -m pip install --upgrade pip 44 | pip install poetry 45 | 46 | - name: Build wheels and source tarball 47 | run: >- 48 | poetry build 49 | 50 | - name: show temporary files 51 | run: >- 52 | ls -l 53 | 54 | - name: publish to PyPI 55 | uses: pypa/gh-action-pypi-publish@release/v1 56 | with: 57 | user: __token__ 58 | password: ${{ secrets.PYPI_API_TOKEN }} 59 | skip_existing: true 60 | - name: Install dependencies 61 | run: | 62 | python -m pip install --upgrade pip 63 | pip install mkdocs mkdocs-material mkdocs-include-markdown-plugin 64 | - name: build docs 65 | run: mkdocs build 66 | - name: Deploy Docs 67 | uses: peaceiris/actions-gh-pages@v3 68 | with: 69 | github_token: ${{ secrets.GITHUB_TOKEN }} 70 | publish_dir: ./site 71 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # macOS 2 | .DS_Store 3 | .AppleDouble 4 | .LSOverride 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | env/ 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | .hypothesis/ 53 | .pytest_cache/ 54 | .test_results/ 55 | codecov 56 | # Translations 57 | *.mo 58 | *.pot 59 | 60 | # Django stuff: 61 | *.log 62 | local_settings.py 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # pyenv 81 | .python-version 82 | 83 | # celery beat schedule file 84 | celerybeat-schedule 85 | 86 | # SageMath parsed files 87 | *.sage.py 88 | 89 | # dotenv 90 | .env 91 | 92 | # virtualenv 93 | .venv 94 | venv/ 95 | universal_data_permissions_scanner_venv/ 96 | ENV/ 97 | 98 | # Spyder project settings 99 | .spyderproject 100 | .spyproject 101 | 102 | # Rope project settings 103 | .ropeproject 104 | 105 | # mkdocs documentation 106 | /site 107 | 108 | # mypy 109 | .mypy_cache/ 110 | 111 | # IDE settings 112 | .vscode/ 113 | .idea/ 114 | 115 | .cache_ggshield 116 | 117 | # mkdocs build dir 118 | site/ 119 | udps-export* 120 | tests/tests_datastores/aws/aws_ptrp/satori_dev_account_iam_entities.json 121 | tests/tests_datastores/aws/aws_ptrp/satori_dev_account_ptrp_result.json 122 | tests/tests_datastores/aws/aws_ptrp/satori_dev_account_ptrp.json 123 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/Lucas-C/pre-commit-hooks 3 | rev: v1.1.9 4 | hooks: 5 | - id: forbid-crlf 6 | - id: remove-crlf 7 | - repo: https://github.com/pre-commit/pre-commit-hooks 8 | rev: v3.4.0 9 | hooks: 10 | - id: trailing-whitespace 11 | - id: end-of-file-fixer 12 | - id: check-merge-conflict 13 | - id: check-yaml 14 | args: [ --unsafe ] 15 | - repo: https://github.com/pre-commit/mirrors-isort 16 | rev: v5.8.0 17 | hooks: 18 | - id: isort 19 | args: [ "--filter-files" ] 20 | - repo: https://github.com/ambv/black 21 | rev: 21.5b1 22 | hooks: 23 | - id: black 24 | language_version: python3.8 25 | - repo: https://github.com/pycqa/flake8 26 | rev: 3.9.2 27 | hooks: 28 | - id: flake8 29 | additional_dependencies: [ flake8-typing-imports==1.10.0 ] 30 | - repo: https://github.com/pre-commit/mirrors-mypy 31 | rev: v0.901 32 | hooks: 33 | - id: mypy 34 | exclude: tests/ 35 | additional_dependencies: 36 | - types-click 37 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | disable=C0114, C0115, C0116, R0913, C0301, E0012, R0801, E0401, E0611, W0719 ; missing-module-docstring , missing-class-docstring, missing-function-docstring, too-many-arguments, line-too-ling, broad-exception-raised, Similar lines 3 | # Please check the comment in ./universal_data_permissions_scanner/datastores/aws/__init__.py 4 | init-hook="import sys; sys.path.insert(0, '.'); sys.path.insert(1, './universal_data_permissions_scanner/datastores/aws/aws_ptrp_package')" 5 | [FORMAT] 6 | max-line-length=120 -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | ## [0.1.1] - 2023-02-14 6 | 7 | ### Added 8 | 9 | - First release on PyPI. 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![codecov](https://codecov.io/gh/SatoriCyber/universal-data-permissions-scanner/branch/main/graph/badge.svg?token=8S85Z0CAEU)](https://codecov.io/gh/SatoriCyber/universal-data-permissions-scanner) 2 | 3 | universal-data-permissions-scanner (AKA udps) helps DevOps and data engineers quickly understand who has access to what data and how. 4 | 5 | DevOps and data engineers are often tasked with managing the security of the databases, data lakes or warehouses they operate. This usually involves setting permissions to enable users to query the data they need. However, as the number of users and use-cases increase, complexity explodes. It's no longer humanly possible to remember who had access to what, how and why, which makes meeting security and compliance requirements impossible. 6 | 7 | The root cause of this problem is that permissions to data are usually stored in normalized form, which is great for evaluating permissions but not so great when you want to clearly understand your permissions landscape. When asked "how come Joe can query that table?", it can be a long process to get to a definitive answer and that's just time we don't have. With so many data stores, each with its own security model, it's not feasible to manage it all manually. 8 | 9 | Identifying this was an issue for many of our customers, the team at [Satori](https://satoricyber.com) decided to build *Universal Data Permissions Scanner*, a service that helps admins to better manage their data store permissions. We believe no one should have to sift through DB system tables to get a clear picture of who can do what with data. 10 | 11 | Universal Data Permissions Scanner is available in two forms: 12 | 1. universal-data-permissions-scanner open source CLI - scan the permissions structure of a database to get the list of all users and data assets they can access. 13 | 2. Satori Posture Manager - a fully managed SaaS solution to periodically scan, store and visualize all users and data assets they can access. Learn more [here](https://satoricyber.com). 14 | 15 | ## Documentation 16 | For more information on the universal-data-permissions-scanner open-source, [go to the docs](https://satoricyber.github.io/universal-data-permissions-scanner/). 17 | 18 | ## Contributing 19 | Please follow the [contributing guidelines](CONTRIBUTING.md). 20 | 21 | ## Credits 22 | This package was created with [Cookiecutter](https://github.com/audreyr/cookiecutter) and the [waynerv/cookiecutter-pypackage](https://github.com/waynerv/cookiecutter-pypackage) project template. 23 | -------------------------------------------------------------------------------- /docs/changelog.md: -------------------------------------------------------------------------------- 1 | {% 2 | include-markdown "../CHANGELOG.md" 3 | %} 4 | -------------------------------------------------------------------------------- /docs/contributing.md: -------------------------------------------------------------------------------- 1 | For more information on how to contribute to authz-analzer, visit [this Github page](https://github.com/SatoriCyber/authz-analyzer/blob/main/CONTRIBUTING.md). -------------------------------------------------------------------------------- /docs/datastores/bigquery.md: -------------------------------------------------------------------------------- 1 | Google BigQuery uses the Google Cloud IAM authorization system to manage access to data assets. GCP IAM implements a role-based access control approach. A GCP IAM role is a set of permissions that allow a principal to perform actions, for example, creating a dataset. To allow principals to perform the actions defined in a role on a resource, an allow policy is created which lists the principals, the role and the resource. 2 | 3 | GCP IAM lets you set allow policies at different levels of the resource hierarchy: organization, folder, project and resource. Allow policies grant access to all resources at lower levels of the hierarchy, for example, when setting an allow policy on a dataset, principals will get be granted the role's permissions on all the tables in the dataset. 4 | 5 | ## Setup Access to Scan Google BigQuery 6 | universal-data-permissions-scanner needs the following permissions: 7 | ``` 8 | bigquery.datasets.get 9 | bigquery.datasets.getIamPolicy 10 | bigquery.tables.get 11 | bigquery.tables.getIamPolicy 12 | bigquery.tables.list 13 | resourcemanager.folders.get 14 | resourcemanager.folders.getIamPolicy 15 | resourcemanager.organizations.get 16 | resourcemanager.organizations.getIamPolicy 17 | resourcemanager.projects.get 18 | resourcemanager.projects.getIamPolicy 19 | iam.roles.get 20 | resourcemanager.folders.list 21 | resourcemanager.projects.list 22 | ``` 23 | 24 | It is recommended to group these permissions into a custom role in GCP. Because universal-data-permissions-scanner required organization-level permissions (i.e. `resourcemanager.organizations.get` and `resourcemanager.organizations.getIamPolicy`), the custom role needs to be created on the organization's Identity and Access Management (IAM) settings. Follow these steps to create a role for universal-data-permissions-scanner: 25 | 26 | 1. Login to the Google Cloud Platform console and navigate to your organization 27 | 2. Navigate to IAM, Roles menu and select the CREATE ROLE button 28 | 3. Fill the general properties of the role like name and description 29 | 4. Use the ADD PERMISSIONS dialog to add the permissions specified above 30 | 5. Click CREATE to create the role 31 | 32 | Now you can assign to the role to the user or service account that will be used to run universal-data-permissions-scanner. 33 | 34 | ## Scanning Google BigQuery 35 | By default, auth-analyzer will use the default application credentials provided by the `gcloud` command line interface. To refresh your credentials, run the following command: 36 | ``` 37 | gcloud auth login --update-dac 38 | ``` 39 | Alternatively, use the `--key-file` option to specify a path to a GCP service account key file. 40 | 41 | ## Known Limitations -------------------------------------------------------------------------------- /docs/datastores/databricks.md: -------------------------------------------------------------------------------- 1 | Databricks has two locations where identity is managed: 2 | * Account 3 | * Workspace (deprecated) 4 | There are two types of identities: 5 | * Users 6 | * service principals 7 | Users and service principals can be assigned to groups. Groups can be assigned to other groups. 8 | Users, service principals and groups are assigned to workspaces, workspaces are assigned to unity-catalog. 9 | Unity-catalog manage access to data assets. 10 | Each asset has ownership, which grants full permission. 11 | Assets are hierarchical, so permissions can be inherited from parent assets. 12 | For example, a users can be granted select on a catalog, all tables which belong to the catalog will inherit the permission. 13 | 14 | ## Setup Access to Scan a Databricks: 15 | ### Azure Setup 16 | 1. Generate a service principal at Azure with access to databricks. 17 | 2. Provide to the service principal ccess to the account, workspace, and unity-catalog. 18 | ## Other setups 19 | 1. Provide a user which has access to the account, workspace, and unity-catalog. 20 | 21 | ## Scanning databricks 22 | ### Azure 23 | ``` 24 | udps databirks \ 25 | --host \ 26 | --client_id \ 27 | --client_secret \ 28 | --tenant-id 29 | ``` 30 | ### Other 31 | ``` 32 | udps databricks \ 33 | --host \ 34 | --username \ 35 | -- password 36 | ``` 37 | -------------------------------------------------------------------------------- /docs/datastores/mongodb.md: -------------------------------------------------------------------------------- 1 | authz-analyzer supports two types of MongoDB implementations: 2 | 3 | * MongoDB Atlas 4 | * Standalone MongoDB Cluster 5 | 6 | ## MongoDB Atlas 7 | 8 | MongoDB Atlas is a managed MongoDB service. It provides managed clusters and a host of related services. authz-analyzer currently supports scanning of MongoDB cluster permissions. MongoDB Atlas implements a role-based access control (RBAC) model to manage access to data assets. 9 | 10 | Atlas has two types of users: 11 | 12 | * Database Users - users that have access to an Atlas-managed MongoDB cluster. Database users may have privileges granted to them or they may be assigned with a built-in or user-defined role. Roles can be organized hierarchically. 13 | * Organization Users - users that have access to the Atlas console. Organization users are assigned to organization roles which define their permissions and resources like projects and clusters they can access. 14 | 15 | ### Setup Access to Scan a MongoDB Atlas Cluster 16 | 17 | To enable universal-data-permissions-scanner to scan the list of users, roles and permissions perform the following steps: 18 | 1. Create an organization API Key in the Atlas management console. 19 | 2. Grant the `Organization Read Only` role to the API key you created 20 | 3. Copy the Public and Private keys and store them for later use. 21 | 22 | To enable universal-data-permissions-scanner to scan the list of databases and collections in a MongoDB cluster perform the following steps: 23 | 1. Create a custom role. 24 | 2. Grant the `listDatabases` action to the role. 25 | 3. Grant the `listCollections` action on each database in the cluster to the role. 26 | 4. Create a database user and assign it to the custom role you created. 27 | 28 | ### Scanning a MongoDB Atlas Cluster 29 | ``` 30 | udps atlas \ 31 | --public_key \ 32 | --private_key \ 33 | --username \ 34 | --password \ 35 | --cluster_name \ 36 | --project 37 | ``` 38 | 39 | ## Standalone MongoDB Cluster 40 | MongoDB implements a role-based access control (RBAC) model to manage access to data assets. Users are assigned with roles which have privileges. Role can be built-it or user-defined, and organized hierarchically. Roles that are assigned to the admin database have access across all databases while roles that are assigned to a specific databases have access only to those databases. 41 | 42 | ### Setup Access to Scan a Standalone MongoDB Cluster 43 | 1. Create a role for universal-data-permissions-scanner using the following command: 44 | ``` 45 | db.createRole( 46 | { 47 | role:"udps", 48 | privileges: [ 49 | { 50 | resource: { 51 | db: "", 52 | collection: "" 53 | }, 54 | actions: ["listDatabases", "listCollections", "viewRole", "viewUser"] 55 | } 56 | ], 57 | roles: [] 58 | } 59 | ) 60 | ``` 61 | 62 | 2. Create a user for universal-data-permissions-scanner using the following command: 63 | ``` 64 | db.createUser( 65 | { 66 | user: "udps_user", 67 | roles: ["udps"], 68 | pwd: "" 69 | } 70 | ) 71 | ``` 72 | 73 | ### Scanning a Standalone MongoDB Cluster 74 | ``` 75 | udps mongodb \ 76 | --host \ 77 | --username \ 78 | --password 79 | ``` 80 | 81 | ## Known Limitations 82 | The following MongoDB features are not currently supported by universal-data-permissions-scanner: 83 | 84 | * Data API 85 | * Cloud users 86 | * LDAP Users 87 | * Federated users 88 | -------------------------------------------------------------------------------- /docs/datastores/postgresql.md: -------------------------------------------------------------------------------- 1 | PostgreSQL implements a role-based access control (RBAC) model to manage access to data assets. In PostgreSQL there is no dedicated user object, instead roles that have the login property are used by users to login to the database. Roles can be organized hierarchically. All users are assigned to the `PUBLIC` role by default. 2 | 3 | ## Setup Access to Scan a PostgreSQL Server: 4 | 1. Create a role for authz-analyzer using the following command: 5 | ``` 6 | CREATE ROLE udps NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT LOGIN NOREPLICATION NOBYPASSRLS PASSWORD ''; 7 | ``` 8 | 2. For each database on the server, grant permissions for the universal-data-permissions-scanner role using the following command: 9 | ``` 10 | GRANT SELECT ON TABLE information_schema.tables TO udps; 11 | GRANT SELECT ON TABLE information_schema.table_privileges TO udps; 12 | ``` 13 | 14 | 3. For deployments which are not AWS RDS add the following permissions: 15 | ``` 16 | GRANT SELECT ON TABLE pg_database TO udps; 17 | GRANT SELECT ON TABLE pg_catalog.pg_roles TO udps; 18 | ``` 19 | 20 | ## Scanning a PostgreSQL Server 21 | ``` 22 | udps postgres \ 23 | --host \ 24 | --username \ 25 | --password \ 26 | --dbname 27 | ``` 28 | -------------------------------------------------------------------------------- /docs/datastores/redshift.md: -------------------------------------------------------------------------------- 1 | Redshift supports [Users](https://docs.aws.amazon.com/redshift/latest/dg/r_Users.html)/[Groups](https://docs.aws.amazon.com/redshift/latest/dg/r_Groups.html) or role-based access control [RBAC](https://docs.aws.amazon.com/redshift/latest/dg/t_Roles.html) model to manage access to data assets.. Roles can be organized hierarchically. All users are assigned to the PUBLIC role by default. 2 | 3 | ## Setup Access to Scan Amazon Redshift 4 | Use the following commands to create a role with the relevant database privileges, then enter them into the Redshift Credentials input fields. 5 | 6 | 7 | ``` sql 8 | -- create role with privileges; 9 | CREATE ROLE satori_scanner_role; 10 | 11 | -- grants the required permissions 12 | GRANT SELECT ON TABLE pg_database,pg_user,pg_group,svv_user_grants,svv_role_grants,svv_relation_privileges TO ROLE satori_scanner_role; 13 | 14 | -- create a dedicated user 15 | CREATE USER satori_scanner_user NOCREATEDB NOCREATEUSER SYSLOG ACCESS UNRESTRICTED password 'REPLACE_WITH_A_STRONG_PASSWORD'; 16 | 17 | -- assign role 'SATORI_SCANNER_ROLE' to the new user 18 | GRANT ROLE satori_scanner_role TO satori_scanner_user; 19 | ``` 20 | 21 | ## Scanning Amazon Redshift 22 | ``` 23 | udps redshift \ 24 | --host \ 25 | --username \ 26 | --password 27 | ``` 28 | 29 | ## Known Limitations 30 | 31 | -------------------------------------------------------------------------------- /docs/datastores/snowflake.md: -------------------------------------------------------------------------------- 1 | Snowflake implements a role-based access control (RBAC) model to manage access to data assets. Roles are granted with privileges on data assets and users are assigned to roles, which can be organized hierarchically. All users are assigned to the `PUBLIC` role by default. 2 | 3 | ## Setup Access to Scan a Snowflake Account 4 | 1. Create a role for universal-data-permissions-scanner using the following command: 5 | ``` 6 | CREATE ROLE UDPS_ROLE; 7 | ``` 8 | 2. Grant privileges to the role you created using the following command: 9 | ``` 10 | GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE UDPS_ROLE; 11 | GRANT USAGE ON WAREHOUSE TO ROLE UDPS_ROLE; 12 | GRANT IMPORT SHARE ON ACCOUNT TO UDPS_ROLE; 13 | ``` 14 | 3. Create a user for universal-data-permissions-scanner and assign it to the role you created using the following commands: 15 | ``` 16 | CREATE USER UDPS password='' default_role = UDPS_ROLE default_warehouse=; 17 | GRANT ROLE UDPS_ROLE TO USER UDPS; 18 | ``` 19 | 20 | ## Scanning Snowflake 21 | ``` 22 | udps snowflake \ 23 | --account \ 24 | --username \ 25 | --password 26 | ``` 27 | 28 | ## Known Limitations 29 | The following Snowflake features are not currently supported by universal-data-permissions-scanner: 30 | 31 | * SNOWFLAKE database roles 32 | * Permissions on objects to a share via a database role -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | Universal Data Permissions Scanner (AKA UDPS) helps DevOps and data engineers quickly understand who has access to what data and how. 2 | 3 | DevOps and data engineers are often tasked with managing the security of the databases, data lakes or warehouses they operate. This usually involves setting permissions to enable users to query the data they need. However, as the number of users and use-cases increase, complexity explodes. It's no longer humanly possible to remember who had access to what, how and why, which makes meeting security and compliance requirements impossible. 4 | 5 | The root cause of this problem is that permissions to data are usually stored in normalized form, which is great for evaluating permissions but not so great when you want to clearly understand your permissions landscape. When asked "how come Joe can query that table?", it can be a long process to get to a definitive answer and that's just time we don't have. With so many data stores, each with its own security model, it's not feasible to manage it all manually. 6 | 7 | Identifying this was an issue for many of our customers, the team at [Satori](https://satoricyber.com) decided to build *Universal Data Permissions Scanner*, a service that helps admins to better manage their data store permissions. We believe no one should have to sift through DB system tables to get a clear picture of who can do what with data. 8 | 9 | ## Using Universal Data Permissions Scanner 10 | Universal Data Permissions Scanner is available in two ways: 11 | 1. universal-data-permissions-scanner - scan the permissions structure of a database to get the list of all users and data assets they can access. 12 | 2. Satori Posture manager - a fully managed SaaS solution to periodically scan, store and visualize all users and data assets they can access. Learn more [here](https://satoricyber.com). 13 | 14 | ## Supported Data Stores 15 | Universal Data Permissions Scanner supports the following data stores, with more on the way: 16 | 17 | * [Amazon Redshift](datastores/redshift.md) 18 | * [Amazon S3](datastores/s3.md) 19 | * [Google BigQuery](datastores/bigquery.md) 20 | * [MongoDB](datastores/mongodb.md) 21 | * [PostgreSQL](datastores/postgresql.md) 22 | * [Snowflake](datastores/snowflake.md) 23 | -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | ## Stable release 4 | 5 | To install udps, run this command in your 6 | terminal: 7 | 8 | ``` console 9 | $ pip install udps 10 | ``` 11 | 12 | This is the preferred method to install universal-data-permissions-scanner, as it will always install the most recent stable release. 13 | 14 | If you don't have [pip][] installed, this [Python installation guide][] 15 | can guide you through the process. 16 | 17 | ## From source 18 | 19 | The source for universal-data-permissions-scanner can be downloaded from 20 | the [Github repo][]. 21 | 22 | You can either clone the public repository: 23 | 24 | ``` console 25 | $ git clone git://github.com/satoricyber/universal-data-permissions-scanner 26 | ``` 27 | 28 | Or download the [tarball][]: 29 | 30 | ``` console 31 | $ curl -OJL https://github.com/satoricyber/universal-data-permissions-scanner/tarball/master 32 | ``` 33 | 34 | In order to isolate the authz-analyzer package from the rest of your system, it is recommended to create a virtualenv. You can find instructions on how to do this in the [Python installation guide][]. 35 | 36 | ``` 37 | python3 -m venv .venv 38 | source .venv/bin/activate 39 | ``` 40 | 41 | Once you have a copy of the source, you can install it with: 42 | 43 | ``` console 44 | $ pip install . 45 | ``` 46 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | sources = universal_data_permissions_scanner 2 | 3 | .PHONY: test format lint unittest coverage pre-commit clean 4 | test: format lint unittest 5 | 6 | format: 7 | isort $(sources) tests 8 | black $(sources) tests 9 | 10 | lint: 11 | flake8 $(sources) tests 12 | mypy $(sources) tests 13 | 14 | unittest: 15 | pytest 16 | 17 | coverage: 18 | pytest --cov=$(sources) --cov-branch --cov-report=term-missing tests 19 | 20 | pre-commit: 21 | pre-commit run --all-files 22 | 23 | clean: 24 | rm -rf .mypy_cache .pytest_cache 25 | rm -rf *.egg-info 26 | rm -rf .tox dist site 27 | rm -rf coverage.xml .coverage 28 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: universal-data-permissions-scanner 2 | site_url: https://satoricyber.github.io/universal-data-permissions-scanner 3 | repo_url: https://github.com/satoricyber/universal-data-permissions-scanner 4 | repo_name: satoricyber/universal-data-permissions-scanner 5 | #strict: true 6 | 7 | theme: 8 | name: material 9 | language: en 10 | #logo: assets/logo.png 11 | palette: 12 | scheme: preference 13 | primary: indigo 14 | accent: indigo 15 | features: 16 | - navigation.indexes 17 | - navigation.instant 18 | - navigation.tabs.sticky 19 | 20 | markdown_extensions: 21 | - pymdownx.emoji: 22 | emoji_index: !!python/name:materialx.emoji.twemoji 23 | emoji_generator: !!python/name:materialx.emoji.to_svg 24 | - pymdownx.critic 25 | - pymdownx.caret 26 | - pymdownx.mark 27 | - pymdownx.tilde 28 | - pymdownx.tabbed 29 | - attr_list 30 | - pymdownx.arithmatex: 31 | generic: true 32 | - pymdownx.highlight: 33 | linenums: false 34 | - pymdownx.superfences 35 | - pymdownx.inlinehilite 36 | - pymdownx.details 37 | - admonition 38 | - toc: 39 | baselevel: 2 40 | permalink: true 41 | slugify: !!python/name:pymdownx.slugs.uslugify 42 | - meta 43 | plugins: 44 | - include-markdown 45 | - search: 46 | lang: en 47 | extra: 48 | homepage: https://satoricyber.github.io/universal-data-permissions-scanner 49 | social: 50 | - icon: fontawesome/brands/twitter 51 | link: https://twitter.com/SatoriCyber 52 | name: Tweet 53 | - icon: fontawesome/brands/facebook 54 | link: https://www.facebook.com/SatoriCyber 55 | name: Facebook 56 | - icon: fontawesome/brands/github 57 | link: https://github.com/satoricyber/universal-data-permissions-scanner 58 | name: Github 59 | - icon: material/email 60 | link: "mailto:contact@satoricyber.com" 61 | google_analytics: 62 | - UA-154128939 63 | - auto 64 | 65 | nav: 66 | - Introduction: index.md 67 | - Installation: installation.md 68 | - Data Stores: 69 | - Amazon Redshift: datastores/redshift.md 70 | - Amazon S3: datastores/s3.md 71 | - Databricks: datastores/databricks.md 72 | - Google BigQuery: datastores/bigquery.md 73 | - MongoDB: datastores/mongodb.md 74 | - PostgreSQL: datastores/postgresql.md 75 | - Snowflake: datastores/snowflake.md 76 | - Contributing: contributing.md 77 | - Changelog: changelog.md 78 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | # Global options: 2 | 3 | [mypy] 4 | warn_return_any = True 5 | warn_unused_configs = True 6 | check_untyped_defs = True 7 | ignore_missing_imports = True 8 | modules = universal_data_permissions_scanner 9 | # Please check the comment in ./universal_data_permissions_scanner/datastores/aws/__init__.py 10 | mypy_path = ./universal_data_permissions_scanner/datastores/aws/aws_ptrp_package 11 | exclude = venv 12 | packages = aws_ptrp -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool] 2 | [tool.poetry] 3 | name = "udps" 4 | version = "0.1.38" 5 | homepage = "https://github.com/satoricyber/universal-data-permissions-scanner" 6 | description = "Analyze authorization." 7 | authors = ["SatoriCyber"] 8 | readme = "README.md" 9 | classifiers=[ 10 | 'Development Status :: 4 - Beta', 11 | 'Intended Audience :: Developers', 12 | 'Natural Language :: English', 13 | 'Programming Language :: Python :: 3', 14 | 'Programming Language :: Python :: 3.8', 15 | 'Programming Language :: Python :: 3.9', 16 | 'Programming Language :: Python :: 3.10', 17 | 'Programming Language :: Python :: 3.11', 18 | ] 19 | packages = [ 20 | { include = "universal_data_permissions_scanner" }, 21 | { include = "tests", format = "sdist" }, 22 | ] 23 | 24 | [tool.poetry.dependencies] 25 | python = ">=3.8.1,<3.12" 26 | click = "8.1.3" 27 | snowflake-connector-python = ">=3.1.0" 28 | google-cloud-bigquery = ">=3.4.2" 29 | google-cloud-resource-manager = ">=1.6.3,<2.0" 30 | google-api-python-client = "~2.66.0" 31 | boto3 = "^1.26.27" 32 | pydantic = "^1.10.2" 33 | networkx = "^2.8.8" 34 | pyserde = "^0.9.6" 35 | psycopg2 = "^2.9.5" 36 | psycopg2-binary = "^2.9.5" 37 | google-cloud-iam = "^2.10.0" 38 | redshift-connector = "^2.0.909" 39 | requests = "^2.28.1" 40 | pymongo = "^4.3.3" 41 | pytest = {version="^7.2.0", optional = true} 42 | pytest-cov = {version="^4.0.0", optional = true} 43 | black = {version="^24.3.0", optional=true} 44 | isort = {version="^5.11.3", optional=true} 45 | twine = {version="^4.0.2", optional=true} 46 | types-psycopg2 = {version="^2.9.21.2", optional=true} 47 | google-api-python-client-stubs = {version="^1.13.0", optional=true} 48 | mypy = {version="^0.991", optional=true} 49 | virtualenv = {version="^20.17.1", optional=true} 50 | pip = {version=">=22.3.1", optional=true} 51 | types-requests = {version="^2.28.11.7", optional=true} 52 | tox = {version="^4.0.14", optional=true} 53 | pre-commit = {version="^2.20.0", optional=true} 54 | bump2version = {version="^1.0.1", optional=true} 55 | mkdocs = {version="^1.4.2", optional=true} 56 | pylint = {version="^2.16.2", optional=true} 57 | pyright = {version="^1.1.293", optional=true} 58 | mkdocs-material = {version="^9.0.12", optional=true} 59 | mkdocs-include-markdown-plugin = {version = "^4.0.3", optional=true} 60 | markdown-it-py = {version = "^2.2.0", optional=true} 61 | databricks-cli = "^0.17.4" 62 | 63 | [tool.poetry.extras] 64 | test = ["pytest", "pytest-cov", "black", "isort", "twine", "mypy", "types-psycopg2", 65 | "google-api-python-client-stubs", "types-requests", "pylint", "pyright", "bump2version"] 66 | release = ["twine", "mkdocs", "mkdocs-material", "mkdocs-include-markdown-plugin"] 67 | 68 | [tool.poetry.scripts] 69 | udps = 'universal_data_permissions_scanner.cli:main' 70 | 71 | 72 | [tool.black] 73 | line-length = 120 74 | skip-string-normalization = true 75 | target-version = ['py36', 'py37', 'py38'] 76 | include = '\.pyi?$' 77 | exclude = ''' 78 | /( 79 | \.eggs 80 | | \.git 81 | | \.hg 82 | | \.mypy_cache 83 | | \.tox 84 | | \.venv 85 | | _build 86 | | buck-out 87 | | build 88 | | dist 89 | )/ 90 | ''' 91 | 92 | [tool.isort] 93 | multi_line_output = 3 94 | include_trailing_comma = true 95 | force_grid_wrap = 0 96 | use_parentheses = true 97 | ensure_newline_before_comments = true 98 | line_length = 120 99 | skip_gitignore = true 100 | # you can skip files as below 101 | #skip_glob = docs/conf.py 102 | 103 | [build-system] 104 | requires = ["poetry-core>=1.0.0"] 105 | build-backend = "poetry.core.masonry.api" 106 | -------------------------------------------------------------------------------- /pyrightconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extraPaths": [ 3 | "./universal_data_permissions_scanner/datastores/aws/aws_ptrp_package" 4 | ], 5 | "stubPath": "" 6 | } -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 88 3 | max-complexity = 18 4 | ignore = E203, E266, W503 5 | docstring-convention = google 6 | per-file-ignores = __init__.py:F401 7 | exclude = .git, 8 | __pycache__, 9 | setup.py, 10 | build, 11 | dist, 12 | docs, 13 | releases, 14 | .venv, 15 | .tox, 16 | .mypy_cache, 17 | .pytest_cache, 18 | .vscode, 19 | .github, 20 | # By default test codes will be linted. 21 | tests 22 | 23 | 24 | [coverage:run] 25 | # uncomment the following to omit files during running 26 | #omit = 27 | [coverage:report] 28 | exclude_lines = 29 | pragma: no cover 30 | def __repr__ 31 | if self.debug: 32 | if settings.DEBUG 33 | raise AssertionError 34 | raise NotImplementedError 35 | if 0: 36 | if __name__ == .__main__.: 37 | def main 38 | 39 | [tox:tox] 40 | isolated_build = true 41 | envlist = py38, py39, py310, py311, format, build, lint, format 42 | 43 | [testenv] 44 | allowlist_externals = pytest 45 | extras = 46 | test 47 | passenv = * 48 | setenv = 49 | PYTHONPATH = {toxinidir}/universal_data_permissions_scanner 50 | PYTHONWARNINGS = ignore 51 | commands = 52 | pytest --cov=universal_data_permissions_scanner --cov-branch --cov-report=xml --cov-report=term-missing --junitxml=.test_results/junit-{envname}.xml --junit-prefix={envname} tests 53 | [testenv:format] 54 | allowlist_externals = 55 | isort 56 | black 57 | extras = 58 | test 59 | commands = 60 | isort universal_data_permissions_scanner 61 | black . --check 62 | 63 | [testenv:lint] 64 | allowlist_externals = 65 | mypy 66 | pylint 67 | pyright 68 | extras = 69 | test 70 | commands = 71 | mypy 72 | pylint universal_data_permissions_scanner 73 | pyright 74 | 75 | 76 | [testenv:build] 77 | allowlist_externals = 78 | poetry 79 | # Remove docs for now 80 | # mkdocs 81 | twine 82 | extras = 83 | doc 84 | dev 85 | commands = 86 | poetry build 87 | # Remove docs for now 88 | # mkdocs build 89 | # Disable twine check for now 90 | # twine check dist/* 91 | 92 | 93 | [gh-actions] 94 | python = 95 | 3.9: py39 96 | 3.10: py310 97 | 3.11: py311 98 | 3.8: py38, format, lint, build 99 | 100 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Unit test package for authz-analyzer.""" 2 | -------------------------------------------------------------------------------- /tests/mocks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/tests/mocks/__init__.py -------------------------------------------------------------------------------- /tests/mocks/mock_writers.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from unittest.mock import MagicMock 3 | 4 | from universal_data_permissions_scanner.models.model import AuthzEntry 5 | 6 | 7 | @dataclass 8 | class MockWriter: 9 | mocked_writer: MagicMock 10 | 11 | @classmethod 12 | def new(cls): 13 | mocked_writer = MagicMock(name="MockWriter") 14 | mocked_writer.write_entry = MagicMock("MockWriteEntry") 15 | return cls(mocked_writer) 16 | 17 | def assert_write_entry_called_once_with(self, entry: AuthzEntry): 18 | self.mocked_writer.write_entry.assert_called_once_with(entry) # type: ignore 19 | 20 | def assert_write_entry_not_called(self): 21 | self.mocked_writer.write_entry.assert_not_called() # type: ignore 22 | 23 | def get(self): 24 | return self.mocked_writer 25 | -------------------------------------------------------------------------------- /tests/test_authz_analyzer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # """Tests for `authz_analyzer` package.""" 3 | 4 | # import pytest 5 | # from click.testing import CliRunner 6 | 7 | # from authz_analyzer import cli 8 | 9 | 10 | # @pytest.fixture 11 | # def response(): 12 | # """Sample pytest fixture. 13 | 14 | # See more at: http://doc.pytest.org/en/latest/fixture.html 15 | # """ 16 | # # import requests 17 | # # return requests.get('https://github.com/audreyr/cookiecutter-pypackage') 18 | 19 | 20 | # def test_content(response): 21 | # """Sample pytest test function with the pytest fixture as an argument.""" 22 | # # from bs4 import BeautifulSoup 23 | # # assert 'GitHub' in BeautifulSoup(response.content).title.string 24 | # del response 25 | 26 | 27 | # def test_command_line_interface(): 28 | # """Test the CLI.""" 29 | # runner = CliRunner() 30 | # result = runner.invoke(cli.main) 31 | # assert result.exit_code == 0 32 | # assert 'authz-analyzer' in result.output 33 | # help_result = runner.invoke(cli.main, ['--help']) 34 | # assert help_result.exit_code == 0 35 | # assert '--help Show this message and exit.' in help_result.output 36 | -------------------------------------------------------------------------------- /tests/test_writers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/tests/test_writers/__init__.py -------------------------------------------------------------------------------- /tests/test_writers/test_csv_writer.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | from unittest.mock import MagicMock 3 | 4 | from universal_data_permissions_scanner.models.model import ( 5 | Asset, 6 | AssetType, 7 | AuthzEntry, 8 | AuthzPathElement, 9 | AuthzPathElementType, 10 | Identity, 11 | IdentityType, 12 | PermissionLevel, 13 | ) 14 | from universal_data_permissions_scanner.writers import CSVWriter 15 | 16 | 17 | @mock.patch('csv.writer') 18 | def test_csv_writer_write_header(_mocked_csv: MagicMock): 19 | """Test the CSV writer write header.""" 20 | mock_fh = MagicMock() 21 | 22 | writer = CSVWriter(mock_fh) 23 | writer.writer.writerow.assert_called_once() # pyright: ignore [reportFunctionMemberAccess] 24 | 25 | 26 | def test_csv_writer_write_entry(): 27 | """Test the CSV writer write entry""" 28 | mocked_csv = MagicMock("MockedCSV") 29 | mocked_write_row = MagicMock("WriteRow") 30 | mocked_csv.writerow = mocked_write_row 31 | mock_fh = MagicMock() 32 | 33 | asset = Asset(name=["table1"], type=AssetType.TABLE) 34 | identity = Identity(id="user1", name="user1", type=IdentityType.USER) 35 | authz_entry_path = AuthzPathElement(id="role1", name="role1", type=AuthzPathElementType.ROLE) 36 | authz_entry = AuthzEntry(asset=asset, path=[authz_entry_path], identity=identity, permission=PermissionLevel.READ) 37 | 38 | writer = CSVWriter(mock_fh) 39 | writer.writer = mocked_csv 40 | writer.write_entry(authz_entry) 41 | mocked_write_row.assert_called_once_with(['USER: user1', 'READ', 'TABLE: table1', 'ROLE role1']) 42 | -------------------------------------------------------------------------------- /tests/test_writers/test_multijson_writer.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock 2 | 3 | from universal_data_permissions_scanner.models.model import ( 4 | Asset, 5 | AssetType, 6 | AuthzEntry, 7 | AuthzPathElement, 8 | AuthzPathElementType, 9 | Identity, 10 | IdentityType, 11 | PermissionLevel, 12 | ) 13 | from universal_data_permissions_scanner.writers import MultiJsonWriter 14 | 15 | 16 | def test_csv_writer_write_entry(): 17 | """Test the MultiJson writer write entry""" 18 | mock_fh = MagicMock() 19 | mock_write = MagicMock("Write") 20 | mock_fh.write = mock_write 21 | 22 | asset = Asset(name=["table1"], type=AssetType.TABLE) 23 | identity = Identity(id="user1", name="user1", type=IdentityType.USER) 24 | authz_entry_path = AuthzPathElement(id="role1", name="role1", type=AuthzPathElementType.ROLE) 25 | authz_entry = AuthzEntry(asset=asset, path=[authz_entry_path], identity=identity, permission=PermissionLevel.READ) 26 | 27 | writer = MultiJsonWriter(mock_fh) 28 | writer.write_entry(authz_entry) 29 | mock_write.assert_called_once() 30 | -------------------------------------------------------------------------------- /tests/tests_datastores/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/tests/tests_datastores/__init__.py -------------------------------------------------------------------------------- /tests/tests_datastores/aws/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tests/tests_datastores/aws/aws_ptrp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/tests/tests_datastores/aws/aws_ptrp/__init__.py -------------------------------------------------------------------------------- /tests/tests_datastores/aws/aws_ptrp/resolve_permissions_test_inputs/iam_user_to_s3_bucket/iam_user_with_inline_policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "input": { 3 | "iam_entities": { 4 | "iam_aws_managed_policies": {}, 5 | "iam_accounts_entities": { 6 | "105246067165": { 7 | "iam_users": { 8 | "arn:aws:iam::105246067165:user/iam_user_1": { 9 | "user_name": "iam_user_1", 10 | "user_id": "AIDA6JM62QPID6GVAJJUK", 11 | "path": "/", 12 | "user_policies": [ 13 | { 14 | "UserName": "iam_user_1", 15 | "PolicyName": "inline-policy-1", 16 | "PolicyDocument": { 17 | "Statement": [ 18 | { 19 | "Effect": "Allow", 20 | "Sid": "VisualEditor", 21 | "Action": [ 22 | "s3:GETObject" 23 | ], 24 | "Resource": [ 25 | "*" 26 | ] 27 | } 28 | ] 29 | } 30 | } 31 | ], 32 | "attached_policies_arn": [], 33 | "arn": "arn:aws:iam::105246067165:user/iam_user_1" 34 | } 35 | }, 36 | "iam_groups": {}, 37 | "iam_roles": {}, 38 | "iam_policies": {} 39 | } 40 | } 41 | }, 42 | "target_account_resources": { 43 | "aws_account_id": "105246067165", 44 | "account_resources": { 45 | "s3": [ 46 | { 47 | "name": "bucket_1", 48 | "aws_account_id": "105246067165", 49 | "policy_document": { 50 | "Statement": [] 51 | } 52 | } 53 | ] 54 | } 55 | } 56 | }, 57 | "output": [ 58 | { 59 | "resource": { 60 | "name": "bucket_1", 61 | "type": "S3_BUCKET", 62 | "notes": [] 63 | }, 64 | "principal": { 65 | "arn": "arn:aws:iam::105246067165:user/iam_user_1", 66 | "name": "iam_user_1", 67 | "type": "IAM_USER", 68 | "notes": [] 69 | }, 70 | "action_permission_level": "READ", 71 | "path_nodes": [ 72 | { 73 | "arn": "arn:aws:iam::105246067165:user/iam_user_1", 74 | "name": "inline-policy-1", 75 | "type": "IAM_INLINE_POLICY", 76 | "notes": [] 77 | } 78 | ], 79 | "action_permissions": [ 80 | "GetObject" 81 | ] 82 | } 83 | ] 84 | } -------------------------------------------------------------------------------- /tests/tests_datastores/aws/aws_ptrp/resolve_permissions_test_inputs/no_resource/iam_user_with_not_resource_in_all_allow.json: -------------------------------------------------------------------------------- 1 | { 2 | "input": { 3 | "iam_entities": { 4 | "iam_aws_managed_policies": {}, 5 | "iam_accounts_entities": { 6 | "105246067165": { 7 | "iam_users": { 8 | "arn:aws:iam::105246067165:user/iam_user_1": { 9 | "user_name": "iam_user_1", 10 | "user_id": "AIDA6JM62QPID6GVAJJUK", 11 | "path": "/", 12 | "user_policies": [ 13 | { 14 | "UserName": "iam_user_1", 15 | "PolicyName": "inline-policy-1", 16 | "PolicyDocument": { 17 | "Statement": [ 18 | { 19 | "Effect": "Allow", 20 | "Sid": "VisualEditor1", 21 | "Action": [ 22 | "s3:GetObject", 23 | "s3:DeleteBucket" 24 | ], 25 | "NotResource": [ 26 | "*" 27 | ] 28 | } 29 | ] 30 | } 31 | } 32 | ], 33 | "attached_policies_arn": [], 34 | "arn": "arn:aws:iam::105246067165:user/iam_user_1" 35 | } 36 | }, 37 | "iam_groups": {}, 38 | "iam_roles": {}, 39 | "iam_policies": {} 40 | } 41 | } 42 | }, 43 | "target_account_resources": { 44 | "aws_account_id": "105246067165", 45 | "account_resources": { 46 | "s3": [ 47 | { 48 | "name": "bucket_1", 49 | "aws_account_id": "105246067165", 50 | "policy_document": { 51 | "Statement": [] 52 | } 53 | } 54 | ] 55 | } 56 | } 57 | }, 58 | "output": [] 59 | } -------------------------------------------------------------------------------- /tests/tests_datastores/aws/aws_ptrp/resolve_permissions_test_inputs/no_resource/iam_user_with_not_resource_on_no_valid_resource_arn_deny.json: -------------------------------------------------------------------------------- 1 | { 2 | "input": { 3 | "iam_entities": { 4 | "iam_aws_managed_policies": {}, 5 | "iam_accounts_entities": { 6 | "105246067165": { 7 | "iam_users": { 8 | "arn:aws:iam::105246067165:user/iam_user_1": { 9 | "user_name": "iam_user_1", 10 | "user_id": "AIDA6JM62QPID6GVAJJUK", 11 | "path": "/", 12 | "user_policies": [ 13 | { 14 | "UserName": "iam_user_1", 15 | "PolicyName": "inline-policy-1", 16 | "PolicyDocument": { 17 | "Statement": [ 18 | { 19 | "_comment": "Verify that all will be denied", 20 | "Effect": "Deny", 21 | "Sid": "VisualEditor1", 22 | "Action": [ 23 | "s3:GetObject", 24 | "s3:DeleteBucket" 25 | ], 26 | "NotResource": [ 27 | "arn:aws:s3:::bucket_1/" 28 | ] 29 | }, 30 | { 31 | "Sid": "VisualEditor1", 32 | "Effect": "Allow", 33 | "Action": [ 34 | "s3:GetObject", 35 | "s3:DeleteBucket" 36 | ], 37 | "Resource": [ 38 | "arn:aws:s3:::*" 39 | ] 40 | } 41 | ] 42 | } 43 | } 44 | ], 45 | "attached_policies_arn": [], 46 | "arn": "arn:aws:iam::105246067165:user/iam_user_1" 47 | } 48 | }, 49 | "iam_groups": {}, 50 | "iam_roles": {}, 51 | "iam_policies": {} 52 | } 53 | } 54 | }, 55 | "target_account_resources": { 56 | "aws_account_id": "105246067165", 57 | "account_resources": { 58 | "s3": [ 59 | { 60 | "name": "bucket_1", 61 | "aws_account_id": "105246067165", 62 | "policy_document": { 63 | "Statement": [] 64 | } 65 | }, 66 | { 67 | "name": "bucket_2", 68 | "aws_account_id": "105246067165", 69 | "policy_document": { 70 | "Statement": [] 71 | } 72 | } 73 | ] 74 | } 75 | } 76 | }, 77 | "output": [] 78 | } -------------------------------------------------------------------------------- /tests/tests_datastores/aws/aws_ptrp/resolve_permissions_test_inputs/not_action/iam_user_allowed_no_action.json: -------------------------------------------------------------------------------- 1 | { 2 | "input": { 3 | "iam_entities": { 4 | "iam_aws_managed_policies": {}, 5 | "iam_accounts_entities": { 6 | "105246067165": { 7 | "iam_users": { 8 | "arn:aws:iam::105246067165:user/iam_user_1": { 9 | "user_name": "iam_user_1", 10 | "user_id": "AIDA6JM62QPID6GVAJJUK", 11 | "path": "/", 12 | "user_policies": [ 13 | { 14 | "UserName": "iam_user_1", 15 | "PolicyName": "inline-policy-1", 16 | "PolicyDocument": { 17 | "Statement": [ 18 | { 19 | "Effect": "Allow", 20 | "Sid": "VisualEditor1", 21 | "NotAction": [ 22 | "*" 23 | ], 24 | "Resource": [ 25 | "arn:aws:s3:::bucket_1/*" 26 | ] 27 | } 28 | ] 29 | } 30 | } 31 | ], 32 | "attached_policies_arn": [], 33 | "arn": "arn:aws:iam::105246067165:user/iam_user_1" 34 | } 35 | }, 36 | "iam_groups": {}, 37 | "iam_roles": {}, 38 | "iam_policies": {} 39 | } 40 | } 41 | }, 42 | "target_account_resources": { 43 | "aws_account_id": "105246067165", 44 | "account_resources": { 45 | "s3": [ 46 | { 47 | "name": "bucket_1", 48 | "aws_account_id": "105246067165", 49 | "policy_document": { 50 | "Statement": [] 51 | } 52 | } 53 | ] 54 | } 55 | } 56 | }, 57 | "output": [] 58 | } -------------------------------------------------------------------------------- /tests/tests_datastores/aws/aws_ptrp/resolve_permissions_test_inputs/s3_resource_regex/deny_and_allow_default.json: -------------------------------------------------------------------------------- 1 | { 2 | "input": { 3 | "iam_entities": { 4 | "iam_aws_managed_policies": {}, 5 | "iam_accounts_entities": { 6 | "105246067165": { 7 | "iam_users": { 8 | "arn:aws:iam::105246067165:user/iam_user_1": { 9 | "user_name": "iam_user_1", 10 | "user_id": "AIDA6JM62QPID6GVAJJUK", 11 | "path": "/", 12 | "user_policies": [ 13 | { 14 | "UserName": "iam_user_1", 15 | "PolicyName": "inline-policy-1", 16 | "PolicyDocument": { 17 | "Statement": [ 18 | { 19 | "Effect": "Allow", 20 | "Sid": "VisualEditor1", 21 | "Action": [ 22 | "s3:ListBucket", 23 | "s3:GetObject" 24 | ], 25 | "Resource": [ 26 | "arn:aws:s3:::bucket_1" 27 | ] 28 | }, 29 | { 30 | "Effect": "Deny", 31 | "Sid": "VisualEditor2", 32 | "Action": [ 33 | "s3:ListBucket", 34 | "s3:GetObject" 35 | ], 36 | "Resource": [ 37 | "arn:aws:s3:::bucket_1*" 38 | ] 39 | } 40 | ] 41 | } 42 | } 43 | ], 44 | "attached_policies_arn": [], 45 | "arn": "arn:aws:iam::105246067165:user/iam_user_1" 46 | } 47 | }, 48 | "iam_groups": {}, 49 | "iam_roles": {}, 50 | "iam_policies": {} 51 | } 52 | } 53 | }, 54 | "target_account_resources": { 55 | "aws_account_id": "105246067165", 56 | "account_resources": { 57 | "s3": [ 58 | { 59 | "name": "bucket_1", 60 | "aws_account_id": "105246067165", 61 | "policy_document": { 62 | "Statement": [] 63 | } 64 | } 65 | ] 66 | } 67 | } 68 | }, 69 | "output": [] 70 | } -------------------------------------------------------------------------------- /tests/tests_datastores/aws/aws_ptrp/resolve_permissions_test_inputs/s3_resource_regex/deny_and_allow_with_one_resource_in_deny_that_is_a_subgroup_of_all_in_allow.json: -------------------------------------------------------------------------------- 1 | { 2 | "input": { 3 | "iam_entities": { 4 | "iam_aws_managed_policies": {}, 5 | "iam_accounts_entities": { 6 | "105246067165": { 7 | "iam_users": { 8 | "arn:aws:iam::105246067165:user/iam_user_1": { 9 | "user_name": "iam_user_1", 10 | "user_id": "AIDA6JM62QPID6GVAJJUK", 11 | "path": "/", 12 | "user_policies": [ 13 | { 14 | "UserName": "iam_user_1", 15 | "PolicyName": "inline-policy-1", 16 | "PolicyDocument": { 17 | "Statement": [ 18 | { 19 | "Effect": "Allow", 20 | "Sid": "VisualEditor1", 21 | "Action": [ 22 | "s3:DeleteObject" 23 | ], 24 | "Resource": [ 25 | "arn:aws:s3:::bucket_1/abc.txt", 26 | "arn:aws:s3:::bucket_1/a.txt" 27 | ] 28 | }, 29 | { 30 | "Effect": "Deny", 31 | "Sid": "VisualEditor2", 32 | "Action": [ 33 | "s3:DeleteObject" 34 | ], 35 | "Resource": [ 36 | "arn:aws:s3:::bucket_1/?.txt", 37 | "arn:aws:s3:::bucket_1/*.txt" 38 | ] 39 | } 40 | ] 41 | } 42 | } 43 | ], 44 | "attached_policies_arn": [], 45 | "arn": "arn:aws:iam::105246067165:user/iam_user_1" 46 | } 47 | }, 48 | "iam_groups": {}, 49 | "iam_roles": {}, 50 | "iam_policies": {} 51 | } 52 | } 53 | }, 54 | "target_account_resources": { 55 | "aws_account_id": "105246067165", 56 | "account_resources": { 57 | "s3": [ 58 | { 59 | "name": "bucket_1", 60 | "aws_account_id": "105246067165", 61 | "policy_document": { 62 | "Statement": [ 63 | { 64 | "Sid": "VisualEditor3", 65 | "Principal": "*", 66 | "Effect": "Deny", 67 | "NotAction": [ 68 | "s3:Delete*" 69 | ], 70 | "Resource": "arn:aws:s3:::bucket_1/*" 71 | } 72 | ] 73 | } 74 | } 75 | ] 76 | } 77 | } 78 | }, 79 | "output": [] 80 | } -------------------------------------------------------------------------------- /tests/tests_datastores/aws/aws_ptrp/test_create_session_with_assume_role.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock, patch 2 | 3 | from aws_ptrp.utils.create_session import create_session_with_assume_role 4 | 5 | TARGET_ACCOUNT_ID = 123456123465 6 | ADDITIONAL_ACCOUNT_ID = 654321654321 7 | target_account_session_params = { 8 | "role_arn": f"arn:aws:iam::{TARGET_ACCOUNT_ID}:role/SatoriScanner", 9 | "external_id": "0000", 10 | "role_session_name": "RoleSessionName", 11 | } 12 | additional_account_session_params = { 13 | "role_arn": f"arn:aws:iam::{TARGET_ACCOUNT_ID}:role/SatoriScanner", 14 | "external_id": "0000", 15 | "role_session_name": "RoleSessionName", 16 | } 17 | target_account_assume_role_called_with = { 18 | 'RoleArn': target_account_session_params['role_arn'], 19 | 'RoleSessionName': target_account_session_params['role_session_name'], 20 | 'ExternalId': target_account_session_params['external_id'], 21 | } 22 | additional_account_assume_role_called_with = { 23 | 'RoleArn': additional_account_session_params['role_arn'], 24 | 'RoleSessionName': additional_account_session_params['role_session_name'], 25 | 'ExternalId': additional_account_session_params['external_id'], 26 | } 27 | target_account_assume_role_response = { 28 | 'Credentials': { 29 | 'AccessKeyId': 'TargetAccount_AccessKeyIdValue', 30 | 'SecretAccessKey': 'TargetAccount_SecretAccessKeyValue', 31 | 'SessionToken': 'TargetAccount_SessionTokenValue', 32 | } 33 | } 34 | additional_account_assume_role_response = { 35 | 'Credentials': { 36 | 'AccessKeyId': 'AdditionalAccount_AccessKeyIdValue', 37 | 'SecretAccessKey': 'AdditionalAccount_SecretAccessKeyValue', 38 | 'SessionToken': 'AdditionalAccount_SessionTokenValue', 39 | } 40 | } 41 | target_account_session_called_with = { 42 | 'aws_access_key_id': target_account_assume_role_response['Credentials']['AccessKeyId'], 43 | 'aws_secret_access_key': target_account_assume_role_response['Credentials']['SecretAccessKey'], 44 | 'aws_session_token': target_account_assume_role_response['Credentials']['SessionToken'], 45 | } 46 | additional_account_session_called_with = { 47 | 'aws_access_key_id': additional_account_assume_role_response['Credentials']['AccessKeyId'], 48 | 'aws_secret_access_key': additional_account_assume_role_response['Credentials']['SecretAccessKey'], 49 | 'aws_session_token': additional_account_assume_role_response['Credentials']['SessionToken'], 50 | } 51 | 52 | 53 | @patch('boto3.client') 54 | @patch('boto3.Session') 55 | def test_create_session_with_assume_role( 56 | mock_session, mock_sts_client 57 | ): # pylint: disable=unused-argument,redefined-outer-name 58 | # Configure the mock sts_client 59 | mock_session.return_value = MagicMock() 60 | mock_sts_client.return_value = MagicMock() 61 | mock_sts_client.return_value.assume_role.side_effect = [ 62 | additional_account_assume_role_response, 63 | target_account_assume_role_response, 64 | ] 65 | 66 | create_session_with_assume_role(**additional_account_session_params) 67 | create_session_with_assume_role(**target_account_session_params) 68 | 69 | # verify input & output of create_session_with_assume_role 70 | mock_sts_client.return_value.assume_role.assert_called() 71 | assert mock_sts_client.return_value.assume_role.call_count == 2 72 | call_args_assume_role = mock_sts_client.return_value.assume_role.call_args_list 73 | assert call_args_assume_role[0] == ((), additional_account_assume_role_called_with) 74 | assert call_args_assume_role[1] == ((), target_account_assume_role_called_with) 75 | 76 | mock_session.assert_called() 77 | assert mock_session.call_count == 2 78 | call_args_session = mock_session.call_args_list 79 | assert call_args_session[0] == ((), additional_account_session_called_with) 80 | assert call_args_session[1] == ((), target_account_session_called_with) 81 | -------------------------------------------------------------------------------- /tests/tests_datastores/aws/aws_ptrp/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/tests/tests_datastores/aws/aws_ptrp/utils/__init__.py -------------------------------------------------------------------------------- /tests/tests_datastores/aws/aws_ptrp/utils/aws_ptrp_load_from_dict.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, Optional, Set 2 | 3 | from aws_ptrp import AwsPtrp 4 | from aws_ptrp.actions.aws_actions import AwsActions 5 | from aws_ptrp.iam.iam_entities import IAMEntities 6 | from aws_ptrp.iam_identity_center.iam_identity_center_entities import IamIdentityCenterEntities 7 | from aws_ptrp.principals.aws_principals import AwsPrincipals 8 | from aws_ptrp.resources.account_resources import AwsAccountResources 9 | from aws_ptrp.services.assume_role.assume_role_service import AssumeRoleService 10 | from aws_ptrp.services.federated_user.federated_user_service import FederatedUserService 11 | from aws_ptrp.services.service_resource_type import ServiceResourceType 12 | from serde.de import from_dict 13 | 14 | from universal_data_permissions_scanner.utils.logger import get_logger 15 | 16 | 17 | def load_aws_ptrp_from_dict( 18 | iam_entities_dict: Dict[Any, Any], 19 | target_account_resources_dict: Dict[Any, Any], 20 | iam_identity_center_entities_dict: Optional[Dict[Any, Any]], 21 | resource_service_types_to_load: Set[ServiceResourceType], 22 | ) -> AwsPtrp: 23 | # Load iam_entities 24 | iam_entities: IAMEntities = from_dict(IAMEntities, iam_entities_dict) # type: ignore 25 | 26 | # Load AWS target account resources 27 | target_account_resources: AwsAccountResources = from_dict(AwsAccountResources, target_account_resources_dict) # type: ignore 28 | target_account_resources.update_services_from_iam_entities( 29 | get_logger(False), iam_entities, resource_service_types_to_load 30 | ) 31 | ## valid that all resources are belong to the target aws account id (except the assume-role, federated-user) 32 | services_not_to_valid = set([AssumeRoleService(), FederatedUserService()]) 33 | for resource_service_type, target_account_service_resources in target_account_resources.account_resources.items(): 34 | if resource_service_type not in services_not_to_valid: 35 | for target_account_service_resource in target_account_service_resources: 36 | assert ( 37 | target_account_service_resource.get_resource_account_id() == target_account_resources.aws_account_id 38 | ) 39 | 40 | # Load AWS actions 41 | aws_actions = AwsActions.load(get_logger(False), resource_service_types_to_load) # type: ignore 42 | 43 | # Load AWS principals 44 | aws_principals = AwsPrincipals.load(get_logger(False), iam_entities, target_account_resources) 45 | 46 | if iam_identity_center_entities_dict: 47 | iam_identity_center_entities = from_dict( 48 | IamIdentityCenterEntities, iam_identity_center_entities_dict 49 | ) # type: ignore 50 | else: 51 | iam_identity_center_entities = None 52 | 53 | return AwsPtrp( 54 | aws_actions=aws_actions, 55 | aws_principals=aws_principals, 56 | iam_entities=iam_entities, 57 | target_account_resources=target_account_resources, 58 | iam_identity_center_entities=iam_identity_center_entities, # type: ignore 59 | ) 60 | -------------------------------------------------------------------------------- /tests/tests_datastores/aws/exporter_test_inputs/ptrp_line_basic_iam_user_and_policy_path.json: -------------------------------------------------------------------------------- 1 | { 2 | "ptrp_line": { 3 | "resource": { 4 | "name": "bucket_1", 5 | "type": "S3_BUCKET", 6 | "notes": [] 7 | }, 8 | "principal": { 9 | "arn": "arn:aws:iam::105246067166:user/iam_user_1", 10 | "name": "iam_user_1", 11 | "type": "IAM_USER", 12 | "notes": [] 13 | }, 14 | "action_permission_level": "READ", 15 | "path_nodes": [ 16 | { 17 | "arn": "arn:aws:iam::105246067165:policy/policy", 18 | "name": "policy", 19 | "type": "IAM_POLICY", 20 | "notes": [] 21 | } 22 | ], 23 | "action_permissions": [ 24 | "GetObject" 25 | ] 26 | }, 27 | "authz_entry": { 28 | "asset": { 29 | "name": [ 30 | "bucket_1" 31 | ], 32 | "type": "S3_BUCKET", 33 | "notes": [] 34 | }, 35 | "path": [ 36 | { 37 | "id": "arn:aws:iam::105246067165:policy/policy", 38 | "name": "policy", 39 | "type": "IAM_POLICY", 40 | "notes": [], 41 | "db_permissions": [ 42 | "GetObject" 43 | ] 44 | } 45 | ], 46 | "identity": { 47 | "id": "arn:aws:iam::105246067166:user/iam_user_1", 48 | "name": "iam_user_1", 49 | "type": "IAM_USER", 50 | "notes": [] 51 | }, 52 | "permission": "READ" 53 | } 54 | } -------------------------------------------------------------------------------- /tests/tests_datastores/aws/exporter_test_inputs/ptrp_line_multiple_roles_in_path_and_all_principals.json: -------------------------------------------------------------------------------- 1 | { 2 | "ptrp_line": { 3 | "resource": { 4 | "name": "bucket_1", 5 | "type": "S3_BUCKET", 6 | "notes": [] 7 | }, 8 | "principal": { 9 | "arn": "Anonymous user", 10 | "name": "Anonymous user", 11 | "type": "ANONYMOUS_USER", 12 | "notes": [] 13 | }, 14 | "action_permission_level": "FULL", 15 | "path_nodes": [ 16 | { 17 | "arn": "arn:aws:iam::105246067165:role/iam_role_all", 18 | "name": "iam_role_all", 19 | "type": "IAM_ROLE", 20 | "notes": [] 21 | }, 22 | { 23 | "arn": "arn:aws:sts::105246067165:assumed-role/iam_role_all/session_name", 24 | "name": "session_name", 25 | "type": "ROLE_SESSION", 26 | "notes": [] 27 | }, 28 | { 29 | "arn": "arn:aws:s3:::bucket_1", 30 | "name": "bucket_1", 31 | "type": "RESOURCE_POLICY", 32 | "notes": [] 33 | } 34 | ], 35 | "action_permissions": [ 36 | "PutBucketObjectLockConfiguration", 37 | "PutBucketOwnershipControls", 38 | "PutEncryptionConfiguration", 39 | "PutIntelligentTieringConfiguration" 40 | ] 41 | }, 42 | "authz_entry": { 43 | "asset": { 44 | "name": [ 45 | "bucket_1" 46 | ], 47 | "type": "S3_BUCKET", 48 | "notes": [] 49 | }, 50 | "path": [ 51 | { 52 | "id": "arn:aws:iam::105246067165:role/iam_role_all", 53 | "name": "iam_role_all", 54 | "type": "IAM_ROLE", 55 | "notes": [], 56 | "db_permissions": [] 57 | }, 58 | { 59 | "id": "arn:aws:sts::105246067165:assumed-role/iam_role_all/session_name", 60 | "name": "session_name", 61 | "type": "ROLE_SESSION", 62 | "notes": [], 63 | "db_permissions": [] 64 | }, 65 | { 66 | "id": "arn:aws:s3:::bucket_1", 67 | "name": "bucket_1", 68 | "type": "RESOURCE_POLICY", 69 | "notes": [], 70 | "db_permissions": [ 71 | "PutBucketObjectLockConfiguration", 72 | "PutBucketOwnershipControls", 73 | "PutEncryptionConfiguration", 74 | "PutIntelligentTieringConfiguration" 75 | ] 76 | } 77 | ], 78 | "identity": { 79 | "id": "Anonymous user", 80 | "name": "Anonymous user", 81 | "type": "ANONYMOUS_USER", 82 | "notes": [] 83 | }, 84 | "permission": "FULL" 85 | } 86 | } -------------------------------------------------------------------------------- /tests/tests_datastores/aws/exporter_test_inputs/ptrp_line_with_federated_user.json: -------------------------------------------------------------------------------- 1 | { 2 | "ptrp_line": { 3 | "resource": { 4 | "name": "bucket_1", 5 | "type": "S3_BUCKET", 6 | "notes": [] 7 | }, 8 | "principal": { 9 | "arn": "arn:aws:iam::105246067165:user/iam_user_1", 10 | "name": "iam_user_1", 11 | "type": "IAM_USER", 12 | "notes": [] 13 | }, 14 | "action_permission_level": "WRITE", 15 | "path_nodes": [ 16 | { 17 | "arn": "arn:aws:iam::105246067165:user/iam_user_1", 18 | "name": "inline-policy-allow", 19 | "type": "IAM_INLINE_POLICY", 20 | "notes": [] 21 | }, 22 | { 23 | "arn": "arn:aws:sts::105246067165:federated-user/federated_user_1", 24 | "name": "federated_user_1", 25 | "type": "FEDERATED_USER", 26 | "notes": [] 27 | }, 28 | { 29 | "arn": "arn:aws:s3:::bucket_1", 30 | "name": "bucket_1", 31 | "type": "RESOURCE_POLICY", 32 | "notes": [] 33 | } 34 | ], 35 | "action_permissions": [ 36 | "PutObject" 37 | ] 38 | }, 39 | "authz_entry": { 40 | "asset": { 41 | "name": [ 42 | "bucket_1" 43 | ], 44 | "type": "S3_BUCKET", 45 | "notes": [] 46 | }, 47 | "path": [ 48 | { 49 | "id": "arn:aws:iam::105246067165:user/iam_user_1", 50 | "name": "inline-policy-allow", 51 | "type": "IAM_INLINE_POLICY", 52 | "notes": [], 53 | "db_permissions": [] 54 | }, 55 | { 56 | "id": "arn:aws:sts::105246067165:federated-user/federated_user_1", 57 | "name": "federated_user_1", 58 | "type": "FEDERATED_USER", 59 | "notes": [], 60 | "db_permissions": [] 61 | }, 62 | { 63 | "id": "arn:aws:s3:::bucket_1", 64 | "name": "bucket_1", 65 | "type": "RESOURCE_POLICY", 66 | "notes": [], 67 | "db_permissions": [ 68 | "PutObject" 69 | ] 70 | } 71 | ], 72 | "identity": { 73 | "id": "arn:aws:iam::105246067165:user/iam_user_1", 74 | "name": "iam_user_1", 75 | "type": "IAM_USER", 76 | "notes": [] 77 | }, 78 | "permission": "WRITE" 79 | } 80 | } -------------------------------------------------------------------------------- /tests/tests_datastores/aws/test_exporter.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import pathlib 4 | from typing import List 5 | 6 | import pytest 7 | from aws_ptrp.ptrp_models import AwsPtrpLine 8 | from serde import to_dict 9 | from serde.de import from_dict 10 | 11 | from universal_data_permissions_scanner.datastores.aws.analyzer.exporter import AWSAuthzAnalyzerExporter 12 | from universal_data_permissions_scanner.models.model import AuthzEntry 13 | from tests.mocks.mock_writers import MockWriter 14 | 15 | RESOURCES_INPUT_DIR = pathlib.Path().joinpath(os.path.dirname(__file__), 'exporter_test_inputs') 16 | 17 | 18 | def get_resolve_permissions_test_inputs() -> List[str]: 19 | ret = [] 20 | assert os.path.isdir(RESOURCES_INPUT_DIR) 21 | for root, _dirs, files in os.walk(RESOURCES_INPUT_DIR): 22 | for file in files: 23 | ret.append(os.path.relpath(os.path.join(root, file), RESOURCES_INPUT_DIR)) 24 | return ret 25 | 26 | 27 | @pytest.mark.parametrize("test_input", get_resolve_permissions_test_inputs()) 28 | def test_exporter(test_input: str): 29 | test_file_path = os.path.join(RESOURCES_INPUT_DIR, test_input) 30 | mocked_writer = MockWriter.new() 31 | exporter: AWSAuthzAnalyzerExporter = AWSAuthzAnalyzerExporter(writer=mocked_writer.get()) 32 | 33 | with open(test_file_path, "r", encoding="utf-8") as json_file_r: 34 | json_loaded = json.load(json_file_r) 35 | ptrp_line: AwsPtrpLine = from_dict(AwsPtrpLine, to_dict(json_loaded["ptrp_line"])) # type: ignore 36 | exporter.export_entry_from_ptrp_line(ptrp_line) 37 | authz_entry: AuthzEntry = from_dict(AuthzEntry, to_dict(json_loaded["authz_entry"])) # type: ignore 38 | mocked_writer.assert_write_entry_called_once_with(authz_entry) 39 | -------------------------------------------------------------------------------- /tests/tests_datastores/aws/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/tests/tests_datastores/aws/utils/__init__.py -------------------------------------------------------------------------------- /tests/tests_datastores/aws/utils/test_aws_regex_full_subset.py: -------------------------------------------------------------------------------- 1 | from aws_ptrp.utils.regex_subset import is_aws_regex_full_subset 2 | 3 | 4 | def test_aws_regex_full_subset(): 5 | assert is_aws_regex_full_subset("*ab", "c*") is False 6 | assert is_aws_regex_full_subset("ab*", "ab") is True 7 | assert is_aws_regex_full_subset("ab", "ab*") is False 8 | assert is_aws_regex_full_subset("*ab", "c") is False 9 | assert is_aws_regex_full_subset("*", "d") is True 10 | assert is_aws_regex_full_subset("bla", "aviv") is False 11 | assert is_aws_regex_full_subset("*bla*", "?blablo") is True 12 | assert is_aws_regex_full_subset("bla*", "bla?") is True 13 | assert is_aws_regex_full_subset("*a*", "ab*") is True 14 | assert is_aws_regex_full_subset("*ab*", "cab*") is True 15 | assert is_aws_regex_full_subset("*ab*", "c?b*") is False 16 | assert is_aws_regex_full_subset("*ab*", "?b*") is False 17 | assert is_aws_regex_full_subset("*aba?caba", "abaccaba") is True 18 | assert is_aws_regex_full_subset("aba?caba", "abaccaba") is True 19 | assert is_aws_regex_full_subset("a*b", "a?b") is True 20 | assert is_aws_regex_full_subset("a*b", "a?c") is False 21 | assert is_aws_regex_full_subset("a*b?", "a?b?") is True 22 | assert is_aws_regex_full_subset("aab*", "aab") is True 23 | assert is_aws_regex_full_subset("a", "*") is False 24 | assert is_aws_regex_full_subset("a*b", "aa") is False 25 | assert is_aws_regex_full_subset("*ba*", "Aviv?ba*") is True 26 | assert is_aws_regex_full_subset("?ba*", "Aviv?ba*") is False 27 | assert is_aws_regex_full_subset("*ba*", "Aviv?ba?") is True 28 | assert is_aws_regex_full_subset("*b?a*", "Aviv?b?a?") is True 29 | assert is_aws_regex_full_subset("*b?a*", "Aviv?c?a?") is False 30 | assert is_aws_regex_full_subset("a", "?") is False 31 | assert is_aws_regex_full_subset("a?", "ab") is True 32 | -------------------------------------------------------------------------------- /tests/tests_datastores/bigquery/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/tests/tests_datastores/bigquery/__init__.py -------------------------------------------------------------------------------- /tests/tests_datastores/databricks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/tests/tests_datastores/databricks/__init__.py -------------------------------------------------------------------------------- /tests/tests_datastores/mongodb/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/tests/tests_datastores/mongodb/__init__.py -------------------------------------------------------------------------------- /tests/tests_datastores/mongodb/test_atlas_build_custom_role_from_response.py: -------------------------------------------------------------------------------- 1 | from universal_data_permissions_scanner.datastores.mongodb.atlas.service_model import ( 2 | CustomRoleEntry, 3 | ) 4 | 5 | from universal_data_permissions_scanner.datastores.mongodb.atlas.model import ( 6 | CustomRole, 7 | Action, 8 | Resource, 9 | Permission, 10 | ) 11 | 12 | 13 | def test_atlas_build_custom_role_from_response(): 14 | cluster_db_to_collections_mapping = { 15 | "test": ["test_collection"], 16 | "test2": ["test_collection2"], 17 | } 18 | 19 | custom_role_entry = CustomRoleEntry( 20 | { # pyright: ignore [reportGeneralTypeIssues] 21 | 'actions': [ # type: ignore 22 | {'action': 'OUT_TO_S3', 'resources': [{'cluster': True}]}, 23 | {'action': 'FIND', 'resources': [{'collection': 'test_collection2', 'db': 'test2'}]}, 24 | {'action': 'DROP_DATABASE', 'resources': [{'collection': '', 'db': 'test'}]}, 25 | ], 26 | 'inheritedRoles': [], 27 | 'roleName': 'bla', 28 | } 29 | ) 30 | 31 | res = CustomRole.build_custom_role_from_response( 32 | entry=custom_role_entry, project_dbs_to_collections=cluster_db_to_collections_mapping 33 | ) 34 | expected = CustomRole( 35 | name='bla', 36 | actions={ 37 | Action(resource=Resource(collection='test_collection', database='test'), permission=Permission.OUT_TO_S3), 38 | Action(resource=Resource(collection='test_collection2', database='test2'), permission=Permission.OUT_TO_S3), 39 | Action(resource=Resource(collection='test_collection2', database='test2'), permission=Permission.FIND), 40 | Action(resource=Resource(collection='test_collection2', database='test2'), permission=Permission.FIND), 41 | Action(resource=Resource(collection='', database='test'), permission=Permission.DROP_DATABASE), 42 | }, 43 | inherited_roles=set(), 44 | ) 45 | 46 | assert res == expected 47 | -------------------------------------------------------------------------------- /tests/tests_datastores/postgres/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/tests/tests_datastores/postgres/__init__.py -------------------------------------------------------------------------------- /tests/tests_datastores/postgres/mocks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/tests/tests_datastores/postgres/mocks/__init__.py -------------------------------------------------------------------------------- /tests/tests_datastores/postgres/mocks/postgres_mock_connector.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from types import TracebackType 3 | from typing import List, NamedTuple, Optional, Type 4 | from unittest.mock import MagicMock 5 | 6 | 7 | class Role(NamedTuple): 8 | username: str 9 | superuser: bool 10 | role: Optional[str] 11 | login: bool 12 | 13 | 14 | class RoleGrant(NamedTuple): 15 | table_name: str 16 | schema: str 17 | type: str 18 | owner: str 19 | relacl: Optional[str] 20 | 21 | 22 | class Table(NamedTuple): 23 | table_catalog: str 24 | table_schema: str 25 | table_name: str 26 | 27 | 28 | @dataclass 29 | class PostgresMockCursor: 30 | roles: List[Role] 31 | role_grants: List[RoleGrant] 32 | all_tables: List[Table] 33 | 34 | def get(self): 35 | postgres_mock = MagicMock(name="PostgresConnectionMock") 36 | fetchall = MagicMock(name="PostgresFetchAllMock", side_effect=[self.roles, self.role_grants, self.all_tables]) 37 | 38 | postgres_mock.fetchall = fetchall 39 | 40 | return postgres_mock 41 | 42 | def __enter__(self): 43 | self.cursor = self.get() 44 | return self.cursor 45 | 46 | def __exit__( 47 | self, 48 | exc_type: Optional[Type[BaseException]], 49 | exc_value: Optional[BaseException], 50 | traceback: Optional[TracebackType], 51 | ): 52 | self.cursor.fetchall.assert_called() # type: ignore 53 | -------------------------------------------------------------------------------- /tests/tests_datastores/redshift/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/tests/tests_datastores/redshift/__init__.py -------------------------------------------------------------------------------- /tests/tests_datastores/snowflake/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/tests/tests_datastores/snowflake/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/__init__.py: -------------------------------------------------------------------------------- 1 | """Top-level package for authz-analyzer.""" 2 | 3 | __author__ = """SatoriCyber""" 4 | __email__ = 'contact@satoricyber.com' 5 | __version__ = '0.1.38' 6 | 7 | from universal_data_permissions_scanner.datastores.aws.analyzer import AwsAssumeRoleInput, AWSAuthzAnalyzer # type: ignore 8 | from universal_data_permissions_scanner.datastores.aws.analyzer.redshift.analyzer import RedshiftAuthzAnalyzer # type: ignore 9 | from universal_data_permissions_scanner.datastores.bigquery.analyzer import BigQueryAuthzAnalyzer # type: ignore 10 | from universal_data_permissions_scanner.datastores.databricks.analyzer import DatabricksAuthzAnalyzer # type: ignore 11 | from universal_data_permissions_scanner.datastores.mongodb.analyzer import MongoDBAuthzAnalyzer # type: ignore 12 | from universal_data_permissions_scanner.datastores.mongodb.atlas.analyzer import MongoDBAtlasAuthzAnalyzer # type: ignore 13 | from universal_data_permissions_scanner.datastores.postgres.analyzer import PostgresAuthzAnalyzer # type: ignore 14 | from universal_data_permissions_scanner.datastores.snowflake.analyzer import SnowflakeAuthzAnalyzer # type: ignore 15 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | # !Temporary solution! 5 | # Until we will create a dedicated repo for the aws_ptrp_package (we don't want to put this package as sibling to authz_analyzer package) 6 | # We want to emphasize that the aws_ptrp_package (AWS Principal to Resource Permissions) is actually a python package and not a inner module in authz_analyzer package 7 | # By adding the below sys, we can work with all internal modules of aws_ptrp as actual separated package 8 | # Both mypy, pylint has valid solution for this workaround (check the .pylintrc, mypy.ini) 9 | sys.path.insert(0, os.path.join(os.path.dirname(__file__), "aws_ptrp_package")) 10 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/analyzer/__init__.py: -------------------------------------------------------------------------------- 1 | from .analyzer import AwsAssumeRoleInput, AWSAuthzAnalyzer 2 | 3 | __all__ = ['AWSAuthzAnalyzer', 'AwsAssumeRoleInput'] 4 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/analyzer/analyzer.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | from dataclasses import dataclass 3 | from logging import Logger 4 | from typing import List, Optional, Set 5 | 6 | # (vs-code) For python auto-complete please add this to your workspace setting.json file 7 | # "python.autoComplete.extraPaths": [ 8 | # "[PATH-TO-AUTHZ-ANALYZER]/authz_analyzer/datastores/aws/aws_ptrp_package/" 9 | # ] 10 | from aws_ptrp import AwsAssumeRole, AwsPtrp 11 | from aws_ptrp.services import ServiceResourceType 12 | from aws_ptrp.services.s3.s3_service import S3Service 13 | 14 | from universal_data_permissions_scanner.datastores.aws.analyzer.exporter import AWSAuthzAnalyzerExporter 15 | from universal_data_permissions_scanner.writers import BaseWriter 16 | from universal_data_permissions_scanner.utils.logger import get_logger 17 | 18 | AwsAssumeRoleInput = namedtuple('AwsAssumeRoleInput', ['role_arn', 'external_id']) 19 | 20 | 21 | @dataclass 22 | class AWSAuthzAnalyzer: 23 | exporter: AWSAuthzAnalyzerExporter 24 | logger: Logger 25 | target_account: AwsAssumeRole 26 | additional_accounts: Optional[List[AwsAssumeRole]] = None 27 | 28 | @classmethod 29 | def connect( 30 | cls, 31 | target_account: AwsAssumeRoleInput, 32 | writer: BaseWriter, 33 | logger: Optional[Logger] = None, 34 | additional_accounts: Optional[List[AwsAssumeRoleInput]] = None, 35 | ): 36 | if logger is None: 37 | logger = get_logger(False) 38 | aws_exporter = AWSAuthzAnalyzerExporter(writer) 39 | target_account_assume_role = AwsAssumeRole( 40 | role_arn=target_account.role_arn, 41 | external_id=target_account.external_id, 42 | ) 43 | if additional_accounts: 44 | additional_accounts_assume_role: Optional[List[AwsAssumeRole]] = [ 45 | AwsAssumeRole( 46 | role_arn=additional_account.role_arn, 47 | external_id=additional_account.external_id, 48 | ) 49 | for additional_account in additional_accounts 50 | ] 51 | else: 52 | additional_accounts_assume_role = None 53 | return cls( 54 | logger=logger, 55 | exporter=aws_exporter, 56 | target_account=target_account_assume_role, 57 | additional_accounts=additional_accounts_assume_role, 58 | ) 59 | 60 | def run_s3(self): 61 | self._run(set([S3Service()])) 62 | 63 | def _run( 64 | self, 65 | resource_service_types: Set[ServiceResourceType], 66 | ): 67 | self.logger.info( 68 | "Starting to analyzed AWS for %s, target account: %s, additional accounts: %s", 69 | resource_service_types, 70 | self.target_account, 71 | self.additional_accounts, 72 | ) 73 | aws_ptrp = AwsPtrp.load_from_role( 74 | logger=self.logger, 75 | resource_service_types_to_load=resource_service_types, 76 | target_account=self.target_account, 77 | additional_accounts=self.additional_accounts, 78 | ) 79 | aws_ptrp.resolve_permissions(self.logger, self.exporter.export_entry_from_ptrp_line) 80 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/analyzer/redshift/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/aws/analyzer/redshift/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/analyzer/redshift/commands/all_databases.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | datname, 3 | datallowconn 4 | FROM 5 | pg_database 6 | where 7 | datistemplate = false; -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/analyzer/redshift/commands/all_tables.sql: -------------------------------------------------------------------------------- 1 | select 2 | table_catalog as db, 3 | table_schema as schema, 4 | table_name as name, 5 | table_type as type 6 | from 7 | information_schema.tables -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/analyzer/redshift/commands/datashare_consumers.sql: -------------------------------------------------------------------------------- 1 | select 2 | share_name, 3 | consumer_account, 4 | consumer_namespace 5 | from 6 | pg_catalog.svv_datashare_consumers -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/analyzer/redshift/commands/datashare_desc.sql: -------------------------------------------------------------------------------- 1 | DESC datashare -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/analyzer/redshift/commands/datashares.sql: -------------------------------------------------------------------------------- 1 | select 2 | share_id, 3 | share_name, 4 | source_database 5 | from 6 | pg_catalog.svv_datashares 7 | where 8 | share_type = 'OUTBOUND' -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/analyzer/redshift/commands/identities_pg_user.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | usesysid AS identity_id, 3 | usename AS identity_name, 4 | 'USER' AS identity_type, 5 | grosysid AS granted_identity_id, 6 | groname AS granted_identity_name, 7 | 'GROUP' AS granted_identity_type, 8 | usesuper as is_admin 9 | FROM 10 | pg_user 11 | LEFT JOIN pg_group ON pg_user.usesysid = ANY (pg_group.grolist); -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/analyzer/redshift/commands/identities_privileges.sql: -------------------------------------------------------------------------------- 1 | -- identities(user,group,role) privilege to asset 2 | SELECT 3 | 'UNKNOWN' AS grantor, 4 | identity_id AS grantee, 5 | namespace_name AS schema_name, 6 | relation_name AS table_name, 7 | privilege_type 8 | FROM svv_relation_privileges 9 | --WHERE identity_name != 'public'; 10 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/analyzer/redshift/commands/identities_svv_role_grants.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | role_id AS identity_id, 3 | role_name AS identity_name, 4 | 'ROLE' AS identity_type, 5 | granted_role_id AS granted_identity_id, 6 | granted_role_name AS granted_identity_name, 7 | 'ROLE' AS granted_identity_type, 8 | FALSE as is_admin 9 | FROM 10 | svv_role_grants 11 | WHERE 12 | identity_name != 'rdsdb'; -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/analyzer/redshift/commands/identities_svv_user_grants.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | user_id AS identity_id, 3 | user_name AS identity_name, 4 | 'USER' AS identity_type, 5 | role_id AS granted_identity_id, 6 | role_name AS granted_identity_name, 7 | 'ROLE' AS granted_identity_type, 8 | FALSE as is_admin 9 | FROM 10 | svv_user_grants; -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/analyzer/redshift/service.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from pathlib import Path 3 | from typing import Any, Optional, Tuple 4 | 5 | import redshift_connector # type: ignore 6 | 7 | 8 | @dataclass 9 | class RedshiftService: 10 | @staticmethod 11 | def get_rows( 12 | redshift_cursor: redshift_connector.Cursor, command_name: Path, params: Optional[str] = None 13 | ) -> Tuple[Any, ...]: 14 | """Get rows from Redshift.""" 15 | command = (Path(__file__).parent / "commands" / command_name).read_text(encoding="utf-8") 16 | if params is not None: 17 | command += " " + params 18 | 19 | redshift_cursor.execute(command) # type: ignore 20 | return redshift_cursor.fetchall() # type: ignore 21 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/__init__.py: -------------------------------------------------------------------------------- 1 | from aws_ptrp.utils.assume_role import AwsAssumeRole 2 | 3 | from .ptrp import AwsPtrp 4 | 5 | __all__ = ['AwsPtrp', 'AwsAssumeRole'] 6 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/actions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/actions/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/actions/actions_resolver.py: -------------------------------------------------------------------------------- 1 | from logging import Logger 2 | from typing import Dict, List, Optional, Set 3 | 4 | from aws_ptrp.actions.aws_actions import AwsActions 5 | from aws_ptrp.services import ServiceActionBase, ServiceActionsResolverBase, ServiceActionType 6 | 7 | 8 | class ActionsResolver: 9 | @staticmethod 10 | def _get_stmt_action_regexes_per_service_type( 11 | _logger: Logger, 12 | stmt_action_regexes: List[str], 13 | service_types_to_resolve: Set[ServiceActionType], 14 | ) -> Dict[ServiceActionType, List[str]]: 15 | ret: Dict[ServiceActionType, List[str]] = dict() 16 | for stmt_action_regex in stmt_action_regexes: 17 | for service_type in service_types_to_resolve: 18 | service_prefix = service_type.get_action_service_prefix() 19 | stmt_relative_id_regex = ( 20 | "*" 21 | if stmt_action_regex == "*" 22 | else ( 23 | stmt_action_regex[len(service_prefix) :] 24 | if stmt_action_regex.startswith(service_prefix) 25 | else None 26 | ) 27 | ) 28 | if stmt_relative_id_regex is None: 29 | continue 30 | 31 | regexes_list: Optional[List[str]] = ret.get(service_type, None) 32 | if regexes_list: 33 | regexes_list.append(stmt_relative_id_regex) 34 | else: 35 | ret[service_type] = [stmt_relative_id_regex] 36 | 37 | return ret 38 | 39 | @classmethod 40 | def resolve_stmt_action_regexes( 41 | cls, 42 | logger: Logger, 43 | stmt_action_regexes: List[str], 44 | not_action_annotated: bool, 45 | aws_actions: AwsActions, 46 | allowed_service_action_types: Optional[Set[ServiceActionType]] = None, 47 | ) -> Optional[Dict[ServiceActionType, ServiceActionsResolverBase]]: 48 | services_action_resolver: Dict[ServiceActionType, ServiceActionsResolverBase] = dict() 49 | 50 | if isinstance(stmt_action_regexes, str): 51 | stmt_action_regexes = [stmt_action_regexes] 52 | 53 | service_types_to_resolve: Set[ServiceActionType] = set(aws_actions.aws_actions.keys()) 54 | if allowed_service_action_types: 55 | service_types_to_resolve.intersection(allowed_service_action_types) 56 | ret: Dict[ServiceActionType, List[str]] = ActionsResolver._get_stmt_action_regexes_per_service_type( 57 | logger, stmt_action_regexes, service_types_to_resolve 58 | ) 59 | for service_type, service_regexes in ret.items(): 60 | service_actions: Optional[Set[ServiceActionBase]] = aws_actions.aws_actions.get(service_type) 61 | if service_actions: 62 | service_action_resolver: ServiceActionsResolverBase = ( 63 | service_type.load_resolver_service_actions_from_single_stmt( 64 | logger, service_regexes, service_actions, not_action_annotated 65 | ) 66 | ) 67 | if not service_action_resolver.is_empty(): 68 | services_action_resolver[service_type] = service_action_resolver 69 | 70 | return services_action_resolver if services_action_resolver else None 71 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/actions/aws_actions.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from logging import Logger 3 | from typing import Any, Dict, List, Optional, Set, Type 4 | 5 | from aws_ptrp.services import ( 6 | ServiceActionBase, 7 | ServiceActionType, 8 | get_service_action_by_name, 9 | get_service_action_type_by_name, 10 | ) 11 | from serde import field, from_dict, serde, to_dict 12 | 13 | 14 | def to_dict_serializer(aws_actions: Dict[ServiceActionType, List[ServiceActionBase]]) -> Dict[str, List[Any]]: 15 | return dict([(k.get_service_name(), to_dict(v)) for (k, v) in aws_actions.items()]) 16 | 17 | 18 | def from_dict_deserializer( 19 | account_actions_from_deserializer: Dict[str, List[Any]], 20 | ) -> Dict[ServiceActionType, List[ServiceActionBase]]: 21 | aws_actions: Dict[ServiceActionType, List[ServiceActionBase]] = dict() 22 | for service_key_name, service_actions_base in account_actions_from_deserializer.items(): 23 | service_type: Optional[Type[ServiceActionType]] = get_service_action_type_by_name(service_key_name) 24 | service_action: Optional[Type[ServiceActionBase]] = get_service_action_by_name(service_key_name) 25 | if service_type and service_action: 26 | value: List[ServiceActionBase] = [ 27 | from_dict(service_action, service_action_base_dict) for service_action_base_dict in service_actions_base 28 | ] # type: ignore 29 | aws_actions[service_type()] = value 30 | 31 | return aws_actions 32 | 33 | 34 | @serde 35 | @dataclass 36 | class AwsActions: 37 | aws_actions: Dict[ServiceActionType, Set[ServiceActionBase]] = field( 38 | serializer=to_dict_serializer, deserializer=from_dict_deserializer 39 | ) 40 | 41 | @classmethod 42 | def load(cls, logger: Logger, service_types_to_load: Set[ServiceActionType]): 43 | logger.info(f"Init AwsActions {service_types_to_load}...") 44 | aws_actions: Dict[ServiceActionType, Set[ServiceActionBase]] = dict() 45 | for service_type_to_load in service_types_to_load: 46 | ret: Set[ServiceActionBase] = service_type_to_load.load_service_actions(logger) 47 | aws_actions[service_type_to_load] = ret 48 | 49 | return cls(aws_actions=aws_actions) 50 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam/iam_policies.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Dict 3 | 4 | from aws_ptrp.iam.policy import Policy 5 | from aws_ptrp.iam.policy.policy_document import PolicyDocument, PolicyDocumentCtx 6 | from aws_ptrp.utils.pagination import paginate_response_list 7 | from boto3 import Session 8 | from serde import from_dict, serde 9 | 10 | 11 | @serde 12 | @dataclass 13 | class IAMPolicy: 14 | policy: Policy 15 | policy_document: PolicyDocument 16 | 17 | def __eq__(self, other): 18 | return self.policy.policy_id == other.policy.policy_id 19 | 20 | def __hash__(self): 21 | return hash(self.policy.policy_id) 22 | 23 | def __repr__(self): 24 | return self.policy.arn 25 | 26 | @staticmethod 27 | def extract_aws_account_id_from_arn_of_iam_entity(arn: str) -> str: 28 | return arn[arn.find(":iam::") + 6 : arn.find(":policy/")] 29 | 30 | def to_policy_document_ctx(self) -> PolicyDocumentCtx: 31 | aws_account_id = IAMPolicy.extract_aws_account_id_from_arn_of_iam_entity(self.policy.arn) 32 | return PolicyDocumentCtx( 33 | policy_document=self.policy_document, 34 | policy_name=self.policy.policy_name, 35 | parent_arn=self.policy.arn, 36 | parent_aws_account_id=aws_account_id, 37 | ) 38 | 39 | 40 | def get_iam_policies(session: Session) -> Dict[str, IAMPolicy]: 41 | iam_client = session.client('iam') 42 | ret: Dict[str, IAMPolicy] = {} 43 | 44 | list_policies = paginate_response_list(iam_client.list_policies, 'Policies', OnlyAttached=True) 45 | for list_policy_response in list_policies: 46 | # Due to the comment in the aws API for list_policies we are using the get_policy for each policy 47 | # "IAM resource-listing operations return a subset of the available attributes for the resource. For example, this operation does not return tags, even though they are an attribute of the returned object. To view all of the information for a customer manged policy, see GetPolicy." 48 | arn = list_policy_response['Arn'] 49 | policy_response = iam_client.get_policy(PolicyArn=arn)['Policy'] 50 | policy: Policy = from_dict(Policy, policy_response) # type: ignore 51 | 52 | policy_version_response = iam_client.get_policy_version(PolicyArn=arn, VersionId=policy.default_version_id) 53 | policy_version_response = policy_version_response['PolicyVersion']['Document'] 54 | policy_document: PolicyDocument = from_dict(PolicyDocument, policy_version_response) # type: ignore 55 | ret[policy.arn] = IAMPolicy( 56 | policy=policy, 57 | policy_document=policy_document, 58 | ) 59 | return ret 60 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam/policy/__init__.py: -------------------------------------------------------------------------------- 1 | from .group_policy import GroupPolicy 2 | from .policy import Policy 3 | from .policy_document import PolicyDocument, PolicyDocumentCtx 4 | from .user_policy import UserPolicy 5 | 6 | __all__ = ['Policy', 'PolicyDocument', 'UserPolicy', 'GroupPolicy', 'PolicyDocumentCtx'] 7 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam/policy/effect.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class Effect(str, Enum): 5 | Deny = "Deny" 6 | Allow = "Allow" 7 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam/policy/group_policy.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from aws_ptrp.iam.policy.policy_document import PolicyDocument 4 | from serde import serde 5 | 6 | 7 | @serde(rename_all="pascalcase") 8 | @dataclass 9 | class GroupPolicy: 10 | group_name: str 11 | policy_name: str 12 | policy_document: PolicyDocument 13 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam/policy/policy.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Optional 3 | 4 | from serde import field, serde 5 | 6 | 7 | @serde(rename_all="pascalcase") 8 | @dataclass 9 | class Policy: 10 | policy_name: str 11 | policy_id: str 12 | arn: str 13 | default_version_id: str 14 | path: str 15 | attachment_count: int 16 | permissions_boundary_usage_count: int 17 | is_attachable: bool 18 | description: Optional[str] = field(default=None, skip_if_default=True) 19 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam/policy/policy_document_utils.py: -------------------------------------------------------------------------------- 1 | def fix_stmt_regex_to_valid_regex(stmt_regex: str, with_case_sensitive: bool) -> str: 2 | # https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-arn-format.html 3 | # aws traits the '?' as regex '.' (any character) 4 | ret = stmt_regex.replace("*", ".*").replace("?", ".") 5 | if not with_case_sensitive: 6 | return f"(?i){ret}$" 7 | else: 8 | return f"{ret}$" 9 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam/policy/user_policy.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from aws_ptrp.iam.policy.policy_document import PolicyDocument 4 | from serde import serde 5 | 6 | 7 | @serde(rename_all="pascalcase") 8 | @dataclass 9 | class UserPolicy: 10 | user_name: str 11 | policy_name: str 12 | policy_document: PolicyDocument 13 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam/public_block_access_config.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from serde import serde 4 | 5 | 6 | @serde(rename_all="pascalcase") 7 | @dataclass 8 | class PublicAccessBlockConfiguration: 9 | block_public_acls: bool 10 | ignore_public_acls: bool 11 | block_public_policy: bool 12 | restrict_public_buckets: bool 13 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam/role/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam/role/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam/role/role_policy.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from aws_ptrp.iam.policy.policy_document import PolicyDocument 4 | from serde import serde 5 | 6 | 7 | @serde(rename_all="pascalcase") 8 | @dataclass 9 | class RolePolicy: 10 | role_name: str 11 | policy_name: str 12 | policy_document: PolicyDocument 13 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam_identity_center/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam_identity_center/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam_identity_center/iam_identity_center_groups.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Dict, List, Set 3 | 4 | from aws_ptrp.iam.policy import PolicyDocumentCtx 5 | from aws_ptrp.ptrp_allowed_lines.allowed_line_nodes_base import PathUserGroupNodeBase 6 | from aws_ptrp.ptrp_models import AwsPtrpPathNodeType 7 | from boto3 import Session 8 | from serde import serde 9 | 10 | 11 | @serde 12 | @dataclass 13 | class IamIdentityCenterGroup(PathUserGroupNodeBase): 14 | group_name: str 15 | group_id: str 16 | group_user_ids: Set[str] 17 | 18 | def get_node_arn(self) -> str: 19 | return self.group_id 20 | 21 | def get_node_name(self) -> str: 22 | return self.group_name 23 | 24 | def __eq__(self, other): 25 | return self.group_id == other.group_id 26 | 27 | def __hash__(self): 28 | return hash(self.group_id) 29 | 30 | def __repr__(self): 31 | return self.group_id 32 | 33 | # PathNodeBase 34 | def get_path_type(self) -> AwsPtrpPathNodeType: 35 | return AwsPtrpPathNodeType.IAM_IDENTITY_CENTER_GROUP 36 | 37 | def get_attached_policies_arn(self) -> List[str]: 38 | return [] 39 | 40 | def get_inline_policies_ctx(self) -> List[PolicyDocumentCtx]: 41 | return [] 42 | 43 | 44 | def get_iam_identity_center_groups( 45 | session: Session, identity_store_id: str, region: str 46 | ) -> Dict[str, IamIdentityCenterGroup]: 47 | identity_store_client = session.client("identitystore", region_name=region) 48 | ret: Dict[str, IamIdentityCenterGroup] = {} 49 | 50 | groups = identity_store_client.list_groups(IdentityStoreId=identity_store_id)['Groups'] 51 | for group in groups: 52 | group_id: str = group['GroupId'] 53 | group_memberships = identity_store_client.list_group_memberships( 54 | IdentityStoreId=identity_store_id, GroupId=group_id 55 | )['GroupMemberships'] 56 | group_user_ids = set([group_membership['MemberId']['UserId'] for group_membership in group_memberships]) 57 | ret[group_id] = IamIdentityCenterGroup( 58 | group_name=group['DisplayName'], 59 | group_id=group_id, 60 | group_user_ids=group_user_ids, 61 | ) 62 | 63 | return ret 64 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam_identity_center/iam_identity_center_users.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Dict, List 3 | 4 | from aws_ptrp.iam.policy import PolicyDocumentCtx 5 | from aws_ptrp.principals.principal import Principal 6 | from aws_ptrp.ptrp_allowed_lines.allowed_line_nodes_base import PrincipalAndPoliciesNodeBase 7 | from aws_ptrp.ptrp_models import AwsPrincipalType 8 | from boto3 import Session 9 | from serde import serde 10 | 11 | 12 | @serde 13 | @dataclass 14 | class IamIdentityCenterUser(PrincipalAndPoliciesNodeBase): 15 | user_name: str 16 | user_id: str 17 | 18 | def get_node_arn(self) -> str: 19 | return self.user_id 20 | 21 | def get_node_name(self) -> str: 22 | return self.user_name 23 | 24 | def __eq__(self, other): 25 | return self.user_id == other.user_id 26 | 27 | def __hash__(self): 28 | return hash(self.user_id) 29 | 30 | def __repr__(self): 31 | return self.user_id 32 | 33 | # PrincipalNodeBase 34 | def get_stmt_principal(self) -> Principal: 35 | return Principal( 36 | principal_type=AwsPrincipalType.IAM_IDENTITY_CENTER_USER, 37 | policy_principal_str=self.user_id, 38 | name=self.user_name, 39 | principal_metadata=None, 40 | ) 41 | 42 | # PoliciesNodeBase 43 | def get_attached_policies_arn(self) -> List[str]: 44 | return [] 45 | 46 | def get_inline_policies_ctx(self) -> List[PolicyDocumentCtx]: 47 | return [] 48 | 49 | 50 | def get_iam_identity_center_users( 51 | session: Session, identity_store_id: str, region: str 52 | ) -> Dict[str, 'IamIdentityCenterUser']: 53 | identity_store_client = session.client("identitystore", region_name=region) 54 | ret: Dict[str, IamIdentityCenterUser] = {} 55 | 56 | users = identity_store_client.list_users(IdentityStoreId=identity_store_id)['Users'] 57 | for user in users: 58 | user_id: str = user['UserId'] 59 | ret[user_id] = IamIdentityCenterUser( 60 | user_name=user['UserName'], 61 | user_id=user_id, 62 | ) 63 | 64 | return ret 65 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/iam_identity_center/permission_sets.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Dict, List, Optional, Set 3 | 4 | from aws_ptrp.ptrp_allowed_lines.allowed_line_nodes_base import PathPermissionSetNodeBase 5 | from aws_ptrp.ptrp_models.ptrp_model import AwsPtrpPathNodeType 6 | from boto3 import Session 7 | from serde import field, serde 8 | 9 | 10 | @serde 11 | @dataclass 12 | class PermissionsSet(PathPermissionSetNodeBase): 13 | name: str 14 | arn: str 15 | accounts_assignments: Dict[str, Set[str]] = field(default_factory=list) # account_id -> set of users and groups ids 16 | 17 | # PathNodeBase 18 | def get_path_type(self) -> AwsPtrpPathNodeType: 19 | return AwsPtrpPathNodeType.PERMISSION_SET 20 | 21 | # NodeBase 22 | def get_node_name(self) -> str: 23 | return self.name 24 | 25 | def get_node_arn(self) -> str: 26 | return self.arn 27 | 28 | def get_account_assignments(self, account_id: str) -> Optional[Set[str]]: 29 | return self.accounts_assignments[account_id] if account_id in self.accounts_assignments else None 30 | 31 | def __eq__(self, other): 32 | return self.arn == other.arn 33 | 34 | def __hash__(self): 35 | return hash(self.arn) 36 | 37 | def __repr__(self): 38 | return self.name 39 | 40 | 41 | def get_permission_sets(session: Session, instance_arn: str, region: str) -> Dict[str, PermissionsSet]: 42 | ret: Dict[str, PermissionsSet] = {} 43 | sso_admin_client = session.client("sso-admin", region_name=region) 44 | permission_sets = sso_admin_client.list_permission_sets(InstanceArn=instance_arn)['PermissionSets'] 45 | for permission_set_arn in permission_sets: 46 | name = sso_admin_client.describe_permission_set(InstanceArn=instance_arn, PermissionSetArn=permission_set_arn)[ 47 | 'PermissionSet' 48 | ]['Name'] 49 | 50 | accounts_assignments: Dict[str, Set[str]] = {} 51 | provisioned_accounts: List[str] = sso_admin_client.list_accounts_for_provisioned_permission_set( 52 | InstanceArn=instance_arn, PermissionSetArn=permission_set_arn 53 | )['AccountIds'] 54 | for account_id in provisioned_accounts: 55 | assignments = sso_admin_client.list_account_assignments( 56 | InstanceArn=instance_arn, 57 | AccountId=account_id, 58 | PermissionSetArn=permission_set_arn, 59 | )['AccountAssignments'] 60 | accounts_assignments[account_id] = set([principal['PrincipalId'] for principal in assignments]) 61 | 62 | ret[permission_set_arn] = PermissionsSet( 63 | name=name, 64 | arn=permission_set_arn, 65 | accounts_assignments=accounts_assignments, 66 | ) 67 | 68 | return ret 69 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Optional 3 | 4 | LOGGER: Optional[logging.Logger] = None 5 | 6 | 7 | def _create_logger(debug: bool) -> logging.Logger: 8 | """Provides a logger in case one is not provided. 9 | 10 | Args: 11 | debug (bool): Should logs be in debug 12 | 13 | Returns: 14 | Logger: Python logger 15 | """ 16 | logger = logging.getLogger('aws_ptrp') 17 | level = logging.INFO if not debug else logging.DEBUG 18 | logger.setLevel(level) 19 | 20 | if logger.handlers: 21 | return logger 22 | 23 | ch = logging.StreamHandler() # pylint: disable=C0103 24 | ch.setLevel(level) 25 | formatter = logging.Formatter('%(asctime)s - %(filename)s:%(lineno)d - %(levelname)s - %(message)s') 26 | ch.setFormatter(formatter) 27 | logger.addHandler(ch) 28 | return logger 29 | 30 | 31 | def get_ptrp_logger() -> logging.Logger: 32 | global LOGGER # pylint: disable=W0603 33 | if LOGGER: 34 | return LOGGER 35 | else: 36 | LOGGER = _create_logger(debug=False) 37 | return LOGGER 38 | 39 | 40 | def set_ptrp_logger(logger: logging.Logger): 41 | global LOGGER # pylint: disable=W0603 42 | LOGGER = logger 43 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/policy_evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | from .policy_evaluation import ( 2 | PolicyEvaluation, 3 | PolicyEvaluationApplyResult, 4 | PolicyEvaluationResult, 5 | PolicyEvaluationsResult, 6 | ) 7 | 8 | __all__ = ['PolicyEvaluation', 'PolicyEvaluationsResult', 'PolicyEvaluationResult', 'PolicyEvaluationApplyResult'] 9 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/principals/__init__.py: -------------------------------------------------------------------------------- 1 | from .principal import Principal, PrincipalBase, is_stmt_principal_relevant_to_resource 2 | 3 | __all__ = ['Principal', 'PrincipalBase', 'is_stmt_principal_relevant_to_resource'] 4 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/principals/no_entity_principal.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List 3 | 4 | from aws_ptrp.iam.policy.policy_document import PolicyDocumentCtx 5 | from aws_ptrp.principals import Principal 6 | from aws_ptrp.ptrp_allowed_lines.allowed_line_nodes_base import PrincipalAndPoliciesNodeBase 7 | 8 | 9 | @dataclass 10 | class NoEntityPrincipal(PrincipalAndPoliciesNodeBase): 11 | stmt_principal: Principal 12 | 13 | def __repr__(self): 14 | return self.get_node_arn() 15 | 16 | def __eq__(self, other): 17 | return self.get_node_arn() == other.get_node_arn() 18 | 19 | def __hash__(self): 20 | return hash(self.get_node_arn()) 21 | 22 | # # impl PrincipalAndPoliciesNodeBase 23 | # def get_permission_boundary(self) -> Optional[PolicyDocument]: 24 | # return None 25 | 26 | # def get_session_policies(self) -> List[PolicyDocument]: 27 | # return [] 28 | 29 | # NodeBase 30 | def get_node_arn(self) -> str: 31 | return self.stmt_principal.get_arn() 32 | 33 | def get_node_name(self) -> str: 34 | return self.stmt_principal.get_name() 35 | 36 | # impl PrincipalNodeBase 37 | def get_stmt_principal(self) -> Principal: 38 | return self.stmt_principal 39 | 40 | # impl PoliciesNodeBase 41 | def get_attached_policies_arn(self) -> List[str]: 42 | return [] 43 | 44 | def get_inline_policies_ctx(self) -> List[PolicyDocumentCtx]: 45 | return [] 46 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/ptrp_allowed_lines/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/ptrp_allowed_lines/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/ptrp_models/__init__.py: -------------------------------------------------------------------------------- 1 | from .ptrp_model import ( 2 | AwsPrincipal, 3 | AwsPrincipalType, 4 | AwsPtrpActionPermissionLevel, 5 | AwsPtrpLine, 6 | AwsPtrpNodeNote, 7 | AwsPtrpNoteType, 8 | AwsPtrpPathNode, 9 | AwsPtrpPathNodeType, 10 | AwsPtrpResource, 11 | AwsPtrpResourceType, 12 | ) 13 | 14 | __all__ = [ 15 | 'AwsPtrpLine', 16 | 'AwsPrincipal', 17 | 'AwsPrincipalType', 18 | 'AwsPtrpNoteType', 19 | 'AwsPtrpNodeNote', 20 | 'AwsPtrpActionPermissionLevel', 21 | 'AwsPtrpResource', 22 | 'AwsPtrpResourceType', 23 | 'AwsPtrpPathNode', 24 | 'AwsPtrpPathNodeType', 25 | ] 26 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/resources/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/resources/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/__init__.py: -------------------------------------------------------------------------------- 1 | from .resolved_stmt import ResolvedSingleStmt, ResolvedSingleStmtGetter, StmtResourcesToResolveCtx 2 | from .service_action_base import ServiceActionBase 3 | from .service_action_type import ( 4 | ServiceActionType, 5 | get_service_action_by_name, 6 | get_service_action_type_by_name, 7 | register_service_action_by_name, 8 | register_service_action_type_by_name, 9 | ) 10 | from .service_actions_resolver_base import ResolvedActionsSingleStmt, ServiceActionsResolverBase 11 | from .service_resource_base import ServiceResourceBase 12 | from .service_resource_type import ( 13 | ServiceResourceType, 14 | get_service_resource_by_name, 15 | get_service_resource_type_by_name, 16 | register_service_resource_by_name, 17 | register_service_resource_type_by_name, 18 | ) 19 | from .service_resources_resolver_base import ( 20 | MethodOnStmtActionsResultType, 21 | MethodOnStmtActionsType, 22 | MethodOnStmtsActionsResult, 23 | ServiceResourcesResolverBase, 24 | ) 25 | 26 | __all__ = [ 27 | 'ServiceActionBase', 28 | 'ServiceActionType', 29 | 'ResolvedActionsSingleStmt', 30 | 'ServiceActionsResolverBase', 31 | 'get_service_action_by_name', 32 | 'register_service_action_by_name', 33 | 'register_service_action_type_by_name', 34 | 'get_service_action_type_by_name', 35 | 'ServiceResourceType', 36 | 'ResolvedSingleStmt', 37 | 'MethodOnStmtsActionsResult', 38 | 'MethodOnStmtActionsType', 39 | 'MethodOnStmtActionsResultType', 40 | 'ServiceResourceBase', 41 | 'ServiceResourcesResolverBase', 42 | 'StmtResourcesToResolveCtx', 43 | 'ResolvedSingleStmtGetter', 44 | 'get_service_resource_by_name', 45 | 'register_service_resource_by_name', 46 | 'register_service_resource_type_by_name', 47 | 'get_service_resource_type_by_name', 48 | ] 49 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/assume_role/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/assume_role/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/assume_role/assume_role_actions.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from enum import Enum, auto 3 | from logging import Logger 4 | from typing import List, Set 5 | 6 | from aws_ptrp.ptrp_models.ptrp_model import AwsPtrpActionPermissionLevel 7 | from aws_ptrp.services import ServiceActionBase, ServiceActionsResolverBase 8 | from aws_ptrp.utils.serde import serde_enum_field 9 | from serde import serde 10 | 11 | 12 | class AssumeRoleActionType(Enum): 13 | ASSUME_ROLE = auto() 14 | ASSUME_ROLE_WITH_SAML = auto() 15 | ASSUME_ROLE_WITH_WEB_IDENTITY = auto() 16 | TAG_SESSION = auto() 17 | SET_SOURCE_IDENTITY = auto() 18 | 19 | 20 | @serde 21 | @dataclass 22 | class AssumeRoleAction(ServiceActionBase): 23 | name: str 24 | is_assumed_role: bool 25 | action_type: AssumeRoleActionType = serde_enum_field(AssumeRoleActionType) 26 | permission_level: AwsPtrpActionPermissionLevel = serde_enum_field(AwsPtrpActionPermissionLevel) 27 | 28 | def __repr__(self): 29 | return self.name 30 | 31 | def __eq__(self, other): 32 | return self.name == other.name 33 | 34 | def __hash__(self): 35 | return hash(self.name) 36 | 37 | def get_action_name(self) -> str: 38 | return self.name 39 | 40 | def get_action_permission_level(self) -> AwsPtrpActionPermissionLevel: 41 | return self.permission_level 42 | 43 | @classmethod 44 | def load_role_trust_actions(cls, _logger: Logger) -> Set[ServiceActionBase]: 45 | return role_trust_actions 46 | 47 | 48 | @dataclass 49 | class AssumeRoleServiceActionsResolver(ServiceActionsResolverBase): 50 | resolved_actions: Set[AssumeRoleAction] 51 | 52 | def get_resolved_actions(self) -> Set[ServiceActionBase]: 53 | return self.resolved_actions # type: ignore[return-value] 54 | 55 | @classmethod 56 | def load_from_single_stmt( 57 | cls, 58 | logger: Logger, 59 | stmt_regexes: List[str], 60 | service_actions: Set[ServiceActionBase], 61 | not_action_annotated: bool, 62 | ) -> 'ServiceActionsResolverBase': 63 | resolved_actions = ServiceActionsResolverBase.resolve_actions_from_single_stmt_regexes( 64 | stmt_regexes, service_actions, not_action_annotated 65 | ) 66 | resolved_assume_actions: Set[AssumeRoleAction] = set( 67 | [s for s in resolved_actions if isinstance(s, AssumeRoleAction)] 68 | ) 69 | return cls(resolved_actions=resolved_assume_actions) 70 | 71 | 72 | role_trust_actions: Set[ServiceActionBase] = { 73 | AssumeRoleAction("AssumeRole", True, AssumeRoleActionType.ASSUME_ROLE, AwsPtrpActionPermissionLevel.FULL), 74 | AssumeRoleAction( 75 | "AssumeRoleWithWebIdentity", 76 | True, 77 | AssumeRoleActionType.ASSUME_ROLE_WITH_WEB_IDENTITY, 78 | AwsPtrpActionPermissionLevel.FULL, 79 | ), 80 | AssumeRoleAction( 81 | "AssumeRoleWithSAML", True, AssumeRoleActionType.ASSUME_ROLE_WITH_SAML, AwsPtrpActionPermissionLevel.FULL 82 | ), 83 | AssumeRoleAction("TagSession", False, AssumeRoleActionType.TAG_SESSION, AwsPtrpActionPermissionLevel.FULL), 84 | AssumeRoleAction("TagSession", False, AssumeRoleActionType.SET_SOURCE_IDENTITY, AwsPtrpActionPermissionLevel.FULL), 85 | } 86 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/assume_role/assume_role_service.py: -------------------------------------------------------------------------------- 1 | from logging import Logger 2 | from typing import Optional, Set, Type 3 | 4 | from aws_ptrp.iam.iam_entities import IAMEntities 5 | from aws_ptrp.ptrp_models import AwsPrincipalType 6 | from aws_ptrp.resources.account_resources import AwsAccountResources 7 | from aws_ptrp.services import ( 8 | ServiceActionBase, 9 | ServiceActionsResolverBase, 10 | ServiceResourceBase, 11 | ServiceResourcesResolverBase, 12 | ServiceResourceType, 13 | ) 14 | from aws_ptrp.services.assume_role.assume_role_actions import AssumeRoleAction, AssumeRoleServiceActionsResolver 15 | from aws_ptrp.services.assume_role.assume_role_resources import AssumeRoleServiceResourcesResolver 16 | from serde import serde 17 | 18 | ROLE_TRUST_SERVICE_NAME = "assume role" 19 | ROLE_TRUST_ACTION_SERVICE_PREFIX = "sts:" 20 | ROLE_TRUST_RESOURCE_SERVICE_PREFIX = "arn:aws:iam::" 21 | 22 | 23 | @serde 24 | class AssumeRoleService(ServiceResourceType): 25 | def get_service_name(self) -> str: 26 | return ROLE_TRUST_SERVICE_NAME 27 | 28 | def get_action_service_prefix(self) -> str: 29 | return ROLE_TRUST_ACTION_SERVICE_PREFIX 30 | 31 | def get_resource_service_prefix(self) -> str: 32 | return ROLE_TRUST_RESOURCE_SERVICE_PREFIX 33 | 34 | @classmethod 35 | def get_service_resources_resolver_type(cls) -> Type[ServiceResourcesResolverBase]: 36 | return AssumeRoleServiceResourcesResolver 37 | 38 | def get_resource_based_policy_irrelevant_principal_types(self) -> Optional[Set[AwsPrincipalType]]: 39 | return {AwsPrincipalType.AWS_STS_FEDERATED_USER_SESSION} 40 | 41 | @classmethod 42 | def get_service_actions_resolver_type(cls) -> Type[ServiceActionsResolverBase]: 43 | return AssumeRoleServiceActionsResolver 44 | 45 | @classmethod 46 | def load_service_actions(cls, logger: Logger) -> Set[ServiceActionBase]: 47 | return AssumeRoleAction.load_role_trust_actions(logger) 48 | 49 | @classmethod 50 | def load_service_resources( 51 | cls, 52 | _logger: Logger, 53 | _aws_account_resources: AwsAccountResources, 54 | iam_entities: IAMEntities, 55 | ) -> Optional[Set[ServiceResourceBase]]: 56 | ret: Set[ServiceResourceBase] = set() 57 | for iam_entities_for_account in iam_entities.iam_accounts_entities.values(): 58 | ret.update(iam_entities_for_account.iam_roles.values()) 59 | return ret 60 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/federated_user/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/federated_user/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/federated_user/federated_user_actions.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from logging import Logger 3 | from typing import List, Set 4 | 5 | from aws_ptrp.ptrp_models.ptrp_model import AwsPtrpActionPermissionLevel 6 | from aws_ptrp.services import ServiceActionBase, ServiceActionsResolverBase 7 | from aws_ptrp.utils.serde import serde_enum_field 8 | from serde import serde 9 | 10 | 11 | @serde 12 | @dataclass 13 | class FederatedUserAction(ServiceActionBase): 14 | name: str 15 | permission_level: AwsPtrpActionPermissionLevel = serde_enum_field(AwsPtrpActionPermissionLevel) 16 | 17 | def __repr__(self): 18 | return self.name 19 | 20 | def __eq__(self, other): 21 | return self.name == other.name 22 | 23 | def __hash__(self): 24 | return hash(self.name) 25 | 26 | def get_action_name(self) -> str: 27 | return self.name 28 | 29 | def get_action_permission_level(self) -> AwsPtrpActionPermissionLevel: 30 | return self.permission_level 31 | 32 | def is_get_federated_token_action(self) -> bool: 33 | return self.get_action_name() == "GetFederationToken" 34 | 35 | @classmethod 36 | def load_federated_user_actions(cls, _logger: Logger) -> Set[ServiceActionBase]: 37 | return federated_user_actions 38 | 39 | 40 | @dataclass 41 | class FederatedUserServiceActionsResolver(ServiceActionsResolverBase): 42 | resolved_actions: Set[FederatedUserAction] 43 | 44 | def get_resolved_actions(self) -> Set[ServiceActionBase]: 45 | return self.resolved_actions # type: ignore[return-value] 46 | 47 | @classmethod 48 | def load_from_single_stmt( 49 | cls, 50 | logger: Logger, 51 | stmt_regexes: List[str], 52 | service_actions: Set[ServiceActionBase], 53 | not_action_annotated: bool, 54 | ) -> 'ServiceActionsResolverBase': 55 | resolved_actions = ServiceActionsResolverBase.resolve_actions_from_single_stmt_regexes( 56 | stmt_regexes, service_actions, not_action_annotated 57 | ) 58 | resolved_federated_user_actions: Set[FederatedUserAction] = set( 59 | [s for s in resolved_actions if isinstance(s, FederatedUserAction)] 60 | ) 61 | return cls(resolved_actions=resolved_federated_user_actions) 62 | 63 | 64 | federated_user_actions: Set[ServiceActionBase] = { 65 | FederatedUserAction("GetFederationToken", AwsPtrpActionPermissionLevel.FULL), 66 | } 67 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/federated_user/federated_user_service.py: -------------------------------------------------------------------------------- 1 | from logging import Logger 2 | from typing import Optional, Set, Type 3 | 4 | from aws_ptrp.ptrp_models import AwsPrincipalType 5 | from aws_ptrp.resources.account_resources import AwsAccountResources 6 | from aws_ptrp.services import ( 7 | ServiceActionBase, 8 | ServiceActionsResolverBase, 9 | ServiceResourceBase, 10 | ServiceResourcesResolverBase, 11 | ServiceResourceType, 12 | ) 13 | from aws_ptrp.services.federated_user.federated_user_actions import ( 14 | FederatedUserAction, 15 | FederatedUserServiceActionsResolver, 16 | ) 17 | from aws_ptrp.services.federated_user.federated_user_resources import ( 18 | FederatedUserResource, 19 | FederatedUserServiceResourcesResolver, 20 | ) 21 | from serde import serde 22 | 23 | FEDERATED_USER_SERVICE_NAME = "federated user" 24 | FEDERATED_USER_ACTION_SERVICE_PREFIX = "sts:" 25 | FEDERATED_USER_RESOURCE_SERVICE_PREFIX = "arn:aws:sts::" 26 | 27 | 28 | @serde 29 | class FederatedUserService(ServiceResourceType): 30 | def get_service_name(self) -> str: 31 | return FEDERATED_USER_SERVICE_NAME 32 | 33 | def get_action_service_prefix(self) -> str: 34 | return FEDERATED_USER_ACTION_SERVICE_PREFIX 35 | 36 | def get_resource_service_prefix(self) -> str: 37 | return FEDERATED_USER_RESOURCE_SERVICE_PREFIX 38 | 39 | def get_resource_based_policy_irrelevant_principal_types(self) -> Optional[Set[AwsPrincipalType]]: 40 | return None 41 | 42 | @classmethod 43 | def get_service_resources_resolver_type(cls) -> Type[ServiceResourcesResolverBase]: 44 | return FederatedUserServiceResourcesResolver 45 | 46 | @classmethod 47 | def get_service_actions_resolver_type(cls) -> Type[ServiceActionsResolverBase]: 48 | return FederatedUserServiceActionsResolver 49 | 50 | @classmethod 51 | def load_service_actions(cls, logger: Logger) -> Set[ServiceActionBase]: 52 | return FederatedUserAction.load_federated_user_actions(logger) 53 | 54 | @classmethod 55 | def load_service_resources( 56 | cls, 57 | _logger: Logger, 58 | aws_account_resources: AwsAccountResources, 59 | _iam_entities, 60 | ) -> Optional[Set[ServiceResourceBase]]: 61 | ret: Set[ServiceResourceBase] = set() 62 | for stmt_principal in aws_account_resources.yield_stmt_principals_from_resource_based_policy( 63 | AwsPrincipalType.AWS_STS_FEDERATED_USER_SESSION 64 | ): 65 | ret.add(FederatedUserResource(federated_principal=stmt_principal)) 66 | return ret 67 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/resolved_stmt.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from dataclasses import dataclass 3 | from typing import Dict, List, Optional, Set 4 | 5 | from aws_ptrp.principals import PrincipalBase 6 | from aws_ptrp.services.service_action_base import ServiceActionBase 7 | from aws_ptrp.services.service_actions_resolver_base import ResolvedActionsSingleStmt 8 | from aws_ptrp.services.service_resource_base import ServiceResourceBase 9 | 10 | 11 | @dataclass 12 | class StmtResourcesToResolveCtx: 13 | service_resources: Set[ServiceResourceBase] 14 | resolved_stmt_principals: Set[PrincipalBase] 15 | resolved_stmt_actions: Set[ServiceActionBase] 16 | stmt_relative_id_resource_regexes: List[str] 17 | is_condition_exists: bool 18 | stmt_name: Optional[str] 19 | stmt_parent_arn: str 20 | policy_name: Optional[str] 21 | 22 | 23 | @dataclass 24 | class ResolvedSingleStmt: 25 | resolved_stmt_principals: Set[PrincipalBase] 26 | resolved_stmt_resources: Dict[ServiceResourceBase, ResolvedActionsSingleStmt] 27 | is_condition_exists: bool 28 | stmt_name: Optional[str] 29 | stmt_parent_arn: str 30 | policy_name: Optional[str] 31 | # add here condition keys, tags, etc.. (single stmt scope) 32 | 33 | def __hash__(self) -> int: 34 | return hash(self.stmt_name) + hash(self.stmt_parent_arn) + hash(self.policy_name) 35 | 36 | def __eq__(self, other): 37 | return ( 38 | self.stmt_parent_arn == other.stmt_parent_arn 39 | and self.policy_name == other.policy_name 40 | and self.stmt_name == other.stmt_name 41 | ) 42 | 43 | @classmethod 44 | def load( 45 | cls, 46 | stmt_ctx: StmtResourcesToResolveCtx, 47 | resolved_stmt_resources: Dict[ServiceResourceBase, ResolvedActionsSingleStmt], 48 | ) -> 'ResolvedSingleStmt': 49 | return cls( 50 | resolved_stmt_principals=stmt_ctx.resolved_stmt_principals, 51 | resolved_stmt_resources=resolved_stmt_resources, 52 | is_condition_exists=stmt_ctx.is_condition_exists, 53 | stmt_name=stmt_ctx.stmt_name, 54 | stmt_parent_arn=stmt_ctx.stmt_parent_arn, 55 | policy_name=stmt_ctx.policy_name, 56 | ) 57 | 58 | def retain_resolved_stmt_resources(self): 59 | # get all service resources with empty resolved stmt actions 60 | service_resources_to_delete = [ 61 | x[0] for x in self.resolved_stmt_resources.items() if not x[1].resolved_stmt_actions 62 | ] 63 | # delete all these keys from the resolved_stmt_resources 64 | for service_resource_to_delete in service_resources_to_delete: 65 | del self.resolved_stmt_resources[service_resource_to_delete] 66 | 67 | 68 | class ResolvedSingleStmtGetter(ABC): 69 | @abstractmethod 70 | def get(self) -> ResolvedSingleStmt: 71 | pass 72 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/s3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/s3/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/s3/bucket.py: -------------------------------------------------------------------------------- 1 | import json 2 | from dataclasses import dataclass 3 | from typing import Optional, Set 4 | 5 | from aws_ptrp.iam.policy import PolicyDocument 6 | from aws_ptrp.iam.public_block_access_config import PublicAccessBlockConfiguration 7 | from aws_ptrp.ptrp_allowed_lines.allowed_line_nodes_base import ResourceNodeBase 8 | from aws_ptrp.ptrp_models.ptrp_model import AwsPtrpResourceType 9 | from aws_ptrp.services import ServiceResourceBase 10 | from aws_ptrp.services.s3.bucket_acl import S3BucketACL 11 | from boto3 import Session 12 | from botocore.exceptions import ClientError 13 | from serde import field, from_dict, serde 14 | 15 | S3_RESOURCE_SERVICE_PREFIX = "arn:aws:s3:::" 16 | 17 | 18 | @serde 19 | @dataclass 20 | class S3Bucket(ResourceNodeBase, ServiceResourceBase): 21 | name: str 22 | aws_account_id: str 23 | acl: Optional[S3BucketACL] = field(default=None, skip_if_default=True) 24 | public_access_block_config: Optional[PublicAccessBlockConfiguration] = field(default=None, skip_if_default=True) 25 | policy_document: Optional[PolicyDocument] = field(default=None, skip_if_default=True) 26 | 27 | def __repr__(self): 28 | return f"S3Bucket({self.name})" 29 | 30 | def __eq__(self, other): 31 | return self.name == other.name 32 | 33 | def __hash__(self): 34 | return hash(self.name) 35 | 36 | # impl ServiceResourceBase 37 | def get_resource_account_id(self) -> str: 38 | return self.aws_account_id 39 | 40 | def get_resource_arn(self) -> str: 41 | return f"{S3_RESOURCE_SERVICE_PREFIX}{self.get_resource_name()}" 42 | 43 | def get_resource_name(self) -> str: 44 | return self.name 45 | 46 | def get_resource_policy(self) -> Optional[PolicyDocument]: 47 | return self.policy_document 48 | 49 | # impl ResourceNodeBase 50 | def get_ptrp_resource_type(self) -> AwsPtrpResourceType: 51 | return AwsPtrpResourceType.S3_BUCKET 52 | 53 | 54 | def get_buckets(session: Session, aws_account_id: str) -> Set[ServiceResourceBase]: 55 | s3_client = session.client('s3') 56 | response = s3_client.list_buckets() 57 | buckets = response['Buckets'] 58 | ret: Set[ServiceResourceBase] = set() 59 | for bucket in buckets: 60 | bucket_name = bucket['Name'] 61 | policy_document: Optional[PolicyDocument] = None 62 | try: 63 | policy_document = from_dict( 64 | PolicyDocument, json.loads(s3_client.get_bucket_policy(Bucket=bucket_name)['Policy']) 65 | ) # type: ignore 66 | except ClientError as error: 67 | if error.response['Error']['Code'] == 'NoSuchBucketPolicy': 68 | pass 69 | else: 70 | raise error 71 | 72 | acl: S3BucketACL = from_dict(S3BucketACL, s3_client.get_bucket_acl(Bucket=bucket_name)) # type: ignore 73 | public_access_block: Optional[PublicAccessBlockConfiguration] = None 74 | try: 75 | public_access_block = from_dict( 76 | PublicAccessBlockConfiguration, 77 | s3_client.get_public_access_block(Bucket=bucket_name)['PublicAccessBlockConfiguration'], 78 | ) # type: ignore 79 | except ClientError as error: 80 | if error.response['Error']['Code'] == 'NoSuchPublicAccessBlockConfiguration': 81 | pass 82 | else: 83 | raise error 84 | 85 | ret.add( 86 | S3Bucket( 87 | aws_account_id=aws_account_id, 88 | name=bucket_name, 89 | policy_document=policy_document, 90 | acl=acl, 91 | public_access_block_config=public_access_block, 92 | ) 93 | ) 94 | 95 | return ret 96 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/s3/bucket_acl.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from enum import Enum 3 | from typing import List, Optional 4 | 5 | from serde import field, serde 6 | 7 | 8 | @serde(rename_all="pascalcase") 9 | @dataclass 10 | class Owner: 11 | id: str = field(rename='ID') # pylint: disable=invalid-name 12 | display_name: Optional[str] = field(default=None, skip_if_default=True) 13 | 14 | 15 | class Permission(str, Enum): 16 | FULL_CONTROL = "FULL_CONTROL" 17 | WRITE = "WRITE" 18 | WRITE_ACP = "WRITE_ACP" 19 | READ = "READ" 20 | READ_ACP = "READ_ACP" 21 | 22 | 23 | class GrantType(str, Enum): 24 | CANONICAL_USER = "CanonicalUser" 25 | AMAZON_CUSTOMER_BY_EMAIL = "AmazonCustomerByEmail" 26 | GROUP = "Group" 27 | 28 | 29 | @serde(rename_all="pascalcase") 30 | @dataclass 31 | class Grantee: 32 | type: GrantType 33 | id: str = field(rename='ID') # pylint: disable=invalid-name 34 | display_name: Optional[str] = field(default=None, skip_if_default=True) 35 | 36 | 37 | @serde(rename_all="pascalcase") 38 | @dataclass 39 | class Grants: 40 | grantee: Grantee 41 | permission: Permission 42 | 43 | 44 | @serde(rename_all="pascalcase") 45 | @dataclass 46 | class S3BucketACL: 47 | owner: Owner 48 | grants: List[Grants] 49 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/s3/s3_service.py: -------------------------------------------------------------------------------- 1 | from logging import Logger 2 | from typing import Optional, Set, Type 3 | 4 | from aws_ptrp.ptrp_models import AwsPrincipalType 5 | from aws_ptrp.services import ( 6 | ServiceActionBase, 7 | ServiceActionsResolverBase, 8 | ServiceResourceBase, 9 | ServiceResourcesResolverBase, 10 | ServiceResourceType, 11 | ) 12 | from aws_ptrp.services.s3.bucket import S3_RESOURCE_SERVICE_PREFIX, get_buckets 13 | from aws_ptrp.services.s3.s3_actions import S3_ACTION_SERVICE_PREFIX, S3Action, S3ServiceActionsResolver 14 | from aws_ptrp.services.s3.s3_resources import S3ServiceResourcesResolver 15 | from boto3 import Session 16 | from serde import serde 17 | 18 | S3_SERVICE_NAME = "s3" 19 | 20 | 21 | @serde 22 | class S3Service(ServiceResourceType): 23 | def get_resource_service_prefix(self) -> str: 24 | return S3_RESOURCE_SERVICE_PREFIX 25 | 26 | def get_action_service_prefix(self) -> str: 27 | return S3_ACTION_SERVICE_PREFIX 28 | 29 | def get_service_name(self) -> str: 30 | return S3_SERVICE_NAME 31 | 32 | def get_resource_based_policy_irrelevant_principal_types(self) -> Optional[Set[AwsPrincipalType]]: 33 | return {AwsPrincipalType.SAML_SESSION, AwsPrincipalType.WEB_IDENTITY_SESSION} 34 | 35 | @classmethod 36 | def get_service_resources_resolver_type(cls) -> Type[ServiceResourcesResolverBase]: 37 | return S3ServiceResourcesResolver 38 | 39 | @classmethod 40 | def get_service_actions_resolver_type(cls) -> Type[ServiceActionsResolverBase]: 41 | return S3ServiceActionsResolver 42 | 43 | @classmethod 44 | def load_service_resources_from_session( 45 | cls, logger: Logger, session: Session, aws_account_id: str 46 | ) -> Optional[Set[ServiceResourceBase]]: 47 | # Get the buckets to analyzed 48 | buckets = get_buckets(session, aws_account_id) 49 | logger.info(f"Got buckets to analyzed: {buckets}") 50 | return buckets 51 | 52 | @classmethod 53 | def load_service_actions(cls, logger: Logger) -> Set[ServiceActionBase]: 54 | return S3Action.load_s3_actions(logger) 55 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/service_action_base.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from dataclasses import dataclass 3 | 4 | from aws_ptrp.ptrp_models.ptrp_model import AwsPtrpActionPermissionLevel 5 | from serde import serde 6 | 7 | 8 | @serde 9 | @dataclass 10 | class ServiceActionBase(ABC): 11 | @abstractmethod 12 | def get_action_name(self) -> str: 13 | pass 14 | 15 | @abstractmethod 16 | def get_action_permission_level(self) -> AwsPtrpActionPermissionLevel: 17 | pass 18 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/service_action_type.py: -------------------------------------------------------------------------------- 1 | from abc import abstractmethod 2 | from logging import Logger 3 | from typing import Dict, List, Optional, Set, Type 4 | 5 | from aws_ptrp.services.service_action_base import ServiceActionBase 6 | from aws_ptrp.services.service_actions_resolver_base import ServiceActionsResolverBase 7 | from aws_ptrp.services.service_base import ServiceType 8 | from serde import serde 9 | 10 | _SERVICE_ACTION_TYPE_BY_NAME: Dict[str, Type['ServiceActionType']] = dict() 11 | 12 | 13 | def register_service_action_type_by_name(service_name: str, service_type: Type['ServiceActionType']): 14 | _SERVICE_ACTION_TYPE_BY_NAME[service_name] = service_type 15 | 16 | 17 | def get_service_action_type_by_name(service_name: str) -> Optional[Type['ServiceActionType']]: 18 | return _SERVICE_ACTION_TYPE_BY_NAME.get(service_name, None) 19 | 20 | 21 | _SERVICE_ACTION_BY_NAME: Dict[str, Type['ServiceActionBase']] = dict() 22 | 23 | 24 | def register_service_action_by_name(service_name: str, service_action: Type['ServiceActionBase']): 25 | _SERVICE_ACTION_BY_NAME[service_name] = service_action 26 | 27 | 28 | def get_service_action_by_name(service_name: str) -> Optional[Type['ServiceActionBase']]: 29 | return _SERVICE_ACTION_BY_NAME.get(service_name, None) 30 | 31 | 32 | @serde 33 | class ServiceActionType(ServiceType): 34 | @abstractmethod 35 | def get_action_service_prefix(self) -> str: 36 | pass 37 | 38 | @classmethod 39 | @abstractmethod 40 | def get_service_actions_resolver_type(cls) -> Type[ServiceActionsResolverBase]: 41 | pass 42 | 43 | @classmethod 44 | @abstractmethod 45 | def load_service_actions(cls, logger: Logger) -> Set[ServiceActionBase]: 46 | pass 47 | 48 | @classmethod 49 | def load_resolver_service_actions_from_single_stmt( 50 | cls, 51 | logger: Logger, 52 | stmt_relative_id_regexes: List[str], 53 | service_actions: Set[ServiceActionBase], 54 | not_action_annotated: bool, 55 | ) -> ServiceActionsResolverBase: 56 | return cls.get_service_actions_resolver_type().load_from_single_stmt( 57 | logger, stmt_relative_id_regexes, service_actions, not_action_annotated 58 | ) 59 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/service_actions_resolver_base.py: -------------------------------------------------------------------------------- 1 | import re 2 | from abc import ABC, abstractmethod 3 | from dataclasses import dataclass 4 | from enum import Enum, auto 5 | from logging import Logger 6 | from typing import List, Set 7 | 8 | from aws_ptrp.iam.policy.policy_document_utils import fix_stmt_regex_to_valid_regex 9 | from aws_ptrp.services.service_action_base import ServiceActionBase 10 | 11 | 12 | class MethodOnStmtActionsType(Enum): 13 | DIFFERENCE = auto() 14 | INTERSECTION = auto() 15 | 16 | def __str__(self) -> str: 17 | return self.name 18 | 19 | def __repr__(self) -> str: 20 | return self.name 21 | 22 | def __hash__(self) -> int: 23 | return hash(self.value) 24 | 25 | 26 | class MethodOnStmtActionsResultType(Enum): 27 | APPLIED = auto() 28 | IGNORE_NO_OVERLAPS_TARGET_RESOURCE = auto() 29 | IGNORE_NO_OVERLAPS_TARGET_PRINCIPAL = auto() 30 | IGNORE_METHOD_DIFFERENCE_CONDITION_EXISTS = auto() 31 | IGNORE_METHOD_DIFFERENCE_WITH_S3_NOT_RESOURCE_OBJECT_REGEX = auto() 32 | 33 | def __str__(self) -> str: 34 | return self.name 35 | 36 | def __repr__(self) -> str: 37 | return self.name 38 | 39 | def __hash__(self) -> int: 40 | return hash(self.value) 41 | 42 | def __eq__(self, other): 43 | return self.value == other.value 44 | 45 | 46 | @dataclass 47 | class ResolvedActionsSingleStmt(ABC): 48 | @property 49 | @abstractmethod 50 | def resolved_stmt_actions(self) -> Set[ServiceActionBase]: 51 | pass 52 | 53 | def difference(self, other: 'ResolvedActionsSingleStmt') -> MethodOnStmtActionsResultType: 54 | self.resolved_stmt_actions.difference_update(other.resolved_stmt_actions) 55 | return MethodOnStmtActionsResultType.APPLIED 56 | 57 | 58 | @dataclass 59 | class ServiceActionsResolverBase(ABC): 60 | @abstractmethod 61 | def get_resolved_actions(self) -> Set[ServiceActionBase]: 62 | pass 63 | 64 | def is_empty(self) -> bool: 65 | return len(self.get_resolved_actions()) == 0 66 | 67 | @classmethod 68 | @abstractmethod 69 | def load_from_single_stmt( 70 | cls, 71 | logger: Logger, 72 | stmt_regexes: List[str], 73 | service_actions: Set[ServiceActionBase], 74 | not_action_annotated: bool, 75 | ) -> 'ServiceActionsResolverBase': 76 | pass 77 | 78 | @staticmethod 79 | def resolve_actions_from_single_stmt_regex( 80 | stmt_regex: str, service_actions: Set[ServiceActionBase] 81 | ) -> Set[ServiceActionBase]: 82 | # actions are case insensitive 83 | regex = re.compile(fix_stmt_regex_to_valid_regex(stmt_regex, with_case_sensitive=False)) 84 | return set([s for s in service_actions if regex.match(s.get_action_name()) is not None]) 85 | 86 | @staticmethod 87 | def resolve_actions_from_single_stmt_regexes( 88 | stmt_regexes: List[str], service_actions: Set[ServiceActionBase], not_action_annotated: bool 89 | ) -> Set[ServiceActionBase]: 90 | resolved_actions: Set[ServiceActionBase] = set() 91 | for stmt_regex in stmt_regexes: 92 | resolved_actions = resolved_actions.union( 93 | ServiceActionsResolverBase.resolve_actions_from_single_stmt_regex(stmt_regex, service_actions) 94 | ) 95 | if not_action_annotated: 96 | resolved_actions = service_actions.difference(resolved_actions) 97 | return resolved_actions 98 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/service_base.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | from serde import serde 4 | 5 | 6 | @serde 7 | class ServiceType(ABC): 8 | @abstractmethod 9 | def get_service_name(self) -> str: 10 | pass 11 | 12 | def __repr__(self): 13 | return self.get_service_name() 14 | 15 | def __eq__(self, other): 16 | return self.get_service_name() == other.get_service_name() 17 | 18 | def __hash__(self): 19 | return hash(self.get_service_name()) 20 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/service_resource_base.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from dataclasses import dataclass 3 | from typing import Optional 4 | 5 | from aws_ptrp.iam.policy.policy_document import PolicyDocument 6 | from serde import serde 7 | 8 | 9 | @serde 10 | @dataclass 11 | class ServiceResourceBase(ABC): 12 | @abstractmethod 13 | def get_resource_arn(self) -> str: 14 | pass 15 | 16 | @abstractmethod 17 | def get_resource_name(self) -> str: 18 | pass 19 | 20 | @abstractmethod 21 | def get_resource_policy(self) -> Optional[PolicyDocument]: 22 | pass 23 | 24 | @abstractmethod 25 | def get_resource_account_id(self) -> str: 26 | pass 27 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/services/service_resource_type.py: -------------------------------------------------------------------------------- 1 | from abc import abstractmethod 2 | from logging import Logger 3 | from typing import Dict, Optional, Set, Type 4 | 5 | from aws_ptrp.ptrp_models import AwsPrincipalType 6 | from aws_ptrp.services.resolved_stmt import StmtResourcesToResolveCtx 7 | from aws_ptrp.services.service_action_type import ServiceActionType 8 | from aws_ptrp.services.service_resource_base import ServiceResourceBase 9 | from aws_ptrp.services.service_resources_resolver_base import ServiceResourcesResolverBase 10 | from boto3 import Session 11 | from serde import serde 12 | 13 | _SERVICE_RESOURCE_TYPE_BY_NAME: Dict[str, Type['ServiceResourceType']] = dict() 14 | 15 | 16 | def register_service_resource_type_by_name(service_name: str, service_type: Type['ServiceResourceType']): 17 | _SERVICE_RESOURCE_TYPE_BY_NAME[service_name] = service_type 18 | 19 | 20 | def get_service_resource_type_by_name(service_name: str) -> Optional[Type['ServiceResourceType']]: 21 | return _SERVICE_RESOURCE_TYPE_BY_NAME.get(service_name, None) 22 | 23 | 24 | _SERVICE_RESOURCE_BY_NAME: Dict[str, Type['ServiceResourceBase']] = dict() 25 | 26 | 27 | def register_service_resource_by_name(service_name: str, service_action: Type['ServiceResourceBase']): 28 | _SERVICE_RESOURCE_BY_NAME[service_name] = service_action 29 | 30 | 31 | def get_service_resource_by_name(service_name: str) -> Optional[Type['ServiceResourceBase']]: 32 | return _SERVICE_RESOURCE_BY_NAME.get(service_name, None) 33 | 34 | 35 | @serde 36 | class ServiceResourceType(ServiceActionType): 37 | @abstractmethod 38 | def get_resource_service_prefix(self) -> str: 39 | pass 40 | 41 | @classmethod 42 | @abstractmethod 43 | def get_service_resources_resolver_type(cls) -> Type[ServiceResourcesResolverBase]: 44 | pass 45 | 46 | @abstractmethod 47 | def get_resource_based_policy_irrelevant_principal_types(self) -> Optional[Set[AwsPrincipalType]]: 48 | pass 49 | 50 | @classmethod 51 | def load_resolver_service_resources_from_single_stmt( 52 | cls, 53 | logger: Logger, 54 | stmt_ctx: StmtResourcesToResolveCtx, 55 | not_resource_annotated: bool, 56 | ) -> ServiceResourcesResolverBase: 57 | return cls.get_service_resources_resolver_type().load_from_single_stmt(logger, stmt_ctx, not_resource_annotated) 58 | 59 | @classmethod 60 | def load_service_resources_from_session( 61 | cls, _logger: Logger, _session: Session, _aws_account_id: str 62 | ) -> Optional[Set[ServiceResourceBase]]: 63 | return None 64 | 65 | @classmethod 66 | def load_service_resources( 67 | cls, 68 | _logger: Logger, 69 | _aws_account_resources, 70 | _iam_entities, 71 | ) -> Optional[Set[ServiceResourceBase]]: 72 | return None 73 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/utils/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/utils/assume_role.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Optional 3 | 4 | from aws_ptrp.principals import Principal 5 | 6 | 7 | @dataclass 8 | class AwsAssumeRole: 9 | role_arn: str 10 | external_id: Optional[str] 11 | 12 | def get_account_id(self) -> str: 13 | principal = Principal.load_from_iam_role(self.role_arn) 14 | account_id = principal.get_account_id() 15 | if not account_id: 16 | raise Exception(f"Unable to extract account id from role_arn {self.role_arn}") 17 | return account_id 18 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/utils/create_session.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import boto3 4 | from boto3 import Session 5 | 6 | 7 | def create_session_with_assume_role( 8 | role_arn: str, external_id: Optional[str], role_session_name: Optional[str] = "AwsPtrpSession" 9 | ) -> Session: 10 | # Create a session with the role you want to assume 11 | sts_client = boto3.client('sts') 12 | params = {'RoleArn': role_arn, 'RoleSessionName': role_session_name} 13 | if external_id: 14 | params['ExternalId'] = external_id 15 | 16 | assumed_role_object = sts_client.assume_role(**params) 17 | 18 | # Use the assumed role's temporary credentials to create a new session 19 | session = boto3.Session( 20 | aws_access_key_id=assumed_role_object['Credentials']['AccessKeyId'], 21 | aws_secret_access_key=assumed_role_object['Credentials']['SecretAccessKey'], 22 | aws_session_token=assumed_role_object['Credentials']['SessionToken'], 23 | ) 24 | return session 25 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/utils/pagination.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | 3 | 4 | def paginate_response_list(method: Callable, key_to_append_list: str, **kwargs): 5 | """ 6 | Paginate through a list of items from the AWS Management Console. 7 | 8 | method: The method to call on the client to get a page of items. 9 | key_to_append_list: the key in the response dictionary to append 10 | kwargs: Additional keyword arguments to pass to the method. 11 | 12 | Returns a list of all the items in the list. 13 | """ 14 | 15 | # Initialize the pagination variables 16 | marker = None 17 | items = [] 18 | 19 | # Loop until there are no more items to paginate 20 | while True: 21 | # Get a page of items 22 | if marker: 23 | kwargs["Marker"] = marker 24 | response = method(**kwargs) 25 | 26 | # Append the items to the list 27 | items += response[key_to_append_list] 28 | 29 | # If there are more items, get the marker for the next page 30 | if response["IsTruncated"]: 31 | marker = response["Marker"] 32 | else: 33 | break 34 | 35 | return items 36 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/utils/regex_subset.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | 4 | def _safe_list_get(lst: List[str], idx: int): 5 | try: 6 | return lst[idx] 7 | except IndexError: 8 | return None 9 | 10 | 11 | def is_aws_regex_full_subset(haystack_aws_regex: str, needle_aws_regex: str) -> bool: 12 | """ 13 | This function checks if the needle_aws_regex is a full subset of the haystack_aws_regex. 14 | Regex 'a' if a full subset of 'b' if any string that matches 'a' also matches 'b'. 15 | AWS regex can have the following regex tokens: 16 | '*' => 0 or more characters(equivalent to .*) 17 | '?' => 1 character(equivalent to .) 18 | For example: 19 | cab* is a full subset of *ab* 20 | c?b* is not a full subset of *ab*, since it can match 'cdb' which is not matched by *ab* 21 | 22 | Returns True if any string that matches the needle_aws_regex also matches the haystack_aws_regex. 23 | """ 24 | i = 0 25 | j = 0 26 | haystack_chars = list(haystack_aws_regex) 27 | needle_chars = list(needle_aws_regex) 28 | while i < len(haystack_chars) and j < len(needle_chars): 29 | haystack_char = haystack_chars[i] 30 | needle_char = needle_chars[j] 31 | if haystack_char == "*" and (needle_char == "*" or needle_char == "?"): 32 | i += 1 33 | j += 1 34 | continue 35 | elif haystack_char == "*": 36 | # We will skip the current haystack_char, treating it as a zero length sequence which the needle will match. 37 | i += 1 38 | continue 39 | elif haystack_char != "*" and needle_char == "*": 40 | return False 41 | elif haystack_char == "?": 42 | # We know that needle_char is not a wildcard, so we can continue 43 | i += 1 44 | j += 1 45 | continue 46 | elif haystack_char == needle_char: 47 | i += 1 48 | j += 1 49 | continue 50 | elif haystack_char != needle_char: 51 | # If previous char in haystack was a wildcard, we can skip the current char in needle, since current needle will match 52 | if _safe_list_get(haystack_chars, i - 1) == "*": 53 | j += 1 54 | continue 55 | return False 56 | 57 | haystack_finished = i >= len(haystack_chars) 58 | needle_finished = j >= len(needle_chars) 59 | 60 | if haystack_finished is False and needle_finished is True: 61 | # We need to check if the rest of the haystack is a wildcard, and if so, needle will match(since * can be treated also as zero sequence of chars) 62 | return haystack_chars[j:] == ['*'] 63 | elif haystack_finished is True and needle_finished is False: 64 | # If we have finished the haystack, but not the needle, then needle subset of haystack if last char in haystack is a wildcard(since it will match any sequence of chars in needle) 65 | return haystack_chars[-1] == "*" 66 | else: 67 | return True 68 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/aws/aws_ptrp_package/aws_ptrp/utils/serde.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from serde import field 4 | 5 | 6 | def serde_enum_field(enum_type: Any) -> Any: 7 | return field(serializer=lambda x: x.name, deserializer=lambda x: enum_type[x]) 8 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/bigquery/__init__.py: -------------------------------------------------------------------------------- 1 | """Package for BigQuery authz-analyzer""" 2 | 3 | __author__ = """SatoriCyber""" 4 | __email__ = 'yoav@satoricyber.com' 5 | __version__ = '0.1.0' 6 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/databricks/__init__.py: -------------------------------------------------------------------------------- 1 | from universal_data_permissions_scanner.datastores.databricks.service.authentication.authentication import Authentication # type: ignore 2 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/databricks/exceptions.py: -------------------------------------------------------------------------------- 1 | class IdentityNotFoundException(Exception): 2 | pass 3 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/databricks/model.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass 4 | from enum import Enum 5 | from typing import List 6 | 7 | 8 | class DBPermissionLevel(Enum): 9 | OWNERSHIP = 3 10 | SELECT = 1 11 | MODIFY = 2 12 | ALL_PRIVILEGES = 3 13 | 14 | def __lt__(self, other: DBPermissionLevel): 15 | return self.value < other.value 16 | 17 | def __str__(self) -> str: 18 | return self.name 19 | 20 | @classmethod 21 | def from_str(cls, permission: str): 22 | if permission == "OWNERSHIP": 23 | return cls.OWNERSHIP 24 | if permission == "SELECT": 25 | return cls.SELECT 26 | if permission == "MODIFY": 27 | return cls.MODIFY 28 | if permission == "ALL_PRIVILEGES": 29 | return cls.ALL_PRIVILEGES 30 | raise ValueError(f"Unknown permission: {permission}") 31 | 32 | 33 | DataBricksIdentityName = str 34 | DataBricksIdentityId = str 35 | 36 | 37 | @dataclass 38 | class Permission: 39 | identity: str 40 | db_permissions: List[DBPermissionLevel] 41 | 42 | 43 | @dataclass 44 | class DatabricksParsedIdentity: 45 | name: str 46 | id: str # pylint: disable=invalid-name 47 | groups: List[ParsedGroup] 48 | type: DataBricksIdentityType 49 | 50 | 51 | @dataclass 52 | class ParsedGroup: 53 | id: str # pylint: disable=invalid-name 54 | name: str 55 | 56 | 57 | class DataBricksIdentityType(Enum): 58 | USER = "USER" 59 | SERVICE_PRINCIPAL = "SERVICE_PRINCIPAL" 60 | 61 | def __str__(self) -> str: 62 | return self.value 63 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/databricks/service/__init__.py: -------------------------------------------------------------------------------- 1 | from universal_data_permissions_scanner.datastores.databricks.service.authentication.authentication import Authentication # type: ignore 2 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/databricks/service/authentication/__init__.py: -------------------------------------------------------------------------------- 1 | from universal_data_permissions_scanner.datastores.databricks.service.authentication.authentication import Authentication # type: ignore 2 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/databricks/service/authentication/authentication.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Union 3 | 4 | from universal_data_permissions_scanner.datastores.databricks.service.authentication.basic import BasicAuthentication 5 | 6 | from universal_data_permissions_scanner.datastores.databricks.service.authentication.oauth import ( 7 | OauthProvider, 8 | OauthProviderAzure, 9 | ) 10 | 11 | 12 | @dataclass 13 | class Authentication: 14 | authentication: Union[BasicAuthentication, OauthProvider] 15 | 16 | @classmethod 17 | def basic(cls, username: str, password: str): 18 | return cls(authentication=BasicAuthentication(username=username, password=password)) 19 | 20 | @classmethod 21 | def oauth_azure(cls, client_id: str, client_secret: str, tenant_id: str): 22 | azure = OauthProviderAzure(tenant_id) 23 | oauth_provider = OauthProvider(client_id=client_id, client_secret=client_secret, provider=azure) 24 | return cls(authentication=oauth_provider) 25 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/databricks/service/authentication/basic.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | 4 | @dataclass 5 | class BasicAuthentication: 6 | username: str 7 | password: str 8 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/databricks/service/authentication/oauth.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from dataclasses import dataclass 3 | from typing import TypedDict 4 | 5 | 6 | import requests 7 | 8 | 9 | class OauthProviderBase(ABC): # pylint: disable=too-few-public-methods 10 | @abstractmethod 11 | def get_token(self, client_id: str, client_secret: str) -> str: 12 | pass 13 | 14 | 15 | class AzureBody(TypedDict): 16 | grant_type: str 17 | client_id: str 18 | client_secret: str 19 | scope: str 20 | 21 | 22 | class AzureHeaders(TypedDict): 23 | content_type: str 24 | 25 | 26 | @dataclass 27 | class OauthProviderAzure(OauthProviderBase): 28 | tenant_id: str 29 | 30 | def _get_url(self): 31 | return f"https://login.microsoftonline.com/{self.tenant_id}/oauth2/v2.0/token" 32 | 33 | @staticmethod 34 | def _build_body(client_id: str, client_secret: str) -> AzureBody: 35 | return AzureBody( 36 | grant_type="client_credentials", 37 | client_id=client_id, 38 | client_secret=client_secret, 39 | scope="2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default", 40 | ) 41 | 42 | @staticmethod 43 | def _get_headers() -> AzureHeaders: 44 | return AzureHeaders(content_type="application/x-www-form-urlencoded") 45 | 46 | @staticmethod 47 | def _get_timeout() -> int: 48 | return 60 49 | 50 | def get_token(self, client_id: str, client_secret: str) -> str: 51 | response = requests.post( 52 | self._get_url(), 53 | data=OauthProviderAzure._build_body(client_id, client_secret), 54 | headers=OauthProviderAzure._get_headers(), # type: ignore 55 | timeout=OauthProviderAzure._get_timeout(), 56 | verify=True, 57 | ) 58 | response.raise_for_status() 59 | access_token: str = response.json()["access_token"] 60 | return access_token 61 | 62 | 63 | @dataclass 64 | class OauthProvider: 65 | client_id: str 66 | client_secret: str 67 | provider: OauthProviderBase 68 | 69 | 70 | def get_authentication_token(oauth_provider: OauthProvider) -> str: 71 | return oauth_provider.provider.get_token(oauth_provider.client_id, oauth_provider.client_secret) 72 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/databricks/service/exceptions.py: -------------------------------------------------------------------------------- 1 | class UnknownCloudProvider(Exception): 2 | pass 3 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/databricks/service/model.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import List, Optional, TypedDict 3 | 4 | 5 | class ParsedUser(TypedDict): 6 | active: bool 7 | id: str 8 | userName: str 9 | 10 | 11 | class DatabricksUserResult(TypedDict): 12 | Resources: List[ParsedUser] 13 | 14 | 15 | class Ref(TypedDict): 16 | ref: str 17 | 18 | 19 | class ResourceType(Enum): 20 | GROUP = "Group" 21 | 22 | 23 | class GroupMeta(TypedDict): 24 | resourceType: ResourceType 25 | 26 | 27 | class Group(TypedDict): 28 | displayName: str 29 | meta: GroupMeta 30 | groups: List[Ref] 31 | id: str 32 | members: List[Ref] 33 | 34 | 35 | class GroupResult(TypedDict): 36 | Resources: List[Group] 37 | 38 | 39 | class TableType(Enum): 40 | MANAGED = "MANAGED" 41 | EXTERNAL = "EXTERNAL" 42 | VIEW = "VIEW" 43 | MATERIALIZED_VIEW = "MATERIALIZED_VIEW" 44 | STREAMING_TABLE = "STREAMING_TABLE" 45 | 46 | def __str__(self) -> str: 47 | return self.value 48 | 49 | 50 | class CatalogList(TypedDict): 51 | """Definition of databricks catalog. 52 | https://docs.databricks.com/api-explorer/workspace/catalogs/list 53 | 54 | owner: can be a user or a group 55 | name: name of the catalog 56 | """ 57 | 58 | name: str 59 | owner: str 60 | metastore_id: str 61 | 62 | 63 | class Schema(TypedDict): 64 | """Definition of databricks schema. 65 | https://docs.databricks.com/api-explorer/workspace/schemas/list 66 | 67 | owner: can be a user or a group 68 | name: name of the schema 69 | """ 70 | 71 | name: str 72 | owner: str 73 | 74 | 75 | class Table(TypedDict): 76 | """Definition of databricks table. 77 | https://docs.databricks.com/api-explorer/workspace/tables/list 78 | 79 | owner: can be a user or a group 80 | name: name of the table 81 | """ 82 | 83 | full_name: str 84 | name: str 85 | table_type: TableType 86 | owner: str 87 | 88 | 89 | class Privilege(TypedDict): 90 | """Definition of databricks privilege. 91 | https://docs.databricks.com/api-explorer/workspace/grants/geteffective 92 | 93 | principal: can be a user or a group 94 | permission: can be READ, WRITE, MANAGE 95 | """ 96 | 97 | inherited_from_name: Optional[str] 98 | inherited_from_type: Optional[str] 99 | privilege: str 100 | 101 | 102 | class PrivilegeAssignments(TypedDict): 103 | """Definition of databricks permission assignment. 104 | https://docs.databricks.com/api-explorer/workspace/grants/geteffective 105 | 106 | principal: can be a user or a group 107 | permission: can be READ, WRITE, MANAGE 108 | """ 109 | 110 | principal: str 111 | privileges: List[Privilege] 112 | 113 | 114 | class ServicePrincipal(TypedDict): 115 | displayName: str 116 | applicationId: str 117 | id: str 118 | active: bool 119 | 120 | 121 | class ServicePrincipals(TypedDict): 122 | Resources: List[ServicePrincipal] 123 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/mongodb/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/mongodb/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/mongodb/atlas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/mongodb/atlas/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/mongodb/atlas/exceptions.py: -------------------------------------------------------------------------------- 1 | from universal_data_permissions_scanner.datastores.mongodb.atlas.service_model import ActionEntry 2 | 3 | from universal_data_permissions_scanner.datastores.mongodb.service_model import ResourceEntry 4 | 5 | 6 | class ActionResourcesNotFoundException(Exception): 7 | def __init__(self, action: ActionEntry) -> None: 8 | self.action = action 9 | super().__init__(f"Resources not found for action: {self.action}") 10 | 11 | 12 | class ActionResourceNotFoundException(Exception): 13 | def __init__(self, resource: ResourceEntry) -> None: 14 | self.resource = resource 15 | super().__init__(f"Collection not found for resource: {self.resource}") 16 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/mongodb/atlas/permission_resolvers.py: -------------------------------------------------------------------------------- 1 | """Atlas Role/Permission resolvers to Authz Permission""" 2 | 3 | from enum import Enum, auto 4 | 5 | from universal_data_permissions_scanner.datastores.mongodb.atlas.model import Permission 6 | from universal_data_permissions_scanner.models.model import PermissionLevel 7 | 8 | 9 | class PermissionScope(Enum): 10 | """Define the scope of the permission.""" 11 | 12 | COLLECTION = auto() 13 | PROJECT = auto() 14 | DATABASE = auto() 15 | 16 | 17 | BUILT_IN_ROLE_MAPPING_ORGANIZATION = { 18 | "ORG_OWNER": PermissionLevel.FULL, 19 | "ORG_READ_ONLY": PermissionLevel.READ, 20 | } 21 | 22 | BUILT_IN_ROLE_MAPPING_PROJECT = { 23 | "GROUP_OWNER": PermissionLevel.FULL, 24 | "GROUP_DATA_ACCESS_ADMIN": PermissionLevel.FULL, 25 | "GROUP_DATA_ACCESS_READ_WRITE": PermissionLevel.FULL, 26 | "GROUP_DATA_ACCESS_READ_ONLY": PermissionLevel.READ, 27 | } 28 | 29 | 30 | BUILT_IN_ROLE_MAPPING_DATABASE = { 31 | "atlasAdmin": (PermissionLevel.FULL, PermissionScope.PROJECT), 32 | "readWriteAnyDatabase": (PermissionLevel.FULL, PermissionScope.PROJECT), 33 | "readAnyDatabase": (PermissionLevel.READ, PermissionScope.PROJECT), 34 | "dbAdmin": (PermissionLevel.FULL, PermissionScope.DATABASE), 35 | "dbAdminAnyDatabase": (PermissionLevel.FULL, PermissionScope.PROJECT), 36 | "read": (PermissionLevel.READ, PermissionScope.COLLECTION), 37 | "readWrite": (PermissionLevel.WRITE, PermissionScope.COLLECTION), 38 | } 39 | 40 | ACTION_MAPPING = { 41 | Permission.FIND: PermissionLevel.READ, 42 | Permission.INSERT: PermissionLevel.WRITE, 43 | Permission.REMOVE: PermissionLevel.WRITE, 44 | Permission.UPDATE: PermissionLevel.WRITE, 45 | Permission.DROP_COLLECTION: PermissionLevel.WRITE, 46 | Permission.DROP_DATABASE: PermissionLevel.WRITE, 47 | Permission.RENAME_COLLECTION_SAME_DB: PermissionLevel.READ, 48 | Permission.LIST_COLLECTIONS: PermissionLevel.READ, 49 | Permission.SQL_GET_SCHEMA: PermissionLevel.READ, 50 | Permission.SQL_SET_SCHEMA: PermissionLevel.WRITE, 51 | Permission.OUT_TO_S3: PermissionLevel.READ, 52 | } 53 | 54 | 55 | def resolve_organization_role(role: str): 56 | """Resolve the permission level for a given organization role.""" 57 | return BUILT_IN_ROLE_MAPPING_ORGANIZATION.get(role) 58 | 59 | 60 | def resolve_project_role(role: str): 61 | """Resolve the permission level for a given project role.""" 62 | return BUILT_IN_ROLE_MAPPING_PROJECT.get(role) 63 | 64 | 65 | def resolve_database_role(role: str): 66 | """Resolve the permission level and scope for a given database role.""" 67 | return BUILT_IN_ROLE_MAPPING_DATABASE.get(role) 68 | 69 | 70 | def resolve_permission(permission: Permission): 71 | """Resolve MongoDB permission to permission level.""" 72 | return ACTION_MAPPING.get(permission) 73 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/mongodb/atlas/service_model.py: -------------------------------------------------------------------------------- 1 | """Defines the objects returned by the Atlas admin API.""" 2 | 3 | from typing import List, TypedDict 4 | 5 | 6 | class DataBaseUserEntryRoleRequired(TypedDict): 7 | """Python doesn't support typed dict with some required fields and some optional. 8 | Only through inheritance. 9 | So those fields. 10 | """ 11 | 12 | databaseName: str 13 | roleName: str 14 | 15 | 16 | class DataBaseUserEntryRole(DataBaseUserEntryRoleRequired, total=False): 17 | """Inherit from DataBaseUserEntryRoleRequired to get required fields.""" 18 | 19 | collectionName: str 20 | 21 | 22 | class DataBaseUserEntryScope(TypedDict): 23 | name: str 24 | type: str 25 | 26 | 27 | class DataBaseUserEntry(TypedDict): 28 | username: str 29 | databaseName: str 30 | scopes: List[DataBaseUserEntryScope] 31 | roles: List[DataBaseUserEntryRole] 32 | 33 | 34 | class OrganizationRole(TypedDict): 35 | """An organization role.""" 36 | 37 | roleName: str 38 | 39 | 40 | class OrganizationUserEntry(TypedDict): 41 | """An organization user entry.""" 42 | 43 | id: str # pylint: disable=invalid-name 44 | username: str 45 | emailAddress: str 46 | databaseName: str 47 | roles: List[OrganizationRole] 48 | teamIds: List[str] 49 | 50 | 51 | class ProjectTeamEntry(TypedDict): 52 | """A project team entry.""" 53 | 54 | teamId: str 55 | roleNames: List[str] 56 | 57 | 58 | class OrganizationTeamEntry(TypedDict): 59 | """An organization team entry.""" 60 | 61 | id: str # pylint: disable=invalid-name 62 | name: str 63 | 64 | 65 | class ClusterConnectionStringEntry(TypedDict): 66 | """Connection string from groups/{groupId}/clusters""" 67 | 68 | standardSrv: str 69 | 70 | 71 | class ClusterEntry(TypedDict): 72 | """Single entry from groups/{groupId}/clusters.""" 73 | 74 | id: str # pylint: disable=invalid-name 75 | name: str 76 | connectionStrings: ClusterConnectionStringEntry 77 | 78 | 79 | class InheritedRoleEntry(TypedDict): 80 | """Inherited role entry.""" 81 | 82 | role: str 83 | db: str 84 | 85 | 86 | class ResourceEntry(TypedDict): 87 | """Resource entry.""" 88 | 89 | cluster: bool 90 | collection: str 91 | db: str 92 | 93 | 94 | class ActionEntry(TypedDict): 95 | """Action entry.""" 96 | 97 | action: str 98 | resources: List[ResourceEntry] 99 | 100 | 101 | class CustomRoleEntry(TypedDict): 102 | """A custom role entry.""" 103 | 104 | roleName: str 105 | actions: List[ActionEntry] 106 | inheritedRoles: List[InheritedRoleEntry] 107 | 108 | 109 | class ProjectInfo(TypedDict): 110 | """A project info entry.""" 111 | 112 | id: str # pylint: disable=invalid-name 113 | name: str 114 | orgId: str 115 | 116 | 117 | class OrganizationEntry(TypedDict): 118 | """An organization entry.""" 119 | 120 | id: str # pylint: disable=invalid-name 121 | name: str 122 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/mongodb/model.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass 4 | from typing import List, Set 5 | 6 | from universal_data_permissions_scanner.datastores.mongodb.service_model import RoleEntry 7 | from universal_data_permissions_scanner.models.model import AuthzPathElement, PermissionLevel 8 | 9 | 10 | @dataclass 11 | class InheritedRole: 12 | """Define a MongoDB inherited role.""" 13 | 14 | name: str 15 | database: str 16 | 17 | def __hash__(self) -> int: 18 | return hash(self.name) + hash(self.database) 19 | 20 | 21 | @dataclass 22 | class Resource: 23 | """Define a MongoDB resource.""" 24 | 25 | database: str 26 | collection: str 27 | 28 | def __hash__(self) -> int: 29 | return hash(self.database) + hash(self.collection) 30 | 31 | 32 | @dataclass 33 | class Privilege: 34 | """Define a MongoDB privilege.""" 35 | 36 | resource: Resource 37 | actions: List[str] 38 | 39 | def __hash__(self) -> int: 40 | return hash(self.resource) + hash(tuple(self.actions)) 41 | 42 | 43 | @dataclass 44 | class Role: 45 | """Define a MongoDB role.""" 46 | 47 | name: str 48 | db: str # pylint: disable=invalid-name 49 | inherited_roles: List[InheritedRole] 50 | privileges: List[Privilege] 51 | 52 | @classmethod 53 | def build_from_response(cls, entry: RoleEntry): 54 | """Build a role from the response.""" 55 | inherited_roles = [InheritedRole(name=role["role"], database=role["db"]) for role in entry["inheritedRoles"]] 56 | privileges = [ 57 | Privilege( 58 | resource=Resource(privilege["resource"]["db"], privilege["resource"]["collection"]), 59 | actions=privilege["actions"], 60 | ) 61 | for privilege in entry["privileges"] 62 | ] 63 | return cls( 64 | name=entry["role"], 65 | db=entry["db"], 66 | inherited_roles=inherited_roles, 67 | privileges=privileges, 68 | ) 69 | 70 | 71 | @dataclass 72 | class AdminRole: 73 | """Define a MongoDB admin role.""" 74 | 75 | name: str 76 | permission_level: PermissionLevel 77 | path: List[AuthzPathElement] 78 | 79 | def __hash__(self) -> int: 80 | return hash(self.name) 81 | 82 | 83 | @dataclass 84 | class AdminUser: 85 | """Define a MongoDB admin user with a single role.""" 86 | 87 | id: str # pylint: disable=invalid-name 88 | name: str 89 | roles: Set[AdminRole] 90 | 91 | def __hash__(self) -> int: 92 | return hash(self.id) 93 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/mongodb/resolvers.py: -------------------------------------------------------------------------------- 1 | from universal_data_permissions_scanner.models.model import PermissionLevel 2 | 3 | BUILT_IN_ROLES_MAP = { 4 | "read": PermissionLevel.READ, 5 | "readWrite": PermissionLevel.WRITE, 6 | "dbAdmin": PermissionLevel.FULL, 7 | "dbOwner": PermissionLevel.FULL, 8 | "userAdmin": PermissionLevel.FULL, 9 | } 10 | 11 | BUILT_IN_CLUSTER_ROLES_MAP = { 12 | "clusterAdmin": PermissionLevel.FULL, 13 | "backup": PermissionLevel.FULL, 14 | "restore": PermissionLevel.FULL, 15 | "readAnyDatabase": PermissionLevel.READ, 16 | "readWriteAnyDatabase": PermissionLevel.WRITE, 17 | "userAdminAnyDatabase": PermissionLevel.FULL, 18 | "dbAdminAnyDatabase": PermissionLevel.FULL, 19 | "root": PermissionLevel.FULL, 20 | "__system": PermissionLevel.FULL, 21 | } 22 | 23 | PRIVILEGE_MAP = { 24 | "find": PermissionLevel.READ, 25 | "insert": PermissionLevel.WRITE, 26 | "remove": PermissionLevel.WRITE, 27 | "update": PermissionLevel.WRITE, 28 | "createRole": PermissionLevel.FULL, 29 | "createUser": PermissionLevel.FULL, 30 | "dropCollection": PermissionLevel.WRITE, 31 | "grantRole": PermissionLevel.FULL, 32 | "dropDatabase": PermissionLevel.WRITE, 33 | "anyAction": PermissionLevel.FULL, 34 | "internal": PermissionLevel.FULL, 35 | } 36 | 37 | 38 | def get_permission_level(role: str): 39 | """Get permission level from role.""" 40 | return BUILT_IN_ROLES_MAP.get(role) 41 | 42 | 43 | def get_permission_level_cluster(role: str): 44 | """Get permission level from role.""" 45 | return BUILT_IN_CLUSTER_ROLES_MAP.get(role) 46 | 47 | 48 | def get_permission_level_privilege(privilege: str): 49 | """Get permission level from privilege.""" 50 | return PRIVILEGE_MAP.get(privilege) 51 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/mongodb/service.py: -------------------------------------------------------------------------------- 1 | """Wrapper for MongoDB client. 2 | 3 | To make shortcuts for repeated logic, use the client directly for all others. 4 | """ 5 | 6 | from dataclasses import dataclass 7 | from typing import Any 8 | 9 | from pymongo import MongoClient # pylint: disable=import-error 10 | from pymongo.database import Database # pylint: disable=import-error 11 | 12 | from universal_data_permissions_scanner.datastores.mongodb.model import Role 13 | from universal_data_permissions_scanner.datastores.mongodb.service_model import RolesInfoEntry, UserInfoResponseEntry 14 | 15 | 16 | @dataclass 17 | class MongoDBService: 18 | client: MongoClient[Any] 19 | 20 | def iter_database_connections(self): 21 | """Iterate over all database connections.""" 22 | for database in self.client.list_databases(): 23 | database_connection: Database[Any] = self.client[database['name']] 24 | name: str = database["name"] 25 | yield (name, database_connection) 26 | 27 | @staticmethod 28 | def get_users(database_connection: Database[Any]): 29 | """Get all users.""" 30 | results: UserInfoResponseEntry = database_connection.command("usersInfo") # type: ignore 31 | return results['users'] 32 | 33 | @staticmethod 34 | def get_custom_roles(database_connection: Database[Any]): 35 | """Get all custom roles.""" 36 | results: RolesInfoEntry = database_connection.command({"rolesInfo": 1, "showPrivileges": True}) # type: ignore 37 | parsed_roles = {role["role"]: Role.build_from_response(role) for role in results['roles']} 38 | return parsed_roles 39 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/mongodb/service_model.py: -------------------------------------------------------------------------------- 1 | """Defines the objects returned by the MongoDB API.""" 2 | 3 | from typing import List, TypedDict 4 | 5 | 6 | class ResourceEntry(TypedDict): 7 | """Define a MongoDB resource. 8 | Returned by the rolesInfo command. 9 | """ 10 | 11 | db: str 12 | collection: str 13 | 14 | 15 | class AssignedRole(TypedDict): 16 | """Define a MongoDB role assignment. 17 | returned by the usersInfo command. 18 | """ 19 | 20 | role: str 21 | db: str 22 | 23 | 24 | class UserEntry(TypedDict): 25 | """Define a MongoDB user. 26 | Returned by the usersInfo command. 27 | """ 28 | 29 | userId: bytes 30 | user: str 31 | db: str 32 | roles: List[AssignedRole] 33 | 34 | 35 | class UserInfoResponseEntry(TypedDict): 36 | """Define the response from the usersInfo command.""" 37 | 38 | users: List[UserEntry] 39 | 40 | 41 | class PrivilegeEntry(TypedDict): 42 | """Define a MongoDB privilege. 43 | Returned by the rolesInfo command. 44 | """ 45 | 46 | resource: ResourceEntry 47 | actions: List[str] 48 | 49 | 50 | class RoleEntry(TypedDict): 51 | """Define a MongoDB role. 52 | Returned by the rolesInfo command. 53 | """ 54 | 55 | role: str 56 | db: str 57 | privileges: List[PrivilegeEntry] 58 | inheritedRoles: List[AssignedRole] 59 | 60 | 61 | class RolesInfoEntry(TypedDict): 62 | """Define the response from the rolesInfo command.""" 63 | 64 | roles: List[RoleEntry] 65 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/postgres/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/postgres/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/postgres/commands/all_databases.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | datname 3 | FROM 4 | pg_database 5 | where 6 | datistemplate = false -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/postgres/commands/all_tables.sql: -------------------------------------------------------------------------------- 1 | with all_db as ( 2 | SELECT 3 | table_schema, 4 | table_name 5 | FROM 6 | information_schema.tables 7 | ORDER BY 8 | table_schema, 9 | table_name 10 | ), 11 | current_db as ( 12 | select 13 | current_database as db 14 | from 15 | current_database() 16 | ) 17 | SELECT 18 | current_db.db, 19 | all_db.table_schema, 20 | all_db.table_name 21 | from 22 | all_db, 23 | current_db -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/postgres/commands/roles.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | r.rolname as username, 3 | r.rolsuper as superuser, 4 | r1.rolname as "role", 5 | r.rolcanlogin as login 6 | FROM 7 | pg_catalog.pg_roles r FULL 8 | OUTER JOIN pg_catalog.pg_auth_members m ON (m.member = r.oid) FULL 9 | OUTER JOIN pg_roles r1 ON (m.roleid = r1.oid) -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/postgres/commands/roles_grants.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | relname as table_name, 3 | nspname as schema_name, 4 | relkind as type, 5 | rolname as owner, 6 | relacl as acl 7 | FROM 8 | pg_namespace 9 | JOIN pg_class ON (relnamespace = pg_namespace.oid) 10 | join pg_roles on (pg_class.relowner = pg_roles.oid) 11 | where 12 | relkind in ('t', 'r', 't', 'v', 'm', 'f', 'p'); -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/postgres/database_query_results.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from enum import Enum 4 | from logging import Logger 5 | from typing import List, NamedTuple, Optional 6 | 7 | 8 | class RoleGrant(NamedTuple): 9 | resource_name: str 10 | schema_name: str 11 | resource_type: str 12 | owner: str 13 | acl: Optional[str] 14 | 15 | 16 | class DataBaseAclPermission(Enum): 17 | SELECT = "r" # pylint: disable=invalid-name 18 | INSERT = "a" # pylint: disable=invalid-name 19 | UPDATE = "w" # pylint: disable=invalid-name 20 | DELETE = "d" # pylint: disable=invalid-name 21 | TRUNCATE = "D" # pylint: disable=invalid-name 22 | REFERENCES = "x" # pylint: disable=invalid-name 23 | 24 | def __ge__(self, other: DataBaseAclPermission): 25 | if self.name in ("INSERT", "UPDATE", "DELETE", "TRUNCATE", "TRIGGER"): 26 | return True 27 | return False 28 | 29 | def __lt__(self, other: DataBaseAclPermission): 30 | if self.name in ("SELECT", "REFERENCES"): 31 | return True 32 | return False 33 | 34 | 35 | class DataBaseAclEntry(NamedTuple): 36 | grantee: str 37 | permissions: List[DataBaseAclPermission] 38 | 39 | def max_permission(self) -> DataBaseAclPermission: 40 | """Get the highest permission level.""" 41 | if len(self.permissions) == 0: 42 | raise ValueError("No permissions") 43 | return max(self.permissions) # type: ignore 44 | 45 | 46 | class DataBaseAcl(NamedTuple): 47 | entries: List[DataBaseAclEntry] 48 | 49 | @classmethod 50 | def serialize_from_str(cls, logger: Logger, src: str): 51 | """Serialize the permission list from a string. 52 | The string format: {=/, =/} 53 | example: 54 | {postgres=arwdDxt/postgres,data_access_west=r/postgres} 55 | """ 56 | entries: List[DataBaseAclEntry] = [] 57 | # remove the curly brackets 58 | src = src[1:-1] 59 | for entry in src.split(","): 60 | grantee, suffix = entry.split("=") 61 | permission_list = suffix.split("/")[0] 62 | result_permission_list: List[DataBaseAclPermission] = [] 63 | for letter in permission_list: 64 | try: 65 | result_permission_list.append(DataBaseAclPermission(letter)) 66 | except ValueError: 67 | logger.debug("Unknown permission letter: %s", letter) 68 | continue 69 | 70 | entries.append(DataBaseAclEntry(grantee=grantee, permissions=result_permission_list)) 71 | return cls(entries=entries) 72 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/postgres/deployment.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from enum import Enum 3 | from typing import Optional 4 | 5 | 6 | class DeploymentType(Enum): 7 | GCP = "gcp" 8 | AWS_RDS = "aws_rds" 9 | # Self hosted, or unknown 10 | OTHER = "other" 11 | 12 | 13 | @dataclass 14 | class Deployment: 15 | deployment_type: DeploymentType 16 | cloud_super_user: Optional[str] 17 | managed: bool 18 | 19 | @classmethod 20 | def aws_rds(cls): 21 | return cls(deployment_type=DeploymentType.AWS_RDS, cloud_super_user="rds_superuser", managed=True) 22 | 23 | @classmethod 24 | def gcp(cls): 25 | return cls(deployment_type=DeploymentType.GCP, cloud_super_user="cloudsqlsuperuser", managed=True) 26 | 27 | @classmethod 28 | def other(cls): 29 | return cls(deployment_type=DeploymentType.OTHER, cloud_super_user=None, managed=False) 30 | 31 | def get_cloud_super_user(self): 32 | return self.cloud_super_user 33 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/postgres/exporter.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Generator, List, Set 2 | 3 | from universal_data_permissions_scanner.datastores.postgres.model import AuthorizationModel, DBRole, ResourceGrant 4 | from universal_data_permissions_scanner.models.model import ( 5 | Asset, 6 | AuthzEntry, 7 | AuthzPathElement, 8 | AuthzPathElementType, 9 | Identity, 10 | IdentityType, 11 | ) 12 | from universal_data_permissions_scanner.writers import BaseWriter 13 | 14 | 15 | def _yield_row(role_name: str, grant: ResourceGrant, roles: List[DBRole]): 16 | auth_path_element = [ 17 | AuthzPathElement( 18 | id=role.name, 19 | name=role.name, 20 | type=AuthzPathElementType.ROLE, 21 | ) 22 | for role in roles 23 | ] 24 | auth_path_element[-1].db_permissions = grant.db_permissions 25 | identity = Identity(id=role_name, name=role_name, type=IdentityType.ROLE_LOGIN) 26 | asset = Asset(name=grant.name, type=grant.type) 27 | yield AuthzEntry( 28 | identity=identity, 29 | asset=asset, 30 | path=auth_path_element, 31 | permission=grant.permission_level, 32 | ) 33 | 34 | 35 | def _iter_role_row( 36 | base_role_name: str, 37 | role: DBRole, 38 | prev_roles: List[DBRole], 39 | roles_to_grants: Dict[str, Set[ResourceGrant]], 40 | role_to_roles: Dict[DBRole, Set[DBRole]], 41 | ) -> Generator[AuthzEntry, None, None]: 42 | grants = roles_to_grants.get(role.name, set()) 43 | prev_roles.append(role) 44 | for grant in grants: 45 | yield from _yield_row(role_name=base_role_name, grant=grant, roles=prev_roles) 46 | 47 | for granted_role in role_to_roles.get(role, set()): 48 | yield from _iter_role_row( 49 | base_role_name=base_role_name, 50 | role=granted_role, 51 | prev_roles=prev_roles, 52 | roles_to_grants=roles_to_grants, 53 | role_to_roles=role_to_roles, 54 | ) 55 | prev_roles.remove(role) 56 | 57 | 58 | def export(model: AuthorizationModel, writer: BaseWriter): 59 | """Export the model to the writer. 60 | 61 | Args: 62 | model (AuthorizationModel): Postgres model which describes the authorization 63 | writer (BaseWriter): Write to write the entries 64 | """ 65 | for role, roles in model.role_to_roles.items(): 66 | if role.can_login is True: 67 | for grant in model.role_to_grants.get(role.name, set()): 68 | for entry in _yield_row(role_name=role.name, grant=grant, roles=[role]): 69 | writer.write_entry(entry) 70 | 71 | for granted_role in roles: 72 | for entry in _iter_role_row( 73 | role.name, 74 | role=granted_role, 75 | prev_roles=[], 76 | roles_to_grants=model.role_to_grants, 77 | role_to_roles=model.role_to_roles, 78 | ): 79 | writer.write_entry(entry) 80 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/postgres/model.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass 4 | from typing import Dict, List, Set 5 | 6 | from universal_data_permissions_scanner.models import PermissionLevel 7 | from universal_data_permissions_scanner.models.model import AssetType 8 | 9 | RoleName = str 10 | 11 | RESOURCE_TYPE_MAP = { 12 | "r": AssetType.TABLE, 13 | "t": AssetType.TOAST_TABLE, 14 | "v": AssetType.VIEW, 15 | "m": AssetType.MATERIALIZED_VIEW, 16 | "f": AssetType.FOREIGN_TABLE, 17 | "p": AssetType.PARTITION_TABLE, 18 | } 19 | 20 | PERMISSION_LEVEL_MAP = { 21 | "SELECT": PermissionLevel.READ, 22 | "REFERENCES": PermissionLevel.READ, 23 | "INSERT": PermissionLevel.WRITE, 24 | "UPDATE": PermissionLevel.WRITE, 25 | "DELETE": PermissionLevel.WRITE, 26 | "TRUNCATE": PermissionLevel.WRITE, 27 | "TRIGGER": PermissionLevel.WRITE, 28 | "SUPER_USER": PermissionLevel.FULL, 29 | } 30 | 31 | 32 | @dataclass 33 | class ResourceGrant: 34 | """Define a resource, e.g. a table, and the permission level. 35 | The list is db.schema.table. 36 | """ 37 | 38 | name: List[str] 39 | permission_level: PermissionLevel 40 | db_permissions: list[str] 41 | type: AssetType 42 | 43 | def __hash__(self) -> int: 44 | return hash(str(self.name)) 45 | 46 | 47 | @dataclass 48 | class DBRole: 49 | """Define a role, e.g. a user, and the roles it has, and if it can login.""" 50 | 51 | name: str 52 | roles: Set[DBRole] 53 | can_login: bool 54 | 55 | @classmethod 56 | def new(cls, name: str, roles: Set[DBRole], can_login: bool): 57 | """Create a new DBRole.""" 58 | return cls(name=name, roles=roles, can_login=can_login) 59 | 60 | def add_role(self, role: DBRole): 61 | self.roles.add(role) 62 | 63 | def __hash__(self) -> int: 64 | return hash(self.name) 65 | 66 | 67 | @dataclass 68 | class AuthorizationModel: 69 | """Define the authorization model. 70 | Map a role to the roles it has, and the grants it has. 71 | Map a role to the grants it has. 72 | """ 73 | 74 | role_to_roles: Dict[DBRole, Set[DBRole]] 75 | role_to_grants: Dict[RoleName, Set[ResourceGrant]] 76 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/snowflake/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/datastores/snowflake/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/snowflake/commands/grants_roles.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | GRANTEE_NAME as grantee_name, 3 | PRIVILEGE as privilege, 4 | TABLE_CATALOG as db, 5 | TABLE_SCHEMA as schema, 6 | name as resource_name, 7 | GRANTED_ON as granted_on 8 | FROM 9 | snowflake.account_usage.grants_to_roles 10 | where 11 | deleted_on is null 12 | and GRANTED_ON in ('TABLE', 'VIEW', 'MATERIALIZED VIEW', 'ROLE') 13 | order by 14 | grantee_name, 15 | db, 16 | schema, 17 | resource_name -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/snowflake/commands/grants_to_share.sql: -------------------------------------------------------------------------------- 1 | show grants to share -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/snowflake/commands/shares.sql: -------------------------------------------------------------------------------- 1 | SHOW SHARES -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/snowflake/commands/user_grants.sql: -------------------------------------------------------------------------------- 1 | with users_grants as ( 2 | SELECT 3 | GRANTEE_NAME as user, 4 | ROLE as role 5 | FROM 6 | snowflake.account_usage.grants_to_users 7 | where 8 | deleted_on is null 9 | ), 10 | users as ( 11 | SELECT 12 | NAME as user, 13 | email, 14 | default_role 15 | from 16 | snowflake.account_usage.users 17 | where 18 | deleted_on is null 19 | ) 20 | select 21 | users.user as user, 22 | users_grants.role as role, 23 | users.email as email, 24 | users.default_role 25 | from 26 | users_grants 27 | right outer join users on users_grants.user = users.user 28 | order by 29 | users_grants.user -------------------------------------------------------------------------------- /universal_data_permissions_scanner/datastores/snowflake/service.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from pathlib import Path 3 | from typing import Any, List, Optional, Tuple 4 | 5 | from snowflake.connector.cursor import SnowflakeCursor 6 | from snowflake.connector.errors import ProgrammingError 7 | 8 | from universal_data_permissions_scanner.errors.snowflake import NoActiveWarehouseException 9 | 10 | COMMANDS_DIR = Path(__file__).parent / "commands" 11 | 12 | 13 | @dataclass 14 | class SnowflakeService: 15 | cursor: SnowflakeCursor 16 | 17 | @classmethod 18 | def connect(cls, cursor: SnowflakeCursor): 19 | return cls(cursor=cursor) 20 | 21 | def get_rows(self, file_name_command: Path, params: Optional[str] = None) -> List[Tuple[Any, ...]]: 22 | """Get rows from Snowflake. 23 | 24 | Args: 25 | file_name_command (Path): File name to load from the commands directory. 26 | params (Optional[Tuple[str, ...]], optional): Parameters to pass to the command. Defaults to None. 27 | 28 | Returns: 29 | List[Tuple[Any, ...]]: results 30 | """ 31 | command = (COMMANDS_DIR / file_name_command).read_text(encoding="utf-8") 32 | if params is not None: 33 | command += " " + params 34 | try: 35 | self.cursor.execute(command=command) 36 | except ProgrammingError as err: 37 | if "No active warehouse selected in the current session" in str(err): 38 | raise NoActiveWarehouseException from err 39 | 40 | return self.cursor.fetchall() # type: ignore 41 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/errors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/errors/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/errors/failed_connection_errors.py: -------------------------------------------------------------------------------- 1 | class ConnectionFailure(Exception): 2 | pass 3 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/errors/snowflake.py: -------------------------------------------------------------------------------- 1 | class NoActiveWarehouseException(BaseException): 2 | pass 3 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/models/__init__.py: -------------------------------------------------------------------------------- 1 | """Contains how the tool is keeping the information internally before the exporting.""" 2 | 3 | from universal_data_permissions_scanner.models.model import PermissionLevel 4 | 5 | __all__ = ["PermissionLevel"] 6 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SatoriCyber/universal-data-permissions-scanner/be6c7b0723b0a04e19ac3b38ed9c55b486dd4b6b/universal_data_permissions_scanner/utils/__init__.py -------------------------------------------------------------------------------- /universal_data_permissions_scanner/utils/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | def get_logger(debug: bool): 5 | """Provides a logger in case one is not provided. 6 | 7 | Args: 8 | debug (bool): Should logs be in debug 9 | 10 | Returns: 11 | Logger: Python logger 12 | """ 13 | logger = logging.getLogger('authz-analyzer') 14 | level = logging.INFO if not debug else logging.DEBUG 15 | logger.setLevel(level) 16 | 17 | if logger.handlers: 18 | return logger 19 | 20 | ch = logging.StreamHandler() # pylint: disable=C0103 21 | ch.setLevel(level) 22 | formatter = logging.Formatter('%(asctime)s - %(filename)s:%(lineno)d - %(levelname)s - %(message)s') 23 | ch.setFormatter(formatter) 24 | logger.addHandler(ch) 25 | return logger 26 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/writers/__init__.py: -------------------------------------------------------------------------------- 1 | """Module for writers.""" 2 | 3 | from universal_data_permissions_scanner.writers.base_writers import BaseWriter, OutputFormat 4 | from universal_data_permissions_scanner.writers.csv_writer import CSVWriter 5 | from universal_data_permissions_scanner.writers.get_writers import open_writer 6 | from universal_data_permissions_scanner.writers.multi_json_exporter import MultiJsonWriter 7 | 8 | __all__ = ["OutputFormat", "BaseWriter", "CSVWriter", "MultiJsonWriter", "open_writer"] 9 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/writers/base_writers.py: -------------------------------------------------------------------------------- 1 | """Module for writers building blocks""" 2 | 3 | from abc import ABC, abstractmethod 4 | from enum import Enum, auto 5 | from contextlib import contextmanager 6 | from typing import BinaryIO, TextIO, Union, Any, Generator 7 | 8 | from universal_data_permissions_scanner.models.model import AuthzEntry 9 | 10 | DEFAULT_OUTPUT_FILE = "authz-analyzer-export" 11 | 12 | 13 | class OutputFormat(Enum): 14 | """The file format to write the output.""" 15 | 16 | CSV = auto() 17 | MULTI_JSON = auto() 18 | 19 | 20 | class BaseWriter(ABC): 21 | """Base class for writers.""" 22 | 23 | def __init__(self, fh: Union[TextIO, BinaryIO]) -> None: # pylint: disable=(invalid-name) 24 | self.fh = fh # pylint: disable=(invalid-name) 25 | self._write_header() 26 | 27 | @abstractmethod 28 | def _write_header(self) -> None: 29 | """Writes header of the file. 30 | Should be called before any write_entry. 31 | """ 32 | 33 | @abstractmethod 34 | def write_entry(self, entry: AuthzEntry) -> None: 35 | """Write a single entry to the file. 36 | 37 | Args: 38 | entry (AuthzEntry): AuthZEntry to write to the file. 39 | """ 40 | 41 | def close(self) -> None: 42 | """Close the writer.""" 43 | self.fh.close() 44 | 45 | @classmethod 46 | @contextmanager 47 | def open(cls, fh: Union[TextIO, BinaryIO]) -> Generator["BaseWriter", Any, None]: # pylint: disable=(invalid-name) 48 | """Context manager to open the writer.""" 49 | writer = cls(fh) 50 | try: 51 | yield writer 52 | finally: 53 | writer.close() 54 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/writers/csv_writer.py: -------------------------------------------------------------------------------- 1 | """Writer for CSV.""" 2 | 3 | import csv 4 | from typing import TextIO 5 | 6 | from universal_data_permissions_scanner.models.model import AuthzEntry 7 | from universal_data_permissions_scanner.writers.base_writers import BaseWriter 8 | 9 | 10 | class CSVWriter(BaseWriter): 11 | """Writer for CSV.""" 12 | 13 | def __init__(self, fh: TextIO): 14 | self.writer = csv.writer(fh, dialect="excel", escapechar="\\", strict=True) 15 | super().__init__(fh) 16 | 17 | def _write_header(self): 18 | self.writer.writerow(["identity", "permission", "asset", "granted_by"]) 19 | 20 | def write_entry(self, entry: AuthzEntry): 21 | path = "->".join([str(x) for x in entry.path]) 22 | self.writer.writerow([str(entry.identity), str(entry.permission), str(entry.asset), path]) 23 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/writers/get_writers.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from io import TextIOWrapper 3 | from pathlib import Path 4 | from contextlib import contextmanager 5 | from typing import Optional, TextIO, Union, Generator, Any 6 | 7 | from universal_data_permissions_scanner.writers.base_writers import BaseWriter, OutputFormat 8 | from universal_data_permissions_scanner.writers.csv_writer import CSVWriter 9 | from universal_data_permissions_scanner.writers.multi_json_exporter import MultiJsonWriter 10 | 11 | 12 | @contextmanager 13 | def open_writer(filename: Optional[Union[Path, str]], output_format: OutputFormat) -> Generator[BaseWriter, Any, None]: 14 | fh = ( # pylint: disable=invalid-name 15 | sys.stdout if filename is None else open(filename, 'w', encoding="utf=8") # pylint: disable=consider-using-with 16 | ) 17 | with _open_writer(fh, output_format) as writer: 18 | yield writer 19 | 20 | 21 | @contextmanager 22 | def _open_writer( 23 | fh: Union[TextIO, TextIOWrapper], output_format: OutputFormat # pylint: disable=(invalid-name) 24 | ) -> Generator[BaseWriter, Any, None]: 25 | if output_format is OutputFormat.MULTI_JSON: 26 | with MultiJsonWriter.open(fh) as writer: 27 | yield writer 28 | elif output_format is OutputFormat.CSV: 29 | with CSVWriter.open(fh) as writer: 30 | yield writer 31 | else: 32 | raise WriterNotFoundException("Output format not support") 33 | 34 | 35 | class WriterNotFoundException(BaseException): 36 | """The writer isn't defined.""" 37 | -------------------------------------------------------------------------------- /universal_data_permissions_scanner/writers/multi_json_exporter.py: -------------------------------------------------------------------------------- 1 | """Exporter for multi-json. 2 | 3 | Each line is a valid json. 4 | Good when there is a need to stream the file to BigQuery. 5 | https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json#loading_semi-structured_json_data 6 | """ 7 | 8 | import json 9 | from typing import Dict, List, Union 10 | 11 | from serde.se import to_dict # pylint: disable=import-error #type: ignore 12 | 13 | from universal_data_permissions_scanner.models.model import AuthzEntry 14 | from universal_data_permissions_scanner.writers.base_writers import BaseWriter 15 | 16 | 17 | class MultiJsonWriter(BaseWriter): 18 | """Writer for multi-json. 19 | Each entry is a valid json, example: 20 | {"identity": {"id": "USER_1", "type": "USER", "name": "USER_1"}, "permission": "Read", "asset": {"name": "db.schema.table", "type": "table"}, "granted_by": [{"type": "ROLE", "id": "super-user", "name": "super-user", "db_permissions": ["SELECT"], "note": "USER_1 has a super-user ROLE"}]} 21 | """ 22 | 23 | def write_entry(self, entry: AuthzEntry): 24 | path: List[Dict[str, Union[str, List[str]]]] = list( 25 | map( 26 | lambda x: { 27 | "type": str(x.type), 28 | "id": x.id, 29 | "name": x.name, 30 | "db_permissions": x.db_permissions, 31 | "notes": [to_dict(note) for note in x.notes], 32 | }, 33 | entry.path, 34 | ) 35 | ) 36 | identity = { 37 | "id": entry.identity.id, 38 | "type": str(entry.identity.type), 39 | "name": entry.identity.name, 40 | "notes": [to_dict(note) for note in entry.identity.notes], 41 | } 42 | asset = { 43 | "name": entry.asset.name, 44 | "type": str(entry.asset.type), 45 | "notes": [to_dict(note) for note in entry.asset.notes], 46 | } 47 | 48 | line = { 49 | "identity": identity, 50 | "permission": str(entry.permission), 51 | "asset": asset, 52 | "granted_by": path, 53 | } 54 | json_line = json.dumps(line, indent=None) 55 | json_line += '\n' 56 | self.fh.write(json_line) # type: ignore 57 | 58 | def _write_header(self): 59 | pass 60 | --------------------------------------------------------------------------------