├── .circleci └── config.yml ├── .codacy.yml ├── .coveragerc ├── .gitignore ├── .pre-commit-config.yaml ├── .prospector.yaml ├── .pylintrc ├── Dockerfile ├── LICENSE ├── README.md ├── bin ├── cloudiscovery └── cloudiscovery.cmd ├── cloudiscovery ├── __init__.py ├── commands │ └── __init__.py ├── locales │ ├── __init__.py │ ├── en_US │ │ ├── LC_MESSAGES │ │ │ ├── __init__.py │ │ │ ├── messages.mo │ │ │ └── messages.po │ │ └── __init__.py │ ├── messages.pot │ └── pt_BR │ │ ├── LC_MESSAGES │ │ ├── __init__.py │ │ ├── messages.mo │ │ └── messages.po │ │ └── __init__.py ├── provider │ ├── __init__.py │ └── aws │ │ ├── __init__.py │ │ ├── all │ │ ├── __init__.py │ │ ├── command.py │ │ ├── data │ │ │ ├── __init__.py │ │ │ ├── omitted_resources.py │ │ │ ├── on_top_policies.py │ │ │ └── required_params_override.py │ │ ├── exception │ │ │ └── __init__.py │ │ └── resource │ │ │ ├── __init__.py │ │ │ └── all.py │ │ ├── command.py │ │ ├── common_aws.py │ │ ├── iot │ │ ├── __init__.py │ │ ├── command.py │ │ ├── diagram.py │ │ └── resource │ │ │ ├── __init__.py │ │ │ ├── certificate.py │ │ │ ├── policy.py │ │ │ └── thing.py │ │ ├── limit │ │ ├── __init__.py │ │ ├── command.py │ │ ├── data │ │ │ ├── __init__.py │ │ │ └── allowed_resources.py │ │ └── resource │ │ │ ├── __init__.py │ │ │ ├── all.py │ │ │ └── ses.py │ │ ├── policy │ │ ├── __init__.py │ │ ├── command.py │ │ ├── diagram.py │ │ └── resource │ │ │ ├── __init__.py │ │ │ ├── general.py │ │ │ └── security.py │ │ ├── security │ │ ├── __init__.py │ │ ├── command.py │ │ ├── data │ │ │ ├── __init__.py │ │ │ └── commands_enabled.py │ │ └── resource │ │ │ ├── __init__.py │ │ │ ├── all.py │ │ │ └── commands │ │ │ ├── CLOUDTRAIL.py │ │ │ ├── DYNAMODB.py │ │ │ ├── EC2.py │ │ │ ├── IAM.py │ │ │ └── __init__.py │ │ └── vpc │ │ ├── __init__.py │ │ ├── command.py │ │ ├── diagram.py │ │ └── resource │ │ ├── __init__.py │ │ ├── analytics.py │ │ ├── application.py │ │ ├── compute.py │ │ ├── containers.py │ │ ├── database.py │ │ ├── enduser.py │ │ ├── identity.py │ │ ├── management.py │ │ ├── mediaservices.py │ │ ├── ml.py │ │ ├── network.py │ │ ├── security.py │ │ └── storage.py ├── shared │ ├── __init__.py │ ├── command.py │ ├── common.py │ ├── diagram.py │ ├── diagramsnet.py │ ├── error_handler.py │ ├── parameters.py │ └── report.py ├── templates │ ├── report_html.html │ └── report_limits.html └── tests │ ├── provider │ └── aws │ │ ├── all │ │ └── resource │ │ │ └── test_all.py │ │ ├── policy │ │ └── test_policy_diagram.py │ │ └── vpc │ │ ├── test_common.py │ │ └── test_diagram.py │ └── shared │ ├── test_shared_command.py │ ├── test_shared_common.py │ └── test_shared_diagram.py ├── dependabot.yml ├── docs ├── .DS_Store └── assets │ ├── aws-all.png │ ├── aws-iot.png │ ├── aws-limit.png │ ├── aws-policy.png │ ├── aws-vpc.png │ └── role-template.json ├── pytest.ini ├── requirements-dev.txt ├── requirements.txt ├── setup.cfg └── setup.py /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | orbs: 4 | python: circleci/python@0.3.0 5 | codecov: codecov/codecov@1.1.0 6 | 7 | jobs: 8 | build: 9 | executor: python/default 10 | steps: 11 | - checkout 12 | - python/load-cache 13 | - python/install-deps 14 | - python/install-deps: 15 | dependency-file: requirements-dev.txt 16 | - python/save-cache 17 | - run: 18 | name: test 19 | command: python setup.py test 20 | - store_test_results: 21 | path: test-results 22 | - store_artifacts: 23 | path: test-results 24 | - codecov/upload 25 | deploy: 26 | executor: python/default 27 | steps: 28 | - checkout 29 | - python/load-cache 30 | - python/install-deps 31 | - python/install-deps: 32 | dependency-file: requirements-dev.txt 33 | - python/save-cache 34 | - run: 35 | name: verify git tag vs. version 36 | command: python setup.py verify 37 | - python/dist 38 | - run: 39 | name: init .pypirc 40 | command: | 41 | echo -e "[pypi]" >> ~/.pypirc 42 | echo -e "username = $PYPI_USER" >> ~/.pypirc 43 | echo -e "password = $PYPI_TOKEN" >> ~/.pypirc 44 | - run: 45 | name: upload to pypi 46 | command: twine upload dist/* 47 | 48 | workflows: 49 | version: 2 50 | build-and-deploy: 51 | jobs: 52 | - build: 53 | filters: # required since `deploy` has tag filters AND requires `build` 54 | tags: 55 | only: /.*/ 56 | - deploy: 57 | requires: 58 | - build 59 | filters: 60 | tags: 61 | only: /[0-9]+(\.[0-9]+)*/ 62 | branches: 63 | ignore: /.*/ -------------------------------------------------------------------------------- /.codacy.yml: -------------------------------------------------------------------------------- 1 | exclude_paths: 2 | - 'cloudiscovery/tests/**' 3 | - 'tests/**' -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = cloudiscovery/tests/*,venv/* -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/python 3 | # Edit at https://www.gitignore.io/?templates=python 4 | 5 | ### Python ### 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | *.pyc 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | pip-wheel-metadata/ 30 | share/python-wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | MANIFEST 35 | assets/diagrams/ 36 | .vscode/ 37 | venv/ 38 | cloudiscovery/assets/ 39 | assets/ 40 | !docs/* 41 | 42 | # PyInstaller 43 | # Usually these files are written by a python script from a template 44 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 45 | *.manifest 46 | *.spec 47 | 48 | # Installer logs 49 | pip-log.txt 50 | pip-delete-this-directory.txt 51 | 52 | # Unit test / coverage reports 53 | htmlcov/ 54 | .tox/ 55 | .nox/ 56 | .coverage 57 | .coverage.* 58 | .cache 59 | nosetests.xml 60 | coverage.xml 61 | *.cover 62 | .hypothesis/ 63 | .pytest_cache/ 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # pipenv 78 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 79 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 80 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 81 | # install all needed dependencies. 82 | #Pipfile.lock 83 | 84 | # celery beat schedule file 85 | celerybeat-schedule 86 | 87 | # SageMath parsed files 88 | *.sage.py 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # Mr Developer 98 | .mr.developer.cfg 99 | .project 100 | .pydevproject 101 | 102 | # mkdocs documentation 103 | /site 104 | 105 | # mypy 106 | .mypy_cache/ 107 | .dmypy.json 108 | dmypy.json 109 | 110 | # Pyre type checker 111 | .pyre/ 112 | 113 | # End of https://www.gitignore.io/api/python 114 | 115 | test-results 116 | 117 | .idea/ 118 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | # black 3 | - repo: https://github.com/ambv/black 4 | rev: stable 5 | hooks: 6 | - id: black 7 | language_version: python3 8 | - repo: git://github.com/guykisel/prospector-mirror 9 | rev: '7ff847e779347033ebbd9e3b88279e7f3a998b45' 10 | hooks: 11 | - id: prospector -------------------------------------------------------------------------------- /.prospector.yaml: -------------------------------------------------------------------------------- 1 | ignore-paths: 2 | - tests 3 | 4 | strictness: medium 5 | 6 | pep8: 7 | disable: 8 | - D100 9 | - D101 10 | - D102 11 | - D103 12 | - D104 13 | - D200 14 | - D212 15 | - D400 16 | - D401 17 | - W503 18 | - E501 19 | options: 20 | max-line-length: 120 21 | 22 | mccabe: 23 | options: 24 | max-complexity: 23 -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | init-hook='import sys; sys.path.append("cloudiscovery")' 3 | 4 | [FORMAT] 5 | max-line-length=120 6 | 7 | [MESSAGES CONTROL] 8 | disable=missing-docstring,useless-suppression,pointless-string-statement,locally-disabled,bad-super-call,unnecessary-lambda,missing-class-docstring,arguments-differ,unused-argument,useless-object-inheritance,too-few-public-methods,missing-module-docstring,import-error,eval-used,bad-continuation,invalid-name,missing-function-docstring,no-self-use,no-name-in-module,too-many-lines,attribute-defined-outside-init,fixme,exec-used,expression-not-assigned,too-many-branches 9 | 10 | [SIMILARITIES] 11 | 12 | # Minimum lines number of a similarity. 13 | min-similarity-lines=6 -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim as cloudiscovery 2 | 3 | LABEL maintainer_1="https://github.com/leandrodamascena/" 4 | LABEL maintainer_2="https://github.com/meshuga" 5 | LABEL Project="https://github.com/Cloud-Architects/cloudiscovery" 6 | 7 | WORKDIR /opt/cloudiscovery 8 | 9 | RUN apt-get update -y 10 | RUN apt-get install -y awscli graphviz 11 | RUN apt-get install -y bash 12 | 13 | COPY . /opt/cloudiscovery 14 | 15 | RUN pip install -r requirements.txt 16 | 17 | RUN bash -------------------------------------------------------------------------------- /bin/cloudiscovery: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import sys 17 | 18 | 19 | import cloudiscovery 20 | 21 | 22 | def main(): 23 | return cloudiscovery.main() 24 | 25 | 26 | if __name__ == "__main__": 27 | sys.exit(main()) 28 | -------------------------------------------------------------------------------- /bin/cloudiscovery.cmd: -------------------------------------------------------------------------------- 1 | @echo OFF 2 | REM=""" 3 | setlocal 4 | set PythonExe="" 5 | set PythonExeFlags= 6 | 7 | for %%i in (cmd bat exe) do ( 8 | for %%j in (python.%%i) do ( 9 | call :SetPythonExe "%%~$PATH:j" 10 | ) 11 | ) 12 | for /f "tokens=2 delims==" %%i in ('assoc .py') do ( 13 | for /f "tokens=2 delims==" %%j in ('ftype %%i') do ( 14 | for /f "tokens=1" %%k in ("%%j") do ( 15 | call :SetPythonExe %%k 16 | ) 17 | ) 18 | ) 19 | %PythonExe% -x %PythonExeFlags% "%~f0" %* 20 | exit /B %ERRORLEVEL% 21 | goto :EOF 22 | 23 | :SetPythonExe 24 | if not ["%~1"]==[""] ( 25 | if [%PythonExe%]==[""] ( 26 | set PythonExe="%~1" 27 | ) 28 | ) 29 | goto :EOF 30 | """ 31 | 32 | # =================================================== 33 | # Python script starts here 34 | # =================================================== 35 | 36 | #!/usr/bin/env python 37 | """ 38 | Licensed under the Apache License, Version 2.0 (the "License"); 39 | 40 | you may not use this file except in compliance with the License. 41 | You may obtain a copy of the License at 42 | 43 | http://www.apache.org/licenses/LICENSE-2.0 44 | 45 | Unless required by applicable law or agreed to in writing, software 46 | distributed under the License is distributed on an "AS IS" BASIS, 47 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 48 | See the License for the specific language governing permissions and 49 | limitations under the License. 50 | """ 51 | 52 | import cloudiscovery 53 | import sys 54 | 55 | 56 | def main(): 57 | return cloudiscovery.main() 58 | 59 | 60 | if __name__ == '__main__': 61 | sys.exit(main()) -------------------------------------------------------------------------------- /cloudiscovery/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | 17 | import gettext 18 | import sys 19 | from os.path import dirname 20 | from typing import List 21 | 22 | import pkg_resources 23 | 24 | """path to pip package""" 25 | sys.path.append(dirname(__file__)) 26 | 27 | # pylint: disable=wrong-import-position 28 | from provider.aws.command import aws_main 29 | from shared.parameters import generate_parser 30 | 31 | 32 | # pylint: disable=wrong-import-position 33 | from shared.common import ( 34 | exit_critical, 35 | Filterable, 36 | parse_filters, 37 | ) 38 | 39 | # Check version 40 | if sys.version_info < (3, 8): 41 | print("Python 3.8 or newer is required", file=sys.stderr) 42 | sys.exit(1) 43 | 44 | __version__ = "2.4.4" 45 | 46 | AVAILABLE_LANGUAGES = ["en_US", "pt_BR"] 47 | 48 | 49 | # pylint: disable=too-many-branches,too-many-statements,too-many-locals 50 | def main(): 51 | # Entry point for the CLI. 52 | # Load commands 53 | parser = generate_parser() 54 | if len(sys.argv) <= 1: 55 | parser.print_help() 56 | return 57 | 58 | args = parser.parse_args() 59 | 60 | if args.language is None or args.language not in AVAILABLE_LANGUAGES: 61 | language = "en_US" 62 | else: 63 | language = args.language 64 | 65 | # Diagram check 66 | if "diagram" not in args: 67 | diagram = False 68 | else: 69 | diagram = args.diagram 70 | 71 | # defining default language to show messages 72 | defaultlanguage = gettext.translation( 73 | "messages", localedir=dirname(__file__) + "/locales", languages=[language] 74 | ) 75 | defaultlanguage.install() 76 | _ = defaultlanguage.gettext 77 | 78 | # diagram version check 79 | check_diagram_version(diagram) 80 | 81 | # filters check 82 | filters: List[Filterable] = [] 83 | if "filters" in args: 84 | if args.filters is not None: 85 | filters = parse_filters(args.filters) 86 | 87 | if args.command.startswith("aws"): 88 | command = aws_main(args) 89 | else: 90 | raise NotImplementedError("Unknown command") 91 | 92 | if "services" in args and args.services is not None: 93 | services = args.services.split(",") 94 | else: 95 | services = [] 96 | 97 | command.run(diagram, args.verbose, services, filters) 98 | 99 | 100 | def check_diagram_version(diagram): 101 | if diagram: 102 | # Checking diagram version. Must be 0.13 or higher 103 | if pkg_resources.get_distribution("diagrams").version < "0.14": 104 | exit_critical( 105 | "You must update diagrams package to 0.14 or higher. " 106 | "- See on https://github.com/mingrammer/diagrams" 107 | ) 108 | 109 | 110 | if __name__ == "__main__": 111 | try: 112 | main() 113 | except KeyboardInterrupt: 114 | print("Finishing script...") 115 | sys.exit(0) 116 | -------------------------------------------------------------------------------- /cloudiscovery/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/commands/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/locales/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/locales/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/locales/en_US/LC_MESSAGES/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/locales/en_US/LC_MESSAGES/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/locales/en_US/LC_MESSAGES/messages.mo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/locales/en_US/LC_MESSAGES/messages.mo -------------------------------------------------------------------------------- /cloudiscovery/locales/en_US/LC_MESSAGES/messages.po: -------------------------------------------------------------------------------- 1 | # SOME DESCRIPTIVE TITLE. 2 | # Copyright (C) YEAR ORGANIZATION 3 | # FIRST AUTHOR , YEAR. 4 | # 5 | msgid "" 6 | msgstr "" 7 | "Project-Id-Version: PACKAGE VERSION\n" 8 | "POT-Creation-Date: 2020-05-04 16:43+0100\n" 9 | "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" 10 | "Last-Translator: Leandro Damascena \n" 11 | "Language-Team: en_US \n" 12 | "MIME-Version: 1.0\n" 13 | "Content-Type: text/plain; charset=cp1252\n" 14 | "Content-Transfer-Encoding: 8bit\n" 15 | "Generated-By: pygettext.py 1.5\n" 16 | 17 | #: .\aws-network-discovery.py:33 18 | msgid "Python 3.6 or newer is required" 19 | msgstr "" 20 | 21 | #: .\commands\vpc.py:20 22 | msgid "Neither region parameter nor region config were passed" 23 | msgstr "" -------------------------------------------------------------------------------- /cloudiscovery/locales/en_US/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/locales/en_US/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/locales/messages.pot: -------------------------------------------------------------------------------- 1 | # SOME DESCRIPTIVE TITLE. 2 | # Copyright (C) YEAR ORGANIZATION 3 | # FIRST AUTHOR , YEAR. 4 | # 5 | msgid "" 6 | msgstr "" 7 | "Project-Id-Version: PACKAGE VERSION\n" 8 | "POT-Creation-Date: 2020-05-04 20:25+0100\n" 9 | "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" 10 | "Last-Translator: FULL NAME \n" 11 | "Language-Team: LANGUAGE \n" 12 | "MIME-Version: 1.0\n" 13 | "Content-Type: text/plain; charset=cp1252\n" 14 | "Content-Transfer-Encoding: 8bit\n" 15 | "Generated-By: pygettext.py 1.5\n" 16 | 17 | #: .\aws-network-discovery.py:33 18 | msgid "Python 3.6 or newer is required" 19 | msgstr "" 20 | 21 | #: .\commands\vpc.py:20 22 | msgid "Neither region parameter nor region config were passed" 23 | msgstr "" -------------------------------------------------------------------------------- /cloudiscovery/locales/pt_BR/LC_MESSAGES/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/locales/pt_BR/LC_MESSAGES/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/locales/pt_BR/LC_MESSAGES/messages.mo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/locales/pt_BR/LC_MESSAGES/messages.mo -------------------------------------------------------------------------------- /cloudiscovery/locales/pt_BR/LC_MESSAGES/messages.po: -------------------------------------------------------------------------------- 1 | # SOME DESCRIPTIVE TITLE. 2 | # Copyright (C) YEAR ORGANIZATION 3 | # FIRST AUTHOR , YEAR. 4 | # 5 | msgid "" 6 | msgstr "" 7 | "Project-Id-Version: PACKAGE VERSION\n" 8 | "POT-Creation-Date: 2020-05-04 16:43+0100\n" 9 | "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" 10 | "Last-Translator: Leandro Damascena \n" 11 | "Language-Team: en_US \n" 12 | "MIME-Version: 1.0\n" 13 | "Content-Type: text/plain; charset=UTF-8\n" 14 | "Content-Transfer-Encoding: 8bit\n" 15 | "Generated-By: pygettext.py 1.5\n" 16 | 17 | #: .\aws-network-discovery.py:33 18 | msgid "Python 3.6 or newer is required" 19 | msgstr "É necessário ter o Python 3.6 ou superior instalado" 20 | 21 | #: .\commands\vpc.py:20 22 | msgid "Neither region parameter nor region config were passed" 23 | msgstr "É preciso informar uma região ou configurar o profile com uma região" -------------------------------------------------------------------------------- /cloudiscovery/locales/pt_BR/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/locales/pt_BR/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/all/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/all/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/all/command.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from provider.aws.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner 4 | from shared.common import Filterable, BaseOptions 5 | from shared.diagram import NoDiagram 6 | 7 | 8 | class AllOptions(BaseAwsOptions, BaseOptions): 9 | services: List[str] 10 | 11 | # pylint: disable=too-many-arguments 12 | def __init__(self, verbose, filters, session, region_name, services: List[str]): 13 | BaseAwsOptions.__init__(self, session, region_name) 14 | BaseOptions.__init__(self, verbose, filters) 15 | self.services = services 16 | 17 | 18 | class All(BaseAwsCommand): 19 | def run( 20 | self, 21 | diagram: bool, 22 | verbose: bool, 23 | services: List[str], 24 | filters: List[Filterable], 25 | ): 26 | for region in self.region_names: 27 | self.init_region_cache(region) 28 | options = AllOptions( 29 | verbose=verbose, 30 | filters=filters, 31 | session=self.session, 32 | region_name=region, 33 | services=services, 34 | ) 35 | 36 | command_runner = AwsCommandRunner(filters=filters) 37 | command_runner.run( 38 | provider="all", 39 | options=options, 40 | diagram_builder=NoDiagram(), 41 | title="AWS Resources - Region {}".format(region), 42 | # pylint: disable=no-member 43 | filename=options.resulting_file_name("all"), 44 | ) 45 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/all/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/all/data/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/all/data/omitted_resources.py: -------------------------------------------------------------------------------- 1 | OMITTED_RESOURCES = [ 2 | "aws_cloudhsm_available_zone", 3 | "aws_cloudhsm_hapg", 4 | "aws_cloudhsm_hsm", 5 | "aws_cloudhsm_luna_client", 6 | "aws_dax_default_parameter", 7 | "aws_dax_parameter_group", 8 | "aws_ec2_reserved_instances_offering", 9 | "aws_ec2_snapshot", 10 | "aws_ec2_spot_price_history", 11 | "aws_ssm_available_patch", 12 | "aws_ssm_document", 13 | "aws_polly_voice", 14 | "aws_lightsail_blueprint", 15 | "aws_lightsail_bundle", 16 | "aws_lightsail_region", 17 | "aws_elastictranscoder_preset", 18 | "aws_ec2_vpc_endpoint_service", 19 | "aws_dms_endpoint_type", 20 | "aws_elasticache_service_update", 21 | "aws_elasticache_cache_parameter_group", 22 | "aws_rds_source_region", 23 | "aws_ssm_association", 24 | "aws_ssm_patch_baseline", 25 | "aws_ec2_prefix", 26 | "aws_ec2_image", 27 | "aws_ec2_region", 28 | "aws_opsworks_operating_system", 29 | "aws_rds_account_attribute", 30 | "aws_route53_geo_location", 31 | "aws_redshift_cluster_track", 32 | "aws_redshift_reserved_node_offering", 33 | "aws_directconnect_location", 34 | "aws_dms_account_attribute", 35 | "aws_securityhub_standard", 36 | "aws_ram_resource_type", 37 | "aws_ram_permission", 38 | "aws_ec2_account_attribute", 39 | "aws_elasticbeanstalk_available_solution_stack", 40 | "aws_redshift_account_attribute", 41 | "aws_opsworks_user_profile", 42 | "aws_directconnect_direct_connect_gateway_association", # DirectConnect resources endpoint are complicated 43 | "aws_directconnect_direct_connect_gateway_attachment", 44 | "aws_directconnect_interconnect", 45 | "aws_dms_replication_task_assessment_result", 46 | "aws_ec2_fpga_image", 47 | "aws_ec2_launch_template_version", 48 | "aws_ec2_reserved_instancesing", 49 | "aws_ec2_spot_datafeed_subscription", 50 | "aws_ec2_transit_gateway_multicast_domain", 51 | "aws_elasticbeanstalk_configuration_option", 52 | "aws_elasticbeanstalk_platform_version", 53 | "aws_iam_credential_report", 54 | "aws_iam_account_password_policy", 55 | "aws_importexport_job", 56 | "aws_iot_o_taupdate", 57 | "aws_iot_default_authorizer", 58 | "aws_workspaces_account", 59 | "aws_workspaces_account_modification", 60 | "aws_rds_export_task", 61 | "aws_rds_custom_availability_zone", 62 | "aws_rds_installation_media", 63 | "aws_rds_d_bsecurity_group", 64 | "aws_rds_reserved_db_instances_offering", 65 | "aws_translate_text_translation_job", 66 | "aws_rekognition_project", 67 | "aws_rekognition_stream_processor", 68 | "aws_sdb_domain", 69 | "aws_redshift_table_restore_status", 70 | "aws_iot_v2_logging_level", 71 | "aws_license_manager_resource_inventory", 72 | "aws_license_manager_license_configuration", 73 | "aws_logs_query_definition", 74 | "aws_autoscaling_scaling_activity", 75 | "aws_autoscaling_auto_scaling_notification_type", 76 | "aws_autoscaling_scaling_process_type", 77 | "aws_autoscaling_termination_policy_type", 78 | "aws_ec2_host_reservation_offering", 79 | "aws_ec2_availability_zone", 80 | "aws_cloudwatch_metric", 81 | "aws_organizations_handshakes_for_organization", 82 | "aws_config_organization_config_rule", 83 | "aws_organizations_root", 84 | "aws_organizations_delegated_administrator", 85 | "aws_organizations_create_account_status", 86 | "aws_config_organization_conformance_pack_status", 87 | "aws_config_organization_conformance_pack", 88 | "aws_ec2_reserved_instances_listing", 89 | "aws_redshift_cluster_security_group", 90 | "aws_guardduty_organization_admin_account", 91 | "aws_elasticache_cache_security_group", 92 | "aws_elasticache_reserved_cache_nodes_offering", 93 | "aws_organizations_aws_service_access_for_organization", 94 | "aws_organizations_account", 95 | "aws_config_organization_config_rule_status", 96 | "aws_dynamodb_backup", 97 | "aws_ec2_prefix_list", 98 | "aws_route53_hosted_zones_by_name", 99 | "aws_es_reserved_elasticsearch_instance_offering", 100 | "aws_ssm_automation_execution", 101 | "aws_route53_checker_ip_range", 102 | ] 103 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/all/data/on_top_policies.py: -------------------------------------------------------------------------------- 1 | ON_TOP_POLICIES = [ 2 | "kafka:ListClusters", 3 | "synthetics:DescribeCanaries", 4 | "medialive:ListInputs", 5 | "cloudhsm:DescribeClusters", 6 | "ssm:GetParametersByPath", 7 | "servicequotas:Get*", 8 | "amplify:ListApps", 9 | "autoscaling-plans:DescribeScalingPlans", 10 | "medialive:ListChannels", 11 | "medialive:ListInputDevices", 12 | "mediapackage:ListChannels", 13 | "qldb:ListLedgers", 14 | "transcribe:ListVocabularies", 15 | "glue:GetDatabases", 16 | "glue:GetUserDefinedFunctions", 17 | "glue:GetSecurityConfigurations", 18 | "glue:GetTriggers", 19 | "glue:GetCrawlers", 20 | "glue:ListWorkflows", 21 | "glue:ListMLTransforms", 22 | "codeguru-reviewer:ListCodeReviews", 23 | "servicediscovery:ListNamespaces", 24 | "apigateway:GET", 25 | "forecast:ListPredictors", 26 | "frauddetector:GetDetectors", 27 | "forecast:ListDatasetImportJobs", 28 | "frauddetector:GetModels", 29 | "frauddetector:GetOutcomes", 30 | "networkmanager:DescribeGlobalNetworks", 31 | "codeartifact:ListDomains", 32 | "ses:GetSendQuota", 33 | "codeguru-profiler:ListProfilingGroups", 34 | ] 35 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/all/data/required_params_override.py: -------------------------------------------------------------------------------- 1 | # Trying to fix documentation errors or its lack made by "happy pirates" at AWS 2 | REQUIRED_PARAMS_OVERRIDE = { 3 | "batch": {"ListJobs": ["jobQueue"]}, 4 | "cloudformation": { 5 | "DescribeStackEvents": ["stackName"], 6 | "DescribeStackResources": ["stackName"], 7 | "GetTemplate": ["stackName"], 8 | "ListTypeVersions": ["arn"], 9 | }, 10 | "codecommit": {"GetBranch": ["repositoryName"]}, 11 | "codedeploy": { 12 | "GetDeploymentTarget": ["deploymentId"], 13 | "ListDeploymentTargets": ["deploymentId"], 14 | }, 15 | "ecs": { 16 | "ListTasks": ["cluster"], 17 | "ListServices": ["cluster"], 18 | "ListContainerInstances": ["cluster"], 19 | "DescribeTasks": ["cluster", "tasks"], 20 | "DescribeServices": ["cluster", "services"], 21 | "DescribeContainerInstances": ["cluster", "containerInstances"], 22 | }, 23 | "elasticbeanstalk": { 24 | "DescribeEnvironmentHealth": ["environmentName"], 25 | "DescribeEnvironmentManagedActionHistory": ["environmentName"], 26 | "DescribeEnvironmentManagedActions": ["environmentName"], 27 | "DescribeEnvironmentResources": ["environmentName"], 28 | "DescribeInstancesHealth": ["environmentName"], 29 | }, 30 | "iam": { 31 | "GetUser": ["userName"], 32 | "ListAccessKeys": ["userName"], 33 | "ListServiceSpecificCredentials": ["userName"], 34 | "ListSigningCertificates": ["userName"], 35 | "ListMFADevices": ["userName"], 36 | "ListSSHPublicKeys": ["userName"], 37 | }, 38 | "iot": {"ListAuditFindings": ["taskId"]}, 39 | "opsworks": { 40 | "ListAuditFindings": ["taskId"], 41 | "DescribeAgentVersions": ["stackId"], 42 | "DescribeApps": ["stackId"], 43 | "DescribeCommands": ["deploymentId"], 44 | "DescribeDeployments": ["appId"], 45 | "DescribeEcsClusters": ["ecsClusterArns"], 46 | "DescribeElasticIps": ["stackId"], 47 | "DescribeElasticLoadBalancers": ["stackId"], 48 | "DescribeInstances": ["stackId"], 49 | "DescribeLayers": ["stackId"], 50 | "DescribePermissions": ["stackId"], 51 | "DescribeRaidArrays": ["stackId"], 52 | "DescribeVolumes": ["stackId"], 53 | }, 54 | "ssm": {"DescribeMaintenanceWindowSchedule": ["windowId"],}, 55 | "shield": {"DescribeProtection": ["protectionId"],}, 56 | "waf": { 57 | "ListActivatedRulesInRuleGroup": ["ruleGroupId"], 58 | "ListLoggingConfigurations": ["limit"], 59 | }, 60 | "waf-regional": { 61 | "ListActivatedRulesInRuleGroup": ["ruleGroupId"], 62 | "ListLoggingConfigurations": ["limit"], 63 | }, 64 | "wafv2": {"ListLoggingConfigurations": ["limit"],}, 65 | } 66 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/all/exception/__init__.py: -------------------------------------------------------------------------------- 1 | import functools 2 | from shared.common import ( 3 | message_handler, 4 | log_critical, 5 | ) 6 | 7 | 8 | def all_exception(func): 9 | # pylint: disable=inconsistent-return-statements 10 | @functools.wraps(func) 11 | def wrapper(*args, **kwargs): 12 | try: 13 | return func(*args, **kwargs) 14 | # pylint: disable=broad-except 15 | except Exception as e: 16 | if func.__qualname__ == "AllResources.analyze_operation": 17 | if not args[0].options.verbose: 18 | return 19 | exception_str = str(e) 20 | if ( 21 | "is not subscribed to AWS Security Hub" in exception_str 22 | or "not enabled for securityhub" in exception_str 23 | or "The subscription does not exist" in exception_str 24 | or "calling the DescribeHub operation" in exception_str 25 | ): 26 | message_handler( 27 | "Operation {} not accessible, AWS Security Hub is not configured... Skipping".format( 28 | args[2] 29 | ), 30 | "WARNING", 31 | ) 32 | elif ( 33 | "not connect to the endpoint URL" in exception_str 34 | or "not available in this region" in exception_str 35 | or "API is not available" in exception_str 36 | ): 37 | message_handler( 38 | "Service {} not available in the selected region... Skipping".format( 39 | args[5] 40 | ), 41 | "WARNING", 42 | ) 43 | elif ( 44 | "Your account is not a member of an organization" in exception_str 45 | or "This action can only be made by accounts in an AWS Organization" 46 | in exception_str 47 | or "The request failed because organization is not in use" 48 | in exception_str 49 | ): 50 | message_handler( 51 | "Service {} only available to account in an AWS Organization... Skipping".format( 52 | args[5] 53 | ), 54 | "WARNING", 55 | ) 56 | elif "is no longer available to new customers" in exception_str: 57 | message_handler( 58 | "Service {} is no longer available to new customers... Skipping".format( 59 | args[5] 60 | ), 61 | "WARNING", 62 | ) 63 | elif ( 64 | "only available to Master account in AWS FM" in exception_str 65 | or "not currently delegated by AWS FM" in exception_str 66 | ): 67 | message_handler( 68 | "Operation {} not accessible, not master account in AWS FM... Skipping".format( 69 | args[2] 70 | ), 71 | "WARNING", 72 | ) 73 | else: 74 | log_critical( 75 | "\nError running operation {}, type {}. Error message {}".format( 76 | args[2], args[1], exception_str 77 | ) 78 | ) 79 | else: 80 | log_critical( 81 | "\nError running method {}. Error message {}".format( 82 | func.__qualname__, str(e) 83 | ) 84 | ) 85 | 86 | return wrapper 87 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/all/resource/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/all/resource/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/command.py: -------------------------------------------------------------------------------- 1 | from provider.aws.all.command import All 2 | from provider.aws.common_aws import generate_session, aws_verbose 3 | from provider.aws.iot.command import Iot 4 | from provider.aws.limit.command import Limit 5 | from provider.aws.policy.command import Policy 6 | from provider.aws.security.command import Security 7 | from provider.aws.vpc.command import Vpc 8 | from shared.common import ( 9 | exit_critical, 10 | message_handler, 11 | BaseCommand, 12 | ) 13 | 14 | DEFAULT_REGION = "us-east-1" 15 | DEFAULT_PARTITION_CODE = "aws" 16 | 17 | 18 | def get_partition(session, region_name): 19 | partition_code = DEFAULT_PARTITION_CODE # assume it's always default partition, even if we can't find a region 20 | partition_name = "AWS Standard" 21 | # pylint: disable=protected-access 22 | loader = session._session.get_component("data_loader") 23 | endpoints = loader.load_data("endpoints") 24 | for partition in endpoints["partitions"]: 25 | for region, _ in partition["regions"].items(): 26 | if region == region_name: 27 | partition_code = partition["partition"] 28 | partition_name = partition["partitionName"] 29 | 30 | if partition_code != DEFAULT_PARTITION_CODE: 31 | message_handler( 32 | "Found non-default partition: {} ({})".format( 33 | partition_code, partition_name 34 | ), 35 | "HEADER", 36 | ) 37 | return partition_code 38 | 39 | 40 | def check_region_profile(arg_region_name, profile_region_name): 41 | if arg_region_name is None and profile_region_name is None: 42 | exit_critical("Neither region parameter nor region config were passed") 43 | 44 | 45 | def check_region(region_parameter, region_name, session, partition_code): 46 | """ 47 | Region us-east-1 as a default region here, if not aws partition, just return asked region 48 | 49 | This is just to list aws regions, doesn't matter default region 50 | """ 51 | if partition_code != "aws": 52 | return [region_name] 53 | 54 | client = session.client("ec2", region_name=DEFAULT_REGION) 55 | 56 | valid_region_names = [ 57 | region["RegionName"] 58 | for region in client.describe_regions(AllRegions=True)["Regions"] 59 | ] 60 | 61 | if region_parameter != "all": 62 | if region_name not in valid_region_names: 63 | message = "There is no region named: {0}".format(region_name) 64 | exit_critical(message) 65 | else: 66 | valid_region_names = [region_name] 67 | 68 | return valid_region_names 69 | 70 | 71 | def aws_main(args) -> BaseCommand: 72 | 73 | # Check if verbose mode is enabled 74 | if args.verbose: 75 | aws_verbose() 76 | 77 | # aws profile check 78 | if "region_name" not in args: 79 | session = generate_session(profile_name=args.profile_name, region_name=None) 80 | else: 81 | session = generate_session( 82 | profile_name=args.profile_name, region_name=args.region_name 83 | ) 84 | 85 | session.get_credentials() 86 | region_name = session.region_name 87 | 88 | partition_code = get_partition(session, region_name) 89 | 90 | if "region_name" not in args: 91 | region_names = [DEFAULT_REGION] 92 | else: 93 | # checking region configuration 94 | check_region_profile( 95 | arg_region_name=args.region_name, profile_region_name=region_name 96 | ) 97 | 98 | # assuming region parameter precedes region configuration 99 | if args.region_name is not None: 100 | region_name = args.region_name 101 | 102 | # get regions 103 | region_names = check_region( 104 | region_parameter=args.region_name, 105 | region_name=region_name, 106 | session=session, 107 | partition_code=partition_code, 108 | ) 109 | 110 | if "threshold" in args: 111 | if args.threshold is not None: 112 | if args.threshold.isdigit() is False: 113 | exit_critical("Threshold must be between 0 and 100") 114 | else: 115 | if int(args.threshold) < 0 or int(args.threshold) > 100: 116 | exit_critical("Threshold must be between 0 and 100") 117 | 118 | if args.command == "aws-vpc": 119 | command = Vpc( 120 | vpc_id=args.vpc_id, 121 | region_names=region_names, 122 | session=session, 123 | partition_code=partition_code, 124 | ) 125 | elif args.command == "aws-policy": 126 | command = Policy( 127 | region_names=region_names, session=session, partition_code=partition_code 128 | ) 129 | elif args.command == "aws-iot": 130 | command = Iot( 131 | thing_name=args.thing_name, 132 | region_names=region_names, 133 | session=session, 134 | partition_code=partition_code, 135 | ) 136 | elif args.command == "aws-all": 137 | command = All( 138 | region_names=region_names, session=session, partition_code=partition_code 139 | ) 140 | elif args.command == "aws-limit": 141 | command = Limit( 142 | region_names=region_names, 143 | session=session, 144 | threshold=args.threshold, 145 | partition_code=partition_code, 146 | ) 147 | elif args.command == "aws-security": 148 | command = Security( 149 | region_names=region_names, 150 | session=session, 151 | commands=args.commands, 152 | partition_code=partition_code, 153 | ) 154 | else: 155 | raise NotImplementedError("Unknown command") 156 | return command 157 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/iot/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/iot/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/iot/command.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from provider.aws.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner 4 | from provider.aws.iot.diagram import IoTDiagram 5 | from shared.common import ResourceDigest, Filterable, BaseOptions 6 | from shared.diagram import NoDiagram, BaseDiagram 7 | 8 | 9 | class IotOptions(BaseAwsOptions, BaseOptions): 10 | thing_name: str 11 | 12 | # pylint: disable=too-many-arguments 13 | def __init__(self, verbose, filters, session, region_name, thing_name): 14 | BaseAwsOptions.__init__(self, session, region_name) 15 | BaseOptions.__init__(self, verbose, filters) 16 | self.thing_name = thing_name 17 | 18 | def iot_digest(self): 19 | return ResourceDigest(id=self.thing_name, type="aws_iot") 20 | 21 | 22 | class Iot(BaseAwsCommand): 23 | # pylint: disable=too-many-arguments 24 | def __init__(self, thing_name, region_names, session, partition_code): 25 | """ 26 | Iot command 27 | 28 | :param thing_name: 29 | :param region_names: 30 | :param session: 31 | :param partition_code: 32 | """ 33 | super().__init__(region_names, session, partition_code) 34 | self.thing_name = thing_name 35 | 36 | def run( 37 | self, 38 | diagram: bool, 39 | verbose: bool, 40 | services: List[str], 41 | filters: List[Filterable], 42 | ): 43 | command_runner = AwsCommandRunner(filters) 44 | 45 | for region_name in self.region_names: 46 | self.init_region_cache(region_name) 47 | 48 | # if thing_name is none, get all things and check 49 | if self.thing_name is None: 50 | client = self.session.client("iot", region_name=region_name) 51 | things = client.list_things() 52 | thing_options = IotOptions( 53 | verbose=verbose, 54 | filters=filters, 55 | session=self.session, 56 | region_name=region_name, 57 | thing_name=things, 58 | ) 59 | diagram_builder: BaseDiagram 60 | if diagram: 61 | diagram_builder = IoTDiagram(thing_name="") 62 | else: 63 | diagram_builder = NoDiagram() 64 | command_runner.run( 65 | provider="iot", 66 | options=thing_options, 67 | diagram_builder=diagram_builder, 68 | title="AWS IoT Resources - Region {}".format(region_name), 69 | filename=thing_options.resulting_file_name("iot"), 70 | ) 71 | else: 72 | things = dict() 73 | things["things"] = [{"thingName": self.thing_name}] 74 | thing_options = IotOptions( 75 | verbose=verbose, 76 | filters=filters, 77 | session=self.session, 78 | region_name=region_name, 79 | thing_name=things, 80 | ) 81 | 82 | if diagram: 83 | diagram_builder = IoTDiagram(thing_name=self.thing_name) 84 | else: 85 | diagram_builder = NoDiagram() 86 | 87 | command_runner.run( 88 | provider="iot", 89 | options=thing_options, 90 | diagram_builder=diagram_builder, 91 | title="AWS IoT {} Resources - Region {}".format( 92 | self.thing_name, region_name 93 | ), 94 | filename=thing_options.resulting_file_name( 95 | self.thing_name + "_iot" 96 | ), 97 | ) 98 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/iot/diagram.py: -------------------------------------------------------------------------------- 1 | from shared.diagram import BaseDiagram 2 | 3 | 4 | class IoTDiagram(BaseDiagram): 5 | def __init__(self, thing_name: str): 6 | """ 7 | Iot diagram 8 | 9 | :param thing_name: 10 | """ 11 | super().__init__() 12 | self.thing_name = thing_name 13 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/iot/resource/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/iot/resource/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/iot/resource/certificate.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from provider.aws.iot.command import IotOptions 4 | 5 | from provider.aws.common_aws import resource_tags 6 | from shared.common import ( 7 | ResourceProvider, 8 | Resource, 9 | message_handler, 10 | ResourceDigest, 11 | ResourceEdge, 12 | ResourceAvailable, 13 | ) 14 | from shared.error_handler import exception 15 | 16 | 17 | class CERTIFICATE(ResourceProvider): 18 | def __init__(self, iot_options: IotOptions): 19 | """ 20 | Iot certificate 21 | 22 | :param iot_options: 23 | """ 24 | super().__init__() 25 | self.iot_options = iot_options 26 | 27 | @exception 28 | @ResourceAvailable(services="iot") 29 | def get_resources(self) -> List[Resource]: 30 | 31 | client = self.iot_options.client("iot") 32 | 33 | resources_found = [] 34 | 35 | if self.iot_options.verbose: 36 | message_handler("Collecting data from IoT Certificates...", "HEADER") 37 | 38 | for thing in self.iot_options.thing_name["things"]: 39 | 40 | response = client.list_thing_principals(thingName=thing["thingName"]) 41 | 42 | for data in response["principals"]: 43 | if "cert/" in data: 44 | lst_cert = data.split("/") 45 | 46 | data_cert = client.describe_certificate(certificateId=lst_cert[1]) 47 | tag_response = client.list_tags_for_resource( 48 | resourceArn=data_cert["certificateDescription"][ 49 | "certificateArn" 50 | ] 51 | ) 52 | 53 | iot_cert_digest = ResourceDigest( 54 | id=data_cert["certificateDescription"]["certificateId"], 55 | type="aws_iot_certificate", 56 | ) 57 | resources_found.append( 58 | Resource( 59 | digest=iot_cert_digest, 60 | name=data_cert["certificateDescription"]["certificateId"], 61 | details="", 62 | group="iot", 63 | tags=resource_tags(tag_response), 64 | ) 65 | ) 66 | 67 | self.relations_found.append( 68 | ResourceEdge( 69 | from_node=iot_cert_digest, 70 | to_node=ResourceDigest( 71 | id=thing["thingName"], type="aws_iot_thing" 72 | ), 73 | ) 74 | ) 75 | 76 | return resources_found 77 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/iot/resource/policy.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from provider.aws.common_aws import resource_tags 4 | from provider.aws.iot.command import IotOptions 5 | from shared.common import ( 6 | ResourceProvider, 7 | Resource, 8 | message_handler, 9 | ResourceDigest, 10 | ResourceEdge, 11 | ResourceAvailable, 12 | ) 13 | from shared.error_handler import exception 14 | 15 | 16 | class POLICY(ResourceProvider): 17 | def __init__(self, iot_options: IotOptions): 18 | """ 19 | Iot policy 20 | 21 | :param iot_options: 22 | """ 23 | super().__init__() 24 | self.iot_options = iot_options 25 | 26 | @exception 27 | @ResourceAvailable(services="iot") 28 | def get_resources(self) -> List[Resource]: 29 | 30 | client = self.iot_options.client("iot") 31 | 32 | resources_found = [] 33 | 34 | if self.iot_options.verbose: 35 | message_handler("Collecting data from IoT Policies...", "HEADER") 36 | 37 | for thing in self.iot_options.thing_name["things"]: 38 | 39 | response = client.list_thing_principals(thingName=thing["thingName"]) 40 | 41 | for data in response["principals"]: 42 | 43 | policies = client.list_principal_policies(principal=data) 44 | 45 | for policy in policies["policies"]: 46 | data_policy = client.get_policy(policyName=policy["policyName"]) 47 | tag_response = client.list_tags_for_resource( 48 | resourceArn=data_policy["policyArn"] 49 | ) 50 | 51 | iot_policy_digest = ResourceDigest( 52 | id=data_policy["policyArn"], type="aws_iot_policy" 53 | ) 54 | resources_found.append( 55 | Resource( 56 | digest=iot_policy_digest, 57 | name=data_policy["policyName"], 58 | details="", 59 | group="iot", 60 | tags=resource_tags(tag_response), 61 | ) 62 | ) 63 | 64 | self.relations_found.append( 65 | ResourceEdge( 66 | from_node=iot_policy_digest, 67 | to_node=ResourceDigest( 68 | id=thing["thingName"], type="aws_iot_thing" 69 | ), 70 | ) 71 | ) 72 | 73 | return resources_found 74 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/iot/resource/thing.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from provider.aws.common_aws import resource_tags 4 | from provider.aws.iot.command import IotOptions 5 | from shared.common import ( 6 | ResourceProvider, 7 | Resource, 8 | message_handler, 9 | ResourceDigest, 10 | ResourceEdge, 11 | ResourceAvailable, 12 | ) 13 | from shared.error_handler import exception 14 | 15 | 16 | class THINGS(ResourceProvider): 17 | def __init__(self, iot_options: IotOptions): 18 | """ 19 | Iot thing 20 | 21 | :param iot_options: 22 | """ 23 | super().__init__() 24 | self.iot_options = iot_options 25 | 26 | @exception 27 | @ResourceAvailable(services="iot") 28 | def get_resources(self) -> List[Resource]: 29 | client = self.iot_options.client("iot") 30 | 31 | resources_found = [] 32 | 33 | if self.iot_options.verbose: 34 | message_handler("Collecting data from IoT Things...", "HEADER") 35 | 36 | for thing in self.iot_options.thing_name["things"]: 37 | client.describe_thing(thingName=thing["thingName"]) 38 | tag_response = client.list_tags_for_resource(resourceArn=thing["thingArn"]) 39 | 40 | resources_found.append( 41 | Resource( 42 | digest=ResourceDigest(id=thing["thingName"], type="aws_iot_thing"), 43 | name=thing["thingName"], 44 | details="", 45 | group="iot", 46 | tags=resource_tags(tag_response), 47 | ) 48 | ) 49 | 50 | return resources_found 51 | 52 | 53 | class TYPE(ResourceProvider): 54 | def __init__(self, iot_options: IotOptions): 55 | """ 56 | Iot type 57 | 58 | :param iot_options: 59 | """ 60 | super().__init__() 61 | self.iot_options = iot_options 62 | 63 | @exception 64 | @ResourceAvailable(services="iot") 65 | def get_resources(self) -> List[Resource]: 66 | 67 | client = self.iot_options.client("iot") 68 | 69 | resources_found = [] 70 | 71 | if self.iot_options.verbose: 72 | message_handler("Collecting data from IoT Things Type...", "HEADER") 73 | 74 | for thing in self.iot_options.thing_name["things"]: 75 | 76 | response = client.describe_thing(thingName=thing["thingName"]) 77 | 78 | thing_types = client.list_thing_types() 79 | 80 | for thing_type in thing_types["thingTypes"]: 81 | 82 | # thingTypeName is not mandatory in IoT Thing 83 | if "thingTypeName" in response: 84 | if thing_type["thingTypeName"] == response["thingTypeName"]: 85 | iot_type_digest = ResourceDigest( 86 | id=thing_type["thingTypeArn"], type="aws_iot_type" 87 | ) 88 | tag_response = client.list_tags_for_resource( 89 | resourceArn=thing_type["thingTypeArn"] 90 | ) 91 | resources_found.append( 92 | Resource( 93 | digest=iot_type_digest, 94 | name=thing_type["thingTypeName"], 95 | details="", 96 | group="iot", 97 | tags=resource_tags(tag_response), 98 | ) 99 | ) 100 | 101 | self.relations_found.append( 102 | ResourceEdge( 103 | from_node=iot_type_digest, 104 | to_node=ResourceDigest( 105 | id=thing["thingName"], type="aws_iot_thing" 106 | ), 107 | ) 108 | ) 109 | 110 | return resources_found 111 | 112 | 113 | class JOB(ResourceProvider): 114 | def __init__(self, iot_options: IotOptions): 115 | """ 116 | Iot job 117 | 118 | :param iot_options: 119 | """ 120 | super().__init__() 121 | self.iot_options = iot_options 122 | 123 | @exception 124 | @ResourceAvailable(services="iot") 125 | def get_resources(self) -> List[Resource]: 126 | 127 | client = self.iot_options.client("iot") 128 | 129 | resources_found = [] 130 | 131 | if self.iot_options.verbose: 132 | message_handler("Collecting data from IoT Jobs...", "HEADER") 133 | 134 | for thing in self.iot_options.thing_name["things"]: 135 | 136 | client.describe_thing(thingName=thing["thingName"]) 137 | 138 | jobs = client.list_jobs() 139 | 140 | for job in jobs["jobs"]: 141 | 142 | data_job = client.describe_job(jobId=job["jobId"]) 143 | 144 | # Find THING name in targets things 145 | for target in data_job["job"]["targets"]: 146 | 147 | if thing["thingName"] in target: 148 | iot_job_digest = ResourceDigest( 149 | id=job["jobId"], type="aws_iot_job" 150 | ) 151 | tag_response = client.list_tags_for_resource( 152 | resourceArn=job["jobArn"] 153 | ) 154 | resources_found.append( 155 | Resource( 156 | digest=iot_job_digest, 157 | name=job["jobId"], 158 | details="", 159 | group="iot", 160 | tags=resource_tags(tag_response), 161 | ) 162 | ) 163 | 164 | self.relations_found.append( 165 | ResourceEdge( 166 | from_node=iot_job_digest, 167 | to_node=ResourceDigest( 168 | id=thing["thingName"], type="aws_iot_thing" 169 | ), 170 | ) 171 | ) 172 | 173 | return resources_found 174 | 175 | 176 | class BILLINGGROUP(ResourceProvider): 177 | def __init__(self, iot_options: IotOptions): 178 | """ 179 | Iot billing group 180 | 181 | :param iot_options: 182 | """ 183 | super().__init__() 184 | self.iot_options = iot_options 185 | 186 | @exception 187 | @ResourceAvailable(services="iot") 188 | def get_resources(self) -> List[Resource]: 189 | 190 | client = self.iot_options.client("iot") 191 | 192 | resources_found = [] 193 | 194 | if self.iot_options.verbose: 195 | message_handler("Collecting data from IoT Billing Group...", "HEADER") 196 | 197 | for thing in self.iot_options.thing_name["things"]: 198 | 199 | response = client.describe_thing(thingName=thing["thingName"]) 200 | 201 | billing_groups = client.list_billing_groups() 202 | 203 | for billing_group in billing_groups["billingGroups"]: 204 | 205 | # billingGroupName is not mandatory in IoT Thing 206 | if "billingGroupName" in response: 207 | 208 | if billing_group["groupName"] == response["billingGroupName"]: 209 | iot_billing_group_digest = ResourceDigest( 210 | id=billing_group["groupArn"], type="aws_iot_billing_group" 211 | ) 212 | tag_response = client.list_tags_for_resource( 213 | resourceArn=billing_group["groupArn"] 214 | ) 215 | resources_found.append( 216 | Resource( 217 | digest=iot_billing_group_digest, 218 | name=billing_group["groupName"], 219 | details="", 220 | group="iot", 221 | tags=resource_tags(tag_response), 222 | ) 223 | ) 224 | 225 | self.relations_found.append( 226 | ResourceEdge( 227 | from_node=iot_billing_group_digest, 228 | to_node=ResourceDigest( 229 | id=thing["thingName"], type="aws_iot_thing" 230 | ), 231 | ) 232 | ) 233 | return resources_found 234 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/limit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/limit/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/limit/command.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from provider.aws.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner 4 | from provider.aws.limit.data.allowed_resources import ( 5 | ALLOWED_SERVICES_CODES, 6 | SPECIAL_RESOURCES, 7 | ) 8 | from shared.common import ( 9 | ResourceCache, 10 | message_handler, 11 | Filterable, 12 | BaseOptions, 13 | log_critical, 14 | ) 15 | from shared.diagram import NoDiagram 16 | 17 | 18 | class LimitOptions(BaseAwsOptions, BaseOptions): 19 | services: List[str] 20 | threshold: str 21 | 22 | # pylint: disable=too-many-arguments 23 | def __init__( 24 | self, 25 | verbose: bool, 26 | filters: List[Filterable], 27 | session, 28 | region_name, 29 | services, 30 | threshold, 31 | ): 32 | BaseAwsOptions.__init__(self, session, region_name) 33 | BaseOptions.__init__(self, verbose, filters) 34 | self.services = services 35 | self.threshold = threshold 36 | 37 | 38 | class LimitParameters: 39 | def __init__(self, session, region: str, services, options: LimitOptions): 40 | self.region = region 41 | self.cache = ResourceCache() 42 | self.session = session 43 | self.options = options 44 | self.services = [] 45 | 46 | if services is None: 47 | for service in ALLOWED_SERVICES_CODES: 48 | self.services.append(service) 49 | for service in SPECIAL_RESOURCES: 50 | self.services.append(service) 51 | else: 52 | self.services = services 53 | 54 | def init_globalaws_limits_cache(self): 55 | """ 56 | AWS has global limit that can be adjustable and others that can't be adjustable 57 | This method make cache for 15 days for aws cache global parameters. AWS don't update limit every time. 58 | Services has differents limit, depending on region. 59 | """ 60 | for service_code in self.services: 61 | if service_code in ALLOWED_SERVICES_CODES: 62 | cache_key = "aws_limits_" + service_code + "_" + self.region 63 | 64 | cache = self.cache.get_key(cache_key) 65 | if cache is not None: 66 | continue 67 | 68 | if self.options.verbose: 69 | message_handler( 70 | "Fetching aws global limit to service {} in region {} to cache...".format( 71 | service_code, self.region 72 | ), 73 | "HEADER", 74 | ) 75 | 76 | cache_codes = dict() 77 | for quota_code in ALLOWED_SERVICES_CODES[service_code]: 78 | 79 | if quota_code != "global": 80 | """ 81 | Impossible to instance once at __init__ method. 82 | Global services such route53 MUST USE us-east-1 region 83 | """ 84 | if ALLOWED_SERVICES_CODES[service_code]["global"]: 85 | service_quota = self.session.client( 86 | "service-quotas", region_name="us-east-1" 87 | ) 88 | else: 89 | service_quota = self.session.client( 90 | "service-quotas", region_name=self.region 91 | ) 92 | 93 | item_to_add = self.get_quota( 94 | quota_code, service_code, service_quota 95 | ) 96 | if item_to_add is None: 97 | continue 98 | 99 | if service_code in cache_codes: 100 | cache_codes[service_code].append(item_to_add) 101 | else: 102 | cache_codes[service_code] = [item_to_add] 103 | 104 | self.cache.set_key(key=cache_key, value=cache_codes, expire=1296000) 105 | 106 | return True 107 | 108 | def get_quota(self, quota_code, service_code, service_quota): 109 | try: 110 | response = service_quota.get_aws_default_service_quota( 111 | ServiceCode=service_code, QuotaCode=quota_code 112 | ) 113 | # pylint: disable=broad-except 114 | except Exception as e: 115 | if self.options.verbose: 116 | log_critical( 117 | "\nCannot take quota {} for {}: {}".format( 118 | quota_code, service_code, str(e) 119 | ) 120 | ) 121 | return None 122 | item_to_add = { 123 | "value": response["Quota"]["Value"], 124 | "adjustable": response["Quota"]["Adjustable"], 125 | "quota_code": quota_code, 126 | "quota_name": response["Quota"]["QuotaName"], 127 | } 128 | return item_to_add 129 | 130 | 131 | class Limit(BaseAwsCommand): 132 | def __init__(self, region_names, session, threshold, partition_code): 133 | """ 134 | All AWS resources 135 | 136 | :param region_names: 137 | :param session: 138 | :param threshold: 139 | :param partition_code: 140 | """ 141 | super().__init__(region_names, session, partition_code) 142 | self.threshold = threshold 143 | 144 | def init_globalaws_limits_cache(self, region, services, options: LimitOptions): 145 | # Cache services global and local services 146 | LimitParameters( 147 | session=self.session, region=region, services=services, options=options 148 | ).init_globalaws_limits_cache() 149 | 150 | def run( 151 | self, 152 | diagram: bool, 153 | verbose: bool, 154 | services: List[str], 155 | filters: List[Filterable], 156 | ): 157 | if not services: 158 | services = [] 159 | for service in ALLOWED_SERVICES_CODES: 160 | services.append(service) 161 | for service in SPECIAL_RESOURCES: 162 | services.append(service) 163 | 164 | for region in self.region_names: 165 | limit_options = LimitOptions( 166 | verbose=verbose, 167 | filters=filters, 168 | session=self.session, 169 | region_name=region, 170 | services=services, 171 | threshold=self.threshold, 172 | ) 173 | self.init_globalaws_limits_cache( 174 | region=region, services=services, options=limit_options 175 | ) 176 | 177 | command_runner = AwsCommandRunner() 178 | command_runner.run( 179 | provider="limit", 180 | options=limit_options, 181 | diagram_builder=NoDiagram(), 182 | title="AWS Limits - Region {}".format(region), 183 | # pylint: disable=no-member 184 | filename=limit_options.resulting_file_name("limit"), 185 | ) 186 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/limit/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/limit/data/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/limit/resource/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/limit/resource/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/limit/resource/all.py: -------------------------------------------------------------------------------- 1 | from concurrent.futures.thread import ThreadPoolExecutor 2 | from typing import List 3 | 4 | from provider.aws.common_aws import get_paginator 5 | from provider.aws.limit.command import LimitOptions 6 | from provider.aws.limit.data.allowed_resources import ( 7 | ALLOWED_SERVICES_CODES, 8 | FILTER_EC2_BIGFAMILY, 9 | SPECIAL_RESOURCES, 10 | ) 11 | from shared.common import ( 12 | ResourceProvider, 13 | Resource, 14 | ResourceDigest, 15 | message_handler, 16 | ResourceCache, 17 | LimitsValues, 18 | ) 19 | from shared.error_handler import exception 20 | 21 | SERVICEQUOTA_TO_BOTO3 = { 22 | "elasticloadbalancing": "elbv2", 23 | "elasticfilesystem": "efs", 24 | "vpc": "ec2", 25 | "codeguru-profiler": "codeguruprofiler", 26 | "AWSCloudMap": "servicediscovery", 27 | "ebs": "ec2", 28 | } 29 | 30 | MAX_EXECUTION_PARALLEL = 2 31 | 32 | 33 | class LimitResources(ResourceProvider): 34 | def __init__(self, options: LimitOptions): 35 | """ 36 | All resources 37 | 38 | :param options: 39 | """ 40 | super().__init__() 41 | self.options = options 42 | self.cache = ResourceCache() 43 | 44 | @exception 45 | # pylint: disable=too-many-locals 46 | def get_resources(self) -> List[Resource]: 47 | 48 | threshold_requested = ( 49 | 0 if self.options.threshold is None else self.options.threshold 50 | ) 51 | 52 | client_quota = self.options.client("service-quotas") 53 | 54 | resources_found = [] 55 | 56 | services = self.options.services 57 | 58 | with ThreadPoolExecutor(MAX_EXECUTION_PARALLEL) as executor: 59 | results = executor.map( 60 | lambda service_name: self.analyze_service( 61 | service_name=service_name, 62 | client_quota=client_quota, 63 | threshold_requested=int(threshold_requested), 64 | ), 65 | services, 66 | ) 67 | 68 | for result in results: 69 | if result is not None: 70 | resources_found.extend(result) 71 | 72 | return resources_found 73 | 74 | @exception 75 | def analyze_service(self, service_name, client_quota, threshold_requested): 76 | 77 | if service_name in SPECIAL_RESOURCES: 78 | return [] 79 | 80 | cache_key = "aws_limits_" + service_name + "_" + self.options.region_name 81 | cache = self.cache.get_key(cache_key) 82 | resources_found = [] 83 | if service_name not in cache: 84 | return [] 85 | 86 | """ 87 | Services that must be enabled in your account. Those services will fail you don't enable 88 | Fraud Detector: https://pages.awscloud.com/amazon-fraud-detector-preview.html# 89 | AWS Organizations: https://console.aws.amazon.com/organizations/ 90 | """ 91 | if service_name in ("frauddetector", "organizations"): 92 | message_handler( 93 | "Attention: Service " 94 | + service_name 95 | + " must be enabled to use API calls.", 96 | "WARNING", 97 | ) 98 | 99 | for data_quota_code in cache[service_name]: 100 | if data_quota_code is None: 101 | continue 102 | resource_found = self.analyze_quota( 103 | client_quota=client_quota, 104 | data_quota_code=data_quota_code, 105 | service=service_name, 106 | threshold_requested=threshold_requested, 107 | ) 108 | if resource_found is not None: 109 | resources_found.append(resource_found) 110 | return resources_found 111 | 112 | @exception 113 | # pylint: disable=too-many-locals,too-many-statements 114 | def analyze_quota( 115 | self, client_quota, data_quota_code, service, threshold_requested 116 | ): 117 | resource_found = None 118 | quota_data = ALLOWED_SERVICES_CODES[service][data_quota_code["quota_code"]] 119 | 120 | value_aws = value = data_quota_code["value"] 121 | 122 | # Quota is adjustable by ticket request, then must override this values. 123 | if bool(data_quota_code["adjustable"]) is True: 124 | try: 125 | response_quota = client_quota.get_service_quota( 126 | ServiceCode=service, QuotaCode=data_quota_code["quota_code"] 127 | ) 128 | if "Value" in response_quota["Quota"]: 129 | value = response_quota["Quota"]["Value"] 130 | else: 131 | value = data_quota_code["value"] 132 | except client_quota.exceptions.NoSuchResourceException: 133 | value = data_quota_code["value"] 134 | 135 | if self.options.verbose: 136 | message_handler( 137 | "Collecting data from Quota: " 138 | + service 139 | + " - " 140 | + data_quota_code["quota_name"] 141 | + "...", 142 | "HEADER", 143 | ) 144 | 145 | # Need to convert some quota-services endpoint 146 | if service in SERVICEQUOTA_TO_BOTO3: 147 | service = SERVICEQUOTA_TO_BOTO3.get(service) 148 | 149 | """ 150 | AWS Networkservice is a global service and just allows region us-west-2 instead us-east-1 151 | Reference https://docs.aws.amazon.com/networkmanager/latest/APIReference/Welcome.html 152 | TODO: If we detect more resources like that, convert it into a dict 153 | """ 154 | if service == "networkmanager": 155 | region_boto3 = "us-west-2" 156 | else: 157 | region_boto3 = self.options.region_name 158 | 159 | client = self.options.session.client(service, region_name=region_boto3) 160 | 161 | usage = 0 162 | 163 | # Check filters by resource 164 | if "filter" in quota_data: 165 | filters = quota_data["filter"] 166 | else: 167 | filters = None 168 | 169 | pages = get_paginator( 170 | client=client, 171 | operation_name=quota_data["method"], 172 | resource_type="aws_limit", 173 | filters=filters, 174 | ) 175 | 176 | if not pages: 177 | if filters: 178 | response = getattr(client, quota_data["method"])(**filters) 179 | else: 180 | response = getattr(client, quota_data["method"])() 181 | 182 | # If fields element is not empty, sum values instead list len 183 | if quota_data["fields"]: 184 | for item in response[quota_data["method"]]: 185 | usage = usage + item[quota_data["fields"]] 186 | else: 187 | usage = len(response[quota_data["key"]]) 188 | else: 189 | for page in pages: 190 | if quota_data["fields"]: 191 | if len(page[quota_data["key"]]) > 0: 192 | usage = usage + page[quota_data["key"]][0][quota_data["fields"]] 193 | else: 194 | usage = usage + len(page[quota_data["key"]]) 195 | 196 | # Value for division 197 | if "divisor" in quota_data: 198 | usage = usage / quota_data["divisor"] 199 | 200 | """ 201 | Hack to workaround boto3 limits of 200 items per filter. 202 | Quota L-1216C47A needs more than 200 items. Not happy with this code 203 | TODO: Refactor this piece of terrible code. 204 | """ 205 | if data_quota_code["quota_code"] == "L-1216C47A": 206 | filters = FILTER_EC2_BIGFAMILY["filter"] 207 | pages = get_paginator( 208 | client=client, 209 | operation_name=quota_data["method"], 210 | resource_type="aws_limit", 211 | filters=filters, 212 | ) 213 | if not pages: 214 | response = getattr(client, quota_data["method"])(**filters) 215 | usage = len(response[quota_data["key"]]) 216 | else: 217 | for page in pages: 218 | usage = usage + len(page[quota_data["key"]]) 219 | 220 | try: 221 | percent = round((usage / value) * 100, 2) 222 | except ZeroDivisionError: 223 | percent = 0 224 | 225 | if percent >= threshold_requested: 226 | resource_found = Resource( 227 | digest=ResourceDigest( 228 | id=data_quota_code["quota_code"], type="aws_limit" 229 | ), 230 | name="", 231 | group="", 232 | limits=LimitsValues( 233 | quota_name=data_quota_code["quota_name"], 234 | quota_code=data_quota_code["quota_code"], 235 | aws_limit=int(value_aws), 236 | local_limit=int(value), 237 | usage=int(usage), 238 | service=service, 239 | percent=percent, 240 | ), 241 | ) 242 | 243 | return resource_found 244 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/limit/resource/ses.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from provider.aws.limit.command import LimitOptions 4 | from shared.common import ResourceProvider, Resource, ResourceDigest, LimitsValues 5 | from shared.error_handler import exception 6 | 7 | 8 | class SesResources(ResourceProvider): 9 | def __init__(self, options: LimitOptions): 10 | """ 11 | SES resources 12 | 13 | :param options: 14 | """ 15 | super().__init__() 16 | self.options = options 17 | 18 | @exception 19 | def get_resources(self) -> List[Resource]: 20 | 21 | services = self.options.services 22 | 23 | if "ses" not in services: 24 | return [] 25 | 26 | client = self.options.client("ses") 27 | 28 | response = client.get_send_quota() 29 | max_send = response["Max24HourSend"] 30 | last_send = response["SentLast24Hours"] 31 | if max_send == -1: 32 | percent = "0" 33 | else: 34 | percent = round((last_send / max_send) * 100, 2) 35 | 36 | return [ 37 | Resource( 38 | digest=ResourceDigest(id="ses-send-quota", type="aws_limit"), 39 | name="", 40 | group="", 41 | limits=LimitsValues( 42 | quota_name="Sending limit for the Amazon SES account per day " 43 | "(limit 200 can mean sandbox mode enabled)", 44 | quota_code="ses-send-quota", 45 | aws_limit=int( 46 | 200 47 | ), # https://forums.aws.amazon.com/thread.jspa?threadID=61090 48 | local_limit=int(max_send), 49 | usage=int(last_send), 50 | service="ses", 51 | percent=percent, 52 | ), 53 | ) 54 | ] 55 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/policy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/policy/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/policy/command.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from provider.aws.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner 4 | from provider.aws.policy.diagram import PolicyDiagram 5 | from shared.common import Filterable, BaseOptions 6 | from shared.diagram import NoDiagram 7 | 8 | 9 | class PolicyOptions(BaseAwsOptions, BaseOptions): 10 | def __init__(self, verbose, filters, session, region_name): 11 | BaseAwsOptions.__init__(self, session, region_name) 12 | BaseOptions.__init__(self, verbose, filters) 13 | 14 | 15 | class Policy(BaseAwsCommand): 16 | def run( 17 | self, 18 | diagram: bool, 19 | verbose: bool, 20 | services: List[str], 21 | filters: List[Filterable], 22 | ): 23 | for region in self.region_names: 24 | self.init_region_cache(region) 25 | options = PolicyOptions( 26 | verbose=verbose, 27 | filters=filters, 28 | session=self.session, 29 | region_name=region, 30 | ) 31 | 32 | command_runner = AwsCommandRunner(filters) 33 | if diagram: 34 | diagram = PolicyDiagram() 35 | else: 36 | diagram = NoDiagram() 37 | command_runner.run( 38 | provider="policy", 39 | options=options, 40 | diagram_builder=diagram, 41 | title="AWS IAM Policies - Region {}".format(region), 42 | filename=options.resulting_file_name("policy"), 43 | ) 44 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/policy/diagram.py: -------------------------------------------------------------------------------- 1 | from typing import List, Dict 2 | 3 | from shared.common import ResourceEdge, Resource, ResourceDigest 4 | from shared.diagram import BaseDiagram, Mapsources, add_resource_to_group 5 | 6 | ROLE_AGGREGATE_PREFIX = "aggregate_" 7 | 8 | 9 | class PolicyDiagram(BaseDiagram): 10 | def __init__(self): 11 | """ 12 | Policy diagram 13 | """ 14 | super().__init__("fdp") 15 | 16 | # pylint: disable=too-many-locals,too-many-branches 17 | def group_by_group( 18 | self, resources: List[Resource], initial_resource_relations: List[ResourceEdge] 19 | ) -> Dict[str, List[Resource]]: 20 | ordered_resources: Dict[str, List[Resource]] = dict() 21 | for resource in resources: 22 | if Mapsources.mapresources.get(resource.digest.type) is not None: 23 | if resource.digest.type == "aws_iam_role": 24 | 25 | got_policy = False 26 | principals = {} 27 | for rel in initial_resource_relations: 28 | if ( 29 | rel.from_node == resource.digest 30 | and rel.to_node.type == "aws_iam_policy" 31 | ): 32 | got_policy = True 33 | if ( 34 | rel.from_node == resource.digest 35 | and rel.label == "assumed by" 36 | ): 37 | principals[rel.to_node.id + "|" + rel.to_node.type] = True 38 | 39 | if got_policy and len(principals) != 0: 40 | add_resource_to_group( 41 | ordered_resources, resource.group, resource 42 | ) 43 | else: 44 | for principal in principals: 45 | add_resource_to_group( 46 | ordered_resources, "to_agg_" + principal, resource 47 | ) 48 | else: 49 | add_resource_to_group(ordered_resources, resource.group, resource) 50 | 51 | keys_to_remove = [] 52 | for key, values in ordered_resources.items(): 53 | if "to_agg_" in key: 54 | names = [] 55 | for value in values: 56 | names.append(value.digest.id) 57 | to_agg_len = len("to_agg_") 58 | principal_id_type = key[to_agg_len:] 59 | principal_id = principal_id_type.rsplit("|", 1)[0] 60 | principal_type = principal_id_type.rsplit("|", 1)[1] 61 | aggregate_digest = ResourceDigest( 62 | id=ROLE_AGGREGATE_PREFIX + principal_id, type="aws_iam_role" 63 | ) 64 | aggregate_role = Resource( 65 | digest=aggregate_digest, 66 | name="Roles for {} ({})".format(principal_id, len(values)), 67 | details=",".join(names), 68 | ) 69 | initial_resource_relations.append( 70 | ResourceEdge( 71 | from_node=aggregate_digest, 72 | to_node=ResourceDigest(id=principal_id, type=principal_type), 73 | label="assumed by", 74 | ) 75 | ) 76 | add_resource_to_group(ordered_resources, "", aggregate_role) 77 | keys_to_remove.append(key) 78 | for key in keys_to_remove: 79 | ordered_resources.pop(key, None) 80 | 81 | return ordered_resources 82 | 83 | def process_relationships( 84 | self, 85 | grouped_resources: Dict[str, List[Resource]], 86 | resource_relations: List[ResourceEdge], 87 | ) -> List[ResourceEdge]: 88 | aggregated_roles = {} 89 | for resource in grouped_resources[""]: 90 | if ( 91 | resource.digest.type == "aws_iam_role" 92 | and resource.digest.id.startswith(ROLE_AGGREGATE_PREFIX) 93 | ): 94 | for role in resource.details.split(","): 95 | aggregated_roles[role] = resource.digest.id 96 | filtered_resources: List[ResourceEdge] = [] 97 | for resource_relation in resource_relations: 98 | if resource_relation.from_node.id not in aggregated_roles: 99 | filtered_resources.append(resource_relation) 100 | 101 | return filtered_resources 102 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/policy/resource/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/policy/resource/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/policy/resource/general.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from provider.aws.common_aws import resource_tags 4 | from provider.aws.policy.command import PolicyOptions 5 | from shared.common import ( 6 | ResourceProvider, 7 | Resource, 8 | message_handler, 9 | ResourceDigest, 10 | ResourceEdge, 11 | ResourceAvailable, 12 | ) 13 | from shared.error_handler import exception 14 | 15 | 16 | class IamUser(ResourceProvider): 17 | @ResourceAvailable(services="iam") 18 | def __init__(self, options: PolicyOptions): 19 | """ 20 | Iam user 21 | 22 | :param options: 23 | """ 24 | super().__init__() 25 | self.options = options 26 | self.client = options.client("iam") 27 | self.users_found: List[Resource] = [] 28 | 29 | @exception 30 | def get_resources(self) -> List[Resource]: 31 | if self.options.verbose: 32 | message_handler("Collecting data from IAM Users...", "HEADER") 33 | paginator = self.client.get_paginator("list_users") 34 | pages = paginator.paginate() 35 | 36 | users_found = [] 37 | for users in pages: 38 | for data in users["Users"]: 39 | tag_response = self.client.list_user_tags(UserName=data["UserName"],) 40 | users_found.append( 41 | Resource( 42 | digest=ResourceDigest(id=data["UserName"], type="aws_iam_user"), 43 | name=data["UserName"], 44 | details="", 45 | group="User", 46 | tags=resource_tags(tag_response), 47 | ) 48 | ) 49 | self.users_found = users_found 50 | return users_found 51 | 52 | @exception 53 | def get_relations(self) -> List[ResourceEdge]: 54 | resources_found = [] 55 | for user in self.users_found: 56 | response = self.client.list_groups_for_user(UserName=user.name) 57 | for group in response["Groups"]: 58 | resources_found.append( 59 | ResourceEdge( 60 | from_node=user.digest, 61 | to_node=ResourceDigest( 62 | id=group["GroupName"], type="aws_iam_group" 63 | ), 64 | ) 65 | ) 66 | 67 | response = self.client.list_attached_user_policies(UserName=user.name) 68 | for policy in response["AttachedPolicies"]: 69 | resources_found.append( 70 | ResourceEdge( 71 | from_node=user.digest, 72 | to_node=ResourceDigest( 73 | id=policy["PolicyArn"], type="aws_iam_policy" 74 | ), 75 | ) 76 | ) 77 | 78 | return resources_found 79 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/security/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/security/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/security/command.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from provider.aws.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner 4 | from shared.common import ( 5 | ResourceCache, 6 | Filterable, 7 | BaseOptions, 8 | ) 9 | from shared.diagram import NoDiagram 10 | 11 | 12 | class SecurityOptions(BaseAwsOptions, BaseOptions): 13 | commands: List[str] 14 | 15 | # pylint: disable=too-many-arguments 16 | def __init__( 17 | self, verbose: bool, filters: List[Filterable], session, region_name, commands, 18 | ): 19 | BaseAwsOptions.__init__(self, session, region_name) 20 | BaseOptions.__init__(self, verbose, filters) 21 | self.commands = commands 22 | 23 | 24 | class SecurityParameters: 25 | def __init__(self, session, region: str, commands, options: SecurityOptions): 26 | self.region = region 27 | self.cache = ResourceCache() 28 | self.session = session 29 | self.options = options 30 | self.commands = commands 31 | 32 | 33 | class Security(BaseAwsCommand): 34 | def __init__(self, region_names, session, commands, partition_code): 35 | """ 36 | All AWS resources 37 | 38 | :param region_names: 39 | :param session: 40 | :param commands: 41 | :param partition_code: 42 | """ 43 | super().__init__(region_names, session, partition_code) 44 | self.commands = commands 45 | 46 | def run( 47 | self, 48 | diagram: bool, 49 | verbose: bool, 50 | services: List[str], 51 | filters: List[Filterable], 52 | ): 53 | 54 | for region in self.region_names: 55 | security_options = SecurityOptions( 56 | verbose=verbose, 57 | filters=filters, 58 | session=self.session, 59 | region_name=region, 60 | commands=self.commands, 61 | ) 62 | 63 | command_runner = AwsCommandRunner() 64 | command_runner.run( 65 | provider="security", 66 | options=security_options, 67 | diagram_builder=NoDiagram(), 68 | title="AWS Security - Region {}".format(region), 69 | # pylint: disable=no-member 70 | filename=security_options.resulting_file_name("security"), 71 | ) 72 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/security/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/security/data/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/security/data/commands_enabled.py: -------------------------------------------------------------------------------- 1 | COMMANDS_ENABLED = { 2 | "access-keys-rotated": { 3 | "parameters": [{"name": "max_age", "default_value": "90", "type": "int"}], 4 | "class": "IAM", 5 | "method": "access_keys_rotated", 6 | "short_description": "Checks whether the active access keys are rotated within the number of days.", 7 | }, 8 | "ebs-encryption": { 9 | "parameters": [ 10 | {"name": "ebs_encryption", "default_value": "no", "type": "bool"} 11 | ], 12 | "class": "EC2", 13 | "method": "ebs_encryption", 14 | "short_description": "Check that Amazon Elastic Block Store (EBS) encryption is enabled by default.", 15 | }, 16 | "restricted-ssh": { 17 | "parameters": [ 18 | {"name": "restricted_ssh", "default_value": "no", "type": "bool"} 19 | ], 20 | "class": "EC2", 21 | "method": "restricted_ssh", 22 | "short_description": "Checks whether SG that are in use disallow unrestricted incoming SSH traffic.", 23 | }, 24 | "imdsv2-check": { 25 | "parameters": [{"name": "imdsv2_check", "default_value": "no", "type": "bool"}], 26 | "class": "EC2", 27 | "method": "imdsv2_check", 28 | "short_description": "Checks Amazon EC2 instance metadata is configured with IMDSv2.", 29 | }, 30 | "pitr-enabled": { 31 | "parameters": [{"name": "pitr_enabled", "default_value": "no", "type": "bool"}], 32 | "class": "DYNAMODB", 33 | "method": "pitr_enabled", 34 | "short_description": "Checks that point in time recovery is enabled for Amazon DynamoDB tables.", 35 | }, 36 | "cloudtrail-enabled": { 37 | "parameters": [ 38 | {"name": "cloudtrail_enabled", "default_value": "no", "type": "bool"} 39 | ], 40 | "class": "CLOUDTRAIL", 41 | "method": "cloudtrail_enabled", 42 | "short_description": "Checks whether AWS CloudTrail is enabled in your AWS account.", 43 | }, 44 | } 45 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/security/resource/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/security/resource/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/security/resource/all.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import importlib 4 | 5 | from provider.aws.security.command import SecurityOptions 6 | from provider.aws.security.data.commands_enabled import COMMANDS_ENABLED 7 | from shared.common import ( 8 | ResourceProvider, 9 | Resource, 10 | message_handler, 11 | ) 12 | from shared.error_handler import exception 13 | 14 | 15 | def build_formatted_commands(): 16 | formatted_commands = [] 17 | for detail_command in COMMANDS_ENABLED: 18 | parameters = COMMANDS_ENABLED[detail_command]["parameters"][0]["name"] 19 | default_value = COMMANDS_ENABLED[detail_command]["parameters"][0][ 20 | "default_value" 21 | ] 22 | formated_command = '{}="{}={}"'.format( 23 | detail_command, parameters, default_value 24 | ) 25 | formatted_commands.append(formated_command) 26 | return formatted_commands 27 | 28 | 29 | class SecuritytResources(ResourceProvider): 30 | def __init__(self, options: SecurityOptions): 31 | """ 32 | All resources 33 | 34 | :param options: 35 | """ 36 | super().__init__() 37 | self.options = options 38 | 39 | @exception 40 | # pylint: disable=too-many-locals 41 | def get_resources(self) -> List[Resource]: 42 | 43 | commands = self.options.commands 44 | 45 | result = [] 46 | 47 | # commands informed, checking for specific commands 48 | if not commands: 49 | commands = build_formatted_commands() 50 | # show all commands to check 51 | if commands[0] == "list": 52 | message_handler("\nFollowing commands are enabled\n", "HEADER") 53 | for detail_command in COMMANDS_ENABLED: 54 | parameters = COMMANDS_ENABLED[detail_command]["parameters"][0]["name"] 55 | default_value = COMMANDS_ENABLED[detail_command]["parameters"][0][ 56 | "default_value" 57 | ] 58 | description = COMMANDS_ENABLED[detail_command]["short_description"] 59 | 60 | formated_command = 'cloudiscovery aws-security -c {}="{}={}"'.format( 61 | detail_command, parameters, default_value 62 | ) 63 | message_handler( 64 | "{} - {} \nExample: {}\n".format( 65 | detail_command, description, formated_command 66 | ), 67 | "OKGREEN", 68 | ) 69 | else: 70 | for command in commands: 71 | command = command.split("=") 72 | 73 | # First position always is command 74 | if command[0] not in COMMANDS_ENABLED: 75 | message_handler( 76 | "Command {} doesn't exists.".format(command[0]), "WARNING" 77 | ) 78 | else: 79 | # Second and thrid parameters are class and method 80 | _class = COMMANDS_ENABLED[command[0]]["class"] 81 | _method = COMMANDS_ENABLED[command[0]]["method"] 82 | _parameter = { 83 | command[1].replace('"', ""): command[2].replace('"', "") 84 | } 85 | 86 | module = importlib.import_module( 87 | "provider.aws.security.resource.commands." + _class 88 | ) 89 | instance = getattr(module, _class)(self.options) 90 | result = result + getattr(instance, _method)(**_parameter) 91 | 92 | return result 93 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/security/resource/commands/CLOUDTRAIL.py: -------------------------------------------------------------------------------- 1 | from provider.aws.security.command import SecurityOptions 2 | 3 | from shared.common import ( 4 | Resource, 5 | ResourceDigest, 6 | SecurityValues, 7 | ) 8 | 9 | 10 | class CLOUDTRAIL: 11 | def __init__(self, options: SecurityOptions): 12 | self.options = options 13 | 14 | def cloudtrail_enabled(self, cloudtrail_enabled): 15 | 16 | client = self.options.client("cloudtrail") 17 | 18 | trails = client.list_trails() 19 | 20 | resources_found = [] 21 | 22 | if not trails["Trails"]: 23 | resources_found.append( 24 | Resource( 25 | digest=ResourceDigest(id="cloudtrail", type="cloudtrail_enabled"), 26 | details="CLOUDTRAIL disabled", 27 | name="cloudtrail", 28 | group="cloudtrail_security", 29 | security=SecurityValues( 30 | status="CRITICAL", 31 | parameter="cloudtrail_enabled", 32 | value="False", 33 | ), 34 | ) 35 | ) 36 | 37 | return resources_found 38 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/security/resource/commands/DYNAMODB.py: -------------------------------------------------------------------------------- 1 | from provider.aws.security.command import SecurityOptions 2 | 3 | from shared.common import ( 4 | Resource, 5 | ResourceDigest, 6 | SecurityValues, 7 | ) 8 | 9 | 10 | class DYNAMODB: 11 | def __init__(self, options: SecurityOptions): 12 | self.options = options 13 | 14 | def pitr_enabled(self, pitr_enabled): 15 | 16 | client = self.options.client("dynamodb") 17 | 18 | tables = client.list_tables()["TableNames"] 19 | 20 | resources_found = [] 21 | 22 | for table in tables: 23 | if ( 24 | client.describe_continuous_backups(TableName=table)[ 25 | "ContinuousBackupsDescription" 26 | ]["PointInTimeRecoveryDescription"]["PointInTimeRecoveryStatus"] 27 | == "DISABLED" 28 | ): 29 | resources_found.append( 30 | Resource( 31 | digest=ResourceDigest(id=table, type="pitr_enabled"), 32 | details="PITR disabled", 33 | name=table, 34 | group="ddb_security", 35 | security=SecurityValues( 36 | status="CRITICAL", parameter="pitr_enabled", value="False", 37 | ), 38 | ) 39 | ) 40 | 41 | return resources_found 42 | 43 | def imdsv2_check(self, imdsv2_check): 44 | 45 | client = self.options.client("ec2") 46 | 47 | instances = client.describe_instances()["Reservations"] 48 | 49 | resources_found = [] 50 | 51 | for instance in instances: 52 | for instance_detail in instance["Instances"]: 53 | if ( 54 | instance_detail["MetadataOptions"]["HttpEndpoint"] == "enabled" 55 | and instance_detail["MetadataOptions"]["HttpTokens"] == "optional" 56 | ): 57 | resources_found.append( 58 | Resource( 59 | digest=ResourceDigest( 60 | id=instance_detail["InstanceId"], type="imdsv2_check" 61 | ), 62 | details="IMDSv2 tokens not enforced", 63 | name=instance_detail["InstanceId"], 64 | group="ddb_security", 65 | security=SecurityValues( 66 | status="CRITICAL", 67 | parameter="imdsv2_check", 68 | value="False", 69 | ), 70 | ) 71 | ) 72 | 73 | return resources_found 74 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/security/resource/commands/EC2.py: -------------------------------------------------------------------------------- 1 | from provider.aws.security.command import SecurityOptions 2 | 3 | from shared.common import ( 4 | Resource, 5 | ResourceDigest, 6 | SecurityValues, 7 | ) 8 | 9 | 10 | class EC2: 11 | def __init__(self, options: SecurityOptions): 12 | self.options = options 13 | 14 | def ebs_encryption(self, ebs_encryption): 15 | 16 | client = self.options.client("ec2") 17 | 18 | volumes = client.describe_volumes()["Volumes"] 19 | 20 | resources_found = [] 21 | 22 | for volume in volumes: 23 | if volume["Encrypted"] is False: 24 | resources_found.append( 25 | Resource( 26 | digest=ResourceDigest( 27 | id=volume["VolumeId"], type="ebs_encryption" 28 | ), 29 | details="This volume is not encypted.", 30 | name=volume["VolumeId"], 31 | group="ec2_security", 32 | security=SecurityValues( 33 | status="CRITICAL", 34 | parameter="ebs_encryption", 35 | value="False", 36 | ), 37 | ) 38 | ) 39 | 40 | return resources_found 41 | 42 | def imdsv2_check(self, imdsv2_check): 43 | 44 | client = self.options.client("ec2") 45 | 46 | instances = client.describe_instances()["Reservations"] 47 | 48 | resources_found = [] 49 | 50 | for instance in instances: 51 | for instance_detail in instance["Instances"]: 52 | if ( 53 | instance_detail["MetadataOptions"]["HttpEndpoint"] == "enabled" 54 | and instance_detail["MetadataOptions"]["HttpTokens"] == "optional" 55 | ): 56 | resources_found.append( 57 | Resource( 58 | digest=ResourceDigest( 59 | id=instance_detail["InstanceId"], type="imdsv2_check" 60 | ), 61 | details="IMDSv2 tokens not enforced", 62 | name=instance_detail["InstanceId"], 63 | group="ec2_security", 64 | security=SecurityValues( 65 | status="CRITICAL", 66 | parameter="imdsv2_check", 67 | value="False", 68 | ), 69 | ) 70 | ) 71 | 72 | return resources_found 73 | 74 | def restricted_ssh(self, restricted_ssh): 75 | 76 | client = self.options.client("ec2") 77 | 78 | security_groups = client.describe_security_groups() 79 | 80 | resources_found = [] 81 | 82 | # pylint: disable=too-many-nested-blocks 83 | for security_group in security_groups["SecurityGroups"]: 84 | for ip_permission in security_group["IpPermissions"]: 85 | if "FromPort" in ip_permission and "ToPort" in ip_permission: 86 | # Port 22 possible opened using port range 87 | if ip_permission["FromPort"] <= 22 >= ip_permission["ToPort"]: 88 | # IPv4 89 | for cidr in ip_permission["IpRanges"]: 90 | if cidr["CidrIp"] == "0.0.0.0/0": 91 | resources_found.append( 92 | Resource( 93 | digest=ResourceDigest( 94 | id=security_group["GroupId"], 95 | type="restricted_ssh", 96 | ), 97 | details="The SSH port of this security group is opened to the world.", 98 | name=security_group["GroupName"], 99 | group="ec2_security", 100 | security=SecurityValues( 101 | status="CRITICAL", 102 | parameter="restricted_ssh", 103 | value="False", 104 | ), 105 | ) 106 | ) 107 | 108 | # IPv6 109 | for cidr in ip_permission["Ipv6Ranges"]: 110 | if cidr["CidrIpv6"] == "::/0": 111 | resources_found.append( 112 | Resource( 113 | digest=ResourceDigest( 114 | id=security_group["GroupId"], 115 | type="restricted_ssh", 116 | ), 117 | details="The SSH port of this security group is opened to the world.", 118 | name=security_group["GroupName"], 119 | group="ec2_security", 120 | security=SecurityValues( 121 | status="CRITICAL", 122 | parameter="restricted_ssh", 123 | value="False", 124 | ), 125 | ) 126 | ) 127 | 128 | return resources_found 129 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/security/resource/commands/IAM.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | import pytz 3 | 4 | from provider.aws.security.command import SecurityOptions 5 | 6 | from shared.common import ( 7 | Resource, 8 | ResourceDigest, 9 | SecurityValues, 10 | ) 11 | 12 | 13 | class IAM: 14 | def __init__(self, options: SecurityOptions): 15 | self.options = options 16 | 17 | def access_keys_rotated(self, max_age): 18 | 19 | client = self.options.client("iam") 20 | 21 | users = client.list_users() 22 | 23 | resources_found = [] 24 | 25 | for user in users["Users"]: 26 | paginator = client.get_paginator("list_access_keys") 27 | for keys in paginator.paginate(UserName=user["UserName"]): 28 | for key in keys["AccessKeyMetadata"]: 29 | 30 | date_compare = datetime.utcnow() - timedelta(days=int(max_age)) 31 | date_compare = date_compare.replace(tzinfo=pytz.utc) 32 | last_rotate = key["CreateDate"] 33 | 34 | if last_rotate < date_compare: 35 | resources_found.append( 36 | Resource( 37 | digest=ResourceDigest( 38 | id=key["AccessKeyId"], type="access_keys_rotated" 39 | ), 40 | details="You must rotate your keys", 41 | name=key["UserName"], 42 | group="iam_security", 43 | security=SecurityValues( 44 | status="CRITICAL", 45 | parameter="max_age", 46 | value=str(max_age), 47 | ), 48 | ) 49 | ) 50 | 51 | return resources_found 52 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/security/resource/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/security/resource/commands/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/vpc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/vpc/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/vpc/command.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from ipaddress import ip_network 4 | 5 | from provider.aws.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner 6 | from provider.aws.vpc.diagram import VpcDiagram 7 | from shared.common import ( 8 | ResourceDigest, 9 | VPCE_REGEX, 10 | SOURCE_IP_ADDRESS_REGEX, 11 | Filterable, 12 | BaseOptions, 13 | ) 14 | from shared.diagram import NoDiagram, BaseDiagram 15 | 16 | 17 | class VpcOptions(BaseAwsOptions, BaseOptions): 18 | vpc_id: str 19 | 20 | # pylint: disable=too-many-arguments 21 | def __init__( 22 | self, verbose: bool, filters: List[Filterable], session, region_name, vpc_id 23 | ): 24 | BaseAwsOptions.__init__(self, session, region_name) 25 | BaseOptions.__init__(self, verbose, filters) 26 | self.vpc_id = vpc_id 27 | 28 | def vpc_digest(self): 29 | return ResourceDigest(id=self.vpc_id, type="aws_vpc") 30 | 31 | 32 | class Vpc(BaseAwsCommand): 33 | # pylint: disable=too-many-arguments 34 | def __init__(self, vpc_id, region_names, session, partition_code): 35 | """ 36 | VPC command 37 | 38 | :param vpc_id: 39 | :param region_names: 40 | :param session: 41 | :param partition_code: 42 | """ 43 | super().__init__(region_names, session, partition_code) 44 | self.vpc_id = vpc_id 45 | 46 | @staticmethod 47 | def check_vpc(vpc_options: VpcOptions): 48 | client = vpc_options.client("ec2") 49 | response = client.describe_vpcs(VpcIds=[vpc_options.vpc_id]) 50 | 51 | dataresponse = response["Vpcs"][0] 52 | message = "------------------------------------------------------\n" 53 | message = ( 54 | message 55 | + "VPC: {} - {}\nCIDR Block: {}\nTenancy: {}\nIs default: {}".format( 56 | vpc_options.vpc_id, 57 | vpc_options.region_name, 58 | dataresponse["CidrBlock"], 59 | dataresponse["InstanceTenancy"], 60 | dataresponse["IsDefault"], 61 | ) 62 | ) 63 | print(message) 64 | 65 | def run( 66 | self, 67 | diagram: bool, 68 | verbose: bool, 69 | services: List[str], 70 | filters: List[Filterable], 71 | ): 72 | # pylint: disable=too-many-branches 73 | command_runner = AwsCommandRunner(filters) 74 | 75 | for region in self.region_names: 76 | self.init_region_cache(region) 77 | 78 | # if vpc is none, get all vpcs and check 79 | if self.vpc_id is None: 80 | client = self.session.client("ec2", region_name=region) 81 | vpcs = client.describe_vpcs() 82 | for data in vpcs["Vpcs"]: 83 | vpc_id = data["VpcId"] 84 | vpc_options = VpcOptions( 85 | verbose=verbose, 86 | filters=filters, 87 | session=self.session, 88 | region_name=region, 89 | vpc_id=vpc_id, 90 | ) 91 | self.check_vpc(vpc_options) 92 | diagram_builder: BaseDiagram 93 | if diagram: 94 | diagram_builder = VpcDiagram(vpc_id=vpc_id) 95 | else: 96 | diagram_builder = NoDiagram() 97 | command_runner.run( 98 | provider="vpc", 99 | options=vpc_options, 100 | diagram_builder=diagram_builder, 101 | title="AWS VPC {} Resources - Region {}".format(vpc_id, region), 102 | filename=vpc_options.resulting_file_name(vpc_id + "_vpc"), 103 | ) 104 | else: 105 | vpc_options = VpcOptions( 106 | verbose=verbose, 107 | filters=filters, 108 | session=self.session, 109 | region_name=region, 110 | vpc_id=self.vpc_id, 111 | ) 112 | 113 | self.check_vpc(vpc_options) 114 | if diagram: 115 | diagram_builder = VpcDiagram(vpc_id=self.vpc_id) 116 | else: 117 | diagram_builder = NoDiagram() 118 | command_runner.run( 119 | provider="vpc", 120 | options=vpc_options, 121 | diagram_builder=diagram_builder, 122 | title="AWS VPC {} Resources - Region {}".format( 123 | self.vpc_id, region 124 | ), 125 | filename=vpc_options.resulting_file_name(self.vpc_id + "_vpc"), 126 | ) 127 | 128 | 129 | # pylint: disable=too-many-branches 130 | def check_ipvpc_inpolicy(document, vpc_options: VpcOptions): 131 | document = document.replace("\\", "").lower() 132 | 133 | # Checking if VPC is inside document, it's a 100% true information 134 | # pylint: disable=no-else-return 135 | if vpc_options.vpc_id in document: 136 | return "direct VPC reference" 137 | else: 138 | # Vpc_id not found, trying to discover if it's a potencial subnet IP or VPCE is allowed 139 | if "aws:sourcevpce" in document: 140 | 141 | # Get VPCE found 142 | aws_sourcevpces = [] 143 | for vpce_tuple in VPCE_REGEX.findall(document): 144 | aws_sourcevpces.append(vpce_tuple[1]) 145 | 146 | # Get all VPCE of this VPC 147 | ec2 = vpc_options.client("ec2") 148 | 149 | filters = [{"Name": "vpc-id", "Values": [vpc_options.vpc_id]}] 150 | 151 | vpc_endpoints = ec2.describe_vpc_endpoints(Filters=filters) 152 | 153 | # iterate VPCEs found found 154 | if len(vpc_endpoints["VpcEndpoints"]) > 0: 155 | matching_vpces = [] 156 | # Iterate VPCE to match vpce in Policy Document 157 | for data in vpc_endpoints["VpcEndpoints"]: 158 | if data["VpcEndpointId"] in aws_sourcevpces: 159 | matching_vpces.append(data["VpcEndpointId"]) 160 | return "VPC Endpoint(s): " + (", ".join(matching_vpces)) 161 | 162 | if "aws:sourceip" in document: 163 | 164 | # Get ip found 165 | aws_sourceips = [] 166 | for vpce_tuple in SOURCE_IP_ADDRESS_REGEX.findall(document): 167 | aws_sourceips.append(vpce_tuple[1]) 168 | # Get subnets cidr block 169 | ec2 = vpc_options.client("ec2") 170 | 171 | filters = [{"Name": "vpc-id", "Values": [vpc_options.vpc_id]}] 172 | 173 | subnets = ec2.describe_subnets(Filters=filters) 174 | overlapping_subnets = [] 175 | # iterate ips found 176 | for ipfound in aws_sourceips: 177 | 178 | # Iterate subnets to match ipaddress 179 | for subnet in list(subnets["Subnets"]): 180 | ipfound = ip_network(ipfound) 181 | network_addres = ip_network(subnet["CidrBlock"]) 182 | 183 | if ipfound.overlaps(network_addres): 184 | overlapping_subnets.append( 185 | "{} ({})".format(str(network_addres), subnet["SubnetId"]) 186 | ) 187 | if len(overlapping_subnets) != 0: 188 | return "source IP(s): {} -> subnet CIDR(s): {}".format( 189 | ", ".join(aws_sourceips), ", ".join(overlapping_subnets) 190 | ) 191 | 192 | return False 193 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/vpc/resource/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/provider/aws/vpc/resource/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/vpc/resource/application.py: -------------------------------------------------------------------------------- 1 | import json 2 | from concurrent.futures.thread import ThreadPoolExecutor 3 | from typing import List 4 | 5 | from provider.aws.common_aws import resource_tags 6 | from provider.aws.vpc.command import VpcOptions, check_ipvpc_inpolicy 7 | from shared.common import ( 8 | datetime_to_string, 9 | ResourceProvider, 10 | Resource, 11 | message_handler, 12 | ResourceDigest, 13 | ResourceEdge, 14 | ResourceAvailable, 15 | ) 16 | from shared.error_handler import exception 17 | 18 | 19 | class SQSPOLICY(ResourceProvider): 20 | def __init__(self, vpc_options: VpcOptions): 21 | """ 22 | Sqs policy 23 | 24 | :param vpc_options: 25 | """ 26 | super().__init__() 27 | self.vpc_options = vpc_options 28 | 29 | @exception 30 | @ResourceAvailable(services="sqs") 31 | def get_resources(self) -> List[Resource]: 32 | 33 | client = self.vpc_options.client("sqs") 34 | 35 | resources_found = [] 36 | 37 | response = client.list_queues() 38 | 39 | if self.vpc_options.verbose: 40 | message_handler("Collecting data from SQS Queue Policy...", "HEADER") 41 | 42 | if "QueueUrls" in response: 43 | 44 | with ThreadPoolExecutor(15) as executor: 45 | results = executor.map( 46 | lambda data: self.analyze_queues(client, data[1]), 47 | enumerate(response["QueueUrls"]), 48 | ) 49 | 50 | for result in results: 51 | 52 | if result[0] is True: 53 | resources_found.append(result[1]) 54 | 55 | return resources_found 56 | 57 | @exception 58 | def analyze_queues(self, client, queue): 59 | 60 | sqs_queue_policy = client.get_queue_attributes( 61 | QueueUrl=queue, AttributeNames=["QueueArn", "Policy"] 62 | ) 63 | 64 | if "Attributes" in sqs_queue_policy: 65 | 66 | if "Policy" in sqs_queue_policy["Attributes"]: 67 | 68 | documentpolicy = sqs_queue_policy["Attributes"]["Policy"] 69 | queuearn = sqs_queue_policy["Attributes"]["QueueArn"] 70 | document = json.dumps(documentpolicy, default=datetime_to_string) 71 | 72 | # check either vpc_id or potencial subnet ip are found 73 | ipvpc_found = check_ipvpc_inpolicy( 74 | document=document, vpc_options=self.vpc_options 75 | ) 76 | 77 | if ipvpc_found is not False: 78 | list_tags_response = client.list_queue_tags(QueueUrl=queue) 79 | resource_digest = ResourceDigest( 80 | id=queuearn, type="aws_sqs_queue_policy" 81 | ) 82 | self.relations_found.append( 83 | ResourceEdge( 84 | from_node=resource_digest, 85 | to_node=self.vpc_options.vpc_digest(), 86 | ) 87 | ) 88 | return ( 89 | True, 90 | Resource( 91 | digest=resource_digest, 92 | name=queue, 93 | details="", 94 | group="application", 95 | tags=resource_tags(list_tags_response), 96 | ), 97 | ) 98 | 99 | return False, None 100 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/vpc/resource/containers.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from provider.aws.common_aws import describe_subnet, resource_tags 4 | from provider.aws.vpc.command import VpcOptions 5 | from shared.common import ( 6 | ResourceProvider, 7 | Resource, 8 | message_handler, 9 | ResourceDigest, 10 | ResourceEdge, 11 | ResourceAvailable, 12 | ) 13 | from shared.error_handler import exception 14 | 15 | 16 | class ECS(ResourceProvider): 17 | def __init__(self, vpc_options: VpcOptions): 18 | """ 19 | Ecs 20 | 21 | :param vpc_options: 22 | """ 23 | super().__init__() 24 | self.vpc_options = vpc_options 25 | 26 | @exception 27 | @ResourceAvailable(services="ecs") 28 | # pylint: disable=too-many-locals,too-many-branches 29 | def get_resources(self) -> List[Resource]: 30 | 31 | client = self.vpc_options.client("ecs") 32 | ec2_client = self.vpc_options.client("ec2") 33 | 34 | resources_found = [] 35 | 36 | clusters_list = client.list_clusters() 37 | response = client.describe_clusters( 38 | clusters=clusters_list["clusterArns"], include=["TAGS"] 39 | ) 40 | 41 | if self.vpc_options.verbose: 42 | message_handler("Collecting data from ECS Cluster...", "HEADER") 43 | 44 | # pylint: disable=too-many-nested-blocks 45 | for data in response["clusters"]: 46 | 47 | # Searching all cluster services 48 | paginator = client.get_paginator("list_services") 49 | pages = paginator.paginate(cluster=data["clusterName"]) 50 | 51 | for services in pages: 52 | if len(services["serviceArns"]) > 0: 53 | service_details = client.describe_services( 54 | cluster=data["clusterName"], services=services["serviceArns"], 55 | ) 56 | 57 | for data_service_detail in service_details["services"]: 58 | if data_service_detail["launchType"] == "FARGATE": 59 | service_subnets = data_service_detail[ 60 | "networkConfiguration" 61 | ]["awsvpcConfiguration"]["subnets"] 62 | 63 | # Using subnet to check VPC 64 | subnets = describe_subnet( 65 | vpc_options=self.vpc_options, 66 | subnet_ids=service_subnets, 67 | ) 68 | 69 | if subnets is not None: 70 | # Iterate subnet to get VPC 71 | for data_subnet in subnets["Subnets"]: 72 | 73 | if data_subnet["VpcId"] == self.vpc_options.vpc_id: 74 | cluster_digest = ResourceDigest( 75 | id=data["clusterArn"], 76 | type="aws_ecs_cluster", 77 | ) 78 | resources_found.append( 79 | Resource( 80 | digest=cluster_digest, 81 | name=data["clusterName"], 82 | details="", 83 | group="container", 84 | tags=resource_tags(data), 85 | ) 86 | ) 87 | self.relations_found.append( 88 | ResourceEdge( 89 | from_node=cluster_digest, 90 | to_node=ResourceDigest( 91 | id=data_subnet["SubnetId"], 92 | type="aws_subnet", 93 | ), 94 | ) 95 | ) 96 | else: 97 | # EC2 services require container instances, list of them should be fine for now 98 | pass 99 | 100 | # Looking for container instances - they are dynamically associated, so manual review is necessary 101 | list_paginator = client.get_paginator("list_container_instances") 102 | list_pages = list_paginator.paginate(cluster=data["clusterName"]) 103 | for list_page in list_pages: 104 | if len(list_page["containerInstanceArns"]) == 0: 105 | continue 106 | 107 | container_instances = client.describe_container_instances( 108 | cluster=data["clusterName"], 109 | containerInstances=list_page["containerInstanceArns"], 110 | ) 111 | ec2_ids = [] 112 | for instance_details in container_instances["containerInstances"]: 113 | ec2_ids.append(instance_details["ec2InstanceId"]) 114 | paginator = ec2_client.get_paginator("describe_instances") 115 | pages = paginator.paginate(InstanceIds=ec2_ids) 116 | for page in pages: 117 | for reservation in page["Reservations"]: 118 | for instance in reservation["Instances"]: 119 | for network_interfaces in instance["NetworkInterfaces"]: 120 | if ( 121 | network_interfaces["VpcId"] 122 | == self.vpc_options.vpc_id 123 | ): 124 | cluster_instance_digest = ResourceDigest( 125 | id=instance["InstanceId"], 126 | type="aws_ecs_cluster", 127 | ) 128 | resources_found.append( 129 | Resource( 130 | digest=cluster_instance_digest, 131 | name=data["clusterName"], 132 | details="Instance in EC2 cluster", 133 | group="container", 134 | tags=resource_tags(data), 135 | ) 136 | ) 137 | self.relations_found.append( 138 | ResourceEdge( 139 | from_node=cluster_instance_digest, 140 | to_node=ResourceDigest( 141 | id=instance["InstanceId"], 142 | type="aws_instance", 143 | ), 144 | ) 145 | ) 146 | 147 | return resources_found 148 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/vpc/resource/enduser.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from provider.aws.common_aws import resource_tags, get_name_tag 4 | from provider.aws.vpc.command import VpcOptions 5 | from shared.common import ( 6 | ResourceProvider, 7 | Resource, 8 | message_handler, 9 | ResourceDigest, 10 | ResourceEdge, 11 | ResourceAvailable, 12 | ) 13 | from shared.error_handler import exception 14 | 15 | 16 | class WORKSPACES(ResourceProvider): 17 | def __init__(self, vpc_options: VpcOptions): 18 | """ 19 | Workspaces 20 | 21 | :param vpc_options: 22 | """ 23 | super().__init__() 24 | self.vpc_options = vpc_options 25 | 26 | @exception 27 | @ResourceAvailable(services="workspaces") 28 | def get_resources(self) -> List[Resource]: 29 | 30 | client = self.vpc_options.client("workspaces") 31 | 32 | resources_found = [] 33 | 34 | response = client.describe_workspaces() 35 | 36 | if self.vpc_options.verbose: 37 | message_handler("Collecting data from Workspaces...", "HEADER") 38 | 39 | for data in response["Workspaces"]: 40 | 41 | # Get tag name 42 | tags = client.describe_tags(ResourceId=data["WorkspaceId"]) 43 | nametag = get_name_tag(tags) 44 | 45 | workspace_name = data["WorkspaceId"] if nametag is None else nametag 46 | 47 | directory_service = self.vpc_options.client("ds") 48 | directories = directory_service.describe_directories( 49 | DirectoryIds=[data["DirectoryId"]] 50 | ) 51 | 52 | for directorie in directories["DirectoryDescriptions"]: 53 | 54 | if "VpcSettings" in directorie: 55 | 56 | if directorie["VpcSettings"]["VpcId"] == self.vpc_options.vpc_id: 57 | workspace_digest = ResourceDigest( 58 | id=data["WorkspaceId"], type="aws_workspaces" 59 | ) 60 | resources_found.append( 61 | Resource( 62 | digest=workspace_digest, 63 | name=workspace_name, 64 | details="", 65 | group="enduser", 66 | tags=resource_tags(tags), 67 | ) 68 | ) 69 | 70 | self.relations_found.append( 71 | ResourceEdge( 72 | from_node=workspace_digest, 73 | to_node=ResourceDigest( 74 | id=directorie["DirectoryId"], type="aws_ds" 75 | ), 76 | ) 77 | ) 78 | 79 | return resources_found 80 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/vpc/resource/identity.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from provider.aws.common_aws import resource_tags 4 | from provider.aws.vpc.command import VpcOptions 5 | from shared.common import ( 6 | ResourceProvider, 7 | Resource, 8 | message_handler, 9 | ResourceDigest, 10 | ResourceEdge, 11 | ResourceAvailable, 12 | ) 13 | from shared.error_handler import exception 14 | 15 | 16 | class DIRECTORYSERVICE(ResourceProvider): 17 | def __init__(self, vpc_options: VpcOptions): 18 | """ 19 | Directory service 20 | 21 | :param vpc_options: 22 | """ 23 | super().__init__() 24 | self.vpc_options = vpc_options 25 | 26 | @exception 27 | @ResourceAvailable(services="ds") 28 | def get_resources(self) -> List[Resource]: 29 | 30 | client = self.vpc_options.client("ds") 31 | 32 | resources_found = [] 33 | 34 | response = client.describe_directories() 35 | 36 | if self.vpc_options.verbose: 37 | message_handler("Collecting data from Directory Services...", "HEADER") 38 | 39 | for data in response["DirectoryDescriptions"]: 40 | 41 | if "VpcSettings" in data: 42 | 43 | if data["VpcSettings"]["VpcId"] == self.vpc_options.vpc_id: 44 | directory_service_digest = ResourceDigest( 45 | id=data["DirectoryId"], type="aws_ds" 46 | ) 47 | resources_found.append( 48 | Resource( 49 | digest=directory_service_digest, 50 | name=data["Name"], 51 | details="", 52 | group="identity", 53 | tags=resource_tags(data), 54 | ) 55 | ) 56 | 57 | for subnet in data["VpcSettings"]["SubnetIds"]: 58 | self.relations_found.append( 59 | ResourceEdge( 60 | from_node=directory_service_digest, 61 | to_node=ResourceDigest(id=subnet, type="aws_subnet"), 62 | ) 63 | ) 64 | 65 | return resources_found 66 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/vpc/resource/management.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from provider.aws.common_aws import resource_tags 4 | from provider.aws.vpc.command import VpcOptions 5 | from shared.common import ( 6 | ResourceProvider, 7 | Resource, 8 | message_handler, 9 | ResourceDigest, 10 | ResourceEdge, 11 | ResourceAvailable, 12 | ) 13 | from shared.error_handler import exception 14 | 15 | 16 | class SYNTHETICSCANARIES(ResourceProvider): 17 | def __init__(self, vpc_options: VpcOptions): 18 | """ 19 | Synthetic canaries 20 | 21 | :param vpc_options: 22 | """ 23 | super().__init__() 24 | self.vpc_options = vpc_options 25 | 26 | @exception 27 | @ResourceAvailable(services="synthetics") 28 | def get_resources(self) -> List[Resource]: 29 | 30 | client = self.vpc_options.client("synthetics") 31 | 32 | resources_found = [] 33 | 34 | response = client.describe_canaries() 35 | 36 | if self.vpc_options.verbose: 37 | message_handler("Collecting data from Synthetic Canaries...", "HEADER") 38 | 39 | for data in response["Canaries"]: 40 | 41 | # Check if VpcConfig is in dict 42 | if "VpcConfig" in data: 43 | 44 | if data["VpcConfig"]["VpcId"] == self.vpc_options.vpc_id: 45 | digest = ResourceDigest(id=data["Id"], type="aws_canaries_function") 46 | resources_found.append( 47 | Resource( 48 | digest=digest, 49 | name=data["Name"], 50 | details="", 51 | group="management", 52 | tags=resource_tags(data), 53 | ) 54 | ) 55 | for subnet_id in data["VpcConfig"]["SubnetIds"]: 56 | self.relations_found.append( 57 | ResourceEdge( 58 | from_node=digest, 59 | to_node=ResourceDigest(id=subnet_id, type="aws_subnet"), 60 | ) 61 | ) 62 | 63 | return resources_found 64 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/vpc/resource/mediaservices.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import List 3 | 4 | from provider.aws.common_aws import describe_subnet, resource_tags 5 | from provider.aws.vpc.command import VpcOptions, check_ipvpc_inpolicy 6 | from shared.common import ( 7 | ResourceProvider, 8 | Resource, 9 | message_handler, 10 | ResourceDigest, 11 | ResourceEdge, 12 | datetime_to_string, 13 | ResourceAvailable, 14 | ) 15 | from shared.error_handler import exception 16 | 17 | 18 | class MEDIACONNECT(ResourceProvider): 19 | def __init__(self, vpc_options: VpcOptions): 20 | """ 21 | Mediaconnect 22 | 23 | :param vpc_options: 24 | """ 25 | super().__init__() 26 | self.vpc_options = vpc_options 27 | 28 | @exception 29 | @ResourceAvailable(services="mediaconnect") 30 | def get_resources(self) -> List[Resource]: 31 | 32 | client = self.vpc_options.client("mediaconnect") 33 | 34 | resources_found = [] 35 | 36 | response = client.list_flows() 37 | 38 | if self.vpc_options.verbose: 39 | message_handler("Collecting data from Media Connect...", "HEADER") 40 | 41 | for data in response["Flows"]: 42 | tags_response = client.list_tags_for_resource(ResourceArn=data["FlowArn"]) 43 | 44 | data_flow = client.describe_flow(FlowArn=data["FlowArn"]) 45 | 46 | if "VpcInterfaces" in data_flow["Flow"]: 47 | 48 | for data_interfaces in data_flow["Flow"]["VpcInterfaces"]: 49 | 50 | # Using subnet to check VPC 51 | subnets = describe_subnet( 52 | vpc_options=self.vpc_options, 53 | subnet_ids=data_interfaces["SubnetId"], 54 | ) 55 | 56 | if subnets is not None: 57 | if subnets["Subnets"][0]["VpcId"] == self.vpc_options.vpc_id: 58 | digest = ResourceDigest( 59 | id=data["FlowArn"], type="aws_media_connect" 60 | ) 61 | resources_found.append( 62 | Resource( 63 | digest=digest, 64 | name=data["Name"], 65 | details="Flow using VPC {} in VPC Interface {}".format( 66 | self.vpc_options.vpc_id, data_interfaces["Name"] 67 | ), 68 | group="mediaservices", 69 | tags=resource_tags(tags_response), 70 | ) 71 | ) 72 | self.relations_found.append( 73 | ResourceEdge( 74 | from_node=digest, 75 | to_node=ResourceDigest( 76 | id=data_interfaces["SubnetId"], 77 | type="aws_subnet", 78 | ), 79 | ) 80 | ) 81 | 82 | return resources_found 83 | 84 | 85 | class MEDIALIVE(ResourceProvider): 86 | def __init__(self, vpc_options: VpcOptions): 87 | """ 88 | Medialive 89 | 90 | :param vpc_options: 91 | """ 92 | super().__init__() 93 | self.vpc_options = vpc_options 94 | 95 | @exception 96 | @ResourceAvailable(services="medialive") 97 | def get_resources(self) -> List[Resource]: 98 | 99 | client = self.vpc_options.client("medialive") 100 | 101 | resources_found = [] 102 | 103 | response = client.list_inputs() 104 | 105 | if self.vpc_options.verbose: 106 | message_handler("Collecting data from Media Live Inputs...", "HEADER") 107 | 108 | for data in response["Inputs"]: 109 | tags_response = client.list_tags_for_resource(ResourceArn=data["Arn"]) 110 | for destinations in data["Destinations"]: 111 | if "Vpc" in destinations: 112 | # describe networkinterface to get VpcId 113 | ec2 = self.vpc_options.client("ec2") 114 | 115 | eni = ec2.describe_network_interfaces( 116 | NetworkInterfaceIds=[destinations["Vpc"]["NetworkInterfaceId"]] 117 | ) 118 | 119 | if eni["NetworkInterfaces"][0]["VpcId"] == self.vpc_options.vpc_id: 120 | digest = ResourceDigest(id=data["Arn"], type="aws_media_live") 121 | resources_found.append( 122 | Resource( 123 | digest=digest, 124 | name="Input " + destinations["Ip"], 125 | details="", 126 | group="mediaservices", 127 | tags=resource_tags(tags_response), 128 | ) 129 | ) 130 | self.relations_found.append( 131 | ResourceEdge( 132 | from_node=digest, to_node=self.vpc_options.vpc_digest(), 133 | ) 134 | ) 135 | return resources_found 136 | 137 | 138 | class MEDIASTORE(ResourceProvider): 139 | def __init__(self, vpc_options: VpcOptions): 140 | """ 141 | Mediastore 142 | 143 | :param vpc_options: 144 | """ 145 | super().__init__() 146 | self.vpc_options = vpc_options 147 | 148 | @exception 149 | @ResourceAvailable(services="mediastore") 150 | def get_resources(self) -> List[Resource]: 151 | 152 | client = self.vpc_options.client("mediastore") 153 | 154 | resources_found = [] 155 | 156 | response = client.list_containers() 157 | 158 | if self.vpc_options.verbose: 159 | message_handler("Collecting data from Media Store...", "HEADER") 160 | 161 | for data in response["Containers"]: 162 | 163 | store_queue_policy = client.get_container_policy(ContainerName=data["Name"]) 164 | 165 | document = json.dumps( 166 | store_queue_policy["Policy"], default=datetime_to_string 167 | ) 168 | 169 | ipvpc_found = check_ipvpc_inpolicy( 170 | document=document, vpc_options=self.vpc_options 171 | ) 172 | 173 | if ipvpc_found is not False: 174 | tags_response = client.list_tags_for_resource(Resource=data["ARN"]) 175 | digest = ResourceDigest(id=data["ARN"], type="aws_mediastore_polocy") 176 | resources_found.append( 177 | Resource( 178 | digest=digest, 179 | name=data["Name"], 180 | details="", 181 | group="mediaservices", 182 | tags=resource_tags(tags_response), 183 | ) 184 | ) 185 | self.relations_found.append( 186 | ResourceEdge( 187 | from_node=digest, to_node=self.vpc_options.vpc_digest() 188 | ) 189 | ) 190 | return resources_found 191 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/vpc/resource/ml.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from provider.aws.common_aws import describe_subnet, resource_tags 4 | from provider.aws.vpc.command import VpcOptions 5 | from shared.common import ( 6 | ResourceProvider, 7 | Resource, 8 | message_handler, 9 | ResourceDigest, 10 | ResourceEdge, 11 | ResourceAvailable, 12 | ) 13 | from shared.error_handler import exception 14 | 15 | 16 | class SAGEMAKERNOTEBOOK(ResourceProvider): 17 | def __init__(self, vpc_options: VpcOptions): 18 | """ 19 | Sagemaker notebook instance 20 | 21 | :param vpc_options: 22 | """ 23 | super().__init__() 24 | self.vpc_options = vpc_options 25 | 26 | @exception 27 | @ResourceAvailable(services="sagemaker") 28 | def get_resources(self) -> List[Resource]: 29 | 30 | client = self.vpc_options.client("sagemaker") 31 | 32 | resources_found = [] 33 | 34 | response = client.list_notebook_instances() 35 | 36 | if self.vpc_options.verbose: 37 | message_handler( 38 | "Collecting data from Sagemaker Notebook instances...", "HEADER" 39 | ) 40 | 41 | for data in response["NotebookInstances"]: 42 | 43 | notebook_instance = client.describe_notebook_instance( 44 | NotebookInstanceName=data["NotebookInstanceName"] 45 | ) 46 | tags_response = client.list_tags(ResourceArn=data["NotebookInstanceArn"],) 47 | 48 | # Using subnet to check VPC 49 | subnets = describe_subnet( 50 | vpc_options=self.vpc_options, subnet_ids=notebook_instance["SubnetId"] 51 | ) 52 | 53 | if subnets is not None: 54 | if subnets["Subnets"][0]["VpcId"] == self.vpc_options.vpc_id: 55 | sagemaker_notebook_digest = ResourceDigest( 56 | id=data["NotebookInstanceArn"], 57 | type="aws_sagemaker_notebook_instance", 58 | ) 59 | resources_found.append( 60 | Resource( 61 | digest=sagemaker_notebook_digest, 62 | name=data["NotebookInstanceName"], 63 | details="", 64 | group="ml", 65 | tags=resource_tags(tags_response), 66 | ) 67 | ) 68 | 69 | self.relations_found.append( 70 | ResourceEdge( 71 | from_node=sagemaker_notebook_digest, 72 | to_node=ResourceDigest( 73 | id=notebook_instance["SubnetId"], type="aws_subnet" 74 | ), 75 | ) 76 | ) 77 | 78 | return resources_found 79 | 80 | 81 | class SAGEMAKERTRAININGOB(ResourceProvider): 82 | def __init__(self, vpc_options: VpcOptions): 83 | """ 84 | Sagemaker training job 85 | 86 | :param vpc_options: 87 | """ 88 | super().__init__() 89 | self.vpc_options = vpc_options 90 | 91 | @exception 92 | @ResourceAvailable(services="sagemaker") 93 | def get_resources(self) -> List[Resource]: 94 | 95 | client = self.vpc_options.client("sagemaker") 96 | 97 | resources_found = [] 98 | 99 | response = client.list_training_jobs() 100 | 101 | if self.vpc_options.verbose: 102 | message_handler("Collecting data from Sagemaker Training Job...", "HEADER") 103 | 104 | for data in response["TrainingJobSummaries"]: 105 | tags_response = client.list_tags(ResourceArn=data["TrainingJobArn"],) 106 | training_job = client.describe_training_job( 107 | TrainingJobName=data["TrainingJobName"] 108 | ) 109 | 110 | if "VpcConfig" in training_job: 111 | 112 | for subnets in training_job["VpcConfig"]["Subnets"]: 113 | 114 | # Using subnet to check VPC 115 | subnet = describe_subnet( 116 | vpc_options=self.vpc_options, subnet_ids=subnets 117 | ) 118 | 119 | if subnet is not None: 120 | 121 | if subnet["Subnets"][0]["VpcId"] == self.vpc_options.vpc_id: 122 | 123 | sagemaker_trainingjob_digest = ResourceDigest( 124 | id=data["TrainingJobArn"], 125 | type="aws_sagemaker_training_job", 126 | ) 127 | resources_found.append( 128 | Resource( 129 | digest=sagemaker_trainingjob_digest, 130 | name=data["TrainingJobName"], 131 | details="", 132 | group="ml", 133 | tags=resource_tags(tags_response), 134 | ) 135 | ) 136 | 137 | self.relations_found.append( 138 | ResourceEdge( 139 | from_node=sagemaker_trainingjob_digest, 140 | to_node=ResourceDigest( 141 | id=subnets, type="aws_subnet" 142 | ), 143 | ) 144 | ) 145 | 146 | return resources_found 147 | 148 | 149 | class SAGEMAKERMODEL(ResourceProvider): 150 | def __init__(self, vpc_options: VpcOptions): 151 | """ 152 | Sagemaker model 153 | 154 | :param vpc_options: 155 | """ 156 | super().__init__() 157 | self.vpc_options = vpc_options 158 | 159 | @exception 160 | @ResourceAvailable(services="sagemaker") 161 | def get_resources(self) -> List[Resource]: 162 | 163 | client = self.vpc_options.client("sagemaker") 164 | 165 | resources_found = [] 166 | 167 | response = client.list_models() 168 | 169 | if self.vpc_options.verbose: 170 | message_handler("Collecting data from Sagemaker Model...", "HEADER") 171 | 172 | for data in response["Models"]: 173 | tags_response = client.list_tags(ResourceArn=data["ModelArn"],) 174 | model = client.describe_training_job(TrainingJobName=data["ModelName"]) 175 | 176 | if "VpcConfig" in model: 177 | 178 | for subnets in model["VpcConfig"]["Subnets"]: 179 | 180 | # Using subnet to check VPC 181 | subnet = describe_subnet( 182 | vpc_options=self.vpc_options, subnet_ids=subnets 183 | ) 184 | 185 | if subnet is not None: 186 | 187 | if subnet["Subnets"][0]["VpcId"] == self.vpc_options.vpc_id: 188 | 189 | sagemaker_model_digest = ResourceDigest( 190 | id=data["ModelArn"], type="aws_sagemaker_model", 191 | ) 192 | resources_found.append( 193 | Resource( 194 | digest=sagemaker_model_digest, 195 | name=data["ModelName"], 196 | details="", 197 | group="ml", 198 | tags=resource_tags(tags_response), 199 | ) 200 | ) 201 | 202 | self.relations_found.append( 203 | ResourceEdge( 204 | from_node=sagemaker_model_digest, 205 | to_node=ResourceDigest( 206 | id=subnets, type="aws_subnet" 207 | ), 208 | ) 209 | ) 210 | 211 | return resources_found 212 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/vpc/resource/security.py: -------------------------------------------------------------------------------- 1 | import json 2 | from concurrent.futures.thread import ThreadPoolExecutor 3 | from typing import List 4 | 5 | from provider.aws.common_aws import resource_tags 6 | from provider.aws.vpc.command import VpcOptions, check_ipvpc_inpolicy 7 | from shared.common import ( 8 | ResourceProvider, 9 | Resource, 10 | message_handler, 11 | ResourceDigest, 12 | ResourceEdge, 13 | datetime_to_string, 14 | ResourceAvailable, 15 | ) 16 | from shared.error_handler import exception 17 | 18 | 19 | class IAMPOLICY(ResourceProvider): 20 | def __init__(self, vpc_options: VpcOptions): 21 | """ 22 | Iam policy 23 | 24 | :param vpc_options: 25 | """ 26 | super().__init__() 27 | self.vpc_options = vpc_options 28 | 29 | @exception 30 | @ResourceAvailable(services="iam") 31 | def get_resources(self) -> List[Resource]: 32 | 33 | client = self.vpc_options.client("iam") 34 | 35 | resources_found = [] 36 | 37 | if self.vpc_options.verbose: 38 | message_handler("Collecting data from IAM Policies...", "HEADER") 39 | paginator = client.get_paginator("list_policies") 40 | pages = paginator.paginate(Scope="Local") 41 | for policies in pages: 42 | with ThreadPoolExecutor(15) as executor: 43 | results = executor.map( 44 | lambda data: self.analyze_policy(client, data), policies["Policies"] 45 | ) 46 | for result in results: 47 | if result[0] is True: 48 | resources_found.append(result[1]) 49 | 50 | return resources_found 51 | 52 | def analyze_policy(self, client, data): 53 | 54 | documentpolicy = client.get_policy_version( 55 | PolicyArn=data["Arn"], VersionId=data["DefaultVersionId"] 56 | ) 57 | 58 | document = json.dumps(documentpolicy, default=datetime_to_string) 59 | 60 | # check either vpc_id or potential subnet ip are found 61 | ipvpc_found = check_ipvpc_inpolicy( 62 | document=document, vpc_options=self.vpc_options 63 | ) 64 | 65 | if ipvpc_found is True: 66 | digest = ResourceDigest(id=data["Arn"], type="aws_iam_policy") 67 | self.relations_found.append( 68 | ResourceEdge(from_node=digest, to_node=self.vpc_options.vpc_digest()) 69 | ) 70 | return ( 71 | True, 72 | Resource( 73 | digest=digest, 74 | name=data["PolicyName"], 75 | details="IAM Policy version {}".format(data["DefaultVersionId"]), 76 | group="security", 77 | ), 78 | ) 79 | 80 | return False, None 81 | 82 | 83 | class CLOUDHSM(ResourceProvider): 84 | def __init__(self, vpc_options: VpcOptions): 85 | """ 86 | Cloud HSM 87 | 88 | :param vpc_options: 89 | """ 90 | super().__init__() 91 | self.vpc_options = vpc_options 92 | 93 | @exception 94 | @ResourceAvailable(services="cloudhsmv2") 95 | def get_resources(self) -> List[Resource]: 96 | 97 | client = self.vpc_options.client("cloudhsmv2") 98 | 99 | resources_found = [] 100 | 101 | response = client.describe_clusters() 102 | 103 | if self.vpc_options.verbose: 104 | message_handler("Collecting data from CloudHSM clusters...", "HEADER") 105 | 106 | for data in response["Clusters"]: 107 | 108 | if data["VpcId"] == self.vpc_options.vpc_id: 109 | cloudhsm_digest = ResourceDigest( 110 | id=data["ClusterId"], type="aws_cloudhsm" 111 | ) 112 | resources_found.append( 113 | Resource( 114 | digest=cloudhsm_digest, 115 | name=data["ClusterId"], 116 | details="", 117 | group="security", 118 | tags=resource_tags(data), 119 | ) 120 | ) 121 | 122 | for subnet in data["SubnetMapping"]: 123 | subnet_id = data["SubnetMapping"][subnet] 124 | self.relations_found.append( 125 | ResourceEdge( 126 | from_node=cloudhsm_digest, 127 | to_node=ResourceDigest(id=subnet_id, type="aws_subnet"), 128 | ) 129 | ) 130 | 131 | return resources_found 132 | -------------------------------------------------------------------------------- /cloudiscovery/provider/aws/vpc/resource/storage.py: -------------------------------------------------------------------------------- 1 | import json 2 | from concurrent.futures.thread import ThreadPoolExecutor 3 | from typing import List 4 | 5 | from botocore.exceptions import ClientError 6 | 7 | from provider.aws.common_aws import describe_subnet, resource_tags, get_name_tag 8 | from provider.aws.vpc.command import VpcOptions, check_ipvpc_inpolicy 9 | from shared.common import ( 10 | ResourceProvider, 11 | Resource, 12 | message_handler, 13 | ResourceDigest, 14 | ResourceEdge, 15 | datetime_to_string, 16 | ResourceAvailable, 17 | ) 18 | from shared.error_handler import exception 19 | 20 | 21 | class EFS(ResourceProvider): 22 | def __init__(self, vpc_options: VpcOptions): 23 | """ 24 | Efs 25 | 26 | :param vpc_options: 27 | """ 28 | super().__init__() 29 | self.vpc_options = vpc_options 30 | 31 | @exception 32 | @ResourceAvailable(services="efs") 33 | def get_resources(self) -> List[Resource]: 34 | 35 | client = self.vpc_options.client("efs") 36 | 37 | resources_found = [] 38 | 39 | # get filesystems available 40 | response = client.describe_file_systems() 41 | 42 | if self.vpc_options.verbose: 43 | message_handler("Collecting data from EFS Mount Targets...", "HEADER") 44 | 45 | for data in response["FileSystems"]: 46 | 47 | filesystem = client.describe_mount_targets( 48 | FileSystemId=data["FileSystemId"] 49 | ) 50 | 51 | nametag = get_name_tag(data) 52 | filesystem_name = data["FileSystemId"] if nametag is None else nametag 53 | 54 | # iterate filesystems to get mount targets 55 | for datafilesystem in filesystem["MountTargets"]: 56 | 57 | # Using subnet to check VPC 58 | subnets = describe_subnet( 59 | vpc_options=self.vpc_options, subnet_ids=datafilesystem["SubnetId"] 60 | ) 61 | 62 | if subnets is not None: 63 | if subnets["Subnets"][0]["VpcId"] == self.vpc_options.vpc_id: 64 | digest = ResourceDigest( 65 | id=data["FileSystemId"], type="aws_efs_file_system" 66 | ) 67 | resources_found.append( 68 | Resource( 69 | digest=digest, 70 | name=filesystem_name, 71 | details="", 72 | group="storage", 73 | tags=resource_tags(data), 74 | ) 75 | ) 76 | self.relations_found.append( 77 | ResourceEdge( 78 | from_node=digest, 79 | to_node=ResourceDigest( 80 | id=datafilesystem["SubnetId"], type="aws_subnet" 81 | ), 82 | ) 83 | ) 84 | 85 | return resources_found 86 | 87 | 88 | class S3POLICY(ResourceProvider): 89 | def __init__(self, vpc_options: VpcOptions): 90 | """ 91 | S3 policy 92 | 93 | :param vpc_options: 94 | """ 95 | super().__init__() 96 | self.vpc_options = vpc_options 97 | 98 | @exception 99 | @ResourceAvailable(services="s3") 100 | def get_resources(self) -> List[Resource]: 101 | 102 | client = self.vpc_options.client("s3") 103 | 104 | resources_found: List[Resource] = [] 105 | 106 | # get buckets available 107 | response = client.list_buckets() 108 | 109 | if self.vpc_options.verbose: 110 | message_handler("Collecting data from S3 Bucket Policies...", "HEADER") 111 | 112 | with ThreadPoolExecutor(15) as executor: 113 | results = executor.map( 114 | lambda data: self.analyze_bucket(client, data), response["Buckets"] 115 | ) 116 | 117 | for result in results: 118 | if result[0] is True: 119 | resources_found.append(result[1]) 120 | 121 | return resources_found 122 | 123 | def analyze_bucket(self, client, data): 124 | try: 125 | documentpolicy = client.get_bucket_policy(Bucket=data["Name"]) 126 | except ClientError: 127 | return False, None 128 | 129 | document = json.dumps(documentpolicy, default=datetime_to_string) 130 | 131 | # check either vpc_id or potential subnet ip are found 132 | ipvpc_found = check_ipvpc_inpolicy( 133 | document=document, vpc_options=self.vpc_options 134 | ) 135 | 136 | if ipvpc_found is True: 137 | tags_response = client.get_bucket_tagging(Bucket=data["Name"]) 138 | digest = ResourceDigest(id=data["Name"], type="aws_s3_bucket_policy") 139 | self.relations_found.append( 140 | ResourceEdge(from_node=digest, to_node=self.vpc_options.vpc_digest()) 141 | ) 142 | return ( 143 | True, 144 | Resource( 145 | digest=digest, 146 | name=data["Name"], 147 | details="", 148 | group="storage", 149 | tags=resource_tags(tags_response), 150 | ), 151 | ) 152 | return False, None 153 | -------------------------------------------------------------------------------- /cloudiscovery/shared/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/cloudiscovery/shared/__init__.py -------------------------------------------------------------------------------- /cloudiscovery/shared/command.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import inspect 3 | from concurrent.futures.thread import ThreadPoolExecutor 4 | from os.path import dirname 5 | from typing import List, Dict 6 | import os 7 | 8 | from shared.common import ( 9 | Resource, 10 | ResourceEdge, 11 | Filterable, 12 | BaseOptions, 13 | message_handler, 14 | ResourceProvider, 15 | ResourceDigest, 16 | ) 17 | from shared.diagram import BaseDiagram 18 | from shared.report import Report 19 | 20 | 21 | class CommandRunner(object): 22 | def __init__(self, provider_name: str, filters: List[Filterable] = None): 23 | """ 24 | Base class command execution 25 | 26 | :param provider_name: 27 | :param filters: 28 | """ 29 | self.provider_name: str = provider_name 30 | self.filters: List[Filterable] = filters 31 | 32 | # pylint: disable=too-many-locals,too-many-arguments 33 | def run( 34 | self, 35 | provider: str, 36 | options: BaseOptions, 37 | diagram_builder: BaseDiagram, 38 | title: str, 39 | filename: str, 40 | ): 41 | """ 42 | Executes a command. 43 | 44 | The project's development pattern is a file with the respective name of the parent 45 | resource (e.g. compute, network), classes of child resources inside this file and run() method to execute 46 | respective check. So it makes sense to load dynamically. 47 | """ 48 | # Iterate to get all modules 49 | message_handler("\nInspecting resources", "HEADER") 50 | providers = [] 51 | for name in os.listdir( 52 | dirname(__file__) 53 | + "/../provider/" 54 | + self.provider_name 55 | + "/" 56 | + provider 57 | + "/resource" 58 | ): 59 | if name.endswith(".py"): 60 | # strip the extension 61 | module = name[:-3] 62 | 63 | # Load and call all run check 64 | for nameclass, cls in inspect.getmembers( 65 | importlib.import_module( 66 | "provider.aws." 67 | + provider.replace("/", ".") 68 | + ".resource." 69 | + module 70 | ), 71 | inspect.isclass, 72 | ): 73 | if ( 74 | issubclass(cls, ResourceProvider) 75 | and cls is not ResourceProvider 76 | ): 77 | providers.append((nameclass, cls)) 78 | providers.sort(key=lambda x: x[0]) 79 | 80 | all_resources: List[Resource] = [] 81 | resource_relations: List[ResourceEdge] = [] 82 | 83 | with ThreadPoolExecutor(15) as executor: 84 | provider_results = executor.map( 85 | lambda data: execute_provider(options, data), providers 86 | ) 87 | 88 | for provider_result in provider_results: 89 | if provider_result[0] is not None: 90 | all_resources.extend(provider_result[0]) 91 | if provider_result[1] is not None: 92 | resource_relations.extend(provider_result[1]) 93 | 94 | unique_resources_dict: Dict[ResourceDigest, Resource] = dict() 95 | for resource in all_resources: 96 | unique_resources_dict[resource.digest] = resource 97 | 98 | unique_resources = list(unique_resources_dict.values()) 99 | 100 | unique_resources.sort(key=lambda x: x.group + x.digest.type + x.name) 101 | resource_relations.sort( 102 | key=lambda x: x.from_node.type 103 | + x.from_node.id 104 | + x.to_node.type 105 | + x.to_node.id 106 | ) 107 | 108 | # Resource filtering and sorting 109 | filtered_resources = filter_resources(unique_resources, self.filters) 110 | filtered_resources.sort(key=lambda x: x.group + x.digest.type + x.name) 111 | 112 | # Relationships filtering and sorting 113 | filtered_relations = filter_relations(filtered_resources, resource_relations) 114 | filtered_relations.sort( 115 | key=lambda x: x.from_node.type 116 | + x.from_node.id 117 | + x.to_node.type 118 | + x.to_node.id 119 | ) 120 | 121 | # Diagram integration 122 | diagram_builder.build( 123 | resources=filtered_resources, 124 | resource_relations=filtered_relations, 125 | title=title, 126 | filename=filename, 127 | ) 128 | 129 | # TODO: Generate reports in json/csv/pdf/xls 130 | report = Report() 131 | report.general_report( 132 | resources=filtered_resources, resource_relations=filtered_relations 133 | ) 134 | report.html_report( 135 | resources=filtered_resources, 136 | resource_relations=filtered_relations, 137 | title=title, 138 | filename=filename, 139 | ) 140 | 141 | # TODO: Export in csv/json/yaml/tf... future... 142 | # ....exporttf(checks).... 143 | 144 | 145 | def execute_provider(options, data) -> (List[Resource], List[ResourceEdge]): 146 | provider_instance = data[1](options) 147 | provider_resources = provider_instance.get_resources() 148 | provider_resource_relations = provider_instance.get_relations() 149 | return provider_resources, provider_resource_relations 150 | 151 | 152 | def filter_resources( 153 | resources: List[Resource], filters: List[Filterable] 154 | ) -> List[Resource]: 155 | if not filters: 156 | return resources 157 | 158 | filtered_resources = [] 159 | for resource in resources: 160 | matches_filter = False 161 | for resource_filter in filters: 162 | if resource_filter.is_tag(): 163 | for resource_tag in resource.tags: 164 | if ( 165 | resource_tag.key == resource_filter.key 166 | and resource_tag.value == resource_filter.value 167 | ): 168 | matches_filter = True 169 | elif resource_filter.is_type(): 170 | if resource.digest.type == resource_filter.type: 171 | matches_filter = True 172 | if matches_filter: 173 | filtered_resources.append(resource) 174 | return filtered_resources 175 | 176 | 177 | def filter_relations( 178 | filtered_resources: List[Resource], resource_relations: List[ResourceEdge] 179 | ): 180 | filtered_relations: List[ResourceEdge] = [] 181 | 182 | for resource_relation in resource_relations: 183 | is_from_present = False 184 | is_to_present = False 185 | for resource in filtered_resources: 186 | if resource_relation.from_node == resource.digest: 187 | is_from_present = True 188 | if resource_relation.to_node == resource.digest: 189 | is_to_present = True 190 | if is_from_present and is_to_present: 191 | filtered_relations.append(resource_relation) 192 | return filtered_relations 193 | -------------------------------------------------------------------------------- /cloudiscovery/shared/common.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import functools 3 | import os.path 4 | import re 5 | import threading 6 | from abc import ABC 7 | from typing import NamedTuple, List, Dict 8 | 9 | from diskcache import Cache 10 | 11 | VPCE_REGEX = re.compile(r'(?<=sourcevpce")(\s*:\s*")(vpce-[a-zA-Z0-9]+)', re.DOTALL) 12 | SOURCE_IP_ADDRESS_REGEX = re.compile( 13 | r'(?<=sourceip")(\s*:\s*")([a-fA-F0-9.:/%]+)', re.DOTALL 14 | ) 15 | FILTER_NAME_PREFIX = "Name=" 16 | FILTER_TAG_NAME_PREFIX = "tags." 17 | FILTER_TYPE_NAME = "type" 18 | FILTER_VALUE_PREFIX = "Value=" 19 | 20 | _LOG_SEMAPHORE = threading.Semaphore() 21 | 22 | 23 | class bcolors: 24 | colors = { 25 | "HEADER": "\033[95m", 26 | "OKBLUE": "\033[94m", 27 | "OKGREEN": "\033[92m", 28 | "WARNING": "\033[93m", 29 | "FAIL": "\033[91m", 30 | "ENDC": "\033[0m", 31 | "BOLD": "\033[1m", 32 | "UNDERLINE": "\033[4m", 33 | } 34 | 35 | 36 | class ResourceDigest(NamedTuple): 37 | id: str 38 | type: str 39 | 40 | def to_string(self): 41 | return f"{self.type}:{self.id}" 42 | 43 | 44 | class ResourceEdge(NamedTuple): 45 | from_node: ResourceDigest 46 | to_node: ResourceDigest 47 | label: str = None 48 | 49 | 50 | # Either key/value is passed or type 51 | class Filterable(NamedTuple): 52 | key: str = None 53 | value: str = None 54 | type: str = None 55 | 56 | def is_type(self): 57 | return self.type is not None 58 | 59 | def is_tag(self): 60 | return self.key is not None and self.value is not None 61 | 62 | 63 | class LimitsValues(NamedTuple): 64 | service: str 65 | quota_name: str 66 | quota_code: str 67 | aws_limit: int 68 | local_limit: int 69 | usage: float 70 | percent: float 71 | 72 | 73 | class SecurityValues(NamedTuple): 74 | status: str 75 | parameter: str 76 | value: str 77 | 78 | 79 | class Resource(NamedTuple): 80 | digest: ResourceDigest 81 | name: str = "" 82 | details: str = "" 83 | group: str = "" 84 | tags: List[Filterable] = [] 85 | limits: LimitsValues = None 86 | security: SecurityValues = None 87 | attributes: Dict[str, object] = {} 88 | 89 | 90 | class ResourceCache: 91 | def __init__(self): 92 | self.cache = Cache( 93 | directory=os.path.dirname(os.path.abspath(__file__)) 94 | + "/../../assets/.cache/" 95 | ) 96 | 97 | def set_key(self, key: str, value: object, expire: int): 98 | self.cache.set(key=key, value=value, expire=expire) 99 | 100 | def get_key(self, key: str): 101 | if key in self.cache: 102 | return self.cache[key] 103 | 104 | return None 105 | 106 | 107 | # Decorator to check services. 108 | class ResourceAvailable(object): 109 | def __init__(self, services): 110 | self.services = services 111 | self.cache = ResourceCache() 112 | 113 | def is_service_available(self, region_name, service_name) -> bool: 114 | cache_key = "aws_paths_" + region_name 115 | cache = self.cache.get_key(cache_key) 116 | return service_name in cache 117 | 118 | def __call__(self, func): 119 | @functools.wraps(func) 120 | def wrapper(*args, **kwargs): 121 | 122 | if "vpc_options" in dir(args[0]): 123 | region_name = args[0].vpc_options.region_name 124 | elif "iot_options" in dir(args[0]): 125 | region_name = args[0].iot_options.region_name 126 | else: 127 | region_name = "us-east-1" 128 | 129 | if self.is_service_available(region_name, self.services): 130 | return func(*args, **kwargs) 131 | 132 | verbose = False 133 | if "vpc_options" in dir(args[0]): 134 | verbose = args[0].vpc_options.verbose 135 | elif "iot_options" in dir(args[0]): 136 | verbose = args[0].iot_options.verbose 137 | elif "options" in dir(args[0]): 138 | verbose = args[0].options.verbose 139 | 140 | if verbose: 141 | message_handler( 142 | "Check " 143 | + func.__qualname__ 144 | + " not available in this region... Skipping", 145 | "WARNING", 146 | ) 147 | 148 | return None 149 | 150 | return wrapper 151 | 152 | 153 | class ResourceProvider: 154 | def __init__(self): 155 | """ 156 | Base provider class that provides resources and relationships. 157 | 158 | The class should be implemented to return resources of the same type 159 | """ 160 | self.relations_found: List[ResourceEdge] = [] 161 | 162 | def get_resources(self) -> List[Resource]: 163 | return [] 164 | 165 | def get_relations(self) -> List[ResourceEdge]: 166 | return self.relations_found 167 | 168 | 169 | def exit_critical(message): 170 | log_critical(message) 171 | raise SystemExit 172 | 173 | 174 | def log_critical(message): 175 | message_handler(message, "FAIL") 176 | 177 | 178 | def message_handler(message, position): 179 | _LOG_SEMAPHORE.acquire() 180 | print(bcolors.colors.get(position), message, bcolors.colors.get("ENDC"), sep="") 181 | _LOG_SEMAPHORE.release() 182 | 183 | 184 | # pylint: disable=inconsistent-return-statements 185 | def datetime_to_string(o): 186 | if isinstance(o, datetime.datetime): 187 | return o.__str__() 188 | 189 | 190 | def _add_filter(filters: List[Filterable], is_tag: bool, full_name: str, value: str): 191 | if is_tag: 192 | name = full_name[len(FILTER_TAG_NAME_PREFIX) :] 193 | filters.append(Filterable(key=name, value=value)) 194 | else: 195 | filters.append(Filterable(type=value)) 196 | 197 | 198 | def parse_filters(arg_filters) -> List[Filterable]: 199 | filters: List[Filterable] = [] 200 | for arg_filter in arg_filters: 201 | filter_parts = arg_filter.split(";") 202 | if len(filter_parts) != 2: 203 | continue 204 | if not filter_parts[0].startswith(FILTER_NAME_PREFIX): 205 | continue 206 | if not filter_parts[1].startswith(FILTER_VALUE_PREFIX): 207 | continue 208 | full_name = filter_parts[0][len(FILTER_NAME_PREFIX) :] 209 | is_tag = False 210 | if full_name.startswith(FILTER_TAG_NAME_PREFIX): 211 | is_tag = True 212 | elif full_name != FILTER_TYPE_NAME: 213 | continue 214 | values = filter_parts[1][len(FILTER_VALUE_PREFIX) :] 215 | 216 | val_buffered = True 217 | wrapped = False 218 | val_buffer = [] 219 | for character in values: 220 | # pylint: disable=no-else-continue 221 | if character == "'": 222 | wrapped = not wrapped 223 | continue 224 | # pylint: disable=no-else-continue 225 | elif character == ":": 226 | if len(val_buffer) == 0: 227 | continue 228 | elif val_buffered and not wrapped: 229 | _add_filter(filters, is_tag, full_name, "".join(val_buffer)) 230 | val_buffered = False 231 | val_buffer = [] 232 | continue 233 | val_buffered = True 234 | val_buffer.append(character) 235 | if len(val_buffer) > 0: 236 | _add_filter(filters, is_tag, full_name, "".join(val_buffer)) 237 | 238 | return filters 239 | 240 | 241 | class BaseCommand(ABC): 242 | def run( 243 | self, 244 | diagram: bool, 245 | verbose: bool, 246 | services: List[str], 247 | filters: List[Filterable], 248 | ): 249 | raise NotImplementedError() 250 | 251 | 252 | class Object(object): 253 | pass 254 | 255 | 256 | class BaseOptions(Object): 257 | verbose: bool 258 | filters: List[Filterable] 259 | 260 | def __init__(self, verbose: bool, filters: List[Filterable]): 261 | self.verbose = verbose 262 | self.filters = filters 263 | -------------------------------------------------------------------------------- /cloudiscovery/shared/error_handler.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import platform 3 | import traceback 4 | import sys 5 | 6 | from shared.common import log_critical 7 | 8 | 9 | # Decorator to catch exceptions and avoid stop script. 10 | 11 | 12 | def exception(func): 13 | @functools.wraps(func) 14 | def wrapper(*args, **kwargs): 15 | # pylint: disable=import-outside-toplevel 16 | from botocore.client import ClientError 17 | from botocore.exceptions import UnknownServiceError 18 | from boto3 import __version__ as boto3_version 19 | 20 | try: 21 | return func(*args, **kwargs) 22 | 23 | except ClientError as e: 24 | exception_str = str(e) 25 | if ( 26 | "Could not connect to the endpoint URL" in exception_str 27 | or "the specified service does not exist" in exception_str 28 | ): 29 | message = "\nThe service {} is not available in this region".format( 30 | func.__qualname__ 31 | ) 32 | else: 33 | message = "\nError running check {}. Error message {}".format( 34 | func.__qualname__, exception_str 35 | ) 36 | log_critical(message) 37 | 38 | except UnknownServiceError: 39 | log_critical("You're running a possible out of date boto3 version.") 40 | log_critical("Please update boto3 to last version.") 41 | 42 | issue_info = "\n".join( 43 | ( 44 | "Python: {0}".format(sys.version), 45 | "boto3 version: {0}".format(boto3_version), 46 | "Platform: {0}".format(platform.platform()), 47 | "", 48 | traceback.format_exc(), 49 | ) 50 | ) 51 | log_critical(issue_info) 52 | 53 | except Exception: # pylint: disable=broad-except 54 | log_critical("You've found a bug! Please, open an issue in GitHub project") 55 | log_critical("https://github.com/Cloud-Architects/cloudiscovery/issues\n") 56 | 57 | issue_info = "\n".join( 58 | ( 59 | "Python: {0}".format(sys.version), 60 | "boto3 version: {0}".format(boto3_version), 61 | "Platform: {0}".format(platform.platform()), 62 | "", 63 | traceback.format_exc(), 64 | ) 65 | ) 66 | log_critical(issue_info) 67 | 68 | return wrapper 69 | -------------------------------------------------------------------------------- /cloudiscovery/shared/parameters.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | 4 | def str2bool(v): 5 | if isinstance(v, bool): 6 | return v 7 | # pylint: disable=no-else-return 8 | if v.lower() in ("yes", "true", "t", "y", "1"): 9 | return True 10 | elif v.lower() in ("no", "false", "f", "n", "0"): 11 | return False 12 | else: 13 | raise argparse.ArgumentTypeError("Boolean value expected.") 14 | 15 | 16 | def generate_parser(): 17 | parser = argparse.ArgumentParser() 18 | 19 | subparsers = parser.add_subparsers(help="commands", dest="command") 20 | 21 | vpc_parser = subparsers.add_parser("aws-vpc", help="Analyze VPCs") 22 | add_default_arguments(vpc_parser) 23 | vpc_parser.add_argument( 24 | "-v", 25 | "--vpc-id", 26 | required=False, 27 | help="Inform VPC to analyze. If not informed, script will check all vpcs.", 28 | ) 29 | 30 | iot_parser = subparsers.add_parser("aws-iot", help="Analyze IoTs") 31 | add_default_arguments(iot_parser) 32 | iot_parser.add_argument( 33 | "-t", 34 | "--thing-name", 35 | required=False, 36 | help="Inform Thing Name to analyze. If not informed, script will check all things inside a region.", 37 | ) 38 | 39 | policy_parser = subparsers.add_parser("aws-policy", help="Analyze policies") 40 | add_default_arguments(policy_parser, is_global=True) 41 | 42 | all_parser = subparsers.add_parser("aws-all", help="Analyze all resources") 43 | add_default_arguments(all_parser, diagram_enabled=False) 44 | add_services_argument(all_parser) 45 | 46 | limit_parser = subparsers.add_parser( 47 | "aws-limit", help="Analyze aws limit resources." 48 | ) 49 | add_default_arguments(limit_parser, diagram_enabled=False, filters_enabled=False) 50 | add_services_argument(limit_parser) 51 | limit_parser.add_argument( 52 | "-t", 53 | "--threshold", 54 | required=False, 55 | help="Select the %% of resource threshold between 0 and 100. \ 56 | For example: --threshold 50 will report all resources with more than 50%% threshold.", 57 | ) 58 | 59 | security_parser = subparsers.add_parser( 60 | "aws-security", help="Analyze aws several security checks." 61 | ) 62 | add_default_arguments(security_parser, diagram_enabled=False, filters_enabled=False) 63 | security_parser.add_argument( 64 | "-c", 65 | "--commands", 66 | action="append", 67 | required=False, 68 | help='Select the security check command that you want to run. \ 69 | To see available commands, please type "-c list". \ 70 | If not passed, command will check all services.', 71 | ) 72 | 73 | return parser 74 | 75 | 76 | def add_services_argument(limit_parser): 77 | limit_parser.add_argument( 78 | "-s", 79 | "--services", 80 | required=False, 81 | help='Define services that you want to check, use "," (comma) to separate multiple names. \ 82 | If not passed, command will check all services.', 83 | ) 84 | 85 | 86 | def add_default_arguments( 87 | parser, is_global=False, diagram_enabled=True, filters_enabled=True 88 | ): 89 | if not is_global: 90 | parser.add_argument( 91 | "-r", 92 | "--region-name", 93 | required=False, 94 | help='Inform REGION NAME to analyze or "all" to check on all regions. \ 95 | If not informed, try to get from config file', 96 | ) 97 | parser.add_argument( 98 | "-p", "--profile-name", required=False, help="Profile to be used" 99 | ) 100 | parser.add_argument( 101 | "-l", "--language", required=False, help="Available languages: pt_BR, en_US" 102 | ) 103 | parser.add_argument( 104 | "--verbose", 105 | "--verbose", 106 | type=str2bool, 107 | nargs="?", 108 | const=True, 109 | default=False, 110 | help="Enable debug mode to sdk calls (default false)", 111 | ) 112 | if filters_enabled: 113 | parser.add_argument( 114 | "-f", 115 | "--filters", 116 | action="append", 117 | required=False, 118 | help="filter resources (tags only for now, you must specify name and values); multiple filters " 119 | "are possible to pass with -f -f approach, values can be separated by : sign; " 120 | "example: Name=tags.costCenter;Value=20000:'20001:1'", 121 | ) 122 | if diagram_enabled: 123 | parser.add_argument( 124 | "-d", 125 | "--diagram", 126 | type=str2bool, 127 | nargs="?", 128 | const=True, 129 | default=True, 130 | help="print diagram with resources (need Graphviz installed). Pass true/y[es] to " 131 | "view image or false/n[o] not to generate image. Default true", 132 | ) 133 | -------------------------------------------------------------------------------- /cloudiscovery/shared/report.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import os 3 | import os.path 4 | from pathlib import Path 5 | from typing import List 6 | 7 | from jinja2 import Environment, FileSystemLoader 8 | 9 | from shared.common import Resource, ResourceEdge, message_handler 10 | from shared.diagram import PATH_DIAGRAM_OUTPUT 11 | from shared.error_handler import exception 12 | 13 | PATH_REPORT_HTML_OUTPUT = "./assets/html_report/" 14 | 15 | 16 | class Report(object): 17 | @staticmethod 18 | def make_directories(): 19 | Path(PATH_REPORT_HTML_OUTPUT).mkdir(parents=True, exist_ok=True) 20 | 21 | @exception 22 | def general_report( 23 | self, resources: List[Resource], resource_relations: List[ResourceEdge] 24 | ): 25 | 26 | message_handler("\n\nFound resources", "HEADER") 27 | 28 | for resource in resources: 29 | # Report to limit 30 | if resource.limits: 31 | usage = ( 32 | str(resource.limits.usage) 33 | + " - " 34 | + str(resource.limits.percent) 35 | + "%" 36 | ) 37 | # pylint: disable=line-too-long 38 | message_handler( 39 | "service: {} - quota code: {} - quota name: {} - aws default quota: {} - applied quota: {} - usage: {}".format( # noqa: E501 40 | resource.limits.service, 41 | resource.limits.quota_code, 42 | resource.limits.quota_name, 43 | resource.limits.aws_limit, 44 | resource.limits.local_limit, 45 | usage, 46 | ), 47 | "OKBLUE", 48 | ) 49 | elif resource.attributes: 50 | # pylint: disable=too-many-format-args 51 | message_handler( 52 | "\nservice: {} - type: {} - id: {} - resource name: {}".format( 53 | resource.group, 54 | resource.digest.type, 55 | resource.digest.id, 56 | resource.name, 57 | ), 58 | "OKBLUE", 59 | ) 60 | for ( 61 | resource_attr_key, 62 | resource_attr_value, 63 | ) in resource.attributes.items(): 64 | message_handler( 65 | "service: {} - type: {} - name: {} -> {}: {}".format( 66 | resource.group, 67 | resource.digest.type, 68 | resource.name, 69 | resource_attr_key, 70 | resource_attr_value, 71 | ), 72 | "OKBLUE", 73 | ) 74 | else: 75 | message_handler( 76 | "type: {} - id: {} - name: {} - details: {}".format( 77 | resource.digest.type, 78 | resource.digest.id, 79 | resource.name, 80 | resource.details, 81 | ), 82 | "OKBLUE", 83 | ) 84 | 85 | if resource_relations: 86 | message_handler("\n\nFound relations", "HEADER") 87 | for resource_relation in resource_relations: 88 | message = "type: {} - id: {} -> type: {} - id: {}".format( 89 | resource_relation.from_node.type, 90 | resource_relation.from_node.id, 91 | resource_relation.to_node.type, 92 | resource_relation.to_node.id, 93 | ) 94 | 95 | message_handler(message, "OKBLUE") 96 | 97 | @exception 98 | def html_report( 99 | self, 100 | resources: List[Resource], 101 | resource_relations: List[ResourceEdge], 102 | title: str, 103 | filename: str, 104 | ): 105 | dir_template = Environment( 106 | loader=FileSystemLoader( 107 | os.path.dirname(os.path.abspath(__file__)) + "/../templates/" 108 | ), 109 | trim_blocks=True, 110 | ) 111 | 112 | """generate image64 to add to report""" 113 | diagram_image = None 114 | if filename is not None: 115 | image_name = PATH_DIAGRAM_OUTPUT + filename + ".png" 116 | if os.path.exists(image_name): 117 | with open(image_name, "rb") as image_file: 118 | diagram_image = base64.b64encode(image_file.read()).decode("utf-8") 119 | 120 | """generate diagrams.net link""" 121 | diagramsnet_image = None 122 | if filename is not None: 123 | image_name = PATH_DIAGRAM_OUTPUT + filename + ".drawio" 124 | if os.path.exists(image_name): 125 | diagramsnet_image = f"..{os.path.sep}..{os.path.sep}" + image_name 126 | 127 | group_title = "Group" 128 | if resources: 129 | if resources[0].limits: 130 | html_output = dir_template.get_template("report_limits.html").render( 131 | default_name=title, resources_found=resources 132 | ) 133 | else: 134 | if resources[0].attributes: 135 | group_title = "Service" 136 | html_output = dir_template.get_template("report_html.html").render( 137 | default_name=title, 138 | resources_found=resources, 139 | resources_relations=resource_relations, 140 | diagram_image=diagram_image, 141 | diagramsnet_image=diagramsnet_image, 142 | group_title=group_title, 143 | ) 144 | 145 | self.make_directories() 146 | 147 | name_output = PATH_REPORT_HTML_OUTPUT + filename + ".html" 148 | 149 | with open(name_output, "w") as file_output: 150 | file_output.write(html_output) 151 | 152 | message_handler("\n\nHTML report generated", "HEADER") 153 | message_handler("Check your HTML report: " + name_output, "OKBLUE") 154 | -------------------------------------------------------------------------------- /cloudiscovery/templates/report_html.html: -------------------------------------------------------------------------------- 1 |

cloudiscovery - A tool to help you discover resources in the cloud environment.

2 |

{{ default_name }}

3 |

Found resources

4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | {% for resource_found in resources_found %} 15 | 16 | 17 | 18 | 19 | 20 | 26 | 31 | 32 | {%- endfor %} 33 | 34 |
Type{{ group_title }}IdNameDetailsTags
{{ resource_found.digest.type}}{{ resource_found.group}}{{ resource_found.digest.id}}{{ resource_found.name}} 21 | {{ resource_found.details}} 22 | {% for attribute_key, attribute_value in resource_found.attributes.items() %} 23 | {{ attribute_key}}: {{ attribute_value}}
24 | {%- endfor %} 25 | 		27 | {% for tag in resource_found.tags %} 28 | {{ tag.key}}: {{ tag.value}}
29 | {%- endfor %} 30 |
35 | {% if resources_relations|length > 0 %} 36 |

Found relations

37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | {% for resource_relations in resources_relations %} 46 | 47 | 48 | 49 | 50 | 51 | 52 | {%- endfor %} 53 | 54 |
From typeFrom idTo typeTo id
{{ resource_relations.from_node.type}}{{ resource_relations.from_node.id}}{{ resource_relations.to_node.type}}{{ resource_relations.to_node.id}}
55 | {%endif %} 56 | {% if diagram_image is not none %} 57 |

Diagram

58 | {% set base64img = "data:image/png;base64," + diagram_image %} 59 | 60 | {%endif %} 61 | {% if diagramsnet_image is not none %} 62 |

Diagram file for diagrams.net

63 | diagrams.net diagram 64 | {%endif %} 65 | 66 |

 

-------------------------------------------------------------------------------- /cloudiscovery/templates/report_limits.html: -------------------------------------------------------------------------------- 1 |

cloudiscovery - A tool to help you discover resources in the cloud environment.

2 |

{{ default_name }}

3 |

4 | An administrator can request a quota increase for a certain limit via Service Quotas console
5 | More on opening a case to Increase Amazon SES Sending Quotas 6 |

7 | 8 | 9 | {% set ns = namespace(oldservice="") %} 10 | 11 | {% for resource_found in resources_found %} 12 | 13 | {% if ns.oldservice != resource_found.limits.service %} 14 | {% if ns.oldservice != "" %} 15 | 16 | 17 | 18 | {% endif %} 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | {% endif %} 31 | 32 | {% set percent = resource_found.limits.percent %} 33 | 34 | {% if percent <= 70 %} 35 | {% set color = "rgb(0,128,0)" %} 36 | {% set message = "OK" %} 37 | {% elif percent > 70 and percent <= 90 %} 38 | {% set color = "rgb(0,0,139)" %} 39 | {% set message = "Attention" %} 40 | {% elif percent > 90 %} 41 | {% set color = "rgb(255,0,0)" %} 42 | {% set message = "Risk" %} 43 | {% endif %} 44 | 45 | 46 | 47 | 48 | 49 | 50 | 56 | 57 | {% set ns.oldservice = resource_found.limits.service %} 58 | {%- endfor %} 59 | 60 |
 
Service - {{ resource_found.limits.service}}
Quota codeQuota nameAWS default quotaApplied quotaUsageUsage percent
{{ resource_found.limits.quota_code}}{{ resource_found.limits.quota_name}}{{ resource_found.limits.aws_limit}}{{ resource_found.limits.local_limit}}{{ resource_found.limits.usage}}{{ percent }}% - {{ message }} 51 | 52 | 53 | Sorry, your browser does not support inline SVG. 54 | 55 |
61 | 62 |

 

-------------------------------------------------------------------------------- /cloudiscovery/tests/provider/aws/all/resource/test_all.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from assertpy import assert_that 4 | 5 | from provider.aws.all.resource.all import ( 6 | retrieve_resource_name, 7 | retrieve_resource_id, 8 | last_singular_name_element, 9 | operation_allowed, 10 | build_resource_type, 11 | ) 12 | 13 | 14 | class TestAllDiagram(TestCase): 15 | def test_last_singular_name_element(self): 16 | assert_that(last_singular_name_element("ListValues")).is_equal_to("Value") 17 | assert_that(last_singular_name_element("DescribeSomeValues")).is_equal_to( 18 | "Value" 19 | ) 20 | 21 | def test_retrieve_resource_name(self): 22 | assert_that( 23 | retrieve_resource_name({"name": "value"}, "ListValues") 24 | ).is_equal_to("value") 25 | 26 | assert_that( 27 | retrieve_resource_name({"ValueName": "value"}, "ListValues") 28 | ).is_equal_to("value") 29 | assert_that( 30 | retrieve_resource_name({"SomeName": "value"}, "ListValues") 31 | ).is_equal_to("value") 32 | 33 | def test_retrieve_resource_id(self): 34 | assert_that( 35 | retrieve_resource_id({"id": "123"}, "ListValues", "value") 36 | ).is_equal_to("123") 37 | 38 | assert_that( 39 | retrieve_resource_id({"arn": "123"}, "ListValues", "value") 40 | ).is_equal_to("123") 41 | 42 | assert_that( 43 | retrieve_resource_id({"ValueName": "value"}, "ListValues", "value") 44 | ).is_equal_to("value") 45 | 46 | assert_that( 47 | retrieve_resource_id({"ValueId": "123"}, "ListValues", "value") 48 | ).is_equal_to("123") 49 | assert_that( 50 | retrieve_resource_id({"ValueArn": "123"}, "ListValues", "value") 51 | ).is_equal_to("123") 52 | assert_that( 53 | retrieve_resource_id({"someId": "123"}, "ListValues", "value") 54 | ).is_equal_to("123") 55 | assert_that( 56 | retrieve_resource_id({"someArn": "123"}, "ListValues", "value") 57 | ).is_equal_to("123") 58 | 59 | def test_operation_allowed(self): 60 | assert_that(operation_allowed(["iam:List*"], "iam", "ListRoles")).is_equal_to( 61 | True 62 | ) 63 | assert_that(operation_allowed(["ecs:List*"], "iam", "ListRoles")).is_equal_to( 64 | False 65 | ) 66 | 67 | def test_build_resource_type(self): 68 | assert_that(build_resource_type("rds", "DescribeDBParameterGroup")).is_equal_to( 69 | "aws_rds_db_parameter_group" 70 | ) 71 | -------------------------------------------------------------------------------- /cloudiscovery/tests/provider/aws/policy/test_policy_diagram.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from assertpy import assert_that 4 | 5 | from provider.aws.policy.diagram import PolicyDiagram, ROLE_AGGREGATE_PREFIX 6 | from shared.common import Resource, ResourceDigest, ResourceEdge 7 | 8 | 9 | class TestPolicyDiagram(TestCase): 10 | def test_role_aggregation(self): 11 | sut = PolicyDiagram() 12 | principal_digest = ResourceDigest( 13 | id="ecs.amazonaws.com", type="aws_ecs_cluster" 14 | ) 15 | role_1_digest = ResourceDigest(id="AWSServiceRoleForECS1", type="aws_iam_role") 16 | role_2_digest = ResourceDigest(id="AWSServiceRoleForECS2", type="aws_iam_role") 17 | role_3_digest = ResourceDigest(id="AWSServiceRoleForECS3", type="aws_iam_role") 18 | policy_digest = ResourceDigest( 19 | id="arn:aws:iam::policy/service-role/AmazonEC2ContainerServiceforEC2Role", 20 | type="aws_iam_policy", 21 | ) 22 | 23 | relations = [ 24 | ResourceEdge( 25 | from_node=role_1_digest, to_node=principal_digest, label="assumed by" 26 | ), 27 | ResourceEdge( 28 | from_node=role_2_digest, to_node=principal_digest, label="assumed by" 29 | ), 30 | ResourceEdge( 31 | from_node=role_3_digest, to_node=principal_digest, label="assumed by" 32 | ), 33 | ResourceEdge(from_node=role_3_digest, to_node=policy_digest), 34 | ] 35 | result = sut.group_by_group( 36 | [ 37 | Resource(digest=principal_digest, name="principal"), 38 | Resource(digest=role_1_digest, name=""), 39 | Resource(digest=role_2_digest, name=""), 40 | Resource(digest=role_3_digest, name=""), 41 | Resource(digest=policy_digest, name=""), 42 | ], 43 | relations, 44 | ) 45 | 46 | assert_that(result).contains_key("") 47 | assert_that(result[""]).is_length(4) 48 | for resource in result[""]: 49 | assert_that(resource.digest).is_not_equal_to(role_1_digest) 50 | assert_that(resource.digest).is_not_equal_to(role_2_digest) 51 | 52 | relationships = sut.process_relationships(result, relations) 53 | assert_that(relationships).is_length(3) 54 | assert_that(relationships).contains( 55 | ResourceEdge( 56 | from_node=ResourceDigest( 57 | id=ROLE_AGGREGATE_PREFIX + principal_digest.id, type="aws_iam_role" 58 | ), 59 | to_node=principal_digest, 60 | label="assumed by", 61 | ) 62 | ) 63 | -------------------------------------------------------------------------------- /cloudiscovery/tests/provider/aws/vpc/test_common.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from unittest.mock import MagicMock 3 | 4 | from provider.aws.vpc.command import check_ipvpc_inpolicy 5 | 6 | 7 | class Test(TestCase): 8 | def test_check_ipvpc_inpolicy(self): 9 | vpce = {"VpcEndpoints": [{"VpcEndpointId": "vpce-1234abcd", "VpcId": "dummy"}]} 10 | policy = """ 11 | {"Version":"2012-10-17","Id":"arn:queue","Statement": 12 | [{"Effect":"Allow","Principal":"*","Action":"SQS:*","Resource":"arn:queue"}, 13 | {"Effect":"Allow","Principal":"*","Action":"sqs:*","Resource":"arn:queue","Condition": 14 | {"StringEquals":{"aws:sourceVpce":"vpce-1234abcd"}}}]} 15 | """ 16 | vpc_options = MagicMock() 17 | vpc_options.vpc_id = "dummy" 18 | vpc_options.client.return_value.describe_vpc_endpoints.return_value = vpce 19 | result = check_ipvpc_inpolicy(policy, vpc_options) 20 | self.assertTrue("vpce-1234abcd" in result) 21 | 22 | def test_check_vpce_inpolicy(self): 23 | subnets = { 24 | "Subnets": [ 25 | { 26 | "CidrBlock": "10.0.64.0/18", 27 | "SubnetId": "subnet-123", 28 | "VpcId": "dummy", 29 | } 30 | ] 31 | } 32 | policy = """ 33 | {"Version":"2012-10-17","Id":"arn:queue","Statement": 34 | [{"Effect":"Allow","Principal":"*","Action":"SQS:*","Resource":"arn:queue"}, 35 | {"Effect":"Allow","Principal":"*","Action":"sqs:*","Resource":"arn:queue","Condition": 36 | {"StringEquals":{"aws:sourceIp": "10.0.0.0/16"}}}]} 37 | """ 38 | vpc_options = MagicMock() 39 | vpc_options.vpc_id = "dummy" 40 | vpc_options.client.return_value.describe_subnets.return_value = subnets 41 | result = check_ipvpc_inpolicy(policy, vpc_options) 42 | self.assertTrue("10.0.0.0/16" in result) 43 | self.assertTrue("10.0.64.0/18" in result) 44 | self.assertTrue("subnet-123" in result) 45 | -------------------------------------------------------------------------------- /cloudiscovery/tests/shared/test_shared_command.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from assertpy import assert_that 4 | 5 | from shared.command import filter_resources, filter_relations 6 | from shared.common import ( 7 | Resource, 8 | ResourceDigest, 9 | ResourceEdge, 10 | Filterable, 11 | ) 12 | 13 | 14 | class TestCommand(TestCase): 15 | def test_no_filters_resource(self): 16 | resources = filter_resources( 17 | [ 18 | Resource( 19 | digest=ResourceDigest(id="1", type="type"), 20 | name="name", 21 | tags=[Filterable(key="key", value="value")], 22 | ) 23 | ], 24 | [], 25 | ) 26 | 27 | assert_that(resources).is_length(1) 28 | assert_that(resources[0].digest).is_equal_to( 29 | ResourceDigest(id="1", type="type") 30 | ) 31 | 32 | def test_one_tag_filter_resource(self): 33 | resources = filter_resources( 34 | [ 35 | Resource( 36 | digest=ResourceDigest(id="1", type="type"), 37 | name="name", 38 | tags=[Filterable(key="key", value="value")], 39 | ), 40 | Resource( 41 | digest=ResourceDigest(id="2", type="type"), 42 | name="name", 43 | tags=[Filterable(key="key", value="wrong")], 44 | ), 45 | ], 46 | [Filterable(key="key", value="value")], 47 | ) 48 | 49 | assert_that(resources).is_length(1) 50 | assert_that(resources[0].digest).is_equal_to( 51 | ResourceDigest(id="1", type="type") 52 | ) 53 | 54 | def test_two_tags_filter_resource(self): 55 | resources = filter_resources( 56 | [ 57 | Resource( 58 | digest=ResourceDigest(id="1", type="type"), 59 | name="name", 60 | tags=[Filterable(key="key", value="value1")], 61 | ), 62 | Resource( 63 | digest=ResourceDigest(id="2", type="type"), 64 | name="name", 65 | tags=[Filterable(key="key", value="value2")], 66 | ), 67 | Resource( 68 | digest=ResourceDigest(id="3", type="type"), 69 | name="name", 70 | tags=[Filterable(key="key", value="wrong")], 71 | ), 72 | ], 73 | [ 74 | Filterable(key="key", value="value1"), 75 | Filterable(key="key", value="value2"), 76 | ], 77 | ) 78 | 79 | assert_that(resources).is_length(2).extracting(0).contains( 80 | ResourceDigest(id="1", type="type"), ResourceDigest(id="2", type="type") 81 | ) 82 | 83 | def test_one_type_filter_resource(self): 84 | resources = filter_resources( 85 | [ 86 | Resource( 87 | digest=ResourceDigest(id="1", type="type1"), 88 | name="name", 89 | tags=[Filterable(key="key", value="value")], 90 | ), 91 | Resource( 92 | digest=ResourceDigest(id="2", type="type2"), 93 | name="name", 94 | tags=[Filterable(key="key", value="wrong")], 95 | ), 96 | ], 97 | [Filterable(type="type1")], 98 | ) 99 | 100 | assert_that(resources).is_length(1) 101 | assert_that(resources[0].digest).is_equal_to( 102 | ResourceDigest(id="1", type="type1") 103 | ) 104 | 105 | def test_no_filters_relation(self): 106 | digest = ResourceDigest(id="1", type="type") 107 | edge = ResourceEdge(from_node=digest, to_node=digest) 108 | relations = filter_relations( 109 | [ 110 | Resource( 111 | digest=digest, 112 | name="name", 113 | tags=[Filterable(key="key", value="value")], 114 | ) 115 | ], 116 | [edge], 117 | ) 118 | 119 | assert_that(relations).is_length(1).contains(edge) 120 | 121 | def test_no_filtered_relation(self): 122 | digest = ResourceDigest(id="1", type="type") 123 | relations = filter_relations( 124 | [], [ResourceEdge(from_node=digest, to_node=digest)] 125 | ) 126 | 127 | assert_that(relations).is_length(0) 128 | -------------------------------------------------------------------------------- /cloudiscovery/tests/shared/test_shared_common.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from assertpy import assert_that 4 | 5 | from shared.common import parse_filters, Filterable 6 | 7 | 8 | class TestCommon(TestCase): 9 | def test_parse_filters_simple_tag_filter(self): 10 | filters = parse_filters(["Name=tags.costCenter;Value=20000"]) 11 | assert_that(filters).is_length(1) 12 | assert_that(filters).contains(Filterable(key="costCenter", value="20000")) 13 | 14 | def test_parse_filters_type_filter(self): 15 | filters = parse_filters(["Name=type;Value=aws_lambda_function"]) 16 | assert_that(filters).is_length(1) 17 | assert_that(filters).contains(Filterable(type="aws_lambda_function")) 18 | 19 | def test_parse_filters_wrong_filter(self): 20 | filters = parse_filters(["Name=wrong;Value=value"]) 21 | assert_that(filters).is_length(0) 22 | 23 | def test_parse_filters_two_values_tag_filter(self): 24 | filters = parse_filters(["Name=tags.costCenter;Value=20000:20001"]) 25 | assert_that(filters).is_length(2) 26 | assert_that(filters).contains(Filterable(key="costCenter", value="20000")) 27 | assert_that(filters).contains(Filterable(key="costCenter", value="20001")) 28 | 29 | def test_parse_filters_two_complex_values_tag_filter(self): 30 | filters = parse_filters(["Name=tags.costCenter;Value=20000:'20000:1'"]) 31 | assert_that(filters).is_length(2) 32 | assert_that(filters).contains(Filterable(key="costCenter", value="20000")) 33 | assert_that(filters).contains(Filterable(key="costCenter", value="20000:1")) 34 | 35 | def test_parse_filters_invalid_tag_parts_filter(self): 36 | filters = parse_filters(["Name=tags.;costCenter;Value=20000"]) 37 | assert_that(filters).is_length(0) 38 | 39 | def test_parse_filters_invalid_tag_name_filter(self): 40 | filters = parse_filters(["nn=tags.costCenter;Value=20000"]) 41 | assert_that(filters).is_length(0) 42 | 43 | def test_parse_filters_invalid_tag_value_filter(self): 44 | filters = parse_filters(["Name=tags.costCenter;vvv20000"]) 45 | assert_that(filters).is_length(0) 46 | -------------------------------------------------------------------------------- /cloudiscovery/tests/shared/test_shared_diagram.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from assertpy import assert_that 4 | 5 | from shared.common import Resource, ResourceDigest 6 | from shared.diagram import VPCDiagramsNetDiagram 7 | from shared.diagramsnet import MX_FILE 8 | 9 | INFLATED_XML = "" 10 | DEFLATED_XML = "s6nIzVHQtwMA" 11 | 12 | 13 | class TestDiagramsNetDiagram(TestCase): 14 | sut = VPCDiagramsNetDiagram() 15 | 16 | def test_deflate_encode(self): 17 | result = VPCDiagramsNetDiagram.deflate_encode(INFLATED_XML) 18 | assert_that(result).is_equal_to(DEFLATED_XML) 19 | 20 | def test_decode_inflate(self): 21 | result = VPCDiagramsNetDiagram.decode_inflate(DEFLATED_XML) 22 | assert_that(result).is_equal_to(INFLATED_XML) 23 | 24 | def test_file_generation(self): 25 | general_resources = [ 26 | Resource( 27 | digest=ResourceDigest(id="123", type="aws_vpc"), 28 | name="name", 29 | details="details", 30 | ) 31 | ] 32 | grouped_resources = {"": general_resources} 33 | relations = [] 34 | result = self.sut.build_diagram(grouped_resources, relations) 35 | assert_that(result).starts_with(MX_FILE[:200]) 36 | -------------------------------------------------------------------------------- /dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "pip" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | -------------------------------------------------------------------------------- /docs/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/docs/.DS_Store -------------------------------------------------------------------------------- /docs/assets/aws-all.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/docs/assets/aws-all.png -------------------------------------------------------------------------------- /docs/assets/aws-iot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/docs/assets/aws-iot.png -------------------------------------------------------------------------------- /docs/assets/aws-limit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/docs/assets/aws-limit.png -------------------------------------------------------------------------------- /docs/assets/aws-policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/docs/assets/aws-policy.png -------------------------------------------------------------------------------- /docs/assets/aws-vpc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cloud-Architects/cloudiscovery/fad132e45f813775eaf0051e628fcbec68291fb4/docs/assets/aws-vpc.png -------------------------------------------------------------------------------- /docs/assets/role-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Setups a role for diagram builder for all resources within an account", 4 | "Resources": { 5 | "cloudiscoveryRole": { 6 | "Type": "AWS::IAM::Role", 7 | "Properties": { 8 | "AssumeRolePolicyDocument" : { 9 | "Statement" : [ 10 | { 11 | "Effect" : "Allow", 12 | "Principal" : { 13 | "AWS": { "Fn::Join" : [ "", [ 14 | "arn:aws:iam::", { "Ref" : "AWS::AccountId" }, ":root" 15 | ]]} 16 | }, 17 | "Action" : [ "sts:AssumeRole" ] 18 | } 19 | ] 20 | }, 21 | "Policies": [{ 22 | "PolicyName": "additional-permissions", 23 | "PolicyDocument": { 24 | "Version": "2012-10-17", 25 | "Statement" : [ 26 | { 27 | "Effect" : "Allow", 28 | "Action" : [ 29 | "kafka:ListClusters", 30 | "synthetics:DescribeCanaries", 31 | "medialive:ListInputs", 32 | "cloudhsm:DescribeClusters", 33 | "ssm:GetParametersByPath", 34 | "servicequotas:Get*", 35 | "amplify:ListApps", 36 | "autoscaling-plans:DescribeScalingPlans", 37 | "medialive:ListChannels", 38 | "medialive:ListInputDevices", 39 | "mediapackage:ListChannels", 40 | "qldb:ListLedgers", 41 | "transcribe:ListVocabularies", 42 | "glue:GetDatabases", 43 | "glue:GetUserDefinedFunctions", 44 | "glue:GetSecurityConfigurations", 45 | "glue:GetTriggers", 46 | "glue:GetCrawlers", 47 | "glue:ListWorkflows", 48 | "glue:ListMLTransforms", 49 | "codeguru-reviewer:ListCodeReviews", 50 | "servicediscovery:ListNamespaces", 51 | "apigateway:GET", 52 | "forecast:ListPredictors", 53 | "frauddetector:GetDetectors", 54 | "forecast:ListDatasetImportJobs", 55 | "frauddetector:GetModels", 56 | "frauddetector:GetOutcomes", 57 | "networkmanager:DescribeGlobalNetworks", 58 | "codeartifact:ListDomains", 59 | "ses:GetSendQuota", 60 | "codeguru-profiler:ListProfilingGroups", 61 | "cloudtrail:ListTrails" 62 | ], 63 | "Resource": [ "*" ] 64 | } 65 | ] 66 | } 67 | }], 68 | "Path" : "/", 69 | "ManagedPolicyArns" : [ 70 | "arn:aws:iam::aws:policy/job-function/ViewOnlyAccess", 71 | "arn:aws:iam::aws:policy/SecurityAudit" 72 | ] 73 | } 74 | } 75 | }, 76 | "Outputs" : { 77 | "cloudiscoveryRoleArn" : { 78 | "Value" : { "Fn::GetAtt": [ "cloudiscoveryRole", "Arn" ]} 79 | } 80 | } 81 | } -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = 3 | # Define where to run (defaults to root level) 4 | --cov . 5 | # HTML test report for docs 6 | --cov-report html:test-results/coverage 7 | # JUnit XML test results 8 | --junitxml=test-results/junit.xml 9 | # Terminal, print missing lines 10 | --cov-report term-missing 11 | # Detailed output 12 | --verbose 13 | # Add distributed testing 14 | --dist=load --numprocesses=auto 15 | testpaths = cloudiscovery 16 | norecursedirs = 17 | dist 18 | build 19 | .tox 20 | venv 21 | .venv 22 | assets 23 | python_paths = cloudiscovery -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | # Install dependencies 2 | -r requirements.txt 3 | 4 | # Install dev dependencies 5 | black 6 | prospector 7 | pytest 8 | pytest-runner 9 | pytest-flake8 10 | pytest-black 11 | pytest-cov 12 | pytest-xdist 13 | pytest-pythonpath 14 | pre-commit 15 | assertpy 16 | wheel 17 | twine -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | ipaddress 3 | jinja2<3.0 4 | diagrams>=0.14 5 | cachetools 6 | diskcache 7 | pytz -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [aliases] 5 | test=pytest 6 | 7 | [metadata] 8 | requires-dist = 9 | boto3>=1.13.20 10 | ipaddress>=1.0.23 11 | diagrams>=0.13 12 | jinja2<3.0 13 | cachetools>=4.1.0 14 | pytz -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import re 5 | import sys 6 | 7 | from setuptools import setup, find_packages 8 | from setuptools.command.install import install 9 | 10 | 11 | ROOT = os.path.dirname(__file__) 12 | VERSION_RE = re.compile(r"""__version__ = ['"]([0-9.]+)['"]""") 13 | 14 | 15 | requires = [ 16 | "boto3", 17 | "ipaddress", 18 | "diagrams>=0.13", 19 | "jinja2<3.0", 20 | "cachetools", 21 | "diskcache", 22 | "pytz", 23 | ] 24 | 25 | 26 | def get_version(): 27 | init = open(os.path.join(ROOT, "cloudiscovery", "__init__.py")).read() 28 | return VERSION_RE.search(init).group(1) 29 | 30 | 31 | class VerifyVersionCommand(install): 32 | """Custom command to verify that the git tag matches our version""" 33 | 34 | description = "verify that the git tag matches our version" 35 | 36 | def run(self): 37 | tag = os.getenv("CIRCLE_TAG") 38 | 39 | if tag != get_version(): 40 | info = "Git tag: {0} does not match the version of this app: {1}".format( 41 | tag, get_version() 42 | ) 43 | sys.exit(info) 44 | 45 | 46 | setup( 47 | name="cloudiscovery", 48 | version=get_version(), 49 | description="The tool to help you discover resources in the cloud environment", 50 | long_description="Long description", 51 | author="Cloud Architects", 52 | url="https://github.com/Cloud-Architects/cloudiscovery", 53 | package_data={ 54 | "": [ 55 | "locales/en_US/LC_MESSAGES/messages.mo", 56 | "locales/pt_BR/LC_MESSAGES/messages.mo", 57 | "templates/report_html.html", 58 | "templates/report_limits.html", 59 | ] 60 | }, 61 | packages=find_packages(exclude=["tests*"]), 62 | install_requires=requires, 63 | setup_requires=["pytest-runner"], 64 | tests_require=[ 65 | "pytest", 66 | "pytest-cov", 67 | "pytest-xdist", 68 | "assertpy", 69 | "pytest-pythonpath", 70 | ], 71 | tests_suite="cloudiscovery", 72 | python_requires=">=3.8", 73 | scripts=["bin/cloudiscovery", "bin/cloudiscovery.cmd"], 74 | license="Apache License 2.0", 75 | classifiers=[ 76 | "Development Status :: 5 - Production/Stable", 77 | "Environment :: Console", 78 | "Intended Audience :: System Administrators", 79 | "Natural Language :: English", 80 | "License :: OSI Approved :: Apache Software License", 81 | "Programming Language :: Python", 82 | "Programming Language :: Python :: 3.8", 83 | ], 84 | cmdclass={"verify": VerifyVersionCommand}, 85 | ) 86 | --------------------------------------------------------------------------------