├── CureIAM ├── helpers │ ├── __init__.py │ ├── hconfigs.py │ ├── hcmd.py │ ├── hlogging.py │ ├── hemails.py │ └── util.py ├── plugins │ ├── __init__.py │ ├── util_plugins.py │ ├── gcp │ │ ├── util_gcp.py │ │ ├── gcpcloud.py │ │ └── gcpcloudiam.py │ ├── files │ │ └── filestore.py │ └── elastic │ │ └── esstore.py ├── alerts │ └── __init__.py ├── models │ ├── __init__.py │ ├── applyrecommendationmodel.py │ └── iamriskscore.py ├── __init__.py ├── __main__.py ├── baseconfig.py ├── ioworkers.py ├── workers.py └── manager.py ├── assets └── images │ ├── command.png │ └── CureIAMLogo.png ├── requirements.txt ├── Makefile ├── .gitignore ├── elastic ├── docker_compose_es.yaml └── .env-ex ├── Dockerfile ├── setup.py ├── test ├── test_es.py └── test_gcp.py ├── SampleCureIAM.yaml ├── README.md └── LICENSE /CureIAM/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | """ A package for Helpers 2 | """ 3 | -------------------------------------------------------------------------------- /CureIAM/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | """A package for CureIAM 2 | """ 3 | -------------------------------------------------------------------------------- /CureIAM/alerts/__init__.py: -------------------------------------------------------------------------------- 1 | """A package for alert plugins packaged with this project. 2 | """ 3 | -------------------------------------------------------------------------------- /assets/images/command.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gojek/CureIAM/HEAD/assets/images/command.png -------------------------------------------------------------------------------- /CureIAM/models/__init__.py: -------------------------------------------------------------------------------- 1 | """A package for models as data store packaged with this project. 2 | """ 3 | -------------------------------------------------------------------------------- /assets/images/CureIAMLogo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gojek/CureIAM/HEAD/assets/images/CureIAMLogo.png -------------------------------------------------------------------------------- /CureIAM/__init__.py: -------------------------------------------------------------------------------- 1 | """CureIAM - Cloud security monitoring framework.""" 2 | 3 | 4 | __version__ = '0.2.0' 5 | -------------------------------------------------------------------------------- /CureIAM/__main__.py: -------------------------------------------------------------------------------- 1 | """Main script for the package.""" 2 | 3 | from CureIAM import manager 4 | 5 | manager.main() 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | elasticsearch==8.7.0 2 | google-api-python-client==2.86.0 3 | PyYAML==6.0 4 | schedule==1.2.0 5 | rich==13.3.5 -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | venv: 2 | python3 -m venv ~/.venv/CureIAM 3 | echo . ~/.venv/CureIAM/bin/activate > venv 4 | 5 | clean: 6 | find . -name "__pycache__" -exec rm -r {} + 7 | find . -name "*.pyc" -exec rm {} + 8 | rm -rf .coverage.* .coverage htmlcov build CureIAM.egg-info dist 9 | 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore automatically generated files. 2 | venv 3 | .venv 4 | .coverage 5 | .coverage.* 6 | htmlcov/ 7 | logs/ 8 | _build/ 9 | *.yaml 10 | *.yml 11 | *.json 12 | .env 13 | 14 | # Ignore files generated due to releases. 15 | uservenv 16 | build/ 17 | IAMRecommending.egg-info/ 18 | dist/ 19 | 20 | # Ignore local development files. 21 | IAMRecommending.yaml 22 | 23 | # Ignore Vim swap files. 24 | *.sw? 25 | 26 | # Ignore Python cache files. 27 | *.pyc 28 | 29 | # Ignore any mac file 30 | *.DS_Store 31 | .DS_Store 32 | 33 | # Not ignoring dockerfle 34 | !elastic/docker_compose_es.yaml -------------------------------------------------------------------------------- /elastic/docker_compose_es.yaml: -------------------------------------------------------------------------------- 1 | version: "3.0" 2 | services: 3 | elasticsearch: 4 | container_name: es-container 5 | image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} 6 | environment: 7 | - xpack.security.enabled=false 8 | - "discovery.type=single-node" 9 | networks: 10 | - es-net 11 | ports: 12 | - ${ES_PORT}:9200 13 | kibana: 14 | container_name: kb-container 15 | image: docker.elastic.co/kibana/kibana:${STACK_VERSION} 16 | environment: 17 | - ELASTICSEARCH_HOSTS=http://es-container:9200 18 | networks: 19 | - es-net 20 | depends_on: 21 | - elasticsearch 22 | ports: 23 | - ${KIBANA_PORT}:5601 24 | networks: 25 | es-net: 26 | driver: bridge -------------------------------------------------------------------------------- /elastic/.env-ex: -------------------------------------------------------------------------------- 1 | # Password for the 'elastic' user (at least 6 characters) 2 | ELASTIC_PASSWORD=<> 3 | 4 | # Password for the 'kibana_system' user (at least 6 characters) 5 | KIBANA_PASSWORD=<> 6 | 7 | # Version of Elastic products 8 | STACK_VERSION=8.7.0 9 | 10 | # Set the cluster name 11 | CLUSTER_NAME=elk 12 | 13 | # Set to 'basic' or 'trial' to automatically start the 30-day trial 14 | LICENSE=basic 15 | #LICENSE=trial 16 | 17 | # Port to expose Elasticsearch HTTP API to the host 18 | ES_PORT=9200 19 | 20 | # Port to expose Kibana to the host 21 | KIBANA_PORT=5601 22 | 23 | # Increase or decrease based on the available host memory (in bytes) 24 | MEM_LIMIT=1073741824 25 | 26 | # Project namespace (defaults to the current folder name if not set) 27 | #COMPOSE_PROJECT_NAME=myproject -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7.11-alpine3.14 2 | 3 | # For seting up the timezone 4 | ENV TZ=Asia/Kolkata 5 | RUN apk update && apk add tzdata 6 | 7 | # Create user 8 | ENV USER=cureiam 9 | ENV UID=1001 10 | 11 | RUN addgroup cureiam 12 | 13 | RUN adduser \ 14 | --disabled-password \ 15 | --gecos "" \ 16 | --ingroup "$USER" \ 17 | --uid "$UID" \ 18 | "$USER" 19 | 20 | # Change workdir 21 | WORKDIR /home/cureiam 22 | 23 | # Copy the necessary files 24 | COPY CureIAM CureIAM 25 | COPY requirements.txt requirements.txt 26 | COPY CureIAM.yaml CureIAM.yaml 27 | COPY cureiamSA.json cureiamSA.json 28 | 29 | 30 | # Set user 31 | USER cureiam 32 | 33 | # Install deps 34 | RUN export PATH=$PATH:/home/cureiam/.local/bin 35 | RUN pip install -r requirements.txt --no-warn-script-location 36 | 37 | ENTRYPOINT ["python"] 38 | CMD ["-m", "CureIAM"] 39 | -------------------------------------------------------------------------------- /CureIAM/models/applyrecommendationmodel.py: -------------------------------------------------------------------------------- 1 | """ IAM Risk score model class implementation 2 | """ 3 | 4 | from CureIAM.helpers import hlogging 5 | 6 | _log = hlogging.get_logger(__name__) 7 | 8 | class IAMApplyRecommendationModel: 9 | """IAMRiskScoreModel plugin for GCP IAM Recommendation records.""" 10 | 11 | def __init__(self, record): 12 | """Create an instance of :class:`IAMRiskScoreModel` plugin. 13 | 14 | This model class generates scores for the recommendation record. 15 | Arguments: 16 | record: dict for GCP record 17 | """ 18 | self._record = record 19 | 20 | self._model = { 21 | 'recommendation_id': record['recommendation_id'], 22 | 'project_id': record['project'], 23 | 'account_type': record['account_type'], 24 | 'account_id': record['account_id'], 25 | 'safe_to_apply_score': None, 26 | 'recommendation_state': None, 27 | 'recommendation_applied_time': None 28 | } 29 | 30 | # There should be three different types of recommendation state 31 | # applied by CureIAM/Claimed/Will not be applied 32 | 33 | 34 | def model(self): 35 | """This function will create model for applyRecommendation 36 | functionality.""" 37 | return self._model -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Setup script.""" 2 | 3 | import setuptools 4 | 5 | import CureIAM 6 | 7 | _description = CureIAM.__doc__.splitlines()[0] 8 | _long_description = open('README.md').read() 9 | _version = CureIAM.__version__ 10 | _requires = open('pkg-requirements.txt').read().splitlines() 11 | 12 | setuptools.setup( 13 | 14 | name='CureIAM', 15 | version=_version, 16 | author='CureIAM Authors and Contributors', 17 | description=_description, 18 | long_description=_long_description, 19 | url='https://github.com/CureIAM/CureIAM', 20 | 21 | install_requires=_requires, 22 | 23 | packages=setuptools.find_packages(exclude=['CureIAM.test']), 24 | 25 | entry_points={ 26 | 'console_scripts': { 27 | 'CureIAM = CureIAM.manager:main' 28 | } 29 | }, 30 | 31 | # Reference for classifiers: https://pypi.org/classifiers/ 32 | classifiers=[ 33 | 'Development Status :: 1 - Pre-Alpha', 34 | 'Intended Audience :: Developers', 35 | 'Intended Audience :: DevOps', 36 | 'Intended Audience :: End Users/Desktop', 37 | 'Intended Audience :: Information Technology', 38 | 'Intended Audience :: System Administrators', 39 | 'License :: OSI Approved :: MIT License', 40 | 'Operating System :: OS Independent', 41 | 'Programming Language :: Python :: 3', 42 | 'Topic :: System :: Monitoring', 43 | ], 44 | 45 | keywords='GCP IAM monitoring framework', 46 | ) 47 | -------------------------------------------------------------------------------- /test/test_es.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from elasticsearch import Elasticsearch 3 | ghost = '127.0.0.1' 4 | gport = 9200 5 | gusername = '' 6 | gpassword = '' 7 | gscheme = 'http' 8 | 9 | def main(argv): 10 | host = argv.host if argv.host != None else ghost 11 | port = argv.port if argv.port != None else gport 12 | username = argv.username if argv.username != None else gusername 13 | password = argv.password if argv.password != None else gpassword 14 | scheme = argv.scheme if argv.scheme != None else gscheme 15 | 16 | es = Elasticsearch([{'host': host, 'port': port, 'scheme': scheme}], http_auth=(username, password)) 17 | 18 | doc = { 19 | 'author': 'author_name', 20 | 'text': 'Interensting content...', 21 | 'timestamp': datetime.now(), 22 | } 23 | resp = es.index(index="test-index", id=1, document=doc) 24 | print(resp['result']) 25 | 26 | 27 | if __name__== "__main__": 28 | import sys, argparse 29 | parser = argparse.ArgumentParser(description='Testing ELK connection') 30 | parser.add_argument('--host','-t', nargs='?', default=None,help='Host') 31 | parser.add_argument('--port','-p', nargs='?', default=None,help='Port') 32 | parser.add_argument('--username','-u', nargs='?', default=None,help='ELK Username') 33 | parser.add_argument('--password','-c', nargs='?', default=None,help='ELK Password') 34 | parser.add_argument('--scheme','-s', nargs='?', default=None,help='http or https') 35 | args = parser.parse_args() 36 | main(args) 37 | -------------------------------------------------------------------------------- /CureIAM/baseconfig.py: -------------------------------------------------------------------------------- 1 | """Base configuration. 2 | 3 | Attributes: 4 | config_yaml (str): Base configuration as YAML code. 5 | config_dict (dict): Base configuration as Python dictionary. 6 | 7 | Here is the complete base configuration present as a string in the 8 | :obj:`config_yaml` attribute:: 9 | 10 | {} 11 | 12 | """ 13 | 14 | import textwrap 15 | 16 | import yaml 17 | 18 | config_yaml = """# Base configuration 19 | logger: 20 | version: 1 21 | 22 | disable_existing_loggers: false 23 | 24 | formatters: 25 | simple: 26 | format: >- 27 | %(asctime)s [%(process)s] [%(processName)s] [%(threadName)s] 28 | %(levelname)s %(name)s:%(lineno)d - %(message)s 29 | datefmt: "%Y-%m-%d %H:%M:%S" 30 | 31 | handlers: 32 | rich_console: 33 | class: rich.logging.RichHandler 34 | formatter: simple 35 | 36 | console: 37 | class: logging.StreamHandler 38 | formatter: simple 39 | stream: ext://sys.stdout 40 | 41 | file: 42 | class: logging.handlers.TimedRotatingFileHandler 43 | formatter: simple 44 | filename: /tmp/CureIAM.log 45 | when: midnight 46 | encoding: utf8 47 | backupCount: 5 48 | 49 | loggers: 50 | adal-python: 51 | level: WARNING 52 | 53 | root: 54 | level: INFO 55 | handlers: 56 | - rich_console 57 | - file 58 | 59 | child: 60 | level: INFO 61 | handlers: 62 | - rich_console 63 | qualname: child 64 | propagate: 0 65 | 66 | 67 | schedule: "00:00" 68 | """ 69 | 70 | 71 | config_dict = yaml.safe_load(config_yaml) 72 | __doc__ = __doc__.format(textwrap.indent(config_yaml, ' ')) 73 | -------------------------------------------------------------------------------- /test/test_gcp.py: -------------------------------------------------------------------------------- 1 | from google.oauth2 import service_account 2 | from googleapiclient import discovery 3 | 4 | def main(argv): 5 | saccount = argv.saccount 6 | project = argv.project 7 | 8 | credentials = service_account.Credentials.from_service_account_file( 9 | saccount) 10 | 11 | service = discovery.build("recommender", 12 | 'v1', 13 | credentials=credentials, 14 | cache_discovery=False) 15 | 16 | ''' 17 | http GET "https://recommender.googleapis.com/v1/projects/${PROJECT_ID}/locations/global/recommenders/google.iam.policy.Recommender/recommendations?pageSize=${PAGE_SIZE}&pageToken=${PAGE_TOKEN}&filter=${FILTER}" "Authorization: Bearer ${ACCESS_TOKEN}" 18 | ''' 19 | # recommender discovery doc location 20 | # res = service.projects().locations().recommenders().recommendations().list( 21 | # parent='projects/565961175665/locations/global/recommenders/google.iam.policy.Recommender' 22 | # ).execute() 23 | 24 | str_project = 'projects/{}/locations/global/recommenders/google.iam.policy.Recommender'.format(project) 25 | 26 | res = service.projects().locations().recommenders().recommendations().list( 27 | parent=str_project 28 | ).execute() 29 | 30 | print(res) 31 | 32 | if __name__== "__main__": 33 | import sys, argparse 34 | parser = argparse.ArgumentParser(description='Testing gcloud credentials & permission') 35 | parser.add_argument('--saccount','-s', nargs='?', default='cureiamSA.json',help='Service account json file') 36 | parser.add_argument('--project','-p', nargs='?', default='something',help='Project name') 37 | args = parser.parse_args() 38 | main(args) 39 | -------------------------------------------------------------------------------- /CureIAM/helpers/hconfigs.py: -------------------------------------------------------------------------------- 1 | import os 2 | import yaml 3 | import typing 4 | from CureIAM.helpers import util, hlogging 5 | 6 | from CureIAM import baseconfig #deprecating soon 7 | 8 | from CureIAM.helpers import hlogging 9 | 10 | _log = hlogging.get_logger(__name__) 11 | 12 | """ 13 | Will deprecated this function soon 14 | """ 15 | def load(config_paths): 16 | """Load configuration from specified configuration paths. 17 | 18 | Arguments: 19 | config_paths (list): Configuration paths. 20 | 21 | Returns: 22 | dict: A dictionary of configuration key-value pairs. 23 | 24 | """ 25 | config = baseconfig.config_dict 26 | 27 | for config_path in config_paths: 28 | config_path = os.path.expanduser(config_path) 29 | _log.info('Looking for %s', config_path) 30 | 31 | if not os.path.isfile(config_path): 32 | continue 33 | 34 | _log.info('Found %s', config_path) 35 | with open(config_path) as f: 36 | new_config = yaml.safe_load(f) 37 | config = util.merge_dicts(config, new_config) 38 | 39 | 40 | return config 41 | 42 | """ 43 | Creating a Singleton Config file to be called everywhere 44 | """ 45 | class Config(object): 46 | 47 | _CONFIG_FILE: typing.Optional[str] = None 48 | _CONFIG: typing.Optional[dict] = None 49 | 50 | @staticmethod 51 | def load(config_files = None) -> dict: 52 | 53 | # Use singleton pattern to store config file location/load config once 54 | Config._CONFIG_FILE = Config.search_configuration_files(config_files) 55 | 56 | with open(Config._CONFIG_FILE, 'r') as f: 57 | Config._CONFIG = yaml.safe_load(f) 58 | # config = util.merge_dicts(config, new_config) 59 | 60 | 61 | return Config._CONFIG 62 | 63 | @staticmethod 64 | def search_configuration_files(config_files): 65 | for config_file in config_files: 66 | config_file = os.path.expanduser(config_file) 67 | 68 | if os.path.isfile(config_file): 69 | _log.info('Found %s', config_file) 70 | return config_file 71 | 72 | return "" 73 | 74 | @staticmethod 75 | def get_config_file() -> str: 76 | return Config._CONFIG_FILE 77 | 78 | @staticmethod 79 | def get_config() -> dict: 80 | return Config._CONFIG 81 | 82 | @staticmethod 83 | def get_config(name) -> dict: 84 | return Config._CONFIG[name] 85 | 86 | @staticmethod 87 | def get_config_logger() -> dict: 88 | return Config._CONFIG["logger"] 89 | 90 | -------------------------------------------------------------------------------- /CureIAM/plugins/util_plugins.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from CureIAM.helpers import hlogging 3 | 4 | _log = hlogging.get_logger(__name__) 5 | 6 | def load(plugin_config): 7 | """Construct an object with specified plugin class and parameters. 8 | 9 | The ``plugin_config`` parameter must be a dictionary with the 10 | following keys: 11 | 12 | - ``plugin``: The value for this key must be a string that 13 | represents the fully qualified class name of the plugin. The 14 | fully qualified class name is in the dotted notation, e.g., 15 | ``pkg.module.ClassName``. 16 | - ``params``: The value for this key must be a :obj:`dict` that 17 | represents the parameters to be passed to the ``__init__`` method 18 | of the plugin class. Each key in the dictionary represents the 19 | parameter name and each value represents the value of the 20 | parameter. 21 | 22 | Example: 23 | Here is an example usage of this function: 24 | 25 | >>> from CureIAM import util 26 | >>> plugin_config = { 27 | ... 'plugin': 'CureIAM.clouds.mockcloud.MockCloud', 28 | ... 'params': { 29 | ... 'record_count': 4, 30 | ... 'record_types': ('baz', 'qux') 31 | ... } 32 | ... } 33 | ... 34 | >>> plugin = util.load_plugin(plugin_config) 35 | >>> print(type(plugin)) 36 | 37 | >>> for record in plugin.read(): 38 | ... print(record['raw']['data'], 39 | ... record['ext']['record_type'], 40 | ... record['com']['record_type']) 41 | ... 42 | 0 baz mock 43 | 1 qux mock 44 | 2 baz mock 45 | 3 qux mock 46 | 47 | Arguments: 48 | plugin_config (dict): Plugin configuration dictionary. 49 | 50 | Returns: 51 | object: An object of type mentioned in the ``plugin`` parameter. 52 | 53 | Raises: 54 | PluginError: If plugin class name is invalid. 55 | 56 | """ 57 | # Split the fully qualified class name into module and class names. 58 | parts = plugin_config['plugin'].rsplit('.', 1) 59 | 60 | # Validate that the fully qualified class name had at least two 61 | # parts: module name and class name. 62 | if len(parts) < 2: 63 | msg = ('Invalid plugin class name: {}; expected format: ' 64 | '[.].'.format(plugin_config['plugin'])) 65 | raise PluginError(msg) 66 | 67 | # Load the specified adapter class from the specified module. 68 | plugin_module = importlib.import_module(parts[0]) 69 | plugin_class = getattr(plugin_module, parts[1]) 70 | 71 | # Initialize params to empty dictionary if none was specified. 72 | plugin_params = plugin_config.get('params', {}) 73 | 74 | # Construct the plugin. 75 | plugin = plugin_class(**plugin_params) 76 | return plugin 77 | 78 | 79 | class PluginError(Exception): 80 | """Represents an error while loading a plugin.""" 81 | -------------------------------------------------------------------------------- /SampleCureIAM.yaml: -------------------------------------------------------------------------------- 1 | # production file 2 | plugins: 3 | gcpCloud: 4 | plugin: CureIAM.plugins.gcp.gcpcloud.GCPCloudIAMRecommendations 5 | params: 6 | key_file_path: cureiamSA.json 7 | filestore: 8 | plugin: CureIAM.plugins.files.filestore.FileStore 9 | gcpIamProcessor: 10 | plugin: CureIAM.plugins.gcp.gcpcloudiam.GCPIAMRecommendationProcessor 11 | params: 12 | mode_scan: true 13 | mode_enforce: false 14 | enforcer: 15 | key_file_path: cureiamSA.json 16 | blocklist_projects: 17 | - project-a 18 | - project-b 19 | - project-c 20 | blocklist_accounts: 21 | - user-a@gmail.com 22 | - user-b@gmail.com 23 | - serviceAccount-a@project-a.iam.gserviceaccount.com 24 | - serviceAccount-b@project-b.iam.gserviceaccount.com 25 | allowlist_account_types: 26 | - user 27 | - group 28 | - serviceAccount 29 | blocklist_account_types: 30 | - None 31 | min_safe_to_apply_score_user: 60 32 | min_safe_to_apply_score_group: 60 33 | min_safe_to_apply_score_SA: 60 34 | esstore: 35 | plugin: CureIAM.plugins.elastic.esstore.EsStore 36 | params: 37 | # Change http to https later if your elastic are using https 38 | scheme: http 39 | host: es-host.com 40 | port: 9200 41 | index: cureiam-stg 42 | username: <> 43 | password: <> 44 | 45 | audits: 46 | IAMAudit: 47 | clouds: 48 | - gcpCloud 49 | processors: 50 | - gcpIamProcessor 51 | stores: 52 | - filestore 53 | - esstore 54 | 55 | run: 56 | - IAMAudit 57 | 58 | logger: 59 | version: 1 60 | disable_existing_loggers: true 61 | formatters: 62 | main: 63 | format: "[%(asctime)s][%(process)s][%(processName)s][%(threadName)s] - %(levelname)s 64 | %(name)s:%(lineno)d - %(message)s" 65 | datefmt: "%Y-%m-%d %H:%M:%S" 66 | standard: 67 | format: "[%(process)s][%(processName)s][%(threadName)s] %(message)s" 68 | datefmt: "%Y-%m-%d %H:%M:%S" 69 | handlers: 70 | main: 71 | formatter: main 72 | class: logging.StreamHandler 73 | stream: ext://sys.stdout 74 | console: 75 | formatter: standard 76 | class: logging.StreamHandler 77 | stream: ext://sys.stdout 78 | rich: 79 | formatter: standard 80 | class: rich.logging.RichHandler 81 | file: 82 | formatter: standard 83 | class: logging.handlers.TimedRotatingFileHandler 84 | filename: "/tmp/CureIAM.log" 85 | when: midnight 86 | encoding: utf8 87 | backupCount: 5 88 | loggers: 89 | '': 90 | handlers: 91 | - file 92 | - main 93 | level: INFO 94 | propagate: false 95 | CureIAM.plugins.gcp.gcpcloudiam: 96 | handlers: 97 | - rich 98 | level: INFO 99 | propagate: false 100 | __main__: 101 | handlers: 102 | - file 103 | - main 104 | level: INFO 105 | propagate: false 106 | 107 | schedule: "14:00" -------------------------------------------------------------------------------- /CureIAM/plugins/gcp/util_gcp.py: -------------------------------------------------------------------------------- 1 | from google.oauth2 import service_account 2 | from googleapiclient import discovery 3 | from CureIAM.helpers import hlogging 4 | 5 | _log = hlogging.get_logger(__name__) 6 | 7 | def set_service_account(key_file_path=None, scopes=[]): 8 | 9 | return service_account.Credentials.from_service_account_file( 10 | key_file_path) 11 | 12 | def get_service_account_class(): 13 | 14 | return service_account.Credentials 15 | 16 | 17 | def build_resource(service_name, key_file_path, version='v1'): 18 | """Create a ``Resource`` object for interacting with Google APIs. 19 | 20 | Arguments: 21 | service_name (str): Name of the service of resource object. 22 | version (str): Version of the API for resource object. 23 | 24 | Returns: 25 | googleapiclient.discovery.Resource: Resource object for 26 | interacting with Google APIs. 27 | """ 28 | 29 | credential = set_service_account(key_file_path) 30 | 31 | # Entire set of service list can be obatinaed from this gcloud command 32 | # gcloud services list --available 33 | 34 | return discovery.build(service_name, 35 | version, 36 | credentials=credential, 37 | cache_discovery=False) 38 | 39 | def get_resource_iterator(resource, key, **list_kwargs): 40 | """Generate resources for specific record types. This function is useful to when API returns 41 | pageToken and there is need to make subsequent calls. 42 | 43 | Arguments: 44 | resource (Resource): GCP resource object. 45 | key (str): The key that we need to look up in the GCP 46 | response JSON to find the list of resources. 47 | key_file_path (str): Path to key file (for logging only). 48 | list_kwargs (dict): Keyword arguments for 49 | ``resource.list()`` call. 50 | Yields: 51 | dict: A GCP configuration record. 52 | """ 53 | try: 54 | request = resource.list(**list_kwargs) 55 | 56 | while request is not None: 57 | response = request.execute() 58 | if key is None: 59 | yield response 60 | else: 61 | for item in response.get(key, []): 62 | yield item 63 | request = resource.list_next(previous_request=request, 64 | previous_response=response) 65 | except Exception as e: 66 | _log.error('Failed to fetch resource list; key: %s; ' 67 | 'list_kwargs: %s; ' 68 | 'error: %s: %s', key, list_kwargs, 69 | type(e).__name__, e) 70 | 71 | def outline_gcp_project(project_index, project, zone, key_file_path): 72 | """Return a summary of a GCP project for logging purpose. 73 | 74 | Arguments: 75 | project_index (int): Project index. 76 | project (Resource): GCP Resource object of the project. 77 | zone (str): Name of the zone for the project. 78 | key_file_path (str): Path of the service account key file 79 | for a project. 80 | 81 | Returns: 82 | str: Return a string that can be used in log messages. 83 | 84 | """ 85 | zone_log = '' if zone is None else 'zone: {}; '.format(zone) 86 | return ('project #{}: {} ({}) ({}); {}key_file_path: {}' 87 | .format(project_index, project.get('projectId'), 88 | project.get('name'), project.get('lifecycleState'), 89 | zone_log, key_file_path)) -------------------------------------------------------------------------------- /CureIAM/plugins/files/filestore.py: -------------------------------------------------------------------------------- 1 | """Filesystem store plugin.""" 2 | 3 | import json 4 | import os 5 | import os.path 6 | 7 | 8 | class FileStore: 9 | """A plugin to store records on the filesystem.""" 10 | 11 | def __init__(self, path='/tmp/CureIAM'): 12 | """Create an instance of :class:`FileStore` plugin. 13 | 14 | Arguments: 15 | path (str): Path of directory where files are written to. 16 | 17 | """ 18 | self._path = os.path.expanduser(path) 19 | self._worker_names = set() 20 | os.makedirs(self._path, exist_ok=True) 21 | 22 | def write(self, record): 23 | """Write JSON records to the file system. 24 | 25 | This method is called once for every ``record`` read from a 26 | cloud. In this example implementation of a store, we simply 27 | write the ``record`` in JSON format to a file. The list of 28 | records is maintained as JSON array in the file. The origin 29 | worker name in ``record['com']['origin_worker']`` is used to 30 | determine the filename. 31 | 32 | The records are written to a ``.tmp`` file because we don't want 33 | to delete the existing complete and useful ``.json`` file 34 | prematurely. 35 | 36 | Note that other implementations of a store may choose to buffer 37 | the records in memory instead of writing each record to the 38 | store immediately. They may then flush the buffer to the store 39 | based on certain conditions such as buffer size, time interval, 40 | etc. 41 | 42 | Arguments: 43 | record (dict): Data to write to the file system. 44 | 45 | """ 46 | worker_name = record.get('com', {}).get('origin_worker', 'no_worker') 47 | 48 | tmp_file_path = os.path.join(self._path, worker_name) + '.tmp' 49 | if worker_name not in self._worker_names: 50 | # If this is the first time we have encountered this 51 | # worker_name, we create a new file for it and write an 52 | # opening bracket to start a JSON array. 53 | with open(tmp_file_path, 'w') as f: 54 | f.write('[\n') 55 | delim = '' 56 | else: 57 | # If this is not the first record of its record type, then 58 | # we need to separate this record from the previous record 59 | # with a comma to form a valid JSON array. 60 | delim = ',\n' 61 | 62 | # Write the record dictionary as JSON object literal. 63 | self._worker_names.add(worker_name) 64 | with open(tmp_file_path, 'a') as f: 65 | f.write(delim + json.dumps(record, indent=2)) 66 | 67 | def done(self): 68 | """Perform final cleanup tasks. 69 | 70 | This method is called after all records have been written. In 71 | this example implementation, we properly terminate the JSON 72 | array in the .tmp file. Then we rename the .tmp file to .json 73 | file. 74 | 75 | Note that other implementations of a store may perform tasks 76 | like closing a connection to a remote store or flushing any 77 | remaining records in a buffer. 78 | 79 | """ 80 | for worker_name in self._worker_names: 81 | # End the JSON array by writing a closing bracket. 82 | tmp_file_path = os.path.join(self._path, worker_name) + '.tmp' 83 | with open(tmp_file_path, 'a') as f: 84 | f.write('\n]\n') 85 | 86 | # Rename the temporary file to a JSON file. 87 | json_file_path = os.path.join(self._path, worker_name) + '.json' 88 | os.replace(tmp_file_path, json_file_path) 89 | -------------------------------------------------------------------------------- /CureIAM/helpers/hcmd.py: -------------------------------------------------------------------------------- 1 | #default library 2 | import argparse 3 | import textwrap 4 | 5 | #custom class 6 | import CureIAM 7 | 8 | from CureIAM.helpers import util 9 | 10 | def parse(args=None): 11 | """Parse command line arguments. 12 | 13 | Arguments: 14 | args (list): List of command line arguments. 15 | 16 | Returns: 17 | argparse.Namespace: Parsed command line arguments. 18 | 19 | """ 20 | default_config_paths = [ 21 | '/etc/CureIAM.yaml', 22 | '~/.CureIAM.yaml', 23 | '~/CureIAM.yaml', 24 | 'CureIAM.yaml', 25 | ] 26 | 27 | description = """ 28 | \033[34m========================================================\033[0m 29 | 30 | \033[91m_____ _____ __ __\033[0m 31 | \033[91m/ ____| |_ _| /\ | \/ |\033[0m 32 | \33[93m| | _ _ _ __ ___ | | / \ | \ / |\033[0m 33 | \33[93m| | | | | | '__/ _ \ | | / /\ \ | |\/| |\033[0m 34 | \33[34m| |___| |_| | | | __/_| |_ / ____ \| | | |\033[0m 35 | \33[34m\_____\__,_|_| \___|_____/_/ \_\_| |_|\033[0m 36 | 37 | \033[34m========================================================\033[0m 38 | \033[91mAudit clouds as specified by configuration.\033[0m 39 | 40 | [*] Audit clouds as specified by configuration - Automated IAM Recommender 41 | [*] Based on Google IAM Cloud Recommender 42 | [*] Credits: Gojek Security Team 43 | [*] Author: Rohit S & Kenny G 44 | [*] Source: https://github.com/gojek/CureIAM 45 | 46 | \033[91mUsage:\033[0m 47 | python3 -m CureIAM [options] 48 | 49 | \033[91mExamples:\033[0m 50 | python3 -m CureIAM -n 51 | python3 -m CureIAM -c CureIAM.yaml 52 | python3 -m CureIAM 53 | 54 | Zero or more config files are specified with the -c/--config option. 55 | The config files specified are merged with a built-in base config. 56 | Use the -p/--print-base-config option to see the built-in base 57 | config. Missing config files are ignored. 58 | 59 | If two or more config files provide conflicting config values, the 60 | config file specified later overrides the built-in base config and 61 | the config files specified earlier. 62 | 63 | If the -c/--config option is specified without any file arguments 64 | following it, then only the built-in base config is used. 65 | 66 | If the -c/--config option is omitted, then the following config 67 | files are searched for and merged with the built-in base config: {}. 68 | Missing config files are ignored. 69 | """ 70 | description = description.format(util.friendly_list(default_config_paths)) 71 | description = description 72 | 73 | # We will use this format to preserve formatting of the description 74 | # above with the newlines and blank lines intact. The default 75 | # formatter line-wraps the entire description after ignoring any 76 | # superfluous whitespace including blank lines, so the paragraph 77 | # breaks are lost, and the usage description looks ugly. 78 | formatter = argparse.RawDescriptionHelpFormatter 79 | 80 | parser = argparse.ArgumentParser(prog='CureIAM', 81 | description=description, 82 | formatter_class=formatter) 83 | 84 | parser.add_argument('-c', '--config', nargs='*', 85 | default=default_config_paths, 86 | help='run audits with specified configuration files') 87 | 88 | parser.add_argument('-n', '--now', action='store_true', 89 | help='ignore configured schedule and run audits now') 90 | 91 | parser.add_argument('-p', '--print-base-config', action='store_true', 92 | help='print base configuration') 93 | 94 | parser.add_argument('-v', '--version', action='version', 95 | version='%(prog)s ' + CureIAM.__version__) 96 | 97 | args = parser.parse_args(args) 98 | return args -------------------------------------------------------------------------------- /CureIAM/helpers/hlogging.py: -------------------------------------------------------------------------------- 1 | import logging.config 2 | import logging 3 | import copy 4 | import json 5 | import typing 6 | from multiprocessing import current_process 7 | 8 | DEFAULT_LOGGING_CONFIG = { #temporary config for logging, will move it to yaml file in the future 9 | 'version': 1, 10 | 'disable_existing_loggers': True, 11 | 'formatters': { 12 | 'main': { 13 | 'format': '[%(asctime)s][%(process)s][%(processName)s][%(threadName)s] - %(levelname)s %(name)s:%(lineno)d - %(message)s', 14 | 'datefmt': '%Y-%m-%d %H:%M:%S' 15 | }, 16 | 'standard': { 17 | 'format': '[%(process)s][%(processName)s][%(threadName)s] %(message)s', 18 | 'datefmt': '%Y-%m-%d %H:%M:%S' 19 | }, 20 | }, 21 | 'handlers': { 22 | 'main': { 23 | 'formatter': 'main', 24 | 'class': 'logging.StreamHandler', 25 | 'stream': 'ext://sys.stdout', # Default is stderr 26 | }, 27 | 'console': { 28 | 'formatter': 'standard', 29 | 'class': 'logging.StreamHandler', 30 | 'stream': 'ext://sys.stdout', # Default is stderr 31 | }, 32 | 'rich': { 33 | 'formatter': 'standard', 34 | 'class': 'rich.logging.RichHandler', 35 | }, 36 | 'file': { 37 | 'formatter': 'standard', 38 | 'class': 'logging.handlers.TimedRotatingFileHandler', 39 | 'filename': '/tmp/CureIAM.log', 40 | 'when': 'midnight', 41 | 'encoding': 'utf8', 42 | 'backupCount': 5, 43 | }, 44 | }, 45 | 'loggers': { 46 | '': { # root logger 47 | 'handlers': ['file','rich'], 48 | 'level': 'INFO', 49 | 'propagate': False 50 | }, 51 | 'CureIAM.plugins.gcp.*': { 52 | 'handlers': ['file','rich'], 53 | 'level': 'INFO', 54 | 'propagate': False 55 | }, 56 | 'CureIAM.plugins.gcp.gcpcloud': { #temporary solution adding file one by one till we've found the way to log properly 57 | 'handlers': ['file','rich'], 58 | 'level': 'INFO', 59 | 'propagate': False 60 | }, 61 | 'CureIAM.plugins.elastic.esstore': { #temporary solution adding file one by one till we've found the way to log properly 62 | 'handlers': ['file','rich'], 63 | 'level': 'INFO', 64 | 'propagate': False 65 | }, 66 | 'CureIAM.plugins.files.filestore': { #temporary solution adding file one by one till we've found the way to log properly 67 | 'handlers': ['file','rich'], 68 | 'level': 'INFO', 69 | 'propagate': False 70 | }, 71 | 'CureIAM.ioworkers': { #temporary solution adding file one by one till we've found the way to log properly 72 | 'handlers': ['file','rich'], 73 | 'level': 'INFO', 74 | 'propagate': False 75 | }, 76 | # 'CureIAM.helpers.hconfigs': { 77 | # 'handlers': ['rich'], 78 | # 'level': 'INFO', 79 | # 'propagate': False 80 | # } 81 | } 82 | } 83 | 84 | """ 85 | Will deprecated this function soon, BETA Testing 86 | """ 87 | def get_logger(name): 88 | # process_name = current_process() 89 | # print (current_process()) 90 | logging_config = None 91 | 92 | if logging_config == None: 93 | logging_config = DEFAULT_LOGGING_CONFIG 94 | 95 | logging.config.dictConfig(logging_config) 96 | 97 | return logging.getLogger(name) 98 | 99 | def obfuscated(data): #it only for demo purposed 100 | len_data = len(data) 101 | idx = data.find('-') 102 | 103 | temp = data[0:2] + ('*'*(len_data-2)) 104 | 105 | return temp 106 | 107 | 108 | """ 109 | Creating a Singleton logging class to be called everywhere 110 | """ 111 | class Logger(object): 112 | """Creating for logging purpose only""" 113 | 114 | _LOG_CONFIG: typing.Optional[dict] = None 115 | 116 | @staticmethod 117 | def set_logger(log_config): 118 | # print (log_config) 119 | Logger._LOG_CONFIG = log_config 120 | 121 | @staticmethod 122 | def get_logger(name): 123 | logging_config = Logger._LOG_CONFIG 124 | 125 | if logging_config == None: 126 | logging_config = DEFAULT_LOGGING_CONFIG 127 | 128 | logging.config.dictConfig(logging_config) 129 | 130 | return logging.getLogger(name) -------------------------------------------------------------------------------- /CureIAM/ioworkers.py: -------------------------------------------------------------------------------- 1 | """Concurrent input/output workers implementation. 2 | """ 3 | 4 | import multiprocessing 5 | import os 6 | import threading 7 | from CureIAM.helpers import hlogging 8 | 9 | _log = hlogging.get_logger(__name__) 10 | 11 | def run(input_func, output_func, processes=0, threads=0, log_tag=''): 12 | """Run concurrent input/output workers with specified functions. 13 | 14 | A two-level hierarchy of workers are created using both 15 | multiprocessing as well as multithreading. At first, ``processes`` 16 | number of worker processes are created. Then within each process 17 | worker, ``threads`` number of worker threads are created. Thus, in 18 | total, ``processes * threads`` number of worker threads are created. 19 | 20 | Arguments: 21 | input_func (callable): A callable which when called yields 22 | tuples. Each tuple must represent arguments to be passed to 23 | ``output_func``. 24 | output_func (callable): A callable that can accept as arguments 25 | an unpacked tuple yielded by ``input_func``. When called, 26 | this callable must work on the arguments and return an 27 | output value. This callable must not return ``None`` for any 28 | input. 29 | processes (int): Number of worker processes to run. If 30 | unspecified or ``0`` or negative integer is specified, then 31 | the number returned by :func:`os.cpu_count` is used. 32 | threads (int): Number of worker threads to run in each process. 33 | If unspecified or ``0`` or negative integer is specified, 34 | then `5` multiplied by the number returned by 35 | :func:`os.cpu_count` is used. 36 | log_tag (str): String to include in every log message. This 37 | helps in differentiating between different workers invoked 38 | by different callers. 39 | 40 | Yields: 41 | Each output value returned by ``output_func``. 42 | 43 | """ 44 | if processes <= 0: 45 | processes = os.cpu_count() 46 | 47 | if threads <= 0: 48 | threads = os.cpu_count() * 5 49 | 50 | if log_tag != '': 51 | log_tag += ': ' 52 | 53 | in_q = multiprocessing.Queue() 54 | out_q = multiprocessing.Queue() 55 | 56 | # Create process workers. 57 | process_workers = [] 58 | for _ in range(processes): 59 | w = multiprocessing.Process(target=_process_worker, 60 | args=(in_q, out_q, threads, 61 | output_func, log_tag)) 62 | w.start() 63 | process_workers.append(w) 64 | 65 | # Get input data for thread workers to work on. 66 | for args in input_func(): 67 | in_q.put(args) 68 | 69 | # Tell each thread worker that there is no more input to work on. 70 | for _ in range(processes * threads): 71 | in_q.put(None) 72 | 73 | # Consume output objects from thread workers and yield them. 74 | yield from _get_output(out_q, processes, threads, log_tag) 75 | 76 | # Wait for process workers to terminate. 77 | for w in process_workers: 78 | w.join() 79 | 80 | def _process_worker(in_q, out_q, threads, output_func, log_tag): 81 | """Process worker.""" 82 | thread_workers = [] 83 | for _ in range(threads): 84 | w = threading.Thread(target=_thread_worker, 85 | args=(in_q, out_q, output_func, log_tag)) 86 | w.start() 87 | thread_workers.append(w) 88 | for w in thread_workers: 89 | w.join() 90 | 91 | 92 | def _thread_worker(in_q, out_q, output_func, log_tag): 93 | """Thread worker.""" 94 | while True: 95 | try: 96 | work = in_q.get() 97 | if work is None: 98 | out_q.put(None) 99 | break 100 | for record in output_func(*work): 101 | out_q.put(record) 102 | except Exception as e: 103 | _log.exception('thread_worker: %sFailed; error: %s: %s', 104 | log_tag, type(e).__name__, e) 105 | 106 | 107 | def _get_output(out_q, processes, threads, log_tag): 108 | """Get output from output queue and yield them.""" 109 | stopped_threads = 0 110 | while True: 111 | try: 112 | record = out_q.get() 113 | if record is None: 114 | stopped_threads += 1 115 | if stopped_threads == processes * threads: 116 | break 117 | continue 118 | yield record 119 | except Exception as e: 120 | _log.exception('%sFailed to get output; error: %s: %s', 121 | log_tag, type(e).__name__, e) 122 | -------------------------------------------------------------------------------- /CureIAM/helpers/hemails.py: -------------------------------------------------------------------------------- 1 | import email 2 | import smtplib 3 | from CureIAM.helpers import hlogging 4 | 5 | _log = hlogging.get_logger(__name__) 6 | 7 | def send(from_addr, to_addrs, subject, content, 8 | host='', port=0, ssl_mode='ssl', 9 | username='', password='', debug=0): 10 | """Send email message. 11 | 12 | When ``ssl_mode` is ``'ssl'`` and ``host`` is uspecified or 13 | specified as ``''`` (the default), the local host is used. When 14 | ``ssl_mode`` is ``'ssl'`` and ``port`` is unspecified or specified 15 | as ``0``, the standard SMTP-over-SSL port, i.e., port 465, is used. 16 | See :class:`smtplib.SMTP_SSL` documentation for more details on 17 | this. 18 | 19 | When ``ssl_mode`` is ``'ssl'` and if ``host`` or ``port`` are 20 | unspecified, i.e., if host or port are ``''`` and/or ``0``, 21 | respectively, the OS default behavior is used. See 22 | :class:`smtplib.SMTP` documentation for more details on this. 23 | 24 | We recommend these parameter values: 25 | 26 | - Leave ``ssl_mode`` unspecified (thus ``'ssl'`` by default) if 27 | your SMTP server supports SSL. 28 | 29 | - Set ``ssl_mode`` to ``'starttls'`` explicitly if your SMTP server 30 | does not support SSL but it supports STARTTLS. 31 | 32 | - Set ``ssl_mode`` to ``'disable'`` explicitly if your SMTP server 33 | supports neither SSL nor STARTTLS. 34 | 35 | - Set ``host`` to the SMTP hostname or address explicitly. 36 | 37 | - Leave ``port`` unspecified (thus ``0`` by default), so that the 38 | appropriate port is chosen automatically. 39 | 40 | With these recommendations, this function should do the right thing 41 | automatically, i.e., connect to port 465 if ``use_ssl`` is 42 | unspecified or ``False`` and port 25 if ``use_ssl`` is ``True``. 43 | 44 | Note that in case of SMTP, there are two different encryption 45 | protocols in use: 46 | 47 | - SSL/TLS (or implicit SSL/TLS): SSL/TLS is used from the beginning 48 | of the connection. This occurs typically on port 465. This is 49 | enabled by default (``ssl_mode`` as ``'ssl'``). 50 | 51 | - STARTTLS (or explicit SSL/TLS): The SMTP session begins as a 52 | plaintext session. Then the client (this function in this case) 53 | makes an explicit request to switch to SSL/TLS by sending the 54 | ``STARTTLS`` command to the server. This occurs typically on port 55 | 25 or port 587. Set ``ssl_mode`` to ``'starttls'`` to enable this 56 | behaviour 57 | 58 | If ``username`` is unspecified or specified as an empty string, no 59 | SMTP authentication is done. If ``username`` is specified as a 60 | non-empty string, then SMTP authentication is done. 61 | 62 | Arguments: 63 | from_addr (str): Sender's email address. 64 | to_addrs (list): A list of :obj:`str` objects where each 65 | :obj:`str` object is a recipient's email address. 66 | subject (str): Email subject. 67 | content (str): Email content. 68 | host (str): SMTP host. 69 | port (int): SMTP port. 70 | ssl_mode (str): SSL mode to use: ``'ssl'`` for SSL/TLS 71 | connection (the default), ``'starttls'`` for STARTTLS, and 72 | ``'disable'`` to disable SSL. 73 | username (str): SMTP username. 74 | password (str): SMTP password. 75 | debug (int or bool): Debug level to pass to 76 | :meth:`SMTP.set_debuglevel` to debug an SMTP session. Set to 77 | ``0`` (the default) or ``False`` to disable debugging. Set 78 | to ``1`` or ``True`` to see SMTP messages. Set to ``2`` to 79 | see timestamped SMTP messages. 80 | 81 | """ 82 | log_data = ('from_addr: {}; to_addrs: {}; subject: {}; host: {}; ' 83 | 'port: {}; ssl_mode: {}' 84 | .format(from_addr, to_addrs, subject, host, port, ssl_mode)) 85 | try: 86 | if ssl_mode == 'ssl': 87 | smtp = smtplib.SMTP_SSL(host, port) 88 | smtp.set_debuglevel(debug) 89 | elif ssl_mode == 'starttls': 90 | smtp = smtplib.SMTP(host, port) 91 | smtp.set_debuglevel(debug) 92 | smtp.starttls() 93 | elif ssl_mode == 'disable': 94 | smtp = smtplib.SMTP(host, port) 95 | smtp.set_debuglevel(debug) 96 | else: 97 | _log.error('Cannot send email; %s; error: %s: %s', log_data, 98 | 'invalid ssl_mode', ssl_mode) 99 | return 100 | 101 | if username: 102 | smtp.login(username, password) 103 | 104 | msg = email.message.EmailMessage() 105 | msg['From'] = from_addr 106 | msg['To'] = ', '.join(to_addrs) 107 | msg['Subject'] = subject 108 | msg.set_content(content) 109 | 110 | smtp.send_message(msg) 111 | smtp.quit() 112 | 113 | _log.info('Sent email successfully; %s', log_data) 114 | 115 | except Exception as e: 116 | _log.error('Failed to send email; %s; error: %s: %s', log_data, 117 | type(e).__name__, e) -------------------------------------------------------------------------------- /CureIAM/plugins/elastic/esstore.py: -------------------------------------------------------------------------------- 1 | """Elasticsearch store plugin.""" 2 | 3 | import json 4 | import datetime 5 | 6 | from elasticsearch import Elasticsearch, ElasticsearchWarning 7 | 8 | from CureIAM.helpers import hlogging 9 | 10 | _log = hlogging.get_logger(__name__) 11 | 12 | class EsStore: 13 | """Elasticsearch adapter to index cloud data in Elasticsearch.""" 14 | 15 | def __init__(self, host='localhost', port=9200, index='iam_recommending', 16 | username=None, 17 | password=None, 18 | scheme='http', 19 | buffer_size=5000000): 20 | """Create an instance of :class:`EsStore` plugin. 21 | 22 | The plugin uses the default port for Elasticsearch if not 23 | specified. 24 | 25 | The ``buffer_size`` for the plugin is the value for the maximum 26 | number of bytes of data to be sent in a bulk API request to 27 | Elasticsearch. 28 | 29 | Arguments: 30 | host (str): Elasticsearch host 31 | port (int): Elasticsearch port 32 | index (str): Elasticsearch index 33 | buffer_size (int): Maximum number of bytes of data to hold 34 | in the in-memory buffer. 35 | 36 | """ 37 | 38 | # _log.info('INIT INDEX ESSTORE') 39 | if username and password: 40 | self._es = Elasticsearch([{'host': host, 'port': port, 'scheme': scheme}], http_auth=(username, password)) 41 | else: 42 | self._es = Elasticsearch([{'host': host, 'port': port, 'scheme': scheme}]) 43 | self._index = index 44 | self._buffer_size = buffer_size 45 | self._buffer = '' 46 | self._cur_buffer_size = 0 47 | 48 | # TODO: Add method to create mapping for efficient indexing of data. 49 | 50 | # TODO: Add method to prune old data. 51 | 52 | # TODO: Add support for multiple indexes 53 | 54 | def _doc_index_body(self, doc, doc_id=None): 55 | """Create the body for a bulk insert API call to Elasticsearch. 56 | 57 | Arguments: 58 | doc (dict): Document 59 | doc_id: Document ID 60 | 61 | Returns: 62 | (str): Request body corresponding to the ``doc``. 63 | 64 | """ 65 | action_def = { 66 | 'index': { 67 | '_index': self._index, 68 | '_id': doc_id 69 | } 70 | } 71 | src_def = doc 72 | return json.dumps(action_def) + '\n' + json.dumps(src_def) + '\n' 73 | 74 | def _flush(self): 75 | """Bulk insert buffered records into Elasticserach.""" 76 | try: 77 | # print (f"=== {self._buffer} ===") 78 | resp = self._es.bulk( 79 | index=self._index, 80 | operations=self._buffer 81 | ) 82 | except ElasticsearchWarning as e: 83 | # Handles exceptions of all types defined here. 84 | # https://github.com/elastic/elasticsearch-py/blob/master/elasticsearch/exceptions.py 85 | _log.error('Bulk Index Error: %s: %s', type(e).__name__, e) 86 | print(self._buffer) 87 | return 88 | 89 | # Read and parse the response. 90 | items = resp['items'] 91 | records_sent = len(items) 92 | fail_count = 0 93 | 94 | # If response code for an item is not 2xx, increment the count of 95 | # failed insertions. 96 | if resp['errors']: 97 | for item in items: 98 | if not 199 < item['index']['status'] < 300: 99 | fail_count += 1 100 | _log.debug('Failed to insert record; ID: %s', 101 | item['index']['_id']) 102 | _log.error('Failed to write %d records', fail_count) 103 | 104 | _log.info('Indexed %d records', records_sent - fail_count) 105 | 106 | # Reset the buffer. 107 | self._cur_buffer_size = 0 108 | self._buffer = '' 109 | 110 | def write(self, record): 111 | """Write JSON records to the Elasticsearch index. 112 | 113 | Flush the buffer by saving its content to Elasticsearch when 114 | the buffer size exceeds the configured size. 115 | 116 | Arguments: 117 | record (dict): Data to save to Elasticsearch. 118 | 119 | """ 120 | # Before writing data to ES add time stamp for indexing. 121 | # Note: For kibana to process timestamp, the field should be in ISO fmt 122 | record.update({ 123 | 'timestamp': str(datetime.datetime.utcnow().isoformat()) 124 | }) 125 | 126 | es_record = self._doc_index_body(record) # TODO: Send valid doc ID 127 | es_record_bytes = len(es_record) 128 | if (self._cur_buffer_size and 129 | es_record_bytes + self._cur_buffer_size > self._buffer_size): 130 | self._flush() 131 | else: 132 | self._buffer += es_record 133 | self._cur_buffer_size += es_record_bytes 134 | 135 | def done(self): 136 | """Flush pending records to Elasticsearch.""" 137 | if self._cur_buffer_size: 138 | self._flush() 139 | -------------------------------------------------------------------------------- /CureIAM/models/iamriskscore.py: -------------------------------------------------------------------------------- 1 | """ IAM Risk score model class implementation 2 | """ 3 | 4 | from CureIAM.helpers import hlogging 5 | 6 | _log = hlogging.get_logger(__name__) 7 | 8 | class IAMRiskScoreModel: 9 | """IAMRiskScoreModel plugin for GCP IAM Recommendation records.""" 10 | 11 | def __init__(self, record): 12 | """Create an instance of :class:`IAMRiskScoreModel` plugin. 13 | 14 | This model class generates scores for the recommendation record. 15 | Arguments: 16 | record: dict for GCP record 17 | """ 18 | self._record = record 19 | 20 | self._score = { 21 | 'safe_to_apply_recommendation_score': None, 22 | 'safe_to_apply_recommendation_score_factors': None, 23 | 'risk_score': None, 24 | 'risk_score_factors': None, 25 | } 26 | 27 | def score(self): 28 | """This function will return the score for a specific record 29 | This will work on the paramters and will create risk score and 30 | safe_to_apply_score 31 | 32 | parameters: 33 | - account_type: 34 | - service_account 35 | - user_account 36 | - group_account 37 | - suggestion_type: 38 | - REMOVE_ROLE 39 | - REPLACE_ROLE 40 | - REPLACE_ROLE_CUSTOMIZABLE 41 | - recommendation_impact_type: 42 | - Security 43 | - inferred_parameters: 44 | - used_permissions 45 | - total_permissions 46 | 47 | inferences: 48 | - helpful_in_collector_enforement: 49 | - safe_to_apply_recommendation_score ∝ (account_type == {user, group}) 50 | - safe_to_apply_recommendation_score ∝ (last time permissions used) 51 | - safe_to_apply_recommendation_score 1/∝ (account_type == {service_account, terraform_account, provisioned throught IaC scripts}) 52 | - safe_to_apply_recommendation_score ∝ (suggestion_type == {REMOVE_ROLE} 53 | - safe_to_apply_recommendation_score 1/∝ (suggestion_type == {REPLACE_ROLE, REPLACE_ROLE_CUSTOMIZABLE}) 54 | - safe_to_apply_recommendation_score 1/∝ (excess_permissions) -- Code Cracking changes ?? 55 | - helpful_in_auditing_dashabord: 56 | - risk_index ∝ (account_type == {account_type == service_account}) 57 | - risk_index 1/∝ (account_type == {user, group}) 58 | - risk_index ∝ {excess_permissions} 59 | - risk_index ∝ {recommendation_impact_type} 60 | - risk_index ∝ {total_permissions} 61 | """ 62 | # print (self._record) 63 | 64 | _account_type = self._record['account_type'] 65 | _suggestion_type = self._record['account_permission_insights_category'] 66 | _used_permissions = int(self._record['account_used_permissions']) 67 | _total_permissions = int(self._record['account_total_permissions'] if self._record['account_total_permissions'] is not None else _used_permissions + 1) 68 | 69 | _excess_permissions = _total_permissions - _used_permissions 70 | # In case excess permissions are 0, make sure the excess permissions are set to 1, other wise this 71 | # will throw error in production. 72 | if _excess_permissions < 1 : 73 | _excess_permissions = 1 74 | 75 | if _total_permissions < 1 : 76 | _total_permissions = 1 77 | 78 | _excess_permissions_percent = _excess_permissions / _total_permissions 79 | 80 | safe_to_apply_recommendation_score = 0 81 | 82 | # Based on the parameters above lets calculate safety 83 | if _account_type == 'user': 84 | safe_to_apply_recommendation_score = 60 85 | elif _account_type == 'group': 86 | safe_to_apply_recommendation_score = 30 87 | else: 88 | safe_to_apply_recommendation_score = 0 89 | 90 | if _suggestion_type == 'REMOVE_ROLE': 91 | safe_to_apply_recommendation_score += 30 92 | elif _suggestion_type == 'REPLACE_ROLE': 93 | safe_to_apply_recommendation_score += 20 94 | else: 95 | safe_to_apply_recommendation_score += 10 96 | 97 | safe_to_apply_recommendation_score /= _excess_permissions_percent 98 | 99 | self._score.update( 100 | { 101 | 'safe_to_apply_recommendation_score': round(safe_to_apply_recommendation_score), 102 | 'safe_to_apply_recommendation_score_factors': 3 103 | } 104 | ) 105 | 106 | risk_score = 0 107 | # Based on the parameters above lets calculate risk_profile 108 | # Risk can be calculated as compound function ?? 109 | # (1+r)^n 110 | n = { 111 | 'user': 2, 112 | 'group': 3, 113 | 'serviceAccount': 5 114 | } 115 | r = _excess_permissions / _total_permissions 116 | risk_score = r**n[_account_type] * 100 117 | 118 | self._score.update( 119 | { 120 | 'risk_score': round(risk_score), 121 | 'risk_score_factors': 2 122 | } 123 | ) 124 | self._score.update( 125 | { 126 | 'over_privilege_score': round(_excess_permissions_percent * 100) 127 | } 128 | ) 129 | return self._score -------------------------------------------------------------------------------- /CureIAM/plugins/gcp/gcpcloud.py: -------------------------------------------------------------------------------- 1 | """Plugin to read the data from the GCP IAM recommendation API 2 | """ 3 | 4 | import json 5 | import logging 6 | from CureIAM.helpers import hlogging 7 | from CureIAM.helpers.hconfigs import Config 8 | 9 | # logging.config.dictConfig() 10 | 11 | # _log = logging.getLogger(__name__) 12 | 13 | _log = hlogging.get_logger(__name__) 14 | 15 | # from google.oauth2 import service_account #reducing the library call 16 | 17 | from CureIAM import ioworkers 18 | from . import util_gcp #call function from same folder 19 | 20 | """OAuth 2.0 scopes for Google APIs required by this plugin. 21 | 22 | See https://developers.google.com/identity/protocols/googlescopes for 23 | more details on OAuth 2.0 scopes for Google APIs.""" 24 | # TODO: Redefine scopes 25 | _GCP_SCOPES = ['https://www.googleapis.com/*'] 26 | 27 | class GCPCloudIAMRecommendations: 28 | """GCP cloud IAM recomendation plugin.""" 29 | 30 | def __init__(self, key_file_path, projects='*', processes=4, threads=10): 31 | """Create an instance of :class:`GCPCloudIAMRecommendations` plugin. 32 | 33 | Arguments: 34 | key_file_path (str): Path of the service account key file for a project. 35 | processes (int): Number of processes to launch. 36 | threads (int): Number of threads to launch in each process. 37 | 38 | """ 39 | self._key_file_path = key_file_path 40 | self._projects = projects 41 | self._processes = processes 42 | self._threads = threads 43 | 44 | # Create credentials for python client from service account key file 45 | # credentials = service_account.Credentials.from_service_account_file( 46 | # self._key_file_path, scopes=_GCP_SCOPES) 47 | 48 | 49 | # testing the use case, will remove it later if it's not work # 50 | credentials = util_gcp.get_service_account_class().from_service_account_file( 51 | self._key_file_path, scopes=_GCP_SCOPES) 52 | 53 | # testing the use case, will remove it later if it's not work # 54 | 55 | # Scan needs to be done for list of projects or all the projects 56 | # projects='*' indicates all projects 57 | if self._projects == '*': 58 | _log.info('Plugin started in Scan-All-Projects-Mode') 59 | # Get the list of all the projects available 60 | self._projects = [] 61 | cloudresourcemanager_service = util_gcp.build_resource( 62 | 'cloudresourcemanager', 63 | self._key_file_path, 64 | 'v1') 65 | for project in util_gcp.get_resource_iterator( 66 | cloudresourcemanager_service.projects(), 67 | 'projects'): 68 | self._projects.append(project['projectId']) 69 | 70 | _log.info('Projects %s', len(self._projects)) 71 | 72 | # Service account key file also has the client email under the key 73 | # client_email. We will use this key file to get the client email for 74 | # this request. 75 | try: 76 | with open(self._key_file_path) as f: 77 | self._client_email = json.loads(f.read()).get('client_email') 78 | except OSError as e: 79 | self._client_email = '' 80 | _log.error('Failed to read client_email from key file: %s; ' 81 | 'error: %s: %s', self._key_file_path, 82 | type(e).__name__, e) 83 | 84 | _log.info('Initialized; key_file_path: %s; processes: %s; threads: %s;' 85 | 'projects to scan: %d', 86 | self._key_file_path, 87 | self._processes, self._threads, 88 | len(self._projects)) 89 | 90 | def read(self): 91 | """Return a GCP cloud infrastructure configuration record. 92 | 93 | Yields: 94 | dict: A GCP cloud infrastructure configuration record. 95 | 96 | """ 97 | yield from ioworkers.run(self._get_projects, 98 | self._get_recommendations, 99 | self._processes, 100 | self._threads, 101 | __name__) 102 | 103 | def _get_projects(self): 104 | """Generate tuples of record types and projects. 105 | 106 | The yielded tuples when unpacked would become arguments for 107 | :meth:`_get_recommendations`. Each such tuple represents a single unit 108 | of work that :meth:`_get_recommendations` can work on independently in 109 | its own worker thread. 110 | 111 | Yields: 112 | tuple: A tuple which when unpacked forms valid arguments for 113 | :meth:`_get_recommendations`. 114 | 115 | """ 116 | try: 117 | 118 | for project in self._projects: 119 | yield ('project_record', '', project, 'global') 120 | 121 | except Exception as e: 122 | _log.error('Failed to fetch projects; key_file_path: %s; ' 123 | 'error: %s: %s', self._key_file_path, 124 | type(e).__name__, e) 125 | 126 | def _get_recommendations(self, record_type, project_index, project, zone=None): 127 | """Generate tuples of record as recommendation for a specific project. 128 | 129 | The yielded tuples when unpacked would become arguments for processor workers 130 | 131 | Yields: 132 | tuple: A tuple which when unpacked forms valid arguments for 133 | :meth:`_get_recommendations`. 134 | 135 | """ 136 | _log.info('Fetching recommendations for project : %s ...', hlogging.obfuscated(project)) 137 | 138 | parent_string = 'projects/{project}/locations/{location}/recommenders/{recommenders}'.format( 139 | project=project, 140 | location='global', 141 | recommenders='google.iam.policy.Recommender' 142 | ) 143 | 144 | recommendations_service = util_gcp.build_resource('recommender', 145 | self._key_file_path, 146 | 'v1') 147 | 148 | recommendations_iterator = util_gcp.get_resource_iterator((recommendations_service 149 | .projects() 150 | .locations() 151 | .recommenders() 152 | .recommendations()), 'recommendations', parent=parent_string) 153 | 154 | for _, recommendation in enumerate(recommendations_iterator): 155 | recommendation.update({'project': project}) 156 | 157 | # Fetch the insights for each recommendation 158 | _insights = [] 159 | 160 | for insight in recommendation.get('associatedInsights', None): 161 | _pattern = insight.get('insight', None) 162 | if _pattern: 163 | i = (recommendations_service 164 | .projects() 165 | .locations() 166 | .insightTypes() 167 | .insights().get(name=_pattern).execute()) 168 | _insights.append(i) 169 | 170 | recommendation.update( 171 | { 'insights': _insights } 172 | ) 173 | 174 | yield { 175 | 'raw': recommendation 176 | } 177 | 178 | _log.info('Fetched recommendations for project: %s', hlogging.obfuscated(project)) 179 | 180 | 181 | def done(self): 182 | """Log a message that this plugin is done.""" 183 | _log.info('GCP IAM Audit done') 184 | -------------------------------------------------------------------------------- /CureIAM/workers.py: -------------------------------------------------------------------------------- 1 | """Worker functions. 2 | """ 3 | 4 | 5 | from CureIAM.helpers import util 6 | from CureIAM.plugins import util_plugins 7 | 8 | from CureIAM.helpers import hlogging 9 | 10 | _log = hlogging.get_logger(__name__) 11 | 12 | def cloud_worker(audit_key, audit_version, plugin_key, plugin_config, 13 | output_queues): 14 | """Worker function for cloud plugins. 15 | 16 | This function instantiates a plugin object from the 17 | ``plugin_config`` dictionary. This function expects the plugin 18 | object to implement a ``read`` method that yields records. This 19 | function calls this ``read`` method to retrieve records and puts 20 | each record into each queue in ``output_queues``. 21 | 22 | Arguments: 23 | audit_key (str): Audit key name in configuration. 24 | audit_version (str): Audit version string. 25 | plugin_key (str): Plugin key name in configuration. 26 | plugin_config (dict): Cloud plugin config dictionary. 27 | output_queues (list): List of :class:`multiprocessing.Queue` 28 | objects to write records to. 29 | 30 | """ 31 | worker_name = audit_key + '_' + plugin_key 32 | _log.info('cloud_worker: %s: Started', worker_name) 33 | 34 | try: 35 | plugin = util_plugins.load(plugin_config) 36 | for record in plugin.read(): 37 | record['com'] = util.merge_dicts(record.get('com', {}), { 38 | 'audit_key': audit_key, 39 | 'audit_version': audit_version, 40 | 'origin_key': plugin_key, 41 | 'origin_class': type(plugin).__name__, 42 | 'origin_worker': worker_name, 43 | 'origin_type': 'cloud', 44 | }) 45 | for q in output_queues: 46 | q.put(record) 47 | 48 | plugin.done() 49 | 50 | except Exception as e: 51 | _log.exception('cloud_worker: %s: Failed; error: %s: %s', 52 | worker_name, type(e).__name__, e) 53 | 54 | _log.info('cloud_worker: %s: Stopped', worker_name) 55 | 56 | 57 | def processor_worker(audit_key, audit_version, plugin_key, plugin_config, 58 | input_queue, output_queues): 59 | """Worker function for processor plugins. 60 | 61 | This function instantiates a plugin object from the 62 | ``plugin_config`` dictionary. This function expects the plugin 63 | object to implement an ``eval`` method that accepts a single record 64 | as a parameter and yields one or more records, and a ``done`` method 65 | to perform cleanup work in the end. 66 | 67 | This function gets records from ``input_queue`` and passes each 68 | record to the ``eval`` method of the plugin object. Then it puts 69 | each record yielded by the ``eval`` method into each queue in 70 | ``output_queues``. 71 | 72 | When there are no more records in the ``input_queue``, i.e., once 73 | ``None`` is found in the ``input_queue``, this function calls the 74 | ``done`` method of the plugin object to indicate that record 75 | processing is over. 76 | 77 | Arguments: 78 | audit_key (str): Audit key name in configuration. 79 | audit_version (str): Audit version string. 80 | plugin_key (str): Plugin key name in configuration. 81 | plugin_config (dict): processor plugin config dictionary. 82 | input_queue (multiprocessing.Queue): Queue to read records from. 83 | output_queues (list): List of :class:`multiprocessing.Queue` 84 | objects to write records to. 85 | 86 | """ 87 | worker_name = audit_key + '_' + plugin_key 88 | _log.info('processor_worker: %s: Started', worker_name) 89 | 90 | try: 91 | plugin = util_plugins.load(plugin_config) 92 | except Exception as e: 93 | _log.exception('processor_worker: %s: Failed; error: %s: %s', 94 | worker_name, type(e).__name__, e) 95 | _log.info('processor_worker: %s: Stopped', worker_name) 96 | return 97 | 98 | while True: 99 | try: 100 | record = input_queue.get() 101 | if record is None: 102 | _log.info('processor_worker: %s: Stopping', worker_name) 103 | plugin.done() 104 | break 105 | 106 | for processor_record in plugin.eval(record): 107 | processor_record['com'] = \ 108 | util.merge_dicts(processor_record.get('com', {}), { 109 | 'audit_key': audit_key, 110 | 'audit_version': audit_version, 111 | 'origin_key': plugin_key, 112 | 'origin_class': type(plugin).__name__, 113 | 'origin_worker': worker_name, 114 | 'origin_type': 'processor', 115 | }) 116 | 117 | for q in output_queues: 118 | q.put(processor_record) 119 | 120 | except Exception as e: 121 | _log.exception('processor_worker: %s: Failed; error: %s: %s', 122 | worker_name, type(e).__name__, e) 123 | 124 | _log.info('processor_worker: %s: Stopped', worker_name) 125 | 126 | 127 | def store_worker(audit_key, audit_version, plugin_key, plugin_config, 128 | input_queue): 129 | """Worker function for store plugins. 130 | 131 | This function instantiates a plugin object from the 132 | ``plugin_config`` dictionary. This function expects the plugin 133 | object to implement a ``write`` method that accepts a single record 134 | as a parameter and a ``done`` method to perform cleanup work in the 135 | end. 136 | 137 | This function gets records from ``input_queue`` and passes each 138 | record to the ``write`` method of the plugin object. 139 | 140 | When there are no more records in the ``input_queue``, i.e., once 141 | ``None`` is found in the ``input_queue``, this function calls the 142 | ``done`` method of the plugin object to indicate that record 143 | processing is over. 144 | 145 | Arguments: 146 | audit_key (str): Audit key name in configuration. 147 | audit_version (str): Audit version string. 148 | plugin_key (str): Plugin key name in configuration. 149 | plugin_config (dict): Store plugin config dictionary. 150 | input_queue (multiprocessing.Queue): Queue to read records from. 151 | 152 | """ 153 | _write_worker(audit_key, audit_version, plugin_key, plugin_config, 154 | input_queue, 'store') 155 | 156 | 157 | def alert_worker(audit_key, audit_version, plugin_key, plugin_config, 158 | input_queue): 159 | """Worker function for alert plugins. 160 | 161 | This function behaves like :func:`CureIAM.workers.store_worker`. 162 | See its documentation for details. 163 | 164 | Arguments: 165 | audit_key (str): Audit key name in configuration. 166 | audit_version (str): Audit version string. 167 | plugin_key (str): Plugin key name in configuration. 168 | plugin_config (dict): Alert plugin config dictionary. 169 | input_queue (multiprocessing.Queue): Queue to read records from. 170 | 171 | """ 172 | _write_worker(audit_key, audit_version, plugin_key, plugin_config, 173 | input_queue, 'alert') 174 | 175 | 176 | def _write_worker(audit_key, audit_version, plugin_key, plugin_config, 177 | input_queue, worker_type): 178 | """Worker function for store and alert plugins. 179 | 180 | Arguments: 181 | audit_key (str): Audit key name in configuration 182 | audit_version (str): Audit version string. 183 | plugin_key (str): Plugin key name in configuration. 184 | plugin_config (dict): Store or alert plugin config dictionary. 185 | input_queue (multiprocessing.Queue): Queue to read records from. 186 | worker_type (str): Either ``'store'`` or ``'alert'``. 187 | 188 | """ 189 | worker_name = audit_key + '_' + plugin_key 190 | _log.info('%s_worker: %s: Started', worker_type, worker_name) 191 | 192 | try: 193 | plugin = util_plugins.load(plugin_config) 194 | except Exception as e: 195 | _log.exception('%s_worker: %s: Failed; error: %s: %s', 196 | worker_type, worker_name, type(e).__name__, e) 197 | _log.info('%s_worker: %s: Stopped', worker_type, worker_name) 198 | return 199 | 200 | while plugin is not None: 201 | try: 202 | record = input_queue.get() 203 | if record is None: 204 | _log.info('%s_worker: %s: Stopping', 205 | worker_type, worker_name) 206 | plugin.done() 207 | break 208 | 209 | record['com'] = util.merge_dicts(record.get('com', {}), { 210 | 'audit_key': audit_key, 211 | 'audit_version': audit_version, 212 | 'target_key': plugin_key, 213 | 'target_class': type(plugin).__name__, 214 | 'target_worker': worker_name, 215 | 'target_type': worker_type, 216 | }) 217 | 218 | plugin.write(record) 219 | 220 | except Exception as e: 221 | _log.exception('%s_worker: %s: Failed; error: %s: %s', 222 | worker_type, worker_name, type(e).__name__, e) 223 | 224 | _log.info('%s_worker: %s: Stopped', worker_type, worker_name) 225 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 | 5 | # CureIAM 6 | 7 |

8 | 9 |

10 | 11 | ### Clean up of over permissioned IAM accounts on GCP infra in an automated way 12 | CureIAM is an easy-to-use, reliable, and performant engine for Least Privilege Principle Enforcement on GCP cloud infra. It enables DevOps and Security team to quickly clean up accounts in GCP infra that have granted permissions of more than what are required. CureIAM fetches the recommendations and insights from GCP IAM recommender, scores them and enforce those recommendations automatically on daily basic. It takes care of scheduling and all other aspects of running these enforcement jobs at scale. It is built on top of GCP IAM recommender APIs and [Cloudmarker](https://github.com/cloudmarker/cloudmarker) framework. 13 | 14 | 15 | ## Key features 16 | Discover what makes CureIAM scalable and production grade. 17 | - **Config driven** : The entire workflow of CureIAM is config driven. Skip to Config section to know more about it. 18 | - **Scalable** : Its is designed to scale because of its plugin driven, multiprocess and multi-threaded approach. 19 | - **Handles Scheduling**: Scheduling part is embedded in CureIAM code itself, configure the time, and CureIAM will run daily at that time note. 20 | - **Plugin driven**: CureIAM codebase is completely plugin oriented, which means, one can plug and play the existing plugins or create new to add more functionality to it. 21 | - **Track actionable insights**: Every action that CureIAM takes, is recorded for audit purpose, It can do that in file store and in elasticsearch store. If you want you can build other store plugins to push that to other stores for tracking purposes. 22 | - **Scoring and Enforcement**: Every recommendation that is fetch by CureIAM is scored against various parameters, after that couple of scores like `safe_to_apply_score`, `risk_score`, `over_privilege_score`. Each score serves a different purpose. For `safe_to_apply_score` identifies the capability to apply recommendation on automated basis, based on the threshold set in `CureIAM.yaml` config file. 23 | 24 | ## Usage 25 | Since CureIAM is built with python, you can run it locally with these commands. Before running make sure to have a configuration file ready in either of `/etc/CureIAM.yaml`, `~/.CureIAM.yaml`, `~/CureIAM.yaml`, or `CureIAM.yaml` and there is Service account JSON file present in current directory with name preferably `cureiamSA.json`. This SA private key can be named anything, but for docker image build, it is preferred to use this name. Make you to reference this file in config for GCP cloud. 26 | 27 | ```bash 28 | # Install necessary dependencies 29 | $ pip install -r requirements.txt 30 | 31 | # Run CureIAM now 32 | $ python -m CureIAM -n 33 | 34 | # Run CureIAM process as schedular 35 | $ python -m CureIAM 36 | 37 | # Check CureIAM help 38 | $ python -m CureIAM --help 39 | ``` 40 | 41 | CureIAM can be also run inside a docker environment, this is completely optional and can be used for CI/CD with K8s cluster deployment. 42 | 43 | ```bash 44 | # Build docker image from dockerfile 45 | $ docker build -t cureiam . 46 | 47 | # Run the image, as schedular 48 | $ docker run -d cureiam 49 | 50 | # Run the image now 51 | $ docker run -f cureiam -m cureiam -n 52 | ``` 53 | 54 | ## Config 55 | `CureIAM.yaml` configuration file is the heart of CureIAM engine. Everything that engine does it does it based on the pipeline configured in this config file. Let's break this down in different sections to make this config look simpler. 56 | 57 | 1. Let's configure first section, which is logging configuration and scheduler configuration. 58 | 59 | ```yaml 60 | logger: 61 | version: 1 62 | 63 | disable_existing_loggers: false 64 | 65 | formatters: 66 | verysimple: 67 | format: >- 68 | [%(process)s] 69 | %(name)s:%(lineno)d - %(message)s 70 | datefmt: "%Y-%m-%d %H:%M:%S" 71 | 72 | handlers: 73 | rich_console: 74 | class: rich.logging.RichHandler 75 | formatter: verysimple 76 | 77 | file: 78 | class: logging.handlers.TimedRotatingFileHandler 79 | formatter: simple 80 | filename: /tmp/CureIAM.log 81 | when: midnight 82 | encoding: utf8 83 | backupCount: 5 84 | 85 | loggers: 86 | adal-python: 87 | level: INFO 88 | 89 | root: 90 | level: INFO 91 | handlers: 92 | - rich_console 93 | - file 94 | 95 | schedule: "16:00" 96 | ``` 97 | This subsection of config uses, `Rich` logging module and schedules CureIAM to run daily at `16:00`. 98 | 99 | 2. Next section is configure different modules, which we MIGHT use in pipeline. This falls under `plugins` section in `CureIAM.yaml`. You can think of this section as declaration for different plugins. 100 | ```yaml 101 | plugins: 102 | gcpCloud: 103 | plugin: CureIAM.plugins.gcp.gcpcloud.GCPCloudIAMRecommendations 104 | params: 105 | key_file_path: cureiamSA.json 106 | 107 | filestore: 108 | plugin: CureIAM.plugins.files.filestore.FileStore 109 | 110 | gcpIamProcessor: 111 | plugin: CureIAM.plugins.gcp.gcpcloudiam.GCPIAMRecommendationProcessor 112 | params: 113 | mode_scan: true 114 | mode_enforce: true 115 | enforcer: 116 | key_file_path: cureiamSA.json 117 | allowlist_projects: 118 | - alpha 119 | blocklist_projects: 120 | - beta 121 | blocklist_accounts: 122 | - foo@bar.com 123 | allowlist_account_types: 124 | - user 125 | - group 126 | - serviceAccount 127 | blocklist_account_types: 128 | - None 129 | min_safe_to_apply_score_user: 0 130 | min_safe_to_apply_score_group: 0 131 | min_safe_to_apply_score_SA: 50 132 | 133 | esstore: 134 | plugin: CureIAM.plugins.elastic.esstore.EsStore 135 | params: 136 | # Change http to https later if your elastic are using https 137 | scheme: http 138 | host: es-host.com 139 | port: 9200 140 | index: cureiam-stg 141 | username: security 142 | password: securepassword 143 | ``` 144 | Each of these plugins declaration has to be of this form: 145 | ```yaml 146 | plugins: 147 | : 148 | plugin: 149 | params: 150 | param1: val1 151 | param2: val2 152 | ``` 153 | For example, for plugins `CureIAM.stores.esstore.EsStore` which is [this file](./stores/esstore) and class `EsStore`. All the params which are defined in yaml has to match the declaration in `__init__()` function of the same plugin class. 154 | 155 | 3. Once plugins are defined , next step is to define how to define pipeline for auditing. And it goes like this: 156 | ```yaml 157 | audits: 158 | IAMAudit: 159 | clouds: 160 | - gcpCloud 161 | processors: 162 | - gcpIamProcessor 163 | stores: 164 | - filestore 165 | - esstore 166 | ``` 167 | Multiple Audits can be created out of this. The one created here is named `IAMAudit` with three plugins in use, `gcpCloud`, `gcpIamProcessor`, `filestores` and `esstore`. Note these are the same plugin names defined in Step 2. Again this is like defining the pipeline, not actually running it. It will be considered for running with definition in next step. 168 | 169 | 4. Tell `CureIAM` to run the Audits defined in previous step. 170 | ```yaml 171 | run: 172 | - IAMAudits 173 | ``` 174 | And this makes the entire configuration for CureIAM, you can find the full sample [here](./SampleCureIAM.yaml), this config driven pipeline concept is inherited from [Cloudmarker](https://github.com/cloudmarker/cloudmarker) framework. 175 | 176 | ## Dashboard 177 | The JSON which is indexed in elasticsearch using Elasticsearch store plugin, can be used to generate dashboard in Kibana. 178 | 179 | ## Contribute 180 | [Please do!] We are looking for any kind of contribution to improve CureIAM's core funtionality and documentation. When in doubt, make a PR! 181 | 182 | ## Credits 183 | Gojek Product Security Team :heart: 184 | 185 | ## Demo 186 | <> 187 | 188 | ============= 189 | # NEW UPDATES May 2023 0.2.0 190 | ## Refactoring 191 | - Breaking down the large code into multiple small function 192 | - Moving all plugins into plugins folder: Esstore, files, Cloud and GCP. 193 | - Adding fixes into zero divide issues 194 | - Migration to new major version of elastic 195 | - Change configuration in CureIAM.yaml file 196 | - Tested in python version 3.9.X 197 | 198 | ## Library Updates 199 | Adding the version in library to avoid any back compatibility issues. 200 | - Elastic==8.7.0 # previously 7.17.9 201 | - elasticsearch==8.7.0 202 | - google-api-python-client==2.86.0 203 | - PyYAML==6.0 204 | - schedule==1.2.0 205 | - rich==13.3.5 206 | 207 | ## Docker Files 208 | - Adding Docker Compose for local Elastic and Kibana in elastic 209 | - Adding .env-ex 210 | change .env-ex to .env to before running the docker 211 | ``` 212 | Running docker compose: docker-compose -f docker_compose_es.yaml up 213 | ``` 214 | 215 | ## Features 216 | - Adding the capability to run scan without applying the recommendation. By default, if mode_scan is false, mode_enforce won't be running. 217 | ``` 218 | mode_scan: true 219 | mode_enforce: false 220 | ``` 221 | - Turn off the email function temporarily. 222 | -------------------------------------------------------------------------------- /CureIAM/helpers/util.py: -------------------------------------------------------------------------------- 1 | """Utility functions.""" 2 | 3 | import argparse 4 | import copy 5 | import importlib 6 | import textwrap 7 | 8 | from CureIAM.helpers import hlogging 9 | 10 | _log = hlogging.get_logger(__name__) 11 | 12 | def wrap_paragraphs(text, width=70): 13 | """Wrap each paragraph in ``text`` to the specified ``width``. 14 | 15 | If the ``text`` is indented with any common leading whitespace, then 16 | that common leading whitespace is removed from every line in text. 17 | Further, any remaining leading and trailing whitespace is removed. 18 | Finally, each paragraph is wrapped to the specified ``width``. 19 | 20 | Arguments: 21 | text (str): String containing paragraphs to be wrapped. 22 | width (int): Maximum length of wrapped lines. 23 | 24 | """ 25 | # Remove any common leading indentation from all lines. 26 | text = textwrap.dedent(text).strip() 27 | 28 | # Split the text into paragraphs. 29 | paragraphs = text.split('\n\n') 30 | 31 | # Wrap each paragraph and join them back into a single string. 32 | wrapped = '\n\n'.join(textwrap.fill(p, width) for p in paragraphs) 33 | return wrapped 34 | 35 | def _merge_dicts(a, b): 36 | """Recursively merge two dictionaries. 37 | 38 | Arguments: 39 | a (dict): First dictionary. 40 | b (dict): Second dictionary. 41 | 42 | Returns: 43 | dict: Merged dictionary. 44 | 45 | """ 46 | c = copy.deepcopy(a) 47 | for k in b: 48 | if (k in a and isinstance(a[k], dict) and isinstance(b[k], dict)): 49 | c[k] = merge_dicts(a[k], b[k]) 50 | else: 51 | c[k] = copy.deepcopy(b[k]) 52 | return c 53 | 54 | 55 | def merge_dicts(*dicts): 56 | """Recursively merge dictionaries. 57 | 58 | The input dictionaries are not modified. Given any 59 | number of dicts, deep copy and merge into a new dict, 60 | precedence goes to key value pairs in latter dicts. 61 | 62 | Example: 63 | Here is an example usage of this function: 64 | 65 | >>> from CureIAM import util 66 | >>> a = {'a': 'apple', 'b': 'ball'} 67 | >>> b = {'b': 'bat', 'c': 'cat'} 68 | >>> c = util.merge_dicts(a, b) 69 | >>> print(c == {'a': 'apple', 'b': 'bat', 'c': 'cat'}) 70 | True 71 | 72 | 73 | Arguments: 74 | *dicts (dict): Variable length dictionary list 75 | 76 | Returns: 77 | dict: Merged dictionary 78 | 79 | """ 80 | result = {} 81 | for dictionary in dicts: 82 | result = _merge_dicts(result, dictionary) 83 | return result 84 | 85 | 86 | def expand_port_ranges(port_ranges): 87 | """Expand ``port_ranges`` to a :obj:`set` of ports. 88 | 89 | Examples: 90 | Here is an example usage of this function: 91 | 92 | >>> from CureIAM import util 93 | >>> ports = util.expand_port_ranges(['22', '3389', '8080-8085']) 94 | >>> print(ports == {22, 3389, 8080, 8081, 8082, 8083, 8084, 8085}) 95 | True 96 | >>> ports = util.expand_port_ranges(['8080-8084', '8082-8086']) 97 | >>> print(ports == {8080, 8081, 8082, 8083, 8084, 8085, 8086}) 98 | True 99 | 100 | Note that in a port range of the form ``m-n``, both ``m`` and 101 | ``n`` are included in the expanded port set. If ``m > n``, we 102 | get an empty port set. 103 | 104 | >>> ports = util.expand_port_ranges(['8085-8080']) 105 | >>> print(ports == set()) 106 | True 107 | 108 | If an invalid port range is found, it is ignored. 109 | 110 | >>> ports = util.expand_port_ranges(['8080', '8081a', '8082']) 111 | >>> print(ports == {8080, 8082}) 112 | True 113 | >>> ports = util.expand_port_ranges(['7070-7075', '8080a-8085']) 114 | >>> print(ports == {7070, 7071, 7072, 7073, 7074, 7075}) 115 | True 116 | 117 | Arguments: 118 | port_ranges (list): A list of strings where each string is a 119 | port number (e.g., ``'80'``) or port range (e.g., ``80-89``). 120 | 121 | Returns: 122 | set: A set of integers that represent the ports specified 123 | by ``port_ranges``. 124 | 125 | """ 126 | # The return value is a set of ports, so that every port number 127 | # occurs only once even if they are found multiple times in 128 | # overlapping port ranges, e.g., ['8080-8084', '8082-8086']. 129 | expanded_port_set = set() 130 | 131 | for port_range in port_ranges: 132 | # If it's just a port number, e.g., '80', add it to the result set. 133 | if port_range.isdigit(): 134 | expanded_port_set.add(int(port_range)) 135 | continue 136 | 137 | # Otherwise, it must look like a port range, e.g., '1024-9999'. 138 | if '-' not in port_range: 139 | continue 140 | 141 | # If it looks like a port range, it must be two numbers 142 | # with a hyphen between them. 143 | start_port, end_port = port_range.split('-', 1) 144 | if not start_port.isdigit() or not end_port.isdigit(): 145 | continue 146 | 147 | # Add the port numbers in the port range to the result set. 148 | expanded_ports = range(int(start_port), int(end_port) + 1) 149 | expanded_port_set.update(expanded_ports) 150 | 151 | return expanded_port_set 152 | 153 | 154 | def friendly_string(technical_string): 155 | """Translate a technical string to a human-friendly phrase. 156 | 157 | In most of our code, we use succint strings to express various 158 | technical details, e.g., ``'gcp'`` to express Google Cloud Platform. 159 | However these technical strings are not ideal while writing 160 | human-friendly messages such as a description of a security issue 161 | detected or a recommendation to remediate such an issue. 162 | 163 | This function helps in converting such technical strings into 164 | human-friendly phrases that can be used in strings intended to be 165 | read by end users (e.g., security analysts responsible for 166 | protecting their cloud infrastructure) of this project. 167 | 168 | Examples: 169 | Here are a few example usages of this function: 170 | 171 | >>> from CureIAM import util 172 | >>> util.friendly_string('azure') 173 | 'Azure' 174 | >>> util.friendly_string('gcp') 175 | 'Google Cloud Platform (GCP)' 176 | 177 | Arguments: 178 | technical_string (str): A technical string. 179 | 180 | Returns: 181 | str: Human-friendly string if a translation from a technical 182 | string to friendly string exists; the same string otherwise. 183 | 184 | """ 185 | phrase_map = { 186 | 'azure': 'Azure', 187 | 'gcp': 'Google Cloud Platform (GCP)', 188 | 'mysql_server': 'MySQL Server', 189 | 'postgresql_server': 'PostgreSQL Server' 190 | } 191 | return phrase_map.get(technical_string, technical_string) 192 | 193 | 194 | def friendly_list(items, conjunction='and'): 195 | """Translate a list of items to a human-friendly list of items. 196 | 197 | Examples: 198 | Here are a few example usages of this function: 199 | 200 | >>> from CureIAM import util 201 | >>> util.friendly_list([]) 202 | 'none' 203 | >>> util.friendly_list(['apple']) 204 | 'apple' 205 | >>> util.friendly_list(['apple', 'ball']) 206 | 'apple and ball' 207 | >>> util.friendly_list(['apple', 'ball', 'cat']) 208 | 'apple, ball, and cat' 209 | >>> util.friendly_list(['apple', 'ball'], 'or') 210 | 'apple or ball' 211 | >>> util.friendly_list(['apple', 'ball', 'cat'], 'or') 212 | 'apple, ball, or cat' 213 | 214 | Arguments: 215 | items (list): List of items. 216 | conjunction (str): Conjunction to be used before the last item 217 | in the list; ``'and'`` by default. 218 | 219 | Returns: 220 | str: Human-friendly list of items with correct placement of 221 | comma and conjunction. 222 | 223 | """ 224 | if not items: 225 | return 'none' 226 | 227 | items = [str(item) for item in items] 228 | 229 | if len(items) == 1: 230 | return items[0] 231 | 232 | if len(items) == 2: 233 | return items[0] + ' ' + conjunction + ' ' + items[1] 234 | 235 | return ', '.join(items[:-1]) + ', ' + conjunction + ' ' + items[-1] 236 | 237 | 238 | def pluralize(count, word, *suffixes): 239 | """Convert ``word`` to plural form if ``count`` is not ``1``. 240 | 241 | Examples: 242 | In the simplest form usage, this function just adds an ``'s'`` 243 | to the input word when the plural form needs to be used. 244 | 245 | >>> from CureIAM import util 246 | >>> util.pluralize(0, 'apple') 247 | 'apples' 248 | >>> util.pluralize(1, 'apple') 249 | 'apple' 250 | >>> util.pluralize(2, 'apple') 251 | 'apples' 252 | 253 | The plural form of some words cannot be formed merely by adding 254 | an ``'s'`` to the word but requires adding a different suffix. 255 | For such cases, provide an additional argument that specifies 256 | the correct suffix. 257 | 258 | >>> util.pluralize(0, 'potato', 'es') 259 | 'potatoes' 260 | >>> util.pluralize(1, 'potato', 'es') 261 | 'potato' 262 | >>> util.pluralize(2, 'potato', 'es') 263 | 'potatoes' 264 | 265 | The plural form of some words cannot be formed merely by adding 266 | a suffix but requires removing a suffix and then adding a new 267 | suffix. For such cases, provide two additional arguments: one 268 | that specifies the suffix to remove from the input word and 269 | another to specify the suffix to add. 270 | 271 | >>> util.pluralize(0, 'sky', 'y', 'ies') 272 | 'skies' 273 | >>> util.pluralize(1, 'sky', 'y', 'ies') 274 | 'sky' 275 | >>> util.pluralize(2, 'sky', 'y', 'ies') 276 | 'skies' 277 | 278 | Returns: 279 | str: The input ``word`` itself if ``count`` is ``1``; plural 280 | form of the ``word`` otherwise. 281 | 282 | """ 283 | if not suffixes: 284 | remove, append = '', 's' 285 | elif len(suffixes) == 1: 286 | remove, append = '', suffixes[0] 287 | elif len(suffixes) == 2: 288 | remove, append = suffixes[0], suffixes[1] 289 | else: 290 | raise PluralizeError('Surplus argument: {!r}'.format(suffixes[2])) 291 | 292 | if count == 1: 293 | return word 294 | 295 | if remove != '' and word.endswith(remove): 296 | word = word[:-len(remove)] 297 | word = word.rstrip(remove) 298 | return word + append 299 | 300 | def outline_az_sub(sub_index, sub, tenant): 301 | """Return a summary of an Azure subscription for logging purpose. 302 | 303 | Arguments: 304 | sub_index (int): Subscription index. 305 | sub (Subscription): Azure subscription model object. 306 | tenant (str): Azure Tenant ID. 307 | 308 | Returns: 309 | str: Return a string that can be used in log messages. 310 | 311 | """ 312 | return ('subscription #{}: {} ({}) ({}); tenant: {}' 313 | .format(sub_index, sub.get('subscription_id'), 314 | sub.get('display_name'), sub.get('state'), tenant)) 315 | 316 | class PluralizeError(Exception): 317 | """Represents an error while converting a word to plural form.""" 318 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /CureIAM/manager.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Manager of worker subprocesses. 4 | 5 | This module invokes the worker subprocesses that perform the cloud 6 | security monitoring tasks. Each worker subprocess wraps around a cloud, 7 | store, processor, or alert plugin and executes the plugin in a separate 8 | subprocess. 9 | """ 10 | 11 | import multiprocessing as mp 12 | 13 | import copy 14 | import textwrap 15 | import time 16 | import json 17 | 18 | #for scheduling 19 | import schedule 20 | 21 | # import pprint 22 | 23 | """ . = CureIAM which is the current module, for now it changes, to remove any depedency name incase if there is any changes folder name in the future 24 | """ 25 | import CureIAM 26 | from CureIAM import baseconfig, workers 27 | from CureIAM.helpers import hconfigs, hemails, hcmd 28 | from CureIAM.helpers.hconfigs import Config 29 | 30 | from CureIAM.helpers import hlogging 31 | # from CureIAM.helpers.hlogging import Logger 32 | 33 | _log = hlogging.get_logger(__name__) 34 | 35 | def main(): 36 | """Run the framework based on the schedule.""" 37 | # Configure the logger as the first thing as per the base 38 | # configuration. We need this to be the first thing, so that 39 | 40 | # Parse the command line arguments and handle the options that can 41 | # be handled immediately. 42 | args = hcmd.parse() 43 | if args.print_base_config: 44 | print(baseconfig.config_yaml.strip()) 45 | return 46 | 47 | # Now load user's configuration files. 48 | # config = hconfigs.load(args.config) 49 | 50 | # print (config) 51 | 52 | # set up the configuration 53 | config = Config.load(args.config) 54 | # Logger.set_logger(Config.get_config_logger()) 55 | 56 | # Then configure the logger once again to honour any logger 57 | # configuration defined in the user's configuration files. 58 | # logging.config.dictConfig(config['logger']) 59 | _log.info('CureIAM %s; configured', CureIAM.__version__) 60 | 61 | # Finally, run the audits, either right now or as per a schedule, 62 | # depending on the command line options. 63 | if args.now: 64 | _log.info('Starting job now') 65 | _run(config) 66 | else: 67 | _log.info('Scheduled to run job everyday at %s', config['schedule']) 68 | schedule.every().day.at(config['schedule']).do(_run, config) 69 | while True: 70 | schedule.run_pending() 71 | time.sleep(60) 72 | 73 | 74 | def _run(config): 75 | """Run the audits. 76 | 77 | Arguments: 78 | config (dict): Configuration dictionary. 79 | 80 | """ 81 | start_time = time.localtime() 82 | _send_email(config.get('email'), 'all audits', start_time) 83 | 84 | # Create an audit object for each audit configured to be run. 85 | audit_version = time.strftime('%Y%m%d_%H%M%S', time.gmtime()) 86 | audits = [] 87 | for audit_key in config['run']: 88 | audits.append(Audit(audit_key, audit_version, config)) 89 | 90 | # Start all audits. 91 | for audit in audits: 92 | audit.start() 93 | 94 | # Wait for all audits to terminate. 95 | for audit in audits: 96 | audit.join() 97 | 98 | end_time = time.localtime() 99 | _send_email(config.get('email'), 'all audits', start_time, end_time) 100 | 101 | 102 | class Audit: 103 | """Audit manager. 104 | 105 | This class encapsulates a set of worker subprocesses and worker 106 | input queues for a single audit configuration. 107 | """ 108 | 109 | def __init__(self, audit_key, audit_version, config): 110 | """Create an instance of :class:`Audit` from configuration. 111 | 112 | A single audit definition (from a list of audit definitions 113 | under the ``audits`` key in the configuration) is instantiated. 114 | Each audit definition contains lists of cloud plugins, store 115 | plugins, processor plugins, and alert plugins. These plugins are 116 | instantiated and multiprocessing queues are set up to take 117 | records from one plugin and feed them to another plugin as per 118 | the audit workflow. 119 | 120 | Arguments: 121 | audit_key (str): Key name for an audit configuration. This 122 | key is looked for in ``config['audits']``. 123 | audit_version (str): Audit version string. 124 | config (dict): Configuration dictionary. This is the 125 | entire configuration dictionary that contains 126 | top-level keys named ``clouds``, ``stores``, ``processors``, 127 | ``alerts``, ``audits``, ``run``, etc. 128 | 129 | """ 130 | self._start_time = time.localtime() 131 | self._audit_key = audit_key 132 | self._audit_version = audit_version 133 | self._config = config 134 | self._audit_config = config['audits'][audit_key] 135 | audit_config = config['audits'][audit_key] 136 | 137 | # We keep all workers in these lists. 138 | self._cloud_workers = [] 139 | self._store_workers = [] 140 | self._processor_workers = [] 141 | self._alert_workers = [] 142 | 143 | # We keep all queues in these lists. 144 | self._store_queues = [] 145 | self._processor_queues = [] 146 | self._alert_queues = [] 147 | 148 | # Create alert workers and queues. 149 | for plugin_key in audit_config.get('alerts', []): 150 | input_queue = mp.Queue() 151 | args = ( 152 | audit_key, 153 | audit_version, 154 | plugin_key, 155 | config['plugins'][plugin_key], 156 | input_queue, 157 | ) 158 | worker = mp.Process(target=workers.alert_worker, args=args) 159 | self._alert_workers.append(worker) 160 | self._alert_queues.append(input_queue) 161 | 162 | # Create processor_workers workers and queues. 163 | for plugin_key in audit_config.get('processors', []): 164 | input_queue = mp.Queue() 165 | args = ( 166 | audit_key, 167 | audit_version, 168 | plugin_key, 169 | config['plugins'][plugin_key], 170 | input_queue, 171 | self._store_queues, 172 | ) 173 | worker = mp.Process(target=workers.processor_worker, args=args) 174 | self._processor_workers.append(worker) 175 | self._processor_queues.append(input_queue) 176 | 177 | # Create store workers and queues. 178 | for plugin_key in audit_config.get('stores', []): 179 | input_queue = mp.Queue() 180 | args = ( 181 | audit_key, 182 | audit_version, 183 | plugin_key, 184 | config['plugins'][plugin_key], 185 | input_queue, 186 | ) 187 | worker = mp.Process(target=workers.store_worker, args=args) 188 | self._store_workers.append(worker) 189 | self._store_queues.append(input_queue) 190 | 191 | # Create cloud workers. 192 | for plugin_key in audit_config.get('clouds', []): 193 | args = ( 194 | audit_key, 195 | audit_version, 196 | plugin_key, 197 | config['plugins'][plugin_key], 198 | self._processor_queues 199 | ) 200 | worker = mp.Process(target=workers.cloud_worker, args=args) 201 | self._cloud_workers.append(worker) 202 | 203 | def start(self): 204 | """Start audit by starting all workers.""" 205 | _send_email(self._config.get('email'), self._audit_key, 206 | self._start_time) 207 | 208 | # Start store and alert workers. 209 | for w in self._store_workers + self._alert_workers: 210 | w.start() 211 | 212 | # Start cloud and processor workers. 213 | for w in self._cloud_workers + self._processor_workers: 214 | w.start() 215 | 216 | # It would have been nice if we could guarantee that the 217 | # begin_audit records are sent to each store and alert before 218 | # any cloud or processor records are sent. But this is difficult to 219 | # achieve because we must avoid forking a multithreaded process. 220 | # 221 | # See https://stackoverflow.com/q/55924761 for more details on 222 | # why a multihreaded process should not be forked. 223 | # 224 | # The q.put() call starts a feeder thread, thus making the 225 | # current process mulithreaded. Therefore, any forking (the 226 | # w.start() calls) must be done prior to q.put() calls. 227 | # 228 | # As a result, we make the q.put() calls *after* starting the 229 | # cloud workers. This means that it is not guaranteed that 230 | # begin_audit record is the very first record that would be 231 | # received by each store and alert plugin. However, the 232 | # begin_audit record would be one of the several earliest 233 | # records received by the plugins. 234 | 235 | def join(self): 236 | """Wait until all workers terminate.""" 237 | # Wait for cloud workers to terminate. 238 | for w in self._cloud_workers: 239 | w.join() 240 | 241 | # Stop processor workers. 242 | for q in self._processor_queues: 243 | q.put(None) 244 | 245 | # Wait for processor workers to terminate. 246 | for w in self._processor_workers: 247 | w.join() 248 | 249 | # Stop store workers. 250 | for q in self._store_queues: 251 | q.put(None) 252 | 253 | # Wait for store workers to terminate. 254 | for w in self._store_workers: 255 | w.join() 256 | 257 | # Stop alert workers. 258 | for q in self._alert_queues: 259 | q.put(None) 260 | 261 | # Wait for alert workers to terminate. 262 | for w in self._alert_workers: 263 | w.join() 264 | 265 | end_time = time.localtime() 266 | _send_email(self._config.get('email'), self._audit_key, 267 | self._start_time, end_time) 268 | 269 | # If Audit results has to be enforced, check if in the 270 | # current audit confi the applyRecommendations is set 271 | # to true 272 | if self._audit_config.get('applyRecommendations', False): 273 | _log.info('Apply recommendations gracefully ...') 274 | 275 | 276 | 277 | def _send_email(email_config, about, start_time, end_time=None): 278 | """Send email about job or audit that is starting or ending. 279 | 280 | Arguments: 281 | email_config (dict): Top-level email configuration dictionary. 282 | about (str): A short string that says what the email 283 | notification is about, e.g., ``'job'`` or ``'audit'``. 284 | start_time (time.struct_time): Start time of job or audit. 285 | end_time (time.struct_time): End time of job or audit. This 286 | argument must not be specified if the job or audit is 287 | starting. 288 | 289 | """ 290 | state = 'starting' if end_time is None else 'ending' 291 | # if email_config is None: 292 | # _log.info('Skipping email notification because email config is ' 293 | # 'missing; about: %s; state: %s', about, state) 294 | # return 295 | 296 | _log.info('Sending email; about: %s; state: %s', about, state) 297 | 298 | # This part of the content is common for both starting and 299 | # ending states. 300 | time_fmt = '%Y-%m-%d %H:%M:%S %z (%Z)' 301 | content = """ 302 | About: {} 303 | Started: {} 304 | """.format(about, time.strftime(time_fmt, start_time)) 305 | content = textwrap.dedent(content).lstrip() 306 | 307 | # This part of the content is added only for ending state. 308 | if state == 'ending': 309 | duration = time.mktime(end_time) - time.mktime(start_time) 310 | mm, ss = divmod(duration, 60) 311 | hh, mm = divmod(mm, 60) 312 | 313 | end_content = """ 314 | Ended: {} 315 | Duration: {:02.0f} h {:02.0f} m {:02.0f} s 316 | """.format(time.strftime(time_fmt, end_time), hh, mm, ss) 317 | 318 | content = content + textwrap.dedent(end_content).lstrip() 319 | 320 | 321 | # hemails.send(content=content, **email_config) 322 | print(content) 323 | _log.info('Sent email; about: %s; state: %s', about, state) 324 | -------------------------------------------------------------------------------- /CureIAM/plugins/gcp/gcpcloudiam.py: -------------------------------------------------------------------------------- 1 | """Plugin to process the data retrieved from `gcpcloud.CureIAM` plugin 2 | """ 3 | 4 | import json 5 | import datetime 6 | 7 | """ . = CureIAM which is the current module, for now it changes, to remove any depedency name incase if there is any changes folder name in the future 8 | """ 9 | 10 | from CureIAM.models.iamriskscore import IAMRiskScoreModel 11 | from CureIAM.models.applyrecommendationmodel import IAMApplyRecommendationModel 12 | from CureIAM.helpers import hlogging 13 | from . import util_gcp #call function from same folder 14 | 15 | # Define module-level logger. 16 | _log = hlogging.get_logger(__name__) 17 | 18 | class GCPIAMRecommendationProcessor: 19 | """SimpleProcessor plugin to perform processing on 20 | gcpcloud.CureIAM IAMRecommendation_record.""" 21 | 22 | """ 23 | SECTION for initiation that has been done only once to generate the list of value 24 | """ 25 | 26 | def __init__(self, mode_scan=False, mode_enforce=False, enforcer=None): 27 | """Create an instance of :class:`GCPIAMRecommendationProcessor` plugin. 28 | """ 29 | self._recommendation_applied = 0 30 | self._recommendation_applied_today = 0 31 | 32 | # Checking the mode, if scan mode is on, then run the scan. If mode enforce is on, then enforce the change according to the recommendation 33 | self._mode_scan = mode_scan 34 | self._mode_enforce = mode_enforce 35 | 36 | # List of enforcement 37 | self._enforcer = enforcer 38 | 39 | if self._enforcer: 40 | # init blocklist 41 | self.init_blocklist(self._enforcer) 42 | 43 | # init whitelist 44 | self.init_whitelist(self._enforcer) 45 | 46 | # init safe score 47 | self.init_safe_score(self._enforcer) 48 | 49 | # init cloud resources 50 | self.init_cloud_resource(self._enforcer) 51 | 52 | def init_blocklist(self, enforcer=None): 53 | # Don't perform operations on these projects 54 | self._apply_recommendation_blocklist_projects = enforcer.get('blocklist_projects', None) 55 | 56 | # Don't perform operations on these accounts_ids 57 | self._apply_recommendation_blocklist_accounts = enforcer.get('blocklist_accounts', None) 58 | self._apply_recommendation_blocklist_account_types = enforcer.get('blocklist_account_types', ['serviceAccount']) 59 | 60 | def init_whitelist(self, enforcer=None): 61 | # Perform allowlist projects 62 | self._apply_recommendation_allowlist_projects = enforcer.get('allowlist_projects', None) 63 | self._apply_recommendation_allowlist_account_types = enforcer.get('allowlist_account_types', ['user', 'group']) 64 | 65 | def init_safe_score(self, enforcer=None): 66 | 67 | # will change into this in the future 68 | self._apply_recommendation_min_score = {} 69 | 70 | # Min recommendation apply score is 60 to default for user 71 | self._apply_recommendation_min_score["user"] = enforcer.get('min_safe_to_apply_score_user', 60) 72 | 73 | # Min recommendation apply score is 60 to default for groups 74 | self._apply_recommendation_min_score["group"] = enforcer.get('min_safe_to_apply_score_group', 60) 75 | 76 | # Min recommendation apply score is 60 to default for SA 77 | self._apply_recommendation_min_score["serviceaccount"] = enforcer.get('min_safe_to_apply_score_SA', 60) 78 | 79 | def init_cloud_resource(self, enforcer=None): 80 | 81 | self._apply_recommendations_svc_acc_key_file = enforcer.get('key_file_path', None) 82 | 83 | self._cloud_resource = util_gcp.build_resource( 84 | service_name='cloudresourcemanager', 85 | key_file_path=self._apply_recommendations_svc_acc_key_file 86 | ) 87 | self._recommender_resource = util_gcp.build_resource( 88 | service_name='recommender', 89 | key_file_path=self._apply_recommendations_svc_acc_key_file 90 | ) 91 | 92 | """ 93 | SECTION for initiation that has been done only once to generate the list of value 94 | """ 95 | 96 | def eval(self, record): 97 | """Function to perform data processing. 98 | 99 | Arguments: 100 | record (dict): Record to evaluate. 101 | { 102 | 'raw': { 103 | "name": "projects/{project-id}/locations/{location}/recommenders/google.iam.policy.Recommender/recommendations/{recommendation-id}", 104 | "description": "Replace the current role with a smaller role to cover the permissions needed.", 105 | "lastRefreshTime": "2021-01-18T08:00:00Z", 106 | "primaryImpact": { 107 | "category": "SECURITY" 108 | }, 109 | "content": { 110 | "operationGroups": [ 111 | { 112 | "operations": [ 113 | { 114 | "action": "add", 115 | "resourceType": "cloudresourcemanager.googleapis.com/Project", 116 | "resource": "//cloudresourcemanager.googleapis.com/projects/565961175665", 117 | "path": "/iamPolicy/bindings/*/members/-", 118 | "value": "user:foo@bar.com", 119 | "pathFilters": { 120 | "/iamPolicy/bindings/*/condition/expression": "", 121 | "/iamPolicy/bindings/*/role": "roles/storage.objectCreator" 122 | } 123 | }, 124 | { 125 | "action": "remove", 126 | "resourceType": "cloudresourcemanager.googleapis.com/Project", 127 | "resource": "//cloudresourcemanager.googleapis.com/projects/565961175665", 128 | "path": "/iamPolicy/bindings/*/members/*", 129 | "pathFilters": { 130 | "/iamPolicy/bindings/*/condition/expression": "", 131 | "/iamPolicy/bindings/*/members/*": "user:@", 132 | "/iamPolicy/bindings/*/role": "roles/storage.objectAdmin" 133 | } 134 | } 135 | } 136 | ] 137 | }, 138 | "stateInfo": { 139 | "state": "ACTIVE" 140 | }, 141 | "etag": "\"ef625ab631b20e49\"", 142 | "recommenderSubtype": "REPLACE_ROLE", 143 | "associatedInsights": [ 144 | { 145 | "insight": "projects/{project-id}/locations/{location}/recommenders/google.iam.policy.Recommender/recommendations/{recommendation-id}" 146 | } 147 | ] 148 | } 149 | } 150 | Yields: 151 | dict: Processed record. 152 | { 153 | 'GCPIAMProcessor': { 154 | 'record_type': 'iam_recommendation' 155 | 'recommendation_name' : name, 156 | 'project': project, 157 | 'recommendation_description' : description, 158 | 'recommendation_action': content.operationGroups.operations[i], 159 | 'recommendetion_recommender_subtype': recommenderSubtype, 160 | 'recommendation_insights': associatedInsights 161 | } 162 | } 163 | """ 164 | # Extract the different `CureIAM_record.recommendation_action.value` 165 | # from the gcpcloud.GCPCloudIAMRecommendations 166 | 167 | iam_raw_record = record.get('raw', {}) 168 | recommendation_dict = dict() 169 | 170 | if iam_raw_record is not None: 171 | recommendation_dict.update( 172 | { 173 | 'project' : iam_raw_record['project'], 174 | 'recommendation_id': iam_raw_record['name'], 175 | 'recommendation_description': iam_raw_record['description'], 176 | 'recommendation_actions' : iam_raw_record['content']['operationGroups'][0]['operations'], 177 | 'recommendetion_recommender_subtype': iam_raw_record['recommenderSubtype'], 178 | 'recommendation_insights': [ i.get('insights') for i in iam_raw_record['associatedInsights']] 179 | } 180 | ) 181 | # Identify the account type on which recommendation is fetched 182 | # If iam_raw_record['recommenderSubtype'] is REPLACE_ROLE, then user 183 | # info will be present as iam_raw_record['content']['operationGroups']['operations'] list 184 | _actor = '' 185 | _actor_total_permissions = 0 186 | _actor_exercised_permissions = 0 187 | _actor_exercised_permissions_category = '' 188 | 189 | for op_grp in iam_raw_record['content']['operationGroups']: 190 | for op in op_grp['operations']: 191 | if op['action'] == 'remove': 192 | _actor = op['pathFilters']['/iamPolicy/bindings/*/members/*'] 193 | # After above parsing _actor would contain something like 194 | # : 195 | _actor_type, _actor = _actor.split(':') 196 | recommendation_dict.update( 197 | { 198 | 'account_type': _actor_type, 199 | 'account_id': _actor 200 | } 201 | ) 202 | 203 | # Get all the Permissions the current actor have 204 | # insights is a list, in case of multiple insights 205 | # all insights will have same `currentTotalPermissionsCount` 206 | # So we are good to include the results from the first one 207 | # only. 208 | insights = iam_raw_record.get('insights', None) 209 | if insights: 210 | _content = insights[0].get('content', None) 211 | if _content: 212 | _actor_exercised_permissions = len(_content.get( 213 | 'exercisedPermissions', 214 | [] 215 | )) + len( 216 | _content.get( 217 | 'inferredPermissions', 218 | [] 219 | ) 220 | ) 221 | _actor_total_permissions = _content.get( 222 | 'currentTotalPermissionsCount', 223 | '0' 224 | ) 225 | _actor_exercised_permissions_category = insights[0].get( 226 | 'category', 227 | '' 228 | ) 229 | 230 | recommendation_dict.update( 231 | { 232 | 'account_total_permissions': int(_actor_total_permissions), 233 | 'account_used_permissions': _actor_exercised_permissions, 234 | 'account_permission_insights_category': _actor_exercised_permissions_category 235 | } 236 | ) 237 | 238 | _res = { 239 | 'raw': iam_raw_record, 240 | 'processor': recommendation_dict , 241 | 'score': IAMRiskScoreModel(recommendation_dict).score(), 242 | 'apply_recommendation': IAMApplyRecommendationModel(recommendation_dict).model() 243 | } 244 | 245 | _res['apply_recommendation'].update( 246 | { 247 | 'safe_to_apply_score': _res['score']['safe_to_apply_recommendation_score'] 248 | } 249 | ) 250 | 251 | # To shorten the logging mechanism 252 | recommendation_info = recommendation_dict['recommendation_id'].split("/") 253 | # print ("{}".format(hlogging.obfuscated(recommendation_info[1]))) 254 | # print ("{}".format(hlogging.obfuscated(recommendation_info[7]))) 255 | 256 | # If recommendation was applied in past 257 | # update the risk score and safe_to_apply_ 258 | # _score to 0 259 | 260 | if _res['raw']['stateInfo']['state']=='SUCCEEDED': 261 | _res['score'].update( 262 | { 263 | 'risk_score': 0, 264 | 'over_privilege_score': 0 265 | } 266 | ) 267 | 268 | self._recommendation_applied += 1 269 | # _log.info('APPLIED in the past, setting score to 0#Project:%s,Recommendations:%s', recommendation_info[1], recommendation_info[7]) 270 | 271 | # enforce the recommendation before saving it in DB. 272 | # Also dont re-apply the recommendation is it is already applied 273 | if self._enforcer and _res['raw']['stateInfo']['state']=='ACTIVE': 274 | _log.info('ENFORCING#Project:%s,Recommendations:%s', hlogging.obfuscated(recommendation_info[1]), hlogging.obfuscated(recommendation_info[7])) 275 | _recomemndation_applied = self._enforce_recommendation(_res) 276 | 277 | if _recomemndation_applied: 278 | _res['raw']['stateInfo']['state'] = 'SUCCEEDED' 279 | _res['apply_recommendation'].update( 280 | { 281 | 'recommendation_state': 'Applied', 282 | 'recommendation_applied_time': str(datetime.datetime.utcnow().isoformat()) 283 | } 284 | ) 285 | _res['score'].update( 286 | { 287 | 'risk_score': 0, 288 | 'over_privilege_score': 0 289 | } 290 | ) 291 | self._recommendation_applied_today += 1 292 | _log.info('APPLIED#Project:%s,Recommendations:%s', hlogging.obfuscated(recommendation_info[1]), hlogging.obfuscated(recommendation_info[7])) 293 | 294 | else: 295 | _log.info('NOT-APPLIED#Project:%s,Recommendations:%s', hlogging.obfuscated(recommendation_info[1]), hlogging.obfuscated(recommendation_info[7])) 296 | 297 | yield _res 298 | 299 | 300 | def _enforce_recommendation(self, record): 301 | """Method to perform Recommendation enforcement 302 | 303 | IAM recommendation doesn't have API to apply the recommendation 304 | directly rather we will have to create IAM resource which will 305 | perform the policy enforcement. This method does the same. 306 | 307 | Arguments: 308 | record(dict): dict record contaning raw + processor record 309 | 310 | Returns: 311 | bool: Indicating if the we were able to successfully apply 312 | recommendation or not. 313 | """ 314 | 315 | """ 316 | Flow: 317 | Apply IAM policy from recommender 318 | - success 319 | - mark recommendation as succeeded 320 | - return True 321 | - no 322 | - dont change the recommendation status 323 | - return False 324 | """ 325 | if not self._mode_scan: 326 | return 327 | 328 | cloud_resource = self._cloud_resource 329 | recommender_resource = self._recommender_resource 330 | 331 | _processor_record = record.get('processor', None) 332 | _score_record = record.get('score', None) 333 | 334 | if _processor_record and _score_record: 335 | _project = _processor_record.get('project', None) 336 | _recommendation_actions = _processor_record.get('recommendation_actions', None) 337 | _recommendation_id = _processor_record.get('recommendation_id', None) 338 | _account_id = _processor_record.get('account_id') 339 | _account_type = _processor_record.get('account_type') 340 | _safety_score = _score_record.get('safe_to_apply_recommendation_score', None) 341 | 342 | _we_want_to_apply_recommendation = False 343 | 344 | # Check blacklist validation 345 | flag_blacklist = self._validate_blacklist(_project=_project,_account_id=_account_id, _account_type=_account_type) 346 | 347 | # Check whitelist validation 348 | flag_whitelist = self._validate_whitelist(_account_type=_account_type) 349 | 350 | if flag_blacklist and flag_whitelist: 351 | # If Recommendation is for SA, apply only for ['REMOVE_ROLE', 'REPLACE_ROLE'] 352 | if ( 353 | _account_type == 'serviceAccount' 354 | and _processor_record.get('recommendetion_recommender_subtype') in ['REMOVE_ROLE', 'REPLACE_ROLE'] 355 | ): 356 | _we_want_to_apply_recommendation = True 357 | 358 | else: 359 | if _account_type != 'serviceAccount': 360 | # If user is owner of any project dont apply recommendation 361 | # this is very bad of detecting owners, need to find better way of doing this. 362 | if not 'owner' in str(record['raw']['content']['operationGroups']): 363 | _we_want_to_apply_recommendation = True 364 | 365 | _we_want_to_apply_recommendation = self._validate_safety_score(_account_type=_account_type, _safety_score=_safety_score) 366 | 367 | 368 | if _we_want_to_apply_recommendation: 369 | # Execute recommendation 370 | _log.info('SHOULD BE APPLIED TO TARGET=project:%s,account:%s,account_type(%s)', 371 | hlogging.obfuscated(_project), 372 | hlogging.obfuscated(_account_id), 373 | _account_type) 374 | if self._mode_enforce: 375 | _log.info('XXX# END of enforcing mode #XXX\n\n') 376 | _status = self._execute_recommendation(cloud_resource=cloud_resource, _project=_project) 377 | else: 378 | _log.info('XXX# END of scan only mode #XXX\n\n') 379 | return False 380 | 381 | # So we have applied recommendation and we are good. 382 | return True 383 | 384 | return False 385 | 386 | 387 | """ 388 | Section for validation that has been break down into a multiple small function 389 | """ 390 | 391 | def _validate_blacklist(self, _project, _account_id, _account_type): # will do the cleanup code later 392 | if (_project in self._apply_recommendation_blocklist_projects): 393 | _log.warn("Project %s is in excluded list", hlogging.obfuscated(_project)) 394 | return False 395 | 396 | if (_account_id in self._apply_recommendation_blocklist_accounts): 397 | _log.warn("Account %s is in excluded list", hlogging.obfuscated(_account_id)) 398 | return False 399 | 400 | if (_account_type in self._apply_recommendation_blocklist_account_types): 401 | _log.warn("Account type %s is in excluded list", _account_type) 402 | return False 403 | 404 | return True 405 | 406 | def _validate_whitelist(self, _account_type): 407 | if (_account_type in self._apply_recommendation_allowlist_account_types): 408 | return True 409 | 410 | return False 411 | 412 | def _validate_safety_score(self, _account_type, _safety_score): # Logic from defining whether we'll remove or not based on the pre-configuration score 413 | 414 | _account_type = _account_type.lower() 415 | std_min_score = self._apply_recommendation_min_score[_account_type] 416 | 417 | _log.info("GCP Recommendation Safety Score(%s) >= Minimum Score Accepted(%s)", _safety_score, std_min_score) 418 | 419 | if _safety_score >= std_min_score: 420 | return True 421 | 422 | return False 423 | 424 | 425 | """ 426 | SECTION for execute in GCP 427 | """ 428 | def _execute_recommendation(self,cloud_resource, _project): #will do more cleanup 429 | 430 | _policies = ( 431 | cloud_resource.projects() 432 | .getIamPolicy( 433 | resource=_project, 434 | body={"options": {"requestedPolicyVersion": "1"}} 435 | ).execute() 436 | ) 437 | 438 | _updated_policies = _policies 439 | 440 | for _recommendation_action in _recommendation_actions: 441 | if _recommendation_action.get('action') == 'remove': 442 | member = ( 443 | _recommendation_action.get('pathFilters') 444 | .get('/iamPolicy/bindings/*/members/*') 445 | ) 446 | role = ( 447 | _recommendation_action.get('pathFilters') 448 | .get('/iamPolicy/bindings/*/role') 449 | ) 450 | _updated_policies = self.modify_policy_remove_member( 451 | _updated_policies, 452 | role, 453 | member 454 | ) 455 | 456 | elif _recommendation_action.get('action') == 'add': 457 | member = _recommendation_action.get('value') 458 | role = ( 459 | _recommendation_action.get('pathFilters') 460 | .get('/iamPolicy/bindings/*/role') 461 | ) 462 | _updated_policies = self.modify_policy_add_member( 463 | _updated_policies, 464 | role, 465 | member 466 | ) 467 | 468 | 469 | #Apply the policies present in recommendations 470 | policy = ( 471 | cloud_resource.projects() 472 | .setIamPolicy(resource=_project, body={'policy': _updated_policies}) 473 | .execute() 474 | ) 475 | # print(policy) 476 | 477 | # Update the recommendation status. 478 | status = ( 479 | recommender_resource 480 | .projects() 481 | .locations() 482 | .recommenders() 483 | .recommendations() 484 | .markSucceeded( 485 | body={ 486 | 'etag': record.get('raw').get('etag'), 487 | 'stateMetadata': { 488 | 'reviewed-by': 'cureiam', 489 | 'owned-by': 'security' 490 | } 491 | }, 492 | name=_recommendation_id) 493 | .execute() 494 | ) 495 | 496 | return _status 497 | 498 | def modify_policy_remove_member(self, policy, role, member): 499 | """Removes a member from a role binding.""" 500 | try: 501 | binding = next(b for b in policy["bindings"] if b["role"] == role) 502 | if "members" in binding and member in binding["members"]: 503 | binding["members"].remove(member) 504 | except StopIteration: 505 | # Policy removed in previous iterations 506 | pass 507 | return policy 508 | 509 | def modify_policy_add_member(self, policy, role, member): 510 | """Adds a new role binding to a policy.""" 511 | binding = {"role": role, "members": [member]} 512 | policy["bindings"].append(binding) 513 | return policy 514 | 515 | def done(self): 516 | """Perform cleanup work. 517 | Since this is a mock plugin, this method does nothing. However, 518 | a typical event plugin may or may not need to perform cleanup 519 | work in this method depending on its nature of work. 520 | """ 521 | _log.info('Recommendation applied: %s; Recommendations applied today: %s', 522 | self._recommendation_applied, self._recommendation_applied_today) --------------------------------------------------------------------------------