├── .coveragerc ├── .gitignore ├── .travis.yml ├── AUTHORS ├── LICENSE ├── MANIFEST.in ├── README.md ├── cloudaux ├── __about__.py ├── __init__.py ├── aws │ ├── README.md │ ├── __init__.py │ ├── autoscaling.py │ ├── decorators.py │ ├── ec2.py │ ├── elb.py │ ├── elbv2.py │ ├── events.py │ ├── events_bus.py │ ├── glacier.py │ ├── iam.py │ ├── lambda_function.py │ ├── resource_groups_tagging.py │ ├── route53.py │ ├── s3.py │ ├── sg.py │ ├── sns.py │ ├── sqs.py │ └── sts.py ├── decorators.py ├── exceptions.py ├── gcp │ ├── README.md │ ├── __init__.py │ ├── auth.py │ ├── config.py │ ├── crm.py │ ├── decorators.py │ ├── gce │ │ ├── __init__.py │ │ ├── address.py │ │ ├── disk.py │ │ ├── firewall.py │ │ ├── forwarding_rule.py │ │ ├── instance.py │ │ ├── network.py │ │ ├── project.py │ │ └── zone.py │ ├── gcpcache.py │ ├── gcs.py │ ├── iam.py │ └── utils.py ├── openstack │ ├── README.md │ ├── __init__.py │ ├── decorators.py │ ├── object_container.py │ └── utils.py ├── orchestration │ ├── __init__.py │ ├── aws │ │ ├── __init__.py │ │ ├── arn.py │ │ ├── elb.py │ │ ├── elbv2.py │ │ ├── events.md │ │ ├── events.py │ │ ├── glacier.md │ │ ├── glacier.py │ │ ├── iam │ │ │ ├── __init__.py │ │ │ ├── group.py │ │ │ ├── managed_policy.md │ │ │ ├── managed_policy.py │ │ │ ├── role.py │ │ │ ├── saml_provider.py │ │ │ ├── server_certificate.py │ │ │ └── user.py │ │ ├── image.md │ │ ├── image.py │ │ ├── lambda_function.md │ │ ├── lambda_function.py │ │ ├── s3.md │ │ ├── s3.py │ │ ├── sg.md │ │ ├── sg.py │ │ ├── sqs.md │ │ ├── sqs.py │ │ ├── vpc.md │ │ └── vpc.py │ ├── gcp │ │ ├── __init__.py │ │ ├── gce │ │ │ ├── __init__.py │ │ │ └── network.py │ │ ├── gcs │ │ │ ├── __init__.py │ │ │ └── bucket.py │ │ └── iam │ │ │ ├── __init__.py │ │ │ └── serviceaccount.py │ └── openstack │ │ ├── __init__.py │ │ ├── security_group.py │ │ └── utils.py └── tests │ ├── __init__.py │ ├── aws │ ├── __init__.py │ ├── conftest.py │ ├── test_arn.py │ ├── test_decorators.py │ ├── test_ec2.py │ ├── test_iam.py │ ├── test_sts.py │ └── test_vpc_orchestration.py │ ├── cloudaux │ ├── __init__.py │ └── test_cloudaux.py │ ├── gcp │ ├── __init__.py │ ├── fixtures │ │ ├── __init__.py │ │ ├── compute.json │ │ └── testkey.json │ ├── test_auth.py │ ├── test_gcpcache.py │ ├── test_integration.py │ └── test_utils.py │ └── openstack │ ├── __init__.py │ ├── fixtures │ └── test-clouds.yaml │ ├── mock_decorators.py │ ├── mock_object_container.py │ ├── mock_utils.py │ └── test_conn.py ├── setup.cfg ├── setup.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | include = cloudaux/*.py 3 | omit = 4 | cloudaux/__about__.py 5 | cloudaux/tests/*.py 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | venv 85 | 86 | # Spyder project settings 87 | .spyderproject 88 | 89 | # Rope project settings 90 | .ropeproject 91 | 92 | # IntelliJ/PyCharm 93 | .idea/ 94 | 95 | .pytest_cache/ 96 | .coverage 97 | test-reports/ 98 | 99 | .DS_Store 100 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - "3.8" 5 | 6 | before_install: 7 | - sudo rm -f /etc/boto.cfg 8 | 9 | cache: 10 | directories: 11 | - .pip_download_cache 12 | 13 | env: 14 | global: 15 | - PIP_DOWNLOAD_CACHE=".pip_download_cache" 16 | - GOOGLE_APPLICATION_CREDENTIALS=$TRAVIS_BUILD_DIR/cloudaux/tests/gcp/fixtures/testkey.json 17 | 18 | 19 | install: 20 | - pip install tox-travis coveralls 21 | 22 | script: 23 | - tox 24 | 25 | after_success: 26 | - coveralls 27 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | - Patrick Kelley 2 | - Kevin Glisson 3 | - Mike Grima 4 | - Josafat Gonzalez 5 | - Curtis Castrapel -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include setup.py README.md MANIFEST.in LICENSE AUTHORS 2 | global-exclude *~ -------------------------------------------------------------------------------- /cloudaux/__about__.py: -------------------------------------------------------------------------------- 1 | __all__ = [ 2 | '__title__', 3 | '__summary__', 4 | '__uri__', 5 | '__version__', 6 | '__author__', 7 | '__email__', 8 | '__license__', 9 | '__copyright__' 10 | ] 11 | 12 | __title__ = 'cloudaux' 13 | __summary__ = 'Cloud Auxiliary is a python wrapper and orchestration module for interacting with cloud providers' 14 | __uri__ = 'https://github.com/Netflix-Skunkworks/cloudaux' 15 | 16 | __version__ = '1.9.6' 17 | 18 | __author__ = 'The Cloudaux Developers' 19 | __email__ = 'oss@netflix.com' 20 | 21 | __license__ = 'Apache License, Version 2.0' 22 | __copyright__ = 'Copyright 2021 %s' % __author__ 23 | -------------------------------------------------------------------------------- /cloudaux/__init__.py: -------------------------------------------------------------------------------- 1 | from cloudaux.aws.sts import sts_conn 2 | 3 | 4 | class CloudAux: 5 | 6 | def __init__(self, **kwargs): 7 | """ 8 | cloudaux = CloudAux( 9 | **{'account_number': '000000000000', 10 | 'assume_role': 'role_name', 11 | }) 12 | """ 13 | self.conn_details = { 14 | 'session_name': 'cloudaux', 15 | 'region': 'us-east-1' 16 | } 17 | self.conn_details.update(kwargs) 18 | 19 | def call(self, function_expr, **kwargs): 20 | """ 21 | cloudaux = CloudAux( 22 | **{'account_number': '000000000000', 23 | 'assume_role': 'role_name', 24 | 'session_name': 'testing', 25 | 'region': 'us-east-1', 26 | 'tech': 'kms', 27 | 'service_type': 'client' 28 | }) 29 | 30 | cloudaux.call("list_aliases") 31 | cloudaux.call("kms.client.list_aliases") 32 | """ 33 | if '.' in function_expr: 34 | tech, service_type, function_name = function_expr.split('.') 35 | else: 36 | tech = self.conn_details.get('tech') 37 | service_type = self.conn_details.get('service_type', 'client') 38 | function_name = function_expr 39 | 40 | @sts_conn(tech, service_type=service_type) 41 | def wrapped_method(function_name, **nargs): 42 | service_type = nargs.pop(nargs.pop('service_type', 'client')) 43 | return getattr(service_type, function_name)(**nargs) 44 | 45 | kwargs.update(self.conn_details) 46 | if 'tech' in kwargs: 47 | del kwargs['tech'] 48 | return wrapped_method(function_name, **kwargs) 49 | 50 | @staticmethod 51 | def go(function_expr, **kwargs): 52 | """ 53 | CloudAux.go( 54 | 'list_aliases', 55 | **{ 56 | 'account_number': '000000000000', 57 | 'assume_role': 'role_name', 58 | 'session_name': 'cloudaux', 59 | 'region': 'us-east-1', 60 | 'tech': 'kms', 61 | 'service_type': 'client' 62 | }) 63 | 64 | CloudAux.go( 65 | 'kms.client.list_aliases', 66 | **{ 67 | 'account_number': '000000000000', 68 | 'assume_role': 'role_name', 69 | 'session_name': 'cloudaux', 70 | 'region': 'us-east-1' 71 | }) 72 | """ 73 | if '.' in function_expr: 74 | tech, service_type, function_name = function_expr.split('.') 75 | else: 76 | tech = kwargs.pop('tech') 77 | service_type = kwargs.get('service_type') 78 | function_name = function_expr 79 | 80 | @sts_conn(tech, service_type=service_type) 81 | def wrapped_method(function_name, **nargs): 82 | service_type = nargs.pop(nargs.pop('service_type', 'client')) 83 | return getattr(service_type, function_name)(**nargs) 84 | 85 | return wrapped_method(function_name, **kwargs) 86 | 87 | 88 | def get_iso_string(input): 89 | """Strips out the microseconds from datetime objects, and returns a proper ISO-format UTC string. 90 | 91 | :param input: Datetime object. 92 | :returns string: A datetime ISO format string with 93 | """ 94 | return input.replace(tzinfo=None, microsecond=0).isoformat() + 'Z' 95 | -------------------------------------------------------------------------------- /cloudaux/aws/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/cloudaux/39d1dc11ccd794f21d95d3d1458abd94425b0927/cloudaux/aws/__init__.py -------------------------------------------------------------------------------- /cloudaux/aws/autoscaling.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.aws.autoscaling 3 | :platform: Unix 4 | :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | .. moduleauthor:: Patrick Kelley 8 | """ 9 | from cloudaux.aws.sts import sts_conn 10 | from cloudaux.aws.decorators import rate_limited 11 | 12 | 13 | @sts_conn('autoscaling') 14 | @rate_limited() 15 | def describe_auto_scaling_groups(account_number=None, region=None, assume_role=None, client=None): 16 | return client.get_paginator('describe_auto_scaling_groups').paginate(PaginationConfig={'MaxItems': 500}) 17 | 18 | 19 | @sts_conn('autoscaling') 20 | @rate_limited() 21 | def describe_launch_configurations(account_number=None, region=None, assume_role=None, client=None): 22 | return client.get_paginator('describe_launch_configurations').paginate() 23 | 24 | 25 | @sts_conn('autoscaling') 26 | @rate_limited() 27 | def create_launch_configuration(**kwargs): 28 | return kwargs.pop('client').create_launch_configuration(**kwargs) 29 | 30 | 31 | @sts_conn('autoscaling') 32 | @rate_limited() 33 | def create_auto_scaling_group(name, launch_config_name, account_number=None, region=None, assume_role=None, client=None): 34 | return client.create_auto_scaling_group( 35 | AutoScalingGroupName=name, 36 | MaxSize=1, 37 | MinSize=1, 38 | LaunchConfigurationName=launch_config_name 39 | ) 40 | 41 | 42 | @sts_conn('autoscaling') 43 | @rate_limited() 44 | def update_auto_scaling_group(**kwargs): 45 | return kwargs.pop('client').update_auto_scaling_group(**kwargs) 46 | -------------------------------------------------------------------------------- /cloudaux/aws/decorators.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.aws.decorators 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Patrick Kelley @monkeysecurity 7 | .. moduleauthor:: Mike Grima 8 | """ 9 | import functools 10 | import time 11 | 12 | import boto 13 | import botocore 14 | 15 | RATE_LIMITING_ERRORS = ['Throttling', 'RequestLimitExceeded', 'SlowDown', 'RequestThrottled'] 16 | 17 | 18 | def rate_limited(max_attempts=None, max_delay=4): 19 | def decorator(f): 20 | metadata = { 21 | 'count': 0, 22 | 'delay': 0 23 | } 24 | 25 | @functools.wraps(f) 26 | def decorated_function(*args, **kwargs): 27 | 28 | def increase_delay(e): 29 | if metadata['delay'] == 0: 30 | metadata['delay'] = 1 31 | elif metadata['delay'] < max_delay: 32 | metadata['delay'] *= 2 33 | 34 | if max_attempts and metadata['count'] > max_attempts: 35 | raise e 36 | 37 | metadata['count'] = 0 38 | while True: 39 | metadata['count'] += 1 40 | if metadata['delay'] > 0: 41 | time.sleep(metadata['delay']) 42 | try: 43 | retval = f(*args, **kwargs) 44 | metadata['delay'] = 0 45 | return retval 46 | except botocore.exceptions.ClientError as e: 47 | if e.response["Error"]["Code"] not in RATE_LIMITING_ERRORS: 48 | raise e 49 | increase_delay(e) 50 | except boto.exception.BotoServerError as e: 51 | if e.error_code not in RATE_LIMITING_ERRORS: 52 | raise e 53 | increase_delay(e) 54 | 55 | return decorated_function 56 | 57 | return decorator 58 | 59 | 60 | def paginated(response_key, request_pagination_marker="Marker", response_pagination_marker="Marker"): 61 | def decorator(func): 62 | @functools.wraps(func) 63 | def decorated_function(*args, **kwargs): 64 | results = [] 65 | 66 | while True: 67 | response = func(*args, **kwargs) 68 | results.extend(response[response_key]) 69 | 70 | # If the "next" pagination marker is in the response, then paginate. Responses may not always have 71 | # items in the response_key, so we should only key off of the response_pagination_marker. 72 | if response.get(response_pagination_marker): 73 | kwargs.update({request_pagination_marker: response[response_pagination_marker]}) 74 | else: 75 | break 76 | return results 77 | return decorated_function 78 | return decorator 79 | -------------------------------------------------------------------------------- /cloudaux/aws/ec2.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.aws.ec2 3 | :platform: Unix 4 | :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Mike Grima 7 | .. moduleauthor:: Kevin Glisson 8 | .. moduleauthor:: Patrick Kelley 9 | """ 10 | from cloudaux.aws.sts import sts_conn 11 | from cloudaux.exceptions import CloudAuxException 12 | from cloudaux.aws.decorators import rate_limited, paginated 13 | 14 | 15 | @sts_conn('ec2') 16 | @rate_limited() 17 | def create_group(group, account_number=None, region=None, assume_role=None, client=None): 18 | if group.vpc_id: 19 | group_id = client.create_security_group( 20 | GroupName=group.name, 21 | Description=group.description, 22 | VpcId=group.vpc_id 23 | )['GroupId'] 24 | else: 25 | group_id = client.create_security_group( 26 | GroupName=group.name, 27 | Description=group.description 28 | )['GroupId'] 29 | 30 | return group_id 31 | 32 | 33 | @sts_conn('ec2') 34 | @rate_limited() 35 | def delete_group(group, account_number=None, region=None, assume_role=None, client=None): 36 | client.delete_security_group( 37 | GroupId=group.aws_group_id 38 | ) 39 | 40 | 41 | @sts_conn('ec2') 42 | @rate_limited() 43 | def authorize_rule(rule, group, account_number=None, region=None, assume_role=None, client=None): 44 | if rule.direction == 'egress': 45 | # response = client.authorize_security_group_egress() 46 | raise CloudAuxException("Modifying egress rules is not yet supported.") 47 | else: 48 | if rule.cidr: 49 | client.authorize_security_group_ingress( 50 | GroupId=group.aws_group_id, 51 | IpProtocol=rule.protocol, 52 | FromPort=rule.from_port, 53 | ToPort=rule.to_port, 54 | CidrIp=rule.cidr 55 | ) 56 | else: 57 | client.authorize_security_group_ingress( 58 | GroupId=group.aws_group_id, 59 | IpProtocol=rule.protocol, 60 | FromPort=rule.from_port, 61 | ToPort=rule.to_port, 62 | SourceSecurityGroupName=rule.source_security_group.name, 63 | # SourceSecurityGroupOwnerId=rule.source_security_group.account.aws_account_id 64 | ) 65 | 66 | 67 | @sts_conn('ec2') 68 | @rate_limited() 69 | def revoke_rule(rule, group, account_number=None, region=None, assume_role=None, client=None): 70 | if rule.direction == 'egress': 71 | # response = client.authorize_security_group_egress() 72 | raise CloudAuxException("Modifying egress rules is not yet supported.") 73 | else: 74 | client.revoke_security_group_ingress( 75 | GroupId=group.aws_group_id, 76 | IpProtocol=rule.protocol, 77 | FromPort=rule.from_port, 78 | ToPort=rule.to_port, 79 | CidrIp=rule.cidr, 80 | ) 81 | 82 | 83 | @sts_conn('ec2') 84 | @rate_limited() 85 | def add_groups_to_instance(instance_id, groups, account_number=None, region=None, assume_role=None, client=None): 86 | client.modify_instance_attribute(InstanceId=instance_id, Groups=groups) 87 | 88 | 89 | @sts_conn('ec2') 90 | @rate_limited() 91 | def describe_instances(**kwargs): 92 | return kwargs.pop('client').get_paginator('describe_instances').paginate() 93 | 94 | 95 | @sts_conn('ec2') 96 | @rate_limited() 97 | def describe_vpn_connections(**kwargs): 98 | return kwargs.pop('client').describe_vpn_connections(**kwargs).get("VpnConnections", []) 99 | 100 | 101 | @sts_conn('ec2') 102 | @rate_limited() 103 | def describe_images(**kwargs): 104 | return kwargs.pop('client').describe_images(**kwargs)['Images'] 105 | 106 | 107 | @sts_conn('ec2') 108 | @rate_limited() 109 | def describe_image_attribute(**kwargs): 110 | return kwargs.pop('client').describe_image_attribute(**kwargs) 111 | 112 | 113 | @sts_conn('ec2') 114 | @rate_limited() 115 | def describe_security_groups(**kwargs): 116 | return kwargs.pop('client').describe_security_groups(**kwargs) 117 | 118 | 119 | @sts_conn('ec2') 120 | @rate_limited() 121 | def create_security_group(**kwargs): 122 | return kwargs.pop('client').create_security_group(**kwargs) 123 | 124 | 125 | @sts_conn('ec2') 126 | @rate_limited() 127 | def authorize_security_group_ingress(**kwargs): 128 | return kwargs.pop('client').authorize_security_group_ingress(**kwargs) 129 | 130 | 131 | @sts_conn('ec2') 132 | @rate_limited() 133 | def authorize_security_group_egress(**kwargs): 134 | return kwargs.pop('client').authorize_security_group_egress(**kwargs) 135 | 136 | 137 | @sts_conn('ec2') 138 | @rate_limited() 139 | def describe_auto_scaling_groups(**kwargs): 140 | return kwargs.pop('client').describe_auto_scaling_groups(**kwargs) 141 | 142 | 143 | # ------------------ # 144 | # VPC is part of EC2 # 145 | # ------------------ # 146 | @sts_conn('ec2') 147 | @rate_limited() 148 | def describe_vpcs(**kwargs): 149 | return kwargs.pop('client').describe_vpcs(**kwargs).get("Vpcs", []) 150 | 151 | 152 | @sts_conn('ec2') 153 | @rate_limited() 154 | def describe_dhcp_options(**kwargs): 155 | return kwargs.pop('client').describe_dhcp_options(**kwargs).get("DhcpOptions", []) 156 | 157 | 158 | @sts_conn('ec2') 159 | @rate_limited() 160 | def describe_internet_gateways(**kwargs): 161 | return kwargs.pop('client').describe_internet_gateways(**kwargs).get("InternetGateways", []) 162 | 163 | 164 | @sts_conn('ec2') 165 | @rate_limited() 166 | def describe_vpc_classic_link(**kwargs): 167 | return kwargs.pop('client').describe_vpc_classic_link(**kwargs).get("Vpcs", []) 168 | 169 | 170 | @paginated('Vpcs', response_pagination_marker='NextToken') 171 | @sts_conn('ec2') 172 | @rate_limited() 173 | def describe_vpc_classic_link_dns_support(**kwargs): 174 | return kwargs.pop('client').describe_vpc_classic_link_dns_support(**kwargs) 175 | 176 | 177 | @sts_conn('ec2') 178 | @rate_limited() 179 | def describe_vpc_peering_connections(**kwargs): 180 | return kwargs.pop('client').describe_vpc_peering_connections(**kwargs).get("VpcPeeringConnections", []) 181 | 182 | 183 | @sts_conn('ec2') 184 | @rate_limited() 185 | def describe_subnets(**kwargs): 186 | return kwargs.pop('client').describe_subnets(**kwargs).get("Subnets", []) 187 | 188 | 189 | @sts_conn('ec2') 190 | @rate_limited() 191 | def describe_route_tables(**kwargs): 192 | return kwargs.pop('client').describe_route_tables(**kwargs).get("RouteTables", []) 193 | 194 | 195 | @sts_conn('ec2') 196 | @rate_limited() 197 | def describe_network_acls(**kwargs): 198 | return kwargs.pop('client').describe_network_acls(**kwargs).get("NetworkAcls", []) 199 | 200 | 201 | @sts_conn('ec2') 202 | @rate_limited() 203 | def describe_vpc_attribute(**kwargs): 204 | return kwargs.pop('client').describe_vpc_attribute(**kwargs) 205 | 206 | 207 | @sts_conn('ec2') 208 | @rate_limited() 209 | def describe_flow_logs(**kwargs): 210 | return kwargs.pop('client').describe_flow_logs(**kwargs).get("FlowLogs", []) 211 | # ------------------ # 212 | 213 | -------------------------------------------------------------------------------- /cloudaux/aws/elb.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.aws.elb 3 | :platform: Unix 4 | :copyright: (c) 2017 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Patrick Kelley 7 | """ 8 | from cloudaux.aws.sts import sts_conn 9 | from cloudaux.aws.decorators import rate_limited 10 | from cloudaux.aws.decorators import paginated 11 | 12 | 13 | @paginated('LoadBalancerDescriptions', response_pagination_marker='NextMarker') 14 | @sts_conn('elb') 15 | @rate_limited() 16 | def describe_load_balancers(client=None, **kwargs): 17 | return client.describe_load_balancers(**kwargs) 18 | 19 | 20 | @sts_conn('elb') 21 | @rate_limited() 22 | def describe_load_balancer_attributes(load_balancer_name, client=None): 23 | return client.describe_load_balancer_attributes( 24 | LoadBalancerName=load_balancer_name)['LoadBalancerAttributes'] 25 | 26 | 27 | @sts_conn('elb') 28 | @rate_limited() 29 | def describe_load_balancer_policies(load_balancer_name, policy_names, client=None): 30 | return client.describe_load_balancer_policies( 31 | LoadBalancerName=load_balancer_name, 32 | PolicyNames=policy_names)['PolicyDescriptions'] 33 | 34 | 35 | @sts_conn('elb') 36 | @rate_limited() 37 | def describe_load_balancer_policy_types(policy_type_names, client=None): 38 | return client.describe_load_balancer_policy_types( 39 | PolicyTypeNames=policy_type_names)['PolicyTypeDescriptions'] 40 | 41 | 42 | @sts_conn('elb') 43 | @rate_limited() 44 | def describe_tags(load_balancer_names, client=None): 45 | return client.describe_tags( 46 | LoadBalancerNames=load_balancer_names)['TagDescriptions'] -------------------------------------------------------------------------------- /cloudaux/aws/elbv2.py: -------------------------------------------------------------------------------- 1 | from cloudaux.aws.sts import sts_conn 2 | from cloudaux.aws.decorators import rate_limited 3 | from cloudaux.aws.decorators import paginated 4 | 5 | 6 | @paginated('LoadBalancers', response_pagination_marker='NextMarker') 7 | @sts_conn('elbv2') 8 | @rate_limited() 9 | def describe_load_balancers(arns=None, names=None, client=None): 10 | """ 11 | Permission: elasticloadbalancing:DescribeLoadBalancers 12 | """ 13 | kwargs = dict() 14 | if arns: 15 | kwargs.update(dict(LoadBalancerArns=arns)) 16 | if names: 17 | kwargs.update(dict(Names=names)) 18 | return client.describe_load_balancers(**kwargs) 19 | 20 | 21 | @paginated('Listeners', response_pagination_marker='NextMarker') 22 | @sts_conn('elbv2') 23 | @rate_limited() 24 | def describe_listeners(load_balancer_arn=None, listener_arns=None, client=None): 25 | """ 26 | Permission: elasticloadbalancing:DescribeListeners 27 | """ 28 | kwargs = dict() 29 | if load_balancer_arn: 30 | kwargs.update(dict(LoadBalancerArn=load_balancer_arn)) 31 | if listener_arns: 32 | kwargs.update(dict(ListenerArns=listener_arns)) 33 | return client.describe_listeners(**kwargs) 34 | 35 | 36 | @sts_conn('elbv2') 37 | @rate_limited() 38 | def describe_load_balancer_attributes(arn, client=None): 39 | """ 40 | Permission: elasticloadbalancing:DescribeLoadBalancerAttributes 41 | """ 42 | return client.describe_load_balancer_attributes( 43 | LoadBalancerArn=arn)['Attributes'] 44 | 45 | 46 | @sts_conn('elbv2') 47 | @rate_limited() 48 | def describe_rules(listener_arn=None, rule_arns=None, client=None): 49 | """ 50 | Permission: elasticloadbalancing:DescribeRules 51 | """ 52 | kwargs = dict() 53 | if listener_arn: 54 | kwargs.update(dict(ListenerArn=listener_arn)) 55 | if rule_arns: 56 | kwargs.update(dict(RuleArns=rule_arns)) 57 | return client.describe_rules(**kwargs)['Rules'] 58 | 59 | 60 | @paginated('SslPolicies', response_pagination_marker='NextMarker') 61 | @sts_conn('elbv2') 62 | @rate_limited() 63 | def describe_ssl_policies(names, client=None): 64 | return client.describe_ssl_policies(Names=names) 65 | 66 | 67 | @sts_conn('elbv2') 68 | @rate_limited() 69 | def describe_tags(arns, client=None): 70 | """ 71 | Permission: elasticloadbalancing:DescribeTags 72 | """ 73 | return client.describe_tags(ResourceArns=arns)['TagDescriptions'] 74 | 75 | 76 | @sts_conn('elbv2') 77 | @rate_limited() 78 | def describe_target_group_attributes(arn, client=None): 79 | """ 80 | Permission: elasticloadbalancing:DescribeTargetGroupAttributes 81 | """ 82 | return client.describe_target_group_attributes(TargetGroupArn=arn)['Attributes'] 83 | 84 | 85 | @paginated('TargetGroups', response_pagination_marker='NextMarker') 86 | @sts_conn('elbv2') 87 | @rate_limited() 88 | def describe_target_groups(load_balancer_arn=None, target_group_arns=None, names=None, client=None): 89 | """ 90 | Permission: elasticloadbalancing:DescribeTargetGroups 91 | """ 92 | kwargs = dict() 93 | if load_balancer_arn: 94 | kwargs.update(LoadBalancerArn=load_balancer_arn) 95 | if target_group_arns: 96 | kwargs.update(TargetGroupArns=target_group_arns) 97 | if names: 98 | kwargs.update(Names=names) 99 | return client.describe_target_groups(**kwargs) 100 | 101 | 102 | @sts_conn('elbv2') 103 | @rate_limited() 104 | def describe_target_health(target_group_arn, targets=None, client=None): 105 | """ 106 | Permission: elasticloadbalancing:DescribeTargetHealth 107 | """ 108 | kwargs = dict(TargetGroupArn=target_group_arn) 109 | if targets: 110 | kwargs.update(Targets=targets) 111 | return client.describe_target_health(**kwargs)['TargetHealthDescriptions'] 112 | -------------------------------------------------------------------------------- /cloudaux/aws/events.py: -------------------------------------------------------------------------------- 1 | from cloudaux.aws.sts import sts_conn 2 | from cloudaux.aws.decorators import rate_limited, paginated 3 | 4 | 5 | @sts_conn('events') 6 | @paginated('Rules', request_pagination_marker="NextToken", 7 | response_pagination_marker="NextToken") 8 | @rate_limited() 9 | def list_rules(client=None, **kwargs): 10 | """ 11 | NamePrefix='string' 12 | """ 13 | result = client.list_rules(**kwargs) 14 | if not result.get("Rules"): 15 | result.update({"Rules": []}) 16 | 17 | return result 18 | 19 | 20 | @sts_conn('events') 21 | @rate_limited() 22 | def describe_rule(client=None, **kwargs): 23 | """ 24 | Name='string' 25 | """ 26 | return client.describe_rule(**kwargs) 27 | 28 | 29 | @sts_conn('events') 30 | @paginated('Targets', request_pagination_marker="NextToken", 31 | response_pagination_marker="NextToken") 32 | @rate_limited() 33 | def list_targets_by_rule(client=None, **kwargs): 34 | """ 35 | Rule='string' 36 | """ 37 | result = client.list_targets_by_rule(**kwargs) 38 | if not result.get("Targets"): 39 | result.update({"Targets": []}) 40 | 41 | return result 42 | -------------------------------------------------------------------------------- /cloudaux/aws/events_bus.py: -------------------------------------------------------------------------------- 1 | from cloudaux.aws.sts import sts_conn 2 | from cloudaux.aws.decorators import rate_limited 3 | 4 | 5 | @sts_conn('events') 6 | @rate_limited() 7 | def describe_event_bus(client=None, **kwargs): 8 | return client.describe_event_bus() 9 | 10 | -------------------------------------------------------------------------------- /cloudaux/aws/glacier.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from cloudaux.aws.decorators import rate_limited, paginated 4 | from cloudaux.aws.sts import sts_conn 5 | 6 | 7 | @sts_conn('glacier') 8 | @paginated('VaultList', request_pagination_marker="marker") 9 | @rate_limited() 10 | def list_vaults(client=None, **kwargs): 11 | return client.list_vaults() 12 | 13 | 14 | @sts_conn('glacier') 15 | @rate_limited() 16 | def describe_vault(vault_name, client=None, **kwargs): 17 | vault_data = client.describe_vault(vaultName=vault_name) 18 | return_obj = dict() 19 | 20 | # some of these fields might not exist, that's ok 21 | for field in ['VaultARN', 'VaultName', 'CreationDate', 'LastInventoryDate', 'NumberOfArchives', 'SizeInBytes']: 22 | try: 23 | return_obj[field] = vault_data[field] 24 | except KeyError: 25 | pass 26 | 27 | return return_obj 28 | 29 | 30 | @sts_conn('glacier') 31 | @rate_limited() 32 | def get_vault_access_policy(vault_name, client=None, **kwargs): 33 | try: 34 | return json.loads(client.get_vault_access_policy(vaultName=vault_name)['policy']['Policy']) 35 | except(client.exceptions.ResourceNotFoundException, KeyError): 36 | return None 37 | 38 | 39 | @sts_conn('glacier') 40 | @rate_limited() 41 | def list_tags_for_vault(vault_name, client=None, **kwargs): 42 | try: 43 | return client.list_tags_for_vault(vaultName=vault_name)['Tags'] 44 | except KeyError: 45 | return None 46 | -------------------------------------------------------------------------------- /cloudaux/aws/lambda_function.py: -------------------------------------------------------------------------------- 1 | from cloudaux.aws.sts import sts_conn 2 | from cloudaux.aws.decorators import rate_limited, paginated 3 | 4 | 5 | @sts_conn('lambda') 6 | @paginated('Aliases') 7 | @rate_limited() 8 | def list_aliases(client=None, **kwargs): 9 | return client.list_aliases(**kwargs) 10 | 11 | 12 | @sts_conn('lambda') 13 | @paginated('Versions') 14 | @rate_limited() 15 | def list_versions_by_function(client=None, **kwargs): 16 | return client.list_versions_by_function(**kwargs) 17 | 18 | 19 | @sts_conn('lambda') 20 | @paginated('EventSourceMappings') 21 | @rate_limited() 22 | def list_event_source_mappings(client=None, **kwargs): 23 | return client.list_event_source_mappings(**kwargs) 24 | 25 | 26 | @sts_conn('lambda') 27 | @rate_limited() 28 | def list_tags(client=None, **kwargs): 29 | return client.list_tags(**kwargs)['Tags'] 30 | 31 | 32 | @sts_conn('lambda') 33 | @rate_limited() 34 | def get_function_configuration(client=None, **kwargs): 35 | return client.get_function_configuration(**kwargs) 36 | 37 | 38 | @sts_conn('lambda') 39 | @rate_limited() 40 | def get_policy(client=None, **kwargs): 41 | return client.get_policy(**kwargs)['Policy'] 42 | 43 | 44 | @sts_conn('lambda') 45 | @paginated('Functions') 46 | @rate_limited() 47 | def list_functions(client=None, **kwargs): 48 | return client.list_functions(**kwargs) -------------------------------------------------------------------------------- /cloudaux/aws/resource_groups_tagging.py: -------------------------------------------------------------------------------- 1 | """ 2 | Funcions for fetching resources using resourcegroupstaggingapi 3 | https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/resourcegroupstaggingapi.html 4 | """ 5 | from cloudaux import sts_conn 6 | from cloudaux.aws.decorators import paginated 7 | 8 | 9 | @sts_conn('resourcegroupstaggingapi', service_type='client') 10 | @paginated('ResourceTagMappingList') 11 | def get_resources(client, **kwargs): 12 | """ 13 | Fetches the paginated list of resources and their tags. 14 | list of supported resources: 15 | https://docs.aws.amazon.com/ARG/latest/userguide/supported-resources.html 16 | """ 17 | return client.get_resources(**kwargs) 18 | -------------------------------------------------------------------------------- /cloudaux/aws/route53.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.aws.route53 3 | :platform: Unix 4 | :copyright: (c) 2016 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Patrick Kelley 7 | """ 8 | 9 | from cloudaux.aws.sts import sts_conn 10 | from cloudaux.aws.decorators import rate_limited 11 | 12 | 13 | @sts_conn('route53', service_type='client') 14 | @rate_limited() 15 | def list_hosted_zones(**kwargs): 16 | client = kwargs.pop('client') 17 | paginator = client.get_paginator('list_hosted_zones') 18 | hosted_zones = [] 19 | for hosted_zone in paginator.paginate(): 20 | hosted_zones.extend(hosted_zone['HostedZones']) 21 | return hosted_zones 22 | 23 | 24 | @sts_conn('route53', service_type='client') 25 | @rate_limited() 26 | def list_resource_record_sets(**kwargs): 27 | client = kwargs.pop('client') 28 | paginator = client.get_paginator('list_resource_record_sets') 29 | resource_record_sets = [] 30 | for resource_record_set in paginator.paginate(HostedZoneId=kwargs.pop('Id')): 31 | resource_record_sets.extend(resource_record_set['ResourceRecordSets']) 32 | return resource_record_sets 33 | 34 | 35 | @sts_conn('route53', service_type='client') 36 | @rate_limited() 37 | def get_hosted_zone(**kwargs): 38 | client = kwargs.pop('client') 39 | return client.get_hosted_zone(**kwargs) 40 | 41 | -------------------------------------------------------------------------------- /cloudaux/aws/s3.py: -------------------------------------------------------------------------------- 1 | from cloudaux.aws.sts import sts_conn 2 | from cloudaux.aws.decorators import rate_limited, paginated 3 | from botocore.exceptions import ClientError 4 | 5 | 6 | S3_REGION_MAPPING = dict( 7 | APNortheast='ap-northeast-1', 8 | APSoutheast='ap-southeast-1', 9 | APSoutheast2='ap-southeast-2', 10 | DEFAULT='', 11 | EU='eu-west-1', 12 | SAEast='sa-east-1', 13 | USWest='us-west-1', 14 | USWest2='us-west-2') 15 | 16 | 17 | @sts_conn('s3') 18 | @rate_limited() 19 | def list_buckets(client=None, **kwargs): 20 | return client.list_buckets() 21 | 22 | 23 | @sts_conn('s3') 24 | @rate_limited() 25 | def get_bucket_location(client=None, **kwargs): 26 | """ 27 | Bucket='string' 28 | """ 29 | return client.get_bucket_location(**kwargs) 30 | 31 | 32 | @sts_conn('s3') 33 | @rate_limited() 34 | def get_bucket_acl(client=None, **kwargs): 35 | """ 36 | Bucket='string' 37 | """ 38 | return client.get_bucket_acl(**kwargs) 39 | 40 | 41 | @sts_conn('s3') 42 | @rate_limited() 43 | def get_bucket_policy(client=None, **kwargs): 44 | """ 45 | Bucket='string' 46 | """ 47 | return client.get_bucket_policy(**kwargs) 48 | 49 | 50 | @sts_conn('s3') 51 | @rate_limited() 52 | def get_bucket_tagging(client=None, **kwargs): 53 | """ 54 | Bucket='string' 55 | """ 56 | return client.get_bucket_tagging(**kwargs) 57 | 58 | 59 | @sts_conn('s3') 60 | @rate_limited() 61 | def get_bucket_versioning(client=None, **kwargs): 62 | """ 63 | Bucket='string' 64 | """ 65 | return client.get_bucket_versioning(**kwargs) 66 | 67 | 68 | @sts_conn('s3') 69 | @rate_limited() 70 | def get_bucket_lifecycle_configuration(client=None, **kwargs): 71 | """ 72 | Bucket='string' 73 | """ 74 | return client.get_bucket_lifecycle_configuration(**kwargs) 75 | 76 | 77 | @sts_conn('s3') 78 | @rate_limited() 79 | def get_bucket_logging(client=None, **kwargs): 80 | """ 81 | Bucket='string' 82 | """ 83 | return client.get_bucket_logging(**kwargs) 84 | 85 | 86 | @sts_conn('s3') 87 | @rate_limited() 88 | def get_bucket_website(client=None, **kwargs): 89 | """ 90 | Bucket='string' 91 | """ 92 | return client.get_bucket_website(**kwargs) 93 | 94 | 95 | @sts_conn('s3') 96 | @rate_limited() 97 | def get_bucket_cors(client=None, **kwargs): 98 | """ 99 | Bucket='string' 100 | """ 101 | return client.get_bucket_cors(**kwargs) 102 | 103 | 104 | @sts_conn('s3') 105 | @rate_limited() 106 | def get_bucket_notification_configuration(client=None, **kwargs): 107 | """ 108 | Bucket='string' 109 | """ 110 | return client.get_bucket_notification_configuration(**kwargs) 111 | 112 | 113 | @sts_conn('s3') 114 | @rate_limited() 115 | def get_bucket_accelerate_configuration(client=None, **kwargs): 116 | """ 117 | Bucket='string' 118 | """ 119 | return client.get_bucket_accelerate_configuration(**kwargs) 120 | 121 | 122 | @sts_conn('s3') 123 | @rate_limited() 124 | def get_bucket_replication(client=None, **kwargs): 125 | """ 126 | Bucket='string' 127 | """ 128 | return client.get_bucket_replication(**kwargs) 129 | 130 | 131 | @sts_conn('s3') 132 | @paginated('AnalyticsConfigurationList', request_pagination_marker="ContinuationToken", 133 | response_pagination_marker="NextContinuationToken") 134 | @rate_limited() 135 | def list_bucket_analytics_configurations(client=None, **kwargs): 136 | """ 137 | Bucket='string' 138 | """ 139 | result = client.list_bucket_analytics_configurations(**kwargs) 140 | if not result.get("AnalyticsConfigurationList"): 141 | result.update({"AnalyticsConfigurationList": []}) 142 | 143 | return result 144 | 145 | 146 | @sts_conn('s3') 147 | @paginated('MetricsConfigurationList', request_pagination_marker="ContinuationToken", 148 | response_pagination_marker="NextContinuationToken") 149 | @rate_limited() 150 | def list_bucket_metrics_configurations(client=None, **kwargs): 151 | """ 152 | Bucket='string' 153 | """ 154 | result = client.list_bucket_metrics_configurations(**kwargs) 155 | if not result.get("MetricsConfigurationList"): 156 | result.update({"MetricsConfigurationList": []}) 157 | 158 | return result 159 | 160 | 161 | @sts_conn('s3') 162 | @paginated('InventoryConfigurationList', request_pagination_marker="ContinuationToken", 163 | response_pagination_marker="NextContinuationToken") 164 | @rate_limited() 165 | def list_bucket_inventory_configurations(client=None, **kwargs): 166 | """ 167 | Bucket='string' 168 | """ 169 | result = client.list_bucket_inventory_configurations(**kwargs) 170 | if not result.get("InventoryConfigurationList"): 171 | result.update({"InventoryConfigurationList": []}) 172 | 173 | return result 174 | 175 | 176 | @sts_conn('s3', service_type='resource') 177 | @rate_limited() 178 | def get_bucket_resource(bucket_name, resource=None, **kwargs): 179 | return resource.Bucket(bucket_name) 180 | 181 | 182 | def get_bucket_region(**kwargs): 183 | # Some s3 buckets do not allow viewing of details. We do not want to 184 | # throw an error in this case because we still want to see that the 185 | # bucket exists 186 | try: 187 | result = get_bucket_location(**kwargs) 188 | location = result['LocationConstraint'] 189 | except ClientError as e: 190 | if 'AccessDenied' not in str(e): 191 | raise e 192 | return None 193 | 194 | if not location: 195 | return 'us-east-1' 196 | 197 | return S3_REGION_MAPPING.get(location, location) 198 | -------------------------------------------------------------------------------- /cloudaux/aws/sg.py: -------------------------------------------------------------------------------- 1 | from cloudaux.aws.decorators import rate_limited, paginated 2 | from cloudaux.aws.sts import sts_conn 3 | 4 | 5 | @sts_conn('ec2') 6 | @paginated('SecurityGroups', request_pagination_marker="nexttoken") 7 | @rate_limited() 8 | def list_security_groups(client=None, **kwargs): 9 | return client.describe_security_groups() 10 | 11 | 12 | @sts_conn('ec2') 13 | @rate_limited() 14 | def describe_security_group(sg_id, client=None, **kwargs): 15 | try: 16 | sg_data = client.describe_security_groups(GroupIds=[sg_id])['SecurityGroups'][0] 17 | except (KeyError, IndexError): 18 | return None 19 | 20 | return_obj = dict() 21 | 22 | for field in ['Description', 'GroupName', 'IpPermissions', 'OwnerId', 'GroupId', 'IpPermissionsEgress', 'VpcId']: 23 | try: 24 | return_obj[field] = sg_data[field] 25 | except KeyError: 26 | pass 27 | 28 | return return_obj 29 | -------------------------------------------------------------------------------- /cloudaux/aws/sns.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.aws.sns 3 | :platform: Unix 4 | :copyright: (c) 2017 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Patrick Kelley 7 | """ 8 | from cloudaux.aws.sts import sts_conn 9 | from cloudaux.aws.decorators import rate_limited, paginated 10 | 11 | 12 | @sts_conn('sns') 13 | @rate_limited() 14 | def add_permission(client=None, **kwargs): 15 | return client.add_permission(**kwargs) 16 | 17 | 18 | @sts_conn('sns') 19 | @rate_limited() 20 | def check_if_phone_number_is_opted_out(client=None, **kwargs): 21 | return client.check_if_phone_number_is_opted_out(**kwargs)['isOptedOut'] 22 | 23 | 24 | @sts_conn('sns') 25 | @rate_limited() 26 | def confirm_subscription(client=None, **kwargs): 27 | return client.confirm_subscription(**kwargs)['SubscriptionArn'] 28 | 29 | 30 | @sts_conn('sns') 31 | @rate_limited() 32 | def create_platform_application(client=None, **kwargs): 33 | return client.create_platform_application(**kwargs)['PlatformApplicationArn'] 34 | 35 | 36 | @sts_conn('sns') 37 | @rate_limited() 38 | def create_platform_endpoint(client=None, **kwargs): 39 | return client.create_platform_endpoint(**kwargs)['EndpointArn'] 40 | 41 | 42 | @sts_conn('sns') 43 | @rate_limited() 44 | def create_topic(client=None, **kwargs): 45 | return client.create_topic(**kwargs)['TopicArn'] 46 | 47 | 48 | @sts_conn('sns') 49 | @rate_limited() 50 | def delete_endpoint(client=None, **kwargs): 51 | return client.delete_endpoint(**kwargs) 52 | 53 | 54 | @sts_conn('sns') 55 | @rate_limited() 56 | def delete_platform_application(client=None, **kwargs): 57 | return client.delete_platform_application(**kwargs) 58 | 59 | 60 | @sts_conn('sns') 61 | @rate_limited() 62 | def delete_topic(client=None, **kwargs): 63 | return client.delete_topic(**kwargs) 64 | 65 | 66 | @sts_conn('sns') 67 | @rate_limited() 68 | def get_endpoint_attributes(client=None, **kwargs): 69 | return client.get_endpoint_attributes(**kwargs)['Attributes'] 70 | 71 | 72 | @sts_conn('sns') 73 | @rate_limited() 74 | def get_platform_application_attributes(client=None, **kwargs): 75 | return client.get_platform_application_attributes(**kwargs)['Attributes'] 76 | 77 | 78 | @sts_conn('sns') 79 | @rate_limited() 80 | def get_sms_attributes(client=None, **kwargs): 81 | return client.get_sms_attributes(**kwargs)['attributes'] 82 | 83 | 84 | @sts_conn('sns') 85 | @rate_limited() 86 | def get_subscription_attributes(client=None, **kwargs): 87 | return client.get_subscription_attributes(**kwargs)['Attributes'] 88 | 89 | 90 | @sts_conn('sns') 91 | @rate_limited() 92 | def get_topic_attributes(client=None, **kwargs): 93 | return client.get_topic_attributes(**kwargs)['Attributes'] 94 | 95 | 96 | @sts_conn('sns') 97 | @paginated('Endpoints', request_pagination_marker="NextToken", response_pagination_marker="NextToken") 98 | @rate_limited() 99 | def list_endpoints_by_platform_application(client=None, **kwargs): 100 | return client.list_endpoints_by_platform_application(**kwargs) 101 | 102 | 103 | @sts_conn('sns') 104 | @paginated('phoneNumbers', request_pagination_marker="nextToken", response_pagination_marker="nextToken") 105 | @rate_limited() 106 | def list_phone_numbers_opted_out(client=None, **kwargs): 107 | return client.list_phone_numbers_opted_out(**kwargs) 108 | 109 | 110 | @sts_conn('sns') 111 | @paginated('PlatformApplications', request_pagination_marker="NextToken", response_pagination_marker="NextToken") 112 | @rate_limited() 113 | def list_platform_applications(client=None, **kwargs): 114 | return client.list_platform_applications(**kwargs) 115 | 116 | 117 | @sts_conn('sns') 118 | @paginated('Subscriptions', request_pagination_marker="NextToken", response_pagination_marker="NextToken") 119 | @rate_limited() 120 | def list_subscriptions(client=None, **kwargs): 121 | return client.list_subscriptions(**kwargs) 122 | 123 | 124 | @sts_conn('sns') 125 | @paginated('Subscriptions', request_pagination_marker="NextToken", response_pagination_marker="NextToken") 126 | @rate_limited() 127 | def list_subscriptions_by_topic(client=None, **kwargs): 128 | return client.list_subscriptions_by_topic(**kwargs) 129 | 130 | 131 | @sts_conn('sns') 132 | @paginated('Topics', request_pagination_marker="NextToken", response_pagination_marker="NextToken") 133 | @rate_limited() 134 | def list_topics(client=None, **kwargs): 135 | return client.list_topics(**kwargs) 136 | 137 | 138 | @sts_conn('sns') 139 | @rate_limited() 140 | def opt_in_phone_number(client=None, **kwargs): 141 | return client.opt_in_phone_number(**kwargs) 142 | 143 | 144 | @sts_conn('sns') 145 | @rate_limited() 146 | def publish(client=None, **kwargs): 147 | return client.publish(**kwargs)['MessageId'] 148 | 149 | 150 | @sts_conn('sns') 151 | @rate_limited() 152 | def remove_permission(client=None, **kwargs): 153 | return client.remove_permission(**kwargs) 154 | 155 | 156 | @sts_conn('sns') 157 | @rate_limited() 158 | def set_endpoint_attributes(client=None, **kwargs): 159 | return client.set_endpoint_attributes(**kwargs) 160 | 161 | 162 | @sts_conn('sns') 163 | @rate_limited() 164 | def set_platform_application_attributes(client=None, **kwargs): 165 | return client.set_platform_application_attributes(**kwargs) 166 | 167 | 168 | @sts_conn('sns') 169 | @rate_limited() 170 | def set_sms_attributes(client=None, **kwargs): 171 | return client.set_sms_attributes(**kwargs) 172 | 173 | 174 | @sts_conn('sns') 175 | @rate_limited() 176 | def set_subscription_attributes(client=None, **kwargs): 177 | return client.set_subscription_attributes(**kwargs) 178 | 179 | 180 | @sts_conn('sns') 181 | @rate_limited() 182 | def set_topic_attributes(client=None, **kwargs): 183 | return client.set_topic_attributes(**kwargs) 184 | 185 | 186 | @sts_conn('sns') 187 | @rate_limited() 188 | def subscribe(client=None, **kwargs): 189 | return client.subscribe(**kwargs)['SubscriptionArn'] 190 | 191 | 192 | @sts_conn('sns') 193 | @rate_limited() 194 | def unsubscribe(client=None, **kwargs): 195 | return client.unsubscribe(**kwargs) -------------------------------------------------------------------------------- /cloudaux/aws/sqs.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.aws.sqs 3 | :platform: Unix 4 | :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | .. moduleauthor:: Patrick Kelley 8 | """ 9 | import json 10 | 11 | from cloudaux.aws.sts import sts_conn 12 | from cloudaux.aws.decorators import rate_limited 13 | 14 | 15 | @sts_conn('sqs') 16 | @rate_limited() 17 | def add_permission(client=None, **kwargs): 18 | return client.add_permission(**kwargs) 19 | 20 | 21 | @sts_conn('sqs') 22 | @rate_limited() 23 | def change_message_visibility(client=None, **kwargs): 24 | return client.change_message_visibility(**kwargs) 25 | 26 | 27 | @sts_conn('sqs') 28 | @rate_limited() 29 | def create_queue(client=None, **kwargs): 30 | return client.create_queue(**kwargs)['QueueUrl'] 31 | 32 | 33 | @sts_conn('sqs') 34 | @rate_limited() 35 | def delete_message(client=None, **kwargs): 36 | return client.delete_message(**kwargs) 37 | 38 | 39 | @sts_conn('sqs') 40 | @rate_limited() 41 | def delete_message_batch(client=None, **kwargs): 42 | return client.delete_message_batch(**kwargs) 43 | 44 | 45 | @sts_conn('sqs') 46 | @rate_limited() 47 | def delete_queue(client=None, **kwargs): 48 | return client.delete_queue(**kwargs) 49 | 50 | 51 | @sts_conn('sqs') 52 | @rate_limited() 53 | def get_queue_attributes(client=None, **kwargs): 54 | attributes = client.get_queue_attributes(**kwargs)['Attributes'] 55 | 56 | if attributes.get("Policy"): 57 | policy = json.loads(attributes["Policy"]) 58 | attributes["Policy"] = policy 59 | 60 | return attributes 61 | 62 | 63 | @sts_conn('sqs') 64 | @rate_limited() 65 | def get_queue_url(client=None, **kwargs): 66 | return client.get_queue_url(**kwargs)['QueueUrl'] 67 | 68 | 69 | @sts_conn('sqs') 70 | @rate_limited() 71 | def list_dead_letter_source_queues(client=None, **kwargs): 72 | return client.list_dead_letter_source_queues(**kwargs).get('queueUrls', []) 73 | 74 | 75 | @sts_conn('sqs') 76 | @rate_limited() 77 | def list_queue_tags(client=None, **kwargs): 78 | return client.list_queue_tags(**kwargs).get('Tags', []) 79 | 80 | 81 | @sts_conn('sqs') 82 | @rate_limited() 83 | def tag_queue(client=None, **kwargs): 84 | client.tag_queue(**kwargs) 85 | 86 | 87 | @sts_conn('sqs') 88 | @rate_limited() 89 | def untag_queue(client=None, **kwargs): 90 | client.untag_queue(**kwargs) 91 | 92 | 93 | @sts_conn('sqs') 94 | @rate_limited() 95 | def list_queues(client=None, **kwargs): 96 | return client.list_queues(**kwargs).get('QueueUrls', []) 97 | 98 | 99 | @sts_conn('sqs') 100 | @rate_limited() 101 | def purge_queue(client=None, **kwargs): 102 | return client.purge_queue(**kwargs) 103 | 104 | 105 | @sts_conn('sqs') 106 | @rate_limited() 107 | def receive_message(client=None, **kwargs): 108 | return client.receive_message(**kwargs)['Messages'] 109 | 110 | 111 | @sts_conn('sqs') 112 | @rate_limited() 113 | def remove_permission(client=None, **kwargs): 114 | return client.remove_permission(**kwargs) 115 | 116 | 117 | @sts_conn('sqs') 118 | @rate_limited() 119 | def send_message(client=None, **kwargs): 120 | return client.send_message(**kwargs) 121 | 122 | 123 | @sts_conn('sqs') 124 | @rate_limited() 125 | def send_message_batch(client=None, **kwargs): 126 | return client.send_message_batch(**kwargs) 127 | 128 | 129 | @sts_conn('sqs') 130 | @rate_limited() 131 | def set_queue_attributes(client=None, **kwargs): 132 | return client.set_queue_attributes(**kwargs) 133 | 134 | 135 | @sts_conn('sqs', service_type='resource') 136 | @rate_limited() 137 | def list_all_existing_queues(resource=None, **kwargs): 138 | return resource.queues.all(**kwargs) 139 | -------------------------------------------------------------------------------- /cloudaux/decorators.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.decorators 3 | :platform: Unix 4 | :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | .. moduleauthor:: Patrick Kelley 8 | .. moduleauthor:: Mike Grima 9 | """ 10 | import functools 11 | from itertools import product 12 | from cloudaux.aws.sts import boto3_cached_conn 13 | from cloudaux.orchestration import modify 14 | 15 | from cloudaux import CloudAux 16 | 17 | 18 | def iter_account_region(service, service_type='client', accounts=None, regions=None, assume_role=None, 19 | session_name='cloudaux', conn_type='cloudaux', external_id=None, arn_partition='aws', read_only=False): 20 | def decorator(func): 21 | @functools.wraps(func) 22 | def decorated_function(*args, **kwargs): 23 | threads = [] 24 | for account, region in product(accounts, regions): 25 | conn_dict = { 26 | 'tech': service, 27 | 'account_number': account, 28 | 'region': region, 29 | 'session_name': session_name, 30 | 'assume_role': assume_role, 31 | 'service_type': service_type, 32 | 'external_id': external_id, 33 | 'arn_partition': arn_partition, 34 | 'read_only': read_only 35 | } 36 | if conn_type == 'cloudaux': 37 | kwargs['cloudaux'] = CloudAux(**conn_dict) 38 | elif conn_type == 'dict': 39 | kwargs['conn_dict'] = conn_dict 40 | elif conn_type == 'boto3': 41 | del conn_dict['tech'] 42 | kwargs['conn'] = boto3_cached_conn(service, **conn_dict) 43 | result = func(*args, **kwargs) 44 | if result: 45 | threads.append(result) 46 | 47 | result = [] 48 | for thread in threads: 49 | result.append(thread) 50 | return result 51 | return decorated_function 52 | return decorator 53 | 54 | 55 | def modify_output(func): 56 | @functools.wraps(func) 57 | def decorated_function(*args, **kwargs): 58 | output = kwargs.pop('output', 'camelized') 59 | result = func(*args, **kwargs) 60 | return modify(result, output=output) 61 | return decorated_function 62 | -------------------------------------------------------------------------------- /cloudaux/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.exceptions 3 | :platform: Unix 4 | :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | .. moduleauthor:: Patrick Kelley 8 | """ 9 | 10 | 11 | class CloudAuxException(Exception): 12 | pass 13 | -------------------------------------------------------------------------------- /cloudaux/gcp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/cloudaux/39d1dc11ccd794f21d95d3d1458abd94425b0927/cloudaux/gcp/__init__.py -------------------------------------------------------------------------------- /cloudaux/gcp/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.gcp.config 3 | :platform: Unix 4 | :copyright: (c) 2016 by Google Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Tom Melendez (@supertom) 7 | """ 8 | 9 | USE_GAX=False 10 | # TODO(supertom): Change: this is not a good default scope. 11 | DEFAULT_SCOPES = ['https://www.googleapis.com/auth/cloud-platform'] 12 | 13 | """ 14 | There are currently two distinct client types for working with Google APIs. 15 | We use the 'client_type' to distinguish between the two: 16 | - cloud: gcloud-python. This is recommended, but not available for all services yet. 17 | - general: google-python-api-client. This is available for many services, but is deprecated. 18 | """ 19 | GOOGLE_CLIENT_MAP = { 20 | 'gcs': 21 | {'client_type': 'cloud', 'module_name': 'storage'}, 22 | 'gce': 23 | {'client_type': 'general', 'module_name': 'compute'}, 24 | 'iam': 25 | {'client_type': 'general', 'module_name': 'iam'}, 26 | 'crm': 27 | {'client_type': 'general', 'module_name': 'cloudresourcemanager'}, 28 | } 29 | -------------------------------------------------------------------------------- /cloudaux/gcp/crm.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.gcp.crm 3 | :platform: Unix 4 | :copyright: (c) 2019 by Fitbit Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Greg Harris 7 | """ 8 | from cloudaux.gcp.utils import service_list 9 | from cloudaux.gcp.decorators import gcp_conn, gcp_stats 10 | from cloudaux.gcp.utils import service_list 11 | 12 | 13 | @gcp_conn('crm') 14 | def get_iam_policy(client=None, **kwargs): 15 | # body={} workaround for https://github.com/googleapis/google-api-python-client/issues/713 16 | req = client.projects().getIamPolicy(resource=kwargs['resource'], body={}) 17 | return req.execute() 18 | 19 | -------------------------------------------------------------------------------- /cloudaux/gcp/decorators.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.gcp.decorators 3 | :platform: Unix 4 | :copyright: (c) 2016 by Google Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Tom Melendez (@supertom) 7 | """ 8 | import time 9 | from functools import wraps 10 | 11 | from six import string_types 12 | 13 | from cloudaux.gcp.gcpcache import GCPCache 14 | from cloudaux.gcp.utils import get_creds_from_kwargs, rewrite_kwargs 15 | 16 | _GCP_STATS = {} 17 | _GCP_CACHE = GCPCache() 18 | 19 | 20 | def _build_key(func_name, args, kwargs): 21 | """Builds key for cache and stats.""" 22 | return "n=%s__args=%s__kwargs=%s" % (func_name, args, kwargs) 23 | 24 | 25 | def gcp_conn(service, service_type='client', future_expiration_minutes=15): 26 | """ 27 | service_type: not currently used. 28 | """ 29 | 30 | def decorator(f): 31 | @wraps(f) 32 | def decorated_function(*args, **kwargs): 33 | # Import here to avoid circular import issue 34 | from cloudaux.gcp.auth import get_client 35 | (conn_args, kwargs) = get_creds_from_kwargs(kwargs) 36 | client_details, client = get_client( 37 | service, service_type=service_type, 38 | future_expiration_minutes=15, **conn_args) 39 | if client_details: 40 | kwargs = rewrite_kwargs(client_details['client_type'], kwargs, 41 | client_details['module_name']) 42 | kwargs['client'] = client 43 | return f(*args, **kwargs) 44 | 45 | return decorated_function 46 | 47 | return decorator 48 | 49 | 50 | def gcp_stats(): 51 | """ 52 | Collect stats 53 | 54 | Specifically, time function calls 55 | :returns: function response 56 | :rtype: varies 57 | """ 58 | 59 | def decorator(f): 60 | @wraps(f) 61 | def decorated_function(*args, **kwargs): 62 | start_time = time.time() 63 | result = f(*args, **kwargs) 64 | end_time = time.time() 65 | strkey = _build_key(f.__name__, args, kwargs) 66 | _GCP_STATS.setdefault(strkey, []).append(end_time - start_time) 67 | return result 68 | 69 | return decorated_function 70 | 71 | return decorator 72 | 73 | 74 | def gcp_cache(future_expiration_minutes=15): 75 | """ 76 | Cache function output 77 | :param future_expiration_minutes: Number of minutes in the future until item 78 | expires. Default is 15. 79 | :returns: function response, optionally from the cache 80 | :rtype: varies 81 | """ 82 | 83 | def decorator(f): 84 | @wraps(f) 85 | def decorated_function(*args, **kwargs): 86 | strkey = _build_key(f.__name__, args, kwargs) 87 | cached_result = _GCP_CACHE.get(strkey) 88 | if cached_result: 89 | return cached_result 90 | else: 91 | result = f(*args, **kwargs) 92 | _GCP_CACHE.insert(strkey, result, future_expiration_minutes) 93 | return result 94 | 95 | return decorated_function 96 | 97 | return decorator 98 | 99 | 100 | def iter_project(projects, key_file=None): 101 | """ 102 | Call decorated function for each item in project list. 103 | 104 | Note: the function 'decorated' is expected to return a value plus a dictionary of exceptions. 105 | 106 | If item in list is a dictionary, we look for a 'project' and 'key_file' entry, respectively. 107 | If item in list is of type string_types, we assume it is the project string. Default credentials 108 | will be used by the underlying client library. 109 | 110 | :param projects: list of project strings or list of dictionaries 111 | Example: {'project':..., 'keyfile':...}. Required. 112 | :type projects: ``list`` of ``str`` or ``list`` of ``dict`` 113 | 114 | :param key_file: path on disk to keyfile, for use with all projects 115 | :type key_file: ``str`` 116 | 117 | :returns: tuple containing a list of function output and an exceptions map 118 | :rtype: ``tuple of ``list``, ``dict`` 119 | """ 120 | 121 | def decorator(func): 122 | @wraps(func) 123 | def decorated_function(*args, **kwargs): 124 | item_list = [] 125 | exception_map = {} 126 | for project in projects: 127 | if isinstance(project, string_types): 128 | kwargs['project'] = project 129 | if key_file: 130 | kwargs['key_file'] = key_file 131 | elif isinstance(project, dict): 132 | kwargs['project'] = project['project'] 133 | kwargs['key_file'] = project['key_file'] 134 | itm, exc = func(*args, **kwargs) 135 | item_list.extend(itm) 136 | exception_map.update(exc) 137 | return (item_list, exception_map) 138 | 139 | return decorated_function 140 | 141 | return decorator 142 | -------------------------------------------------------------------------------- /cloudaux/gcp/gce/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/cloudaux/39d1dc11ccd794f21d95d3d1458abd94425b0927/cloudaux/gcp/gce/__init__.py -------------------------------------------------------------------------------- /cloudaux/gcp/gce/address.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.gcp.gce.address 3 | :platform: Unix 4 | :copyright: (c) 2019 by Fitbit Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Greg Harris 7 | """ 8 | from cloudaux.gcp.utils import gce_list, gce_list_aggregated 9 | from cloudaux.gcp.decorators import gcp_conn 10 | 11 | @gcp_conn('gce') 12 | def list_addresses(client=None, **kwargs): 13 | """ 14 | :rtype: ``list`` 15 | """ 16 | 17 | return gce_list_aggregated(service=client.addresses(), **kwargs) 18 | 19 | 20 | @gcp_conn('gce') 21 | def list_global_addresses(client=None, **kwargs): 22 | """ 23 | :rtype: ``list`` 24 | """ 25 | 26 | return gce_list(service=client.globalAddresses(), **kwargs) 27 | 28 | 29 | @gcp_conn('gce') 30 | def get_address(client=None, **kwargs): 31 | service = client.addresses() 32 | req = service.get(project=kwargs['project'], 33 | address=kwargs['Address'], 34 | region=kwargs['Region']) 35 | return req.execute() 36 | 37 | 38 | @gcp_conn('gce') 39 | def get_global_address(client=None, **kwargs): 40 | service = client.globalAddresses() 41 | req = service.get(project=kwargs['project'], 42 | address=kwargs['GlobalAddress']) 43 | return req.execute() 44 | -------------------------------------------------------------------------------- /cloudaux/gcp/gce/disk.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.gcp.gce.disk 3 | :platform: Unix 4 | :copyright: (c) 2019 by Fitbit Inc., see AUTHORS for more 5 | :license: Apache, see LCIENSE for more details. 6 | .. moduleauthor:: Greg Harris 7 | """ 8 | from cloudaux.gcp.utils import gce_list 9 | from cloudaux.gcp.decorators import gcp_conn 10 | 11 | @gcp_conn('gce') 12 | def list_disks(client=None, **kwargs): 13 | """ 14 | :rtype: ``list`` 15 | """ 16 | return gce_list(service=client.disks(), 17 | **kwargs) 18 | 19 | @gcp_conn('gce') 20 | def get_disk(client=None, **kwargs): 21 | service = client.disks() 22 | req = service.get(project=kwargs['project'], name=kwargs['disk']) 23 | resp = req.execute() 24 | return resp 25 | -------------------------------------------------------------------------------- /cloudaux/gcp/gce/firewall.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.gcp.gce.firewall 3 | :platform: Unix 4 | :copyright: (c) 2016 by Google Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Tom Melendez (@supertom) 7 | """ 8 | from cloudaux.gcp.utils import gce_list 9 | from cloudaux.gcp.decorators import gcp_conn 10 | 11 | @gcp_conn('gce') 12 | def list_firewall_rules(client=None, **kwargs): 13 | """ 14 | :rtype: ``list`` 15 | """ 16 | return gce_list(service=client.firewalls(), 17 | **kwargs) 18 | 19 | @gcp_conn('gce') 20 | def get_firewall_rule(client=None, **kwargs): 21 | service = client.firewalls() 22 | req = service.get(project=kwargs['project'], firewall=kwargs['Firewall']) 23 | resp = req.execute() 24 | return resp 25 | -------------------------------------------------------------------------------- /cloudaux/gcp/gce/forwarding_rule.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.gcp.gce.forwarding_rule 3 | :platform: Unix 4 | :copyright: (c) 2019 by Fitbit Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Greg Harris 7 | """ 8 | from cloudaux.gcp.utils import gce_list, gce_list_aggregated 9 | from cloudaux.gcp.decorators import gcp_conn 10 | 11 | 12 | @gcp_conn('gce') 13 | def list_forwarding_rules(client=None, **kwargs): 14 | """ 15 | :rtype: ``list`` 16 | """ 17 | 18 | return gce_list_aggregated(service=client.forwardingRules(), **kwargs) 19 | 20 | 21 | @gcp_conn('gce') 22 | def list_global_forwarding_rules(client=None, **kwargs): 23 | """ 24 | :rtype: ``list`` 25 | """ 26 | 27 | return gce_list(service=client.globalForwardingRules(), **kwargs) 28 | 29 | 30 | @gcp_conn('gce') 31 | def get_forwarding_rule(client=None, **kwargs): 32 | service = client.forwardingRules() 33 | req = service.get(project=kwargs['project'], 34 | forwardingRule=kwargs['ForwardingRule'], 35 | region=kwargs['Region']) 36 | return req.execute() 37 | 38 | 39 | @gcp_conn('gce') 40 | def get_global_forwarding_rule(client=None, **kwargs): 41 | service = client.globalForwardingRules() 42 | req = service.get(project=kwargs['project'], 43 | forwardingRule=kwargs['GlobalForwardingRule']) 44 | return req.execute() 45 | -------------------------------------------------------------------------------- /cloudaux/gcp/gce/instance.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.gcp.gce.instance 3 | :platform: Unix 4 | :copyright: (c) 2019 by Fitbit Inc., see AUTHORS for more 5 | :license: Apache, see LCIENSE for more details. 6 | .. moduleauthor:: Greg Harris 7 | """ 8 | from cloudaux.gcp.utils import gce_list 9 | from cloudaux.gcp.decorators import gcp_conn 10 | 11 | @gcp_conn('gce') 12 | def list_instances(client=None, **kwargs): 13 | """ 14 | :rtype: ``list`` 15 | """ 16 | return gce_list(service=client.instances(), 17 | **kwargs) 18 | 19 | @gcp_conn('gce') 20 | def get_instance(client=None, **kwargs): 21 | service = client.instances() 22 | req = service.get(project=kwargs['project'], name=kwargs['instance']) 23 | resp = req.execute() 24 | return resp 25 | -------------------------------------------------------------------------------- /cloudaux/gcp/gce/network.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.gcp.gce.network 3 | :platform: Unix 4 | :copyright: (c) 2016 by Google Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Tom Melendez (@supertom) 7 | """ 8 | from cloudaux.gcp.utils import gce_list, gce_list_aggregated 9 | from cloudaux.gcp.decorators import gcp_conn 10 | 11 | @gcp_conn('gce') 12 | def list_networks(client=None, **kwargs): 13 | """ 14 | :rtype: ``list`` 15 | """ 16 | return gce_list(service=client.networks(), 17 | **kwargs) 18 | 19 | 20 | @gcp_conn('gce') 21 | def list_subnetworks(client=None, **kwargs): 22 | """ 23 | :rtype: ``list`` 24 | """ 25 | 26 | return gce_list_aggregated(service=client.subnetworks(), 27 | key_name='subnetworks', **kwargs) 28 | 29 | 30 | @gcp_conn('gce') 31 | def get_network(client=None, **kwargs): 32 | service = client.networks() 33 | req = service.get(project=kwargs['project'], network=kwargs['Network']) 34 | resp = req.execute() 35 | return resp 36 | 37 | 38 | @gcp_conn('gce') 39 | def get_subnetwork(client=None, **kwargs): 40 | service = client.subnetworks() 41 | req = service.get(project=kwargs['project'], 42 | subnetwork=kwargs['Subnetwork'], region=kwargs['Region']) 43 | resp = req.execute() 44 | return resp 45 | -------------------------------------------------------------------------------- /cloudaux/gcp/gce/project.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.gcp.gce.project 3 | :platform: Unix 4 | :copyright: (c) 2019 by Fitbit Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Greg Harris 7 | """ 8 | from cloudaux.gcp.utils import service_list 9 | from cloudaux.gcp.decorators import gcp_conn, gcp_stats 10 | from cloudaux.gcp.utils import service_list 11 | 12 | @gcp_conn('gce') 13 | def get_project(client=None, **kwargs): 14 | req = client.projects().get(project=kwargs['project']) 15 | resp = req.execute() 16 | return resp 17 | -------------------------------------------------------------------------------- /cloudaux/gcp/gce/zone.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.gcp.gce.zone 3 | :platform: Unix 4 | :copyright: (c) 2019 by Fitbit Inc., see AUTHORS for more 5 | :license: Apache, see LCIENSE for more details. 6 | .. moduleauthor:: Greg Harris 7 | """ 8 | from cloudaux.gcp.utils import gce_list 9 | from cloudaux.gcp.decorators import gcp_conn 10 | 11 | @gcp_conn('gce') 12 | def list_zones(client=None, **kwargs): 13 | """ 14 | :rtype: ``list`` 15 | """ 16 | return gce_list(service=client.zones(), 17 | **kwargs) 18 | 19 | @gcp_conn('gce') 20 | def get_zone(client=None, **kwargs): 21 | service = client.zones() 22 | req = service.get(project=kwargs['project'], name=kwargs['zone']) 23 | resp = req.execute() 24 | return resp 25 | -------------------------------------------------------------------------------- /cloudaux/gcp/gcpcache.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.gcp.gcpcache 3 | :platform: Unix 4 | :copyright: (c) 2016 by Google Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Tom Melendez (@supertom) 7 | """ 8 | import dateutil.tz 9 | import datetime 10 | 11 | class GCPCache(object): 12 | def __init__(self): 13 | self._CACHE = {} 14 | self._CACHE_STATS = {'access_stats': {}} 15 | 16 | def get(self, key, delete_if_expired=True): 17 | """ 18 | Retrieve key from Cache. 19 | 20 | :param key: key to look up in cache. 21 | :type key: ``object`` 22 | 23 | :param delete_if_expired: remove value from cache if it is expired. 24 | Default is True. 25 | :type delete_if_expired: ``bool`` 26 | 27 | :returns: value from cache or None 28 | :rtype: varies or None 29 | """ 30 | self._update_cache_stats(key, None) 31 | 32 | if key in self._CACHE: 33 | (expiration, obj) = self._CACHE[key] 34 | if expiration > self._now(): 35 | self._update_cache_stats(key, 'hit') 36 | return obj 37 | else: 38 | if delete_if_expired: 39 | self.delete(key) 40 | self._update_cache_stats(key, 'expired') 41 | return None 42 | 43 | self._update_cache_stats(key, 'miss') 44 | return None 45 | 46 | def insert(self, key, obj, future_expiration_minutes=15): 47 | """ 48 | Insert item into cache. 49 | 50 | :param key: key to look up in cache. 51 | :type key: ``object`` 52 | 53 | :param obj: item to store in cache. 54 | :type obj: varies 55 | 56 | :param future_expiration_minutes: number of minutes item is valid 57 | :type param: ``int`` 58 | 59 | :returns: True 60 | :rtype: ``bool`` 61 | """ 62 | expiration_time = self._calculate_expiration(future_expiration_minutes) 63 | self._CACHE[key] = (expiration_time, obj) 64 | return True 65 | 66 | def delete(self, key): 67 | del self._CACHE[key] 68 | return True 69 | 70 | def _now(self): 71 | return datetime.datetime.now(dateutil.tz.tzutc()) 72 | 73 | def _calculate_expiration(self, future_expiration_minutes=15): 74 | return self._now() + datetime.timedelta( 75 | minutes=future_expiration_minutes) 76 | 77 | def _update_cache_stats(self, key, result): 78 | """ 79 | Update the cache stats. 80 | 81 | If no cache-result is specified, we iniitialize the key. 82 | Otherwise, we increment the correct cache-result. 83 | 84 | Note the behavior for expired. A client can be expired and the key 85 | still exists. 86 | """ 87 | if result is None: 88 | self._CACHE_STATS['access_stats'].setdefault(key, 89 | {'hit': 0, 'miss': 0, 'expired': 0}) 90 | else: 91 | self._CACHE_STATS['access_stats'][key][result] +=1 92 | 93 | def get_access_details(self, key=None): 94 | """Get access details in cache.""" 95 | if key in self._CACHE_STATS: 96 | return self._CACHE_STATS['access_stats'][key] 97 | else: 98 | return self._CACHE_STATS['access_stats'] 99 | 100 | def get_stats(self): 101 | """Get general stats for the cache.""" 102 | expired = sum([x['expired'] for _, x in 103 | self._CACHE_STATS['access_stats'].items()]) 104 | miss = sum([x['miss'] for _, x in 105 | self._CACHE_STATS['access_stats'].items()]) 106 | 107 | hit = sum([x['hit'] for _, x in 108 | self._CACHE_STATS['access_stats'].items()]) 109 | return { 110 | 'totals': { 111 | 'keys': len(self._CACHE_STATS['access_stats']), 112 | 'expired': expired, 113 | 'miss': miss, 114 | 'hit': hit, 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /cloudaux/gcp/gcs.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.gcp.gcs 3 | :platform: Unix 4 | :copyright: (c) 2016 by Google Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Tom Melendez (@supertom) 7 | """ 8 | from cloudaux.gcp.decorators import gcp_conn 9 | 10 | @gcp_conn('gcs') 11 | def list_buckets(client=None, **kwargs): 12 | """ 13 | List buckets for a project. 14 | 15 | :param client: client object to use. 16 | :type client: Google Cloud Storage client 17 | 18 | :returns: list of dictionary reprsentation of Bucket 19 | :rtype: ``list`` of ``dict`` 20 | """ 21 | buckets = client.list_buckets(**kwargs) 22 | return [b.__dict__ for b in buckets] 23 | 24 | @gcp_conn('gcs') 25 | def get_bucket(client=None, **kwargs): 26 | """ 27 | Get bucket object. 28 | 29 | :param client: client object to use. 30 | :type client: Google Cloud Storage client 31 | 32 | :returns: Bucket object 33 | :rtype: ``object`` 34 | """ 35 | bucket = client.lookup_bucket(kwargs['Bucket']) 36 | return bucket 37 | 38 | def get_bucket_field(**kwargs): 39 | """ 40 | Get value from member field of bucket object. 41 | 42 | :param Field: name of member of Bucket object to return. 43 | :type Field: ``str`` 44 | 45 | :returns: value contained by the specified member field. 46 | :rtype: varies 47 | """ 48 | bucket = get_bucket(**kwargs) 49 | if bucket: 50 | return getattr(bucket, kwargs['Field'], None) 51 | else: 52 | return None 53 | 54 | def list_objects_in_bucket(**kwargs): 55 | """ 56 | List objects in bucket. 57 | 58 | :param Bucket: name of bucket 59 | :type Bucket: ``str`` 60 | 61 | :returns list of objects in bucket 62 | :rtype: ``list`` 63 | """ 64 | bucket = get_bucket(**kwargs) 65 | if bucket: 66 | return [o for o in bucket.list_blobs()] 67 | else: 68 | return None 69 | 70 | def get_object_in_bucket(**kwargs): 71 | """ 72 | Retrieve object from Bucket. 73 | 74 | :param Bucket: name of bucket 75 | :type Bucket: ``str`` 76 | 77 | :returns: object from bucket or None 78 | :rtype ``object`` or None 79 | """ 80 | bucket = get_bucket(**kwargs) 81 | if bucket: 82 | return bucket.get_blob(kwargs['Object']) 83 | else: 84 | return None 85 | -------------------------------------------------------------------------------- /cloudaux/gcp/iam.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.gcp.iam 3 | :platform: Unix 4 | :copyright: (c) 2016 by Google Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Tom Melendez (@supertom) 7 | """ 8 | from cloudaux.gcp.utils import service_list 9 | from cloudaux.gcp.decorators import gcp_conn, gcp_stats 10 | from cloudaux.gcp.utils import service_list 11 | 12 | @gcp_conn('iam') 13 | def list_serviceaccounts(client=None, **kwargs): 14 | return service_list(service=client.projects().serviceAccounts(), 15 | key_name='accounts', **kwargs) 16 | @gcp_conn('iam') 17 | def get_serviceaccount(client=None, **kwargs): 18 | """ 19 | service_account='string' 20 | """ 21 | service_account=kwargs.pop('service_account') 22 | resp = client.projects().serviceAccounts().get( 23 | name=service_account).execute() 24 | return resp 25 | 26 | @gcp_conn('iam') 27 | def get_serviceaccount_keys(client=None, **kwargs): 28 | """ 29 | service_account='string' 30 | """ 31 | service_account=kwargs.pop('service_account') 32 | kwargs['name'] = service_account 33 | return service_list(client.projects().serviceAccounts().keys(), 34 | key_name='keys', **kwargs) 35 | @gcp_conn('iam') 36 | def get_iam_policy(client=None, **kwargs): 37 | """ 38 | service_account='string' 39 | """ 40 | service_account=kwargs.pop('service_account') 41 | resp = client.projects().serviceAccounts().getIamPolicy( 42 | resource=service_account).execute() 43 | # TODO(supertom): err handling, check if 'bindings' is correct 44 | if 'bindings' in resp: 45 | return resp['bindings'] 46 | else: 47 | return None 48 | 49 | @gcp_conn('crm') 50 | def get_project_iam_policy(client=None, **kwargs): 51 | # body={} is a workaround for a bug in older version of the google libraries 52 | req = client.projects().getIamPolicy(resource=kwargs['resource'], body={}) 53 | return req.execute() 54 | -------------------------------------------------------------------------------- /cloudaux/gcp/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.gcp.utils 3 | :platform: Unix 4 | :copyright: (c) 2016 by Google Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Tom Melendez (@supertom) 7 | """ 8 | 9 | 10 | def strdate(dte): 11 | return dte.strftime('%Y-%m-%dT%H:%M:%SZ') 12 | 13 | 14 | def get_creds_from_kwargs(kwargs): 15 | """Helper to get creds out of kwargs.""" 16 | creds = { 17 | 'key_file': kwargs.pop('key_file', None), 18 | 'http_auth': kwargs.pop('http_auth', None), 19 | 'project': kwargs.get('project', None), 20 | 'user_agent': kwargs.pop('user_agent', None), 21 | 'api_version': kwargs.pop('api_version', 'v1') 22 | } 23 | return (creds, kwargs) 24 | 25 | 26 | def rewrite_kwargs(conn_type, kwargs, module_name=None): 27 | """ 28 | Manipulate connection keywords. 29 | 30 | Modifieds keywords based on connection type. 31 | 32 | There is an assumption here that the client has 33 | already been created and that these keywords are being 34 | passed into methods for interacting with various services. 35 | 36 | Current modifications: 37 | - if conn_type is not cloud and module is 'compute', 38 | then rewrite project as name. 39 | - if conn_type is cloud and module is 'storage', 40 | then remove 'project' from dict. 41 | 42 | :param conn_type: E.g. 'cloud' or 'general' 43 | :type conn_type: ``str`` 44 | 45 | :param kwargs: Dictionary of keywords sent in by user. 46 | :type kwargs: ``dict`` 47 | 48 | :param module_name: Name of specific module that will be loaded. 49 | Default is None. 50 | :type conn_type: ``str`` or None 51 | 52 | :returns kwargs with client and module specific changes 53 | :rtype: ``dict`` 54 | """ 55 | if conn_type != 'cloud' and module_name != 'compute': 56 | if 'project' in kwargs: 57 | kwargs['name'] = 'projects/%s' % kwargs.pop('project') 58 | if conn_type == 'cloud' and module_name == 'storage': 59 | if 'project' in kwargs: 60 | del kwargs['project'] 61 | return kwargs 62 | 63 | 64 | def gce_list_aggregated(service=None, key_name='name', **kwargs): 65 | """General aggregated list function for the GCE service.""" 66 | resp_list = [] 67 | req = service.aggregatedList(**kwargs) 68 | 69 | while req is not None: 70 | resp = req.execute() 71 | for location, item in resp['items'].items(): 72 | if key_name in item: 73 | resp_list.extend(item[key_name]) 74 | 75 | req = service.aggregatedList_next(previous_request=req, 76 | previous_response=resp) 77 | return resp_list 78 | 79 | 80 | def gce_list(service=None, **kwargs): 81 | """General list function for the GCE service.""" 82 | resp_list = [] 83 | req = service.list(**kwargs) 84 | 85 | while req is not None: 86 | resp = req.execute() 87 | for item in resp.get('items', []): 88 | resp_list.append(item) 89 | req = service.list_next(previous_request=req, previous_response=resp) 90 | return resp_list 91 | 92 | 93 | def service_list(service=None, key_name=None, **kwargs): 94 | """General list function for Google APIs.""" 95 | resp_list = [] 96 | req = service.list(**kwargs) 97 | 98 | while req is not None: 99 | resp = req.execute() 100 | if key_name and key_name in resp: 101 | resp_list.extend(resp[key_name]) 102 | else: 103 | resp_list.append(resp) 104 | # Not all list calls have a list_next 105 | if hasattr(service, 'list_next'): 106 | req = service.list_next(previous_request=req, 107 | previous_response=resp) 108 | else: 109 | req = None 110 | return resp_list 111 | 112 | 113 | def get_cache_stats(): 114 | """Helper to retrieve stats cache.""" 115 | from cloudaux.gcp.decorators import _GCP_CACHE 116 | return _GCP_CACHE.get_stats() 117 | 118 | 119 | def get_cache_access_details(key=None): 120 | """Retrieve detailed cache information.""" 121 | from cloudaux.gcp.decorators import _GCP_CACHE 122 | return _GCP_CACHE.get_access_details(key=key) 123 | 124 | 125 | def get_gcp_stats(): 126 | """Retrieve stats, such as function timings.""" 127 | from cloudaux.gcp.decorators import _GCP_STATS 128 | return _GCP_STATS 129 | 130 | 131 | def get_user_agent_default(pkg_name='cloudaux'): 132 | """ 133 | Get default User Agent String. 134 | 135 | Try to import pkg_name to get an accurate version number. 136 | 137 | return: string 138 | """ 139 | version = '0.0.1' 140 | try: 141 | import pkg_resources 142 | version = pkg_resources.get_distribution(pkg_name).version 143 | except pkg_resources.DistributionNotFound: 144 | pass 145 | except ImportError: 146 | pass 147 | 148 | return 'cloudaux/%s' % (version) 149 | 150 | 151 | def get_user_agent(**kwargs): 152 | """ 153 | If there is a useragent, find it. 154 | 155 | Look in the keywords for user_agent. If not found, 156 | return get_user_agent_default 157 | """ 158 | user_agent = kwargs.get('user_agent', None) 159 | if not user_agent: 160 | return get_user_agent_default() 161 | return user_agent 162 | -------------------------------------------------------------------------------- /cloudaux/openstack/README.md: -------------------------------------------------------------------------------- 1 | # cloudaux OpenStack support 2 | 3 | Cloud Auxiliary has support for Openstack. 4 | 5 | ## Documentation 6 | 7 | - [CloudAux](../../README.md "CloudAux Readme") 8 | - [OpenStack](README.md "OpenStack Docs") [THIS FILE] 9 | - [AWS](../aws/README.md "Amazon Web Services Docs") 10 | - [GCP](../gcp/README.md "Google Cloud Platform Docs") 11 | 12 | ## Features 13 | 14 | - intelligent connection caching. 15 | - generalized OpenStack SDK generator usage. 16 | - orchestrates all the calls required to fully describe an item. 17 | - control which attributes are returned flags. 18 | 19 | ## Orchestration Supported Technologies 20 | 21 | - Openstack SDK supported services 22 | 23 | ## Install 24 | 25 | pip install cloudaux 26 | 27 | For OpenStack support run: 28 | 29 | pip install cloudaux\[openstack\] 30 | 31 | ## Authentication/Authorization 32 | 33 | - Keystone clouds.yaml 34 | 35 | ## Example 36 | 37 | from cloudaux.openstack.decorators import _connect 38 | conn = _connect(cloud_name, region, yaml_file): 39 | 40 | # Over your entire environment: 41 | from cloudaux.openstack.decorators import iter_account_region, get_regions 42 | 43 | @iter_account_region(account_regions=get_regions()) 44 | def list_networks(conn=None, service='network', generator='security_groups'): 45 | from cloudaux.openstack.utils import list_items 46 | list_items(**kwargs) 47 | 48 | ## Orchestration Example 49 | 50 | ### Security Group 51 | 52 | from cloudaux.orchestration.openstack.security_group import get_security_group, FLAGS 53 | 54 | secgroup = get_security_group(result, flags=flags, **kwargs) 55 | 56 | # The flags parameter is optional but allows the user to indicate that 57 | # only a subset of the full item description is required. 58 | # Security Group Flag Options: 59 | # RULES, INSTANCES (default) 60 | # For instance: flags=FLAGS.RULES | FLAGS.INSTANCES 61 | 62 | print(json.dumps(secgroup, indent=4, sort_keys=True)) 63 | 64 | { 65 | "assigned_to": [ 66 | { 67 | "instance_id": "..." 68 | } 69 | ], 70 | "created_at": "...", 71 | "description": "...", 72 | "id": "...", 73 | "location": "...", 74 | "name": "...", 75 | "project_id": "...", 76 | "revision_number": 3, 77 | "rules": [ 78 | { 79 | "rule_type": "...", 80 | "remote_group_id": "...", 81 | "from_port": "...", 82 | "description": "...", 83 | "tags": [], 84 | "to_port": "...", 85 | "ethertype": "...", 86 | "created_at": "...", 87 | "updated_at": "...", 88 | "security_group_id": "...", 89 | "revision_number": 0, 90 | "tenant_id": "...", 91 | "project_id": "..."", 92 | "id": "...", 93 | "cidr_ip": "...", 94 | "ip_protocol": "..." 95 | }, 96 | ], 97 | "updated_at": "..." 98 | } -------------------------------------------------------------------------------- /cloudaux/openstack/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/cloudaux/39d1dc11ccd794f21d95d3d1458abd94425b0927/cloudaux/openstack/__init__.py -------------------------------------------------------------------------------- /cloudaux/openstack/decorators.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.openstack.decorators 3 | :platform: Unix 4 | :copyright: Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. See AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Michael Stair 7 | """ 8 | import os 9 | from functools import wraps 10 | 11 | from openstack import connect 12 | from openstack.config.loader import OpenStackConfig 13 | from openstack.exceptions import HttpException 14 | 15 | """ this is mix of the aws and gcp decorator conventions """ 16 | 17 | CACHE = {} 18 | 19 | def _connect(cloud_name, region, yaml_file): 20 | os.environ["OS_CLIENT_CONFIG_FILE"] = yaml_file 21 | return connect(cloud=cloud_name, region_name=region) 22 | 23 | 24 | def get_regions(cloud_name, yaml_file): 25 | config = OpenStackConfig(config_files=[yaml_file]) 26 | return config._get_regions(cloud_name) 27 | 28 | 29 | def keystone_cached_conn(cloud_name, region, yaml_file): 30 | key = ( 31 | cloud_name, 32 | region ) 33 | 34 | if key in CACHE: 35 | """ check the token to see if our connection is still valid """ 36 | _cloud_name, conn = CACHE[key] 37 | try: 38 | conn.authorize() 39 | except HttpException: 40 | del CACHE[key] 41 | else: 42 | return conn 43 | try: 44 | conn = _connect(cloud_name, region, yaml_file) 45 | except Exception as e: 46 | raise e 47 | 48 | CACHE[key] = (conn.name, conn) 49 | return conn 50 | 51 | def openstack_conn(): 52 | def decorator(f): 53 | @wraps(f) 54 | def decorated_function(*args, **kwargs): 55 | kwargs['conn'] = keystone_cached_conn( 56 | kwargs.pop('cloud_name'), kwargs.pop('region'), kwargs.pop('yaml_file') ) 57 | return f(*args, **kwargs) 58 | 59 | return decorated_function 60 | 61 | return decorator 62 | 63 | 64 | def iter_account_region(account_regions): 65 | def decorator(func): 66 | @wraps(func) 67 | def decorated_function(*args, **kwargs): 68 | threads = [] 69 | for account_creds, regions in account_regions.iteritems(): 70 | account_name, cloud_name, yaml_file = account_creds 71 | for region in regions: 72 | kwargs['account_name'] = account_name 73 | kwargs['cloud_name'] = cloud_name 74 | kwargs['yaml_file'] = yaml_file 75 | kwargs['region'] = region 76 | result = func(*args, **kwargs) 77 | if result: 78 | threads.append(result) 79 | result = [] 80 | for thread in threads: 81 | result.append(thread) 82 | return result 83 | return decorated_function 84 | return decorator 85 | -------------------------------------------------------------------------------- /cloudaux/openstack/object_container.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.openstack.object_container 3 | :platform: Unix 4 | :copyright: Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. See AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Michael Stair 7 | """ 8 | from cloudaux.openstack.decorators import openstack_conn 9 | 10 | @openstack_conn() 11 | def get_container_metadata(conn=None, **kwargs): 12 | return conn.object_store.get_container_metadata(**kwargs) 13 | -------------------------------------------------------------------------------- /cloudaux/openstack/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.openstack.utils 3 | :platform: Unix 4 | :copyright: Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. See AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Michael Stair 7 | """ 8 | from cloudaux.openstack.decorators import openstack_conn 9 | 10 | @openstack_conn() 11 | def list_items(conn=None, **kwargs): 12 | """ 13 | :rtype: ``list`` 14 | """ 15 | return [x for x in getattr( getattr( conn, kwargs.pop('service') ), 16 | kwargs.pop('generator'))(**kwargs)] 17 | -------------------------------------------------------------------------------- /cloudaux/orchestration/__init__.py: -------------------------------------------------------------------------------- 1 | from inflection import camelize, underscore 2 | 3 | 4 | def _modify(item, func): 5 | """ 6 | Modifies each item.keys() string based on the func passed in. 7 | Often used with inflection's camelize or underscore methods. 8 | 9 | :param item: dictionary representing item to be modified 10 | :param func: function to run on each key string 11 | :return: dictionary where each key has been modified by func. 12 | """ 13 | result = dict() 14 | for key in item: 15 | result[func(key)] = item[key] 16 | return result 17 | 18 | 19 | def modify(item, output='camelized'): 20 | """ 21 | Calls _modify and either passes the inflection.camelize method or the inflection.underscore method. 22 | 23 | :param item: dictionary representing item to be modified 24 | :param output: string 'camelized' or 'underscored' 25 | :return: 26 | """ 27 | if output == 'camelized': 28 | return _modify(item, camelize) 29 | elif output == 'underscored': 30 | return _modify(item, underscore) -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/__init__.py: -------------------------------------------------------------------------------- 1 | from cloudaux.orchestration.aws.arn import ARN 2 | from cloudaux.exceptions import CloudAuxException 3 | from cloudaux.orchestration.aws.iam import MissingFieldException 4 | 5 | def _conn_from_args(item, conn): 6 | if item.get('Arn'): 7 | conn.update(_conn_from_arn(item.get('Arn'))) 8 | elif item.get('AccountNumber'): 9 | conn.update({'account_number': item['AccountNumber']}) 10 | del item['AccountNumber'] 11 | 12 | 13 | def _conn_from_arn(arn): 14 | """ 15 | Extracts the account number from an ARN. 16 | :param arn: Amazon ARN containing account number. 17 | :return: dictionary with a single account_number key that can be merged with an existing 18 | connection dictionary containing fields such as assume_role, session_name, region. 19 | """ 20 | arn = ARN(arn) 21 | if arn.error: 22 | raise CloudAuxException('Bad ARN: {arn}'.format(arn=arn)) 23 | return dict( 24 | account_number=arn.account_number, 25 | ) 26 | 27 | 28 | def _get_name_from_structure(item, default): 29 | """ 30 | Given a possibly sparsely populated item dictionary, try to retrieve the item name. 31 | First try the default field. If that doesn't exist, try to parse the from the ARN. 32 | :param item: dict containing (at the very least) item_name and/or arn 33 | :return: item name 34 | """ 35 | if item.get(default): 36 | return item.get(default) 37 | 38 | if item.get('Arn'): 39 | arn = item.get('Arn') 40 | item_arn = ARN(arn) 41 | if item_arn.error: 42 | raise CloudAuxException('Bad ARN: {arn}'.format(arn=arn)) 43 | return item_arn.parsed_name 44 | 45 | raise MissingFieldException('Cannot extract item name from input: {input}.'.format(input=item)) 46 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/arn.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.orchestration.aws.arn 3 | :platform: Unix 4 | .. version:: $$VERSION$$ 5 | .. moduleauthor:: Patrick Kelley 6 | """ 7 | 8 | import re 9 | 10 | 11 | class ARN(object): 12 | tech = None 13 | region = None 14 | account_number = None 15 | name = None 16 | parsed_name = None 17 | partition = None 18 | error = False 19 | root = False 20 | service = False 21 | 22 | def __init__(self, input): 23 | arn_match = re.search('^arn:([^:]*):([^:]*):([^:]*):(|\*|[\d]{12}):(.+)$', input) 24 | if arn_match: 25 | if arn_match.group(2) == "iam" and arn_match.group(5) == "root": 26 | self.root = True 27 | 28 | return self._from_arn(arn_match, input) 29 | 30 | acct_number_match = re.search('^(\d{12})+$', input) 31 | if acct_number_match: 32 | return self._from_account_number(input) 33 | 34 | aws_service_match = re.search('^([^.]*)\.amazonaws\.com$', input) 35 | if aws_service_match: 36 | return self._from_aws_service(input, aws_service_match.group(1)) 37 | 38 | self.error = True 39 | 40 | def _from_arn(self, arn_match, input): 41 | self.partition = arn_match.group(1) 42 | self.tech = arn_match.group(2) 43 | self.region = arn_match.group(3) 44 | self.account_number = arn_match.group(4) 45 | self.name = arn_match.group(5) 46 | 47 | resource_list = arn_match.group(5).split('/') 48 | 49 | # Handle the longer service level primitives like service roles 50 | # arn:aws:iam::123456789123:role/aws-service-role/elasticache.amazonaws.com/AWSServiceRoleForElastiCache 51 | if len(resource_list) == 2: 52 | self.resource_type = resource_list[0] 53 | self.resource = resource_list[-1] 54 | else: 55 | self.resource_type = resource_list[0] 56 | self.resource = '/'.join(resource_list[1:]) 57 | 58 | self.parsed_name = self.name.split('/')[-1] 59 | 60 | def _from_account_number(self, input): 61 | self.account_number = input 62 | 63 | def _from_aws_service(self, input, service): 64 | self.tech = service 65 | self.service = True 66 | 67 | @staticmethod 68 | def extract_arns_from_statement_condition(condition): 69 | condition_subsection \ 70 | = condition.get('ArnEquals', {}) or \ 71 | condition.get('ForAllValues:ArnEquals', {}) or \ 72 | condition.get('ForAnyValue:ArnEquals', {}) or \ 73 | condition.get('ArnLike', {}) or \ 74 | condition.get('ForAllValues:ArnLike', {}) or \ 75 | condition.get('ForAnyValue:ArnLike', {}) or \ 76 | condition.get('StringLike', {}) or \ 77 | condition.get('ForAllValues:StringLike', {}) or \ 78 | condition.get('ForAnyValue:StringLike', {}) or \ 79 | condition.get('StringEquals', {}) or \ 80 | condition.get('ForAllValues:StringEquals', {}) or \ 81 | condition.get('ForAnyValue:StringEquals', {}) 82 | 83 | # aws:sourcearn can be found with in lowercase or camelcase or other cases... 84 | condition_arns = [] 85 | for key, value in condition_subsection.iteritems(): 86 | if key.lower() == 'aws:sourcearn' or key.lower() == 'aws:sourceowner': 87 | if isinstance(value, list): 88 | condition_arns.extend(value) 89 | else: 90 | condition_arns.append(value) 91 | 92 | if not isinstance(condition_arns, list): 93 | return [condition_arns] 94 | return condition_arns 95 | 96 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/elbv2.py: -------------------------------------------------------------------------------- 1 | from cloudaux.aws.elbv2 import * 2 | from cloudaux.decorators import modify_output 3 | from flagpole import FlagRegistry, Flags 4 | 5 | registry = FlagRegistry() 6 | FLAGS = Flags('BASE', 'LISTENERS', 'RULES', 'ATTRIBUTES', 'TAGS', 7 | 'TARGET_GROUPS', 'TARGET_GROUP_ATTRIBUTES', 'TARGET_GROUP_HEALTH') 8 | 9 | 10 | @registry.register(flag=FLAGS.LISTENERS, depends_on=FLAGS.BASE, key='listeners') 11 | def get_listeners(alb, **conn): 12 | return describe_listeners(load_balancer_arn=alb['LoadBalancerArn'], **conn) 13 | 14 | 15 | @registry.register(flag=FLAGS.RULES, depends_on=FLAGS.LISTENERS, key='rules') 16 | def get_rules(alb, **conn): 17 | rules = list() 18 | for listener in alb['listeners']: 19 | rules.extend(describe_rules(listener_arn=listener['ListenerArn'], **conn)) 20 | return rules 21 | 22 | 23 | @registry.register(flag=FLAGS.ATTRIBUTES, depends_on=FLAGS.BASE, key='attributes') 24 | def get_attributes(alb, **conn): 25 | return describe_load_balancer_attributes(alb['LoadBalancerArn'], **conn) 26 | 27 | 28 | @registry.register(flag=FLAGS.TAGS, depends_on=FLAGS.BASE, key='tags') 29 | def get_tags(alb, **conn): 30 | return describe_tags([alb['LoadBalancerArn']], **conn) 31 | 32 | 33 | @registry.register(flag=FLAGS.TARGET_GROUPS, depends_on=FLAGS.BASE, key='target_groups') 34 | def get_target_groups(alb, **conn): 35 | return describe_target_groups(load_balancer_arn=alb['LoadBalancerArn'], **conn) 36 | 37 | 38 | @registry.register(flag=FLAGS.TARGET_GROUP_ATTRIBUTES, depends_on=FLAGS.TARGET_GROUPS, key='target_group_attributes') 39 | def _get_target_group_attributes(alb, **conn): 40 | target_group_attributes = list() 41 | for target_group in alb['target_groups']: 42 | target_group_attributes.extend( 43 | describe_target_group_attributes(target_group['TargetGroupArn'], **conn)) 44 | return target_group_attributes 45 | 46 | 47 | @registry.register(flag=FLAGS.TARGET_GROUP_HEALTH, depends_on=FLAGS.TARGET_GROUPS, key='target_group_health') 48 | def _get_target_group_health(alb, **conn): 49 | target_group_health = list() 50 | for target_group in alb['target_groups']: 51 | target_group_health.extend(describe_target_health(target_group['TargetGroupArn'], **conn)) 52 | return target_group_health 53 | 54 | 55 | @registry.register(flag=FLAGS.BASE) 56 | def get_base(alb, **conn): 57 | base_fields = frozenset( 58 | ['LoadBalancerArn', 'State', 'DNSName', 'CreatedTime', 'Scheme', 'Type', 'IpAddressType', 'VpcId', 59 | 'CanonicalHostedZoneId', 'SecurityGroups', 'LoadBalancerName', 'AvailabilityZones']) 60 | needs_base = False 61 | 62 | for field in base_fields: 63 | if field not in alb: 64 | needs_base = True 65 | break 66 | 67 | if needs_base: 68 | if 'LoadBalancerName' in alb: 69 | alb = describe_load_balancers(names=[alb['LoadBalancerName']], **conn) 70 | elif 'LoadBalancerArn' in alb: 71 | alb = describe_load_balancers(arns=[alb['LoadBalancerArn']], **conn) 72 | alb = alb[0] 73 | 74 | # Python 2 and 3 support: 75 | try: 76 | basestring 77 | except NameError as _: 78 | basestring = str 79 | 80 | if not isinstance(alb['CreatedTime'], basestring): 81 | alb['CreatedTime'] = str(alb['CreatedTime']) 82 | 83 | # Copy LoadBalancerArn to just Arn 84 | alb['Arn'] = alb.get('LoadBalancerArn') 85 | alb.update({ 86 | '_version': 2, 87 | 'region': conn.get('region')}) 88 | return alb 89 | 90 | 91 | @modify_output 92 | def get_elbv2(alb, flags=FLAGS.ALL, **conn): 93 | """ 94 | Fully describes an ALB (ELBv2). 95 | 96 | :param alb: Could be an ALB Name, ALB ARN, or a dictionary. Likely the return value from a previous call to describe_load_balancers. At a minimum, must contain a key titled 'LoadBalancerArn'. 97 | :param flags: Flags describing which sections should be included in the return value. Default is FLAGS.ALL. 98 | :return: Returns a dictionary describing the ALB with the fields described in the flags parameter. 99 | """ 100 | # Python 2 and 3 support: 101 | try: 102 | basestring 103 | except NameError as _: 104 | basestring = str 105 | 106 | if isinstance(alb, basestring): 107 | from cloudaux.orchestration.aws.arn import ARN 108 | alb_arn = ARN(alb) 109 | if alb_arn.error: 110 | alb = dict(LoadBalancerName=alb) 111 | else: 112 | alb = dict(LoadBalancerArn=alb) 113 | 114 | return registry.build_out(flags, start_with=alb, pass_datastructure=True, **conn) 115 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/events.md: -------------------------------------------------------------------------------- 1 | # CloudAux CloudWatch Events 2 | 3 | CloudAux can build out a JSON object describing an AWS CloudWatch Event configuration. 4 | 5 | ## Example 6 | 7 | from cloudaux.orchestration.aws.events import get_event 8 | 9 | conn = dict( 10 | account_number='111111111111', 11 | assume_role='SecurityMonkey') 12 | 13 | event = get_event('SomeCloudWatchEventRule', **conn) 14 | 15 | print(json.dumps(event, indent=2, sort_keys=True)) 16 | 17 | { 18 | "Region": "us-east-1", 19 | "_version": 1, 20 | "State": "ENABLED", 21 | "Description": "ScheduledRule", 22 | "Targets": [ 23 | { 24 | "Arn": "arn:aws:lambda:us-east-1:111111111111:function:MyLambdaFunction", 25 | "Id": "SomeFunction" 26 | } 27 | ], 28 | "Name": "SomeCloudWatchEventRule", 29 | "Arn": "arn:aws:events:us-east-1:111111111111:rule/SomeCloudWatchEventRule", 30 | "Rule": "rate(4 hours)" 31 | } 32 | 33 | 34 | ## Flags 35 | 36 | The `get_event` command accepts flags describing what parts of the structure to build out. 37 | 38 | from cloudaux.orchestration.aws.events import FLAGS 39 | 40 | some_fields = FLAGS.DESCRIBE | FLAGS.TARGETS 41 | vault = get_event('SomeCloudWatchEventRule', flags=desired_flags, **conn) 42 | 43 | If not provided, `get_event` assumes `FLAGS.ALL`, which recommended for most use cases. 44 | 45 | - `BASE` - ARN, Name, Region, and Cloudaux implementation number 46 | - `DESCRIPTION` - The event Description, State, and the rule Schedule Expression --or-- the Event Pattern. 47 | - `TARGETS` - The CloudWatch Event Targets 48 | - `ALL` - all items 49 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/events.py: -------------------------------------------------------------------------------- 1 | from cloudaux.aws.events import describe_rule 2 | from cloudaux.aws.events import list_targets_by_rule 3 | from cloudaux.decorators import modify_output 4 | from flagpole import FlagRegistry, Flags 5 | 6 | from cloudaux.orchestration.aws import ARN 7 | 8 | registry = FlagRegistry() 9 | FLAGS = Flags('BASE', 'DESCRIBE', 'TARGETS') 10 | 11 | 12 | @registry.register(flag=FLAGS.TARGETS, key='targets') 13 | def list_targets(rule, **conn): 14 | return list_targets_by_rule(Rule=rule['Name'], **conn) 15 | 16 | 17 | @registry.register(flag=FLAGS.DESCRIBE) 18 | def get_rule_description(rule, **conn): 19 | if rule.get('ScheduleExpression', None): 20 | rule_detail = rule['ScheduleExpression'] 21 | else: 22 | rule_detail = rule['EventPattern'] 23 | 24 | return { 25 | 'description': rule['Description'], 26 | 'state': rule['State'], 27 | 'rule': rule_detail 28 | } 29 | 30 | 31 | @registry.register(flag=FLAGS.BASE) 32 | def get_base(rule, **conn): 33 | return { 34 | 'arn': rule["Arn"], 35 | 'name': rule['Name'], 36 | 'region': conn.get('region'), 37 | '_version': 1 38 | } 39 | 40 | 41 | @modify_output 42 | def get_event(rule, flags=FLAGS.ALL, **conn): 43 | """ 44 | Orchestrates all the calls required to fully build out a CloudWatch Event Rule in the following format: 45 | 46 | { 47 | "Arn": ..., 48 | "Name": ..., 49 | "Region": ..., 50 | "Description": ..., 51 | "State": ..., 52 | "Rule": ..., 53 | "Targets" ..., 54 | "_version": 1 55 | } 56 | 57 | :param rule: str cloudwatch event name 58 | :param flags: By default, set to ALL fields 59 | :param conn: dict containing enough information to make a connection to the desired account. 60 | Must at least have 'assume_role' key. 61 | :return: dict containing a fully built out event rule with targets. 62 | """ 63 | # Python 2 and 3 support: 64 | try: 65 | basestring 66 | except NameError as _: 67 | basestring = str 68 | 69 | # If string is passed in, determine if it's a name or ARN. Build a dict. 70 | if isinstance(rule, basestring): 71 | rule_arn = ARN(rule) 72 | if rule_arn.error: 73 | rule_name = rule 74 | else: 75 | rule_name = rule_arn.name 76 | 77 | rule = describe_rule(Name=rule_name, **conn) 78 | 79 | return registry.build_out(flags, rule, **conn) 80 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/glacier.md: -------------------------------------------------------------------------------- 1 | # CloudAux AWS Lambda Function 2 | 3 | CloudAux can build out a JSON object describing an AWS Glacier Vault. 4 | 5 | ## Example 6 | 7 | from cloudaux.orchestration.aws.glacier import get_vault 8 | 9 | conn = dict( 10 | account_number='111111111111', 11 | assume_role='SecurityMonkey') 12 | 13 | vault = get_vault('MyVault', **conn) 14 | 15 | print(json.dumps(provider, indent=2, sort_keys=True)) 16 | 17 | { 18 | "ARN": "arn:aws:glacier:us-east-1:111111111111:vaults/MyVault", 19 | "CreationDate": "2014-01-27T18:31:49.143Z", 20 | "LastInventoryDate": "", 21 | "NumberOfArchives": 0, 22 | "Policy": "", 23 | "SizeInBytes": 0, 24 | "Tags": {}, 25 | "VaultARN": "arn:aws:glacier:us-east-1:111111111111:vaults/MyVault", 26 | "VaultName": "MyVault" 27 | } 28 | 29 | ## Flags 30 | 31 | The `get_vault` command accepts flags describing what parts of the structure to build out. 32 | 33 | from cloudaux.orchestration.aws.glacier import FLAGS 34 | 35 | desired_fields = FLAGS.DESCRIBE | FLAGS.TAGS 36 | vault = get_vault('MyVaultName', flags=desired_flags, **conn) 37 | 38 | If not provided, `get_vault` assumes `FLAGS.ALL`. 39 | 40 | - BASE - VaultARN, VaultName, CreationDate, SizeInBytes, NumberOfArchives, and Cloudaux implementation number 41 | - TAGS - any tags applied to the vault 42 | - POLICY - a Vault access policy (if defined) 43 | - ALL - all items 44 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/glacier.py: -------------------------------------------------------------------------------- 1 | from cloudaux.aws.glacier import describe_vault, get_vault_access_policy, list_tags_for_vault 2 | from cloudaux.decorators import modify_output 3 | from cloudaux.orchestration.aws.arn import ARN 4 | from flagpole import FlagRegistry, Flags 5 | from six import string_types 6 | 7 | registry = FlagRegistry() 8 | FLAGS = Flags('BASE', 'POLICY', 'TAGS') 9 | 10 | 11 | @registry.register(flag=FLAGS.BASE) 12 | def _get_base(vault_obj, **conn): 13 | base_fields = ['VaultARN', 'VaultName', 'CreationDate', 'NumberOfArchives', 'SizeInBytes'] 14 | 15 | if not all(field in vault_obj for field in base_fields): 16 | vault_obj = describe_vault(vault_name=vault_obj['VaultName'], **conn) 17 | 18 | vault_obj['_version'] = 1 19 | # sometimes it's expected that the item contains 'Arn' and not 'VaultARN' 20 | vault_obj['Arn'] = vault_obj['VaultARN'] 21 | return vault_obj 22 | 23 | 24 | @registry.register(flag=FLAGS.POLICY, key='Policy') 25 | def _get_vault_access_policy(vault_obj, **conn): 26 | return get_vault_access_policy(vault_name=vault_obj['VaultName'], **conn) 27 | 28 | 29 | @registry.register(flag=FLAGS.TAGS, key='Tags') 30 | def _list_tags_for_vault(vault_obj, **conn): 31 | return list_tags_for_vault(vault_name=vault_obj['VaultName'], **conn) 32 | 33 | 34 | @modify_output 35 | def get_vault(vault_obj, flags=FLAGS.ALL, **conn): 36 | """ 37 | Orchestrates calls to build a Glacier Vault in the following format: 38 | 39 | { 40 | "VaultARN": ..., 41 | "VaultName": ..., 42 | "CreationDate" ..., 43 | "LastInventoryDate" ..., 44 | "NumberOfArchives" ..., 45 | "SizeInBytes" ..., 46 | "Policy" ..., 47 | "Tags" ... 48 | } 49 | Args: 50 | vault_obj: name, ARN, or dict of Glacier Vault 51 | flags: Flags describing which sections should be included in the return value. Default ALL 52 | 53 | Returns: 54 | dictionary describing the requested Vault 55 | """ 56 | if isinstance(vault_obj, string_types): 57 | vault_arn = ARN(vault_obj) 58 | if vault_arn.error: 59 | vault_obj = {'VaultName': vault_obj} 60 | else: 61 | vault_obj = {'VaultName': vault_arn.parsed_name} 62 | 63 | return registry.build_out(flags, vault_obj, **conn) 64 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/iam/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | class MissingFieldException(Exception): 3 | pass 4 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/iam/group.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.orchestration.aws.iam.group 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Mike Grima 7 | """ 8 | from cloudaux import get_iso_string 9 | from cloudaux.aws.iam import get_group as get_group_api, list_group_policies, get_group_policy_document, \ 10 | list_attached_group_managed_policies 11 | from cloudaux.decorators import modify_output 12 | from flagpole import FlagRegistry, Flags 13 | 14 | from cloudaux.orchestration import modify 15 | from cloudaux.orchestration.aws import _conn_from_args 16 | from cloudaux.orchestration.aws.iam import MissingFieldException 17 | 18 | registry = FlagRegistry() 19 | FLAGS = Flags('BASE', 'INLINE_POLICIES', 'MANAGED_POLICIES', 'USERS') 20 | 21 | 22 | @registry.register(flag=FLAGS.INLINE_POLICIES, key='inline_policies') 23 | def get_inline_policies(group, **conn): 24 | """Get the inline policies for the group.""" 25 | policy_list = list_group_policies(group['GroupName'], **conn) 26 | 27 | policy_documents = {} 28 | 29 | for policy in policy_list: 30 | policy_documents[policy] = get_group_policy_document(group['GroupName'], policy, **conn) 31 | 32 | return policy_documents 33 | 34 | 35 | @registry.register(flag=FLAGS.MANAGED_POLICIES, key='managed_policies') 36 | def get_managed_policies(group, **conn): 37 | """Get a list of the managed policy names that are attached to the group.""" 38 | managed_policies = list_attached_group_managed_policies(group['GroupName'], **conn) 39 | 40 | managed_policy_names = [] 41 | 42 | for policy in managed_policies: 43 | managed_policy_names.append(policy['PolicyName']) 44 | 45 | return managed_policy_names 46 | 47 | 48 | @registry.register(flag=FLAGS.USERS, key='users') 49 | def get_users(group, **conn): 50 | """Gets a list of the usernames that are a part of this group.""" 51 | group_details = get_group_api(group['GroupName'], **conn) 52 | 53 | user_list = [] 54 | for user in group_details.get('Users', []): 55 | user_list.append(user['UserName']) 56 | 57 | return user_list 58 | 59 | 60 | @registry.register(flag=FLAGS.BASE) 61 | def _get_base(group, **conn): 62 | """Fetch the base IAM Group.""" 63 | group['_version'] = 1 64 | 65 | # Get the initial group details (only needed if we didn't grab the users): 66 | group.update(get_group_api(group['GroupName'], users=False, **conn)['Group']) 67 | 68 | # Cast CreateDate from a datetime to something JSON serializable. 69 | group['CreateDate'] = get_iso_string(group['CreateDate']) 70 | return group 71 | 72 | 73 | @modify_output 74 | def get_group(group, flags=FLAGS.BASE | FLAGS.INLINE_POLICIES | FLAGS.MANAGED_POLICIES, **conn): 75 | """ 76 | Orchestrates all the calls required to fully build out an IAM Group in the following format: 77 | 78 | { 79 | "Arn": ..., 80 | "GroupName": ..., 81 | "Path": ..., 82 | "GroupId": ..., 83 | "CreateDate": ..., # str 84 | "InlinePolicies": ..., 85 | "ManagedPolicies": ..., # These are just the names of the Managed Policies. 86 | "Users": ..., # False by default -- these are just the names of the users. 87 | "_version": 1 88 | } 89 | 90 | :param flags: By default, Users is disabled. This is somewhat expensive as it has to call the `get_group` call 91 | multiple times. 92 | :param group: dict MUST contain the GroupName and also a combination of either the ARN or the account_number. 93 | :param output: Determines whether keys should be returned camelized or underscored. 94 | :param conn: dict containing enough information to make a connection to the desired account. 95 | Must at least have 'assume_role' key. 96 | :return: dict containing fully built out Group. 97 | """ 98 | if not group.get('GroupName'): 99 | raise MissingFieldException('Must include GroupName.') 100 | 101 | group = modify(group, output='camelized') 102 | _conn_from_args(group, conn) 103 | return registry.build_out(flags, start_with=group, pass_datastructure=True, **conn) 104 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/iam/managed_policy.md: -------------------------------------------------------------------------------- 1 | # CloudAux AWS IAM Managed Policies 2 | 3 | CloudAux can build out a JSON object describing an IAM Managed Policy. 4 | 5 | ## Example 6 | 7 | from cloudaux.orchestration.aws.iam.managed_policy import get_managed_policy, FLAGS 8 | 9 | conn = dict( 10 | account_number='123456789012', 11 | assume_role='SecurityMonkey') 12 | 13 | managed_policy = get_managed_policy( 14 | {'Arn': 'arn:aws:iam::123456789012:policy/testCloudAuxPolicy'}, flags=FLAGS.ALL, **conn) 15 | 16 | # The flags parameter is optional. It presently doesn't have any other specified items at this time. 17 | 18 | print(json.dumps(managed_policy, indent=2, sort_keys=True)) 19 | 20 | { 21 | "Arn": "arn:aws:iam::123456789012:policy/testCloudAuxPolicy", 22 | "AttachmentCount": 0, 23 | "CreateDate": "2018-10-16T00:24:27Z", 24 | "DefaultVersionId": "v1", 25 | "Description": "Test CloudAux Policy", 26 | "Document": { 27 | "Statement": [ 28 | { 29 | "Action": "s3:ListBucket", 30 | "Effect": "Allow", 31 | "Resource": "*" 32 | } 33 | ], 34 | "Version": "2012-10-17" 35 | }, 36 | "Path": "/", 37 | "PolicyId": "AVJIT34BWVH14CIACU1CM", 38 | "PolicyName": "testCloudAuxPolicy", 39 | "UpdateDate": "2018-10-16T00:24:27Z", 40 | "_version": 1 41 | } 42 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/iam/managed_policy.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.orchestration.aws.iam.managed_policy 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Mike Grima 7 | """ 8 | from cloudaux import get_iso_string 9 | from cloudaux.aws.iam import get_policy, get_managed_policy_document 10 | from cloudaux.decorators import modify_output 11 | from flagpole import FlagRegistry, Flags 12 | 13 | from cloudaux.orchestration.aws import _conn_from_args 14 | from cloudaux.orchestration.aws import _get_name_from_structure 15 | from cloudaux.orchestration.aws.iam import MissingFieldException 16 | 17 | registry = FlagRegistry() 18 | FLAGS = Flags('BASE') 19 | 20 | 21 | @registry.register(flag=FLAGS.BASE) 22 | def get_base(managed_policy, **conn): 23 | """Fetch the base Managed Policy. 24 | 25 | This includes the base policy and the latest version document. 26 | 27 | :param managed_policy: 28 | :param conn: 29 | :return: 30 | """ 31 | managed_policy['_version'] = 1 32 | 33 | arn = _get_name_from_structure(managed_policy, 'Arn') 34 | policy = get_policy(arn, **conn) 35 | document = get_managed_policy_document(arn, policy_metadata=policy, **conn) 36 | 37 | managed_policy.update(policy['Policy']) 38 | managed_policy['Document'] = document 39 | 40 | # Fix the dates: 41 | managed_policy['CreateDate'] = get_iso_string(managed_policy['CreateDate']) 42 | managed_policy['UpdateDate'] = get_iso_string(managed_policy['UpdateDate']) 43 | 44 | return managed_policy 45 | 46 | 47 | @modify_output 48 | def get_managed_policy(managed_policy, flags=FLAGS.ALL, **conn): 49 | """ 50 | Orchestrates all of the calls required to fully build out an IAM Managed Policy in the following format: 51 | 52 | { 53 | "Arn": "...", 54 | "PolicyName": "...", 55 | "PolicyId": "...", 56 | "Path": "...", 57 | "DefaultVersionId": "...", 58 | "AttachmentCount": 123, 59 | "PermissionsBoundaryUsageCount": 123, 60 | "IsAttachable": ..., 61 | "Description": "...", 62 | "CreateDate": "...", 63 | "UpdateDate": "...", 64 | "Document": "...", 65 | "_version": 1 66 | } 67 | 68 | :param managed_policy: dict MUST contain the ARN. 69 | :param flags: 70 | :param conn: 71 | :return: 72 | """ 73 | if not managed_policy.get('Arn'): 74 | raise MissingFieldException('Must include Arn.') 75 | 76 | _conn_from_args(managed_policy, conn) 77 | return registry.build_out(flags, start_with=managed_policy, pass_datastructure=True, **conn) 78 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/iam/role.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.orchestration.aws.iam.role 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Patrick Kelley @monkeysecurity 7 | .. moduleauthor:: Will Bengtson 8 | """ 9 | from cloudaux import CloudAux, get_iso_string 10 | from cloudaux.aws.iam import get_role_managed_policies, get_role_inline_policies, get_role_instance_profiles, \ 11 | get_account_authorization_details, list_role_tags 12 | from cloudaux.orchestration.aws import _get_name_from_structure, _conn_from_args 13 | from cloudaux.orchestration import modify 14 | from cloudaux.decorators import modify_output 15 | from flagpole import FlagRegistry, Flags 16 | 17 | 18 | registry = FlagRegistry() 19 | FLAGS = Flags('BASE', 'MANAGED_POLICIES', 'INLINE_POLICIES', 'INSTANCE_PROFILES', 'TAGS') 20 | 21 | 22 | @registry.register(flag=FLAGS.TAGS, depends_on=FLAGS.BASE, key='tags') 23 | def get_tags(role, **conn): 24 | tags = list_role_tags(role, **conn) 25 | # AWS Returns a funky format for tags: 26 | # [{ 27 | # "Key": "owner", 28 | # "Value": "bandersnatch" 29 | # }] 30 | 31 | # reformat into a single dict. 32 | # { "owner": "bandersnatch" } 33 | return {t.get('Key'): t.get('Value') for t in tags} 34 | 35 | 36 | @registry.register(flag=FLAGS.MANAGED_POLICIES, depends_on=FLAGS.BASE, key='managed_policies') 37 | def get_managed_policies(role, **conn): 38 | return get_role_managed_policies(role, **conn) 39 | 40 | 41 | @registry.register(flag=FLAGS.INLINE_POLICIES, depends_on=FLAGS.BASE, key='inline_policies') 42 | def get_inline_policies(role, **conn): 43 | return get_role_inline_policies(role, **conn) 44 | 45 | 46 | @registry.register(flag=FLAGS.INSTANCE_PROFILES, depends_on=FLAGS.BASE, key='instance_profiles') 47 | def get_instance_profiles(role, **conn): 48 | return get_role_instance_profiles(role, **conn) 49 | 50 | 51 | @registry.register(flag=FLAGS.BASE) 52 | def _get_base(role, **conn): 53 | """ 54 | Determine whether the boto get_role call needs to be made or if we already have all that data 55 | in the role object. 56 | :param role: dict containing (at the very least) role_name and/or arn. 57 | :param conn: dict containing enough information to make a connection to the desired account. 58 | :return: Camelized dict describing role containing all all base_fields. 59 | """ 60 | base_fields = frozenset(['Arn', 'AssumeRolePolicyDocument', 'Path', 'RoleId', 'RoleName', 'CreateDate']) 61 | needs_base = False 62 | 63 | for field in base_fields: 64 | if field not in role: 65 | needs_base = True 66 | break 67 | 68 | if needs_base: 69 | role_name = _get_name_from_structure(role, 'RoleName') 70 | role = CloudAux.go('iam.client.get_role', RoleName=role_name, **conn) 71 | role = role['Role'] 72 | 73 | # cast CreateDate from a datetime to something JSON serializable. 74 | role.update(dict(CreateDate=get_iso_string(role['CreateDate']))) 75 | role['_version'] = 3 76 | 77 | return role 78 | 79 | 80 | @modify_output 81 | def get_role(role, flags=FLAGS.ALL, **conn): 82 | """ 83 | Orchestrates all the calls required to fully build out an IAM Role in the following format: 84 | 85 | { 86 | "Arn": ..., 87 | "AssumeRolePolicyDocument": ..., 88 | "CreateDate": ..., # str 89 | "InlinePolicies": ..., 90 | "InstanceProfiles": ..., 91 | "ManagedPolicies": ..., 92 | "Path": ..., 93 | "RoleId": ..., 94 | "RoleName": ..., 95 | "Tags": {}, 96 | "_version": 3 97 | } 98 | 99 | :param role: dict containing (at the very least) role_name and/or arn. 100 | :param output: Determines whether keys should be returned camelized or underscored. 101 | :param conn: dict containing enough information to make a connection to the desired account. 102 | Must at least have 'assume_role' key. 103 | :return: dict containing a fully built out role. 104 | """ 105 | role = modify(role, output='camelized') 106 | _conn_from_args(role, conn) 107 | return registry.build_out(flags, start_with=role, pass_datastructure=True, **conn) 108 | 109 | 110 | def get_all_roles(**conn): 111 | """ 112 | Returns a List of Roles represented as the dictionary below: 113 | 114 | { 115 | "Arn": ..., 116 | "AssumeRolePolicyDocument": ..., 117 | "CreateDate": ..., # str 118 | "InlinePolicies": ..., 119 | "InstanceProfiles": ..., 120 | "ManagedPolicies": ..., 121 | "Path": ..., 122 | "RoleId": ..., 123 | "RoleName": ..., 124 | } 125 | 126 | :param conn: dict containing enough information to make a connection to the desired account. 127 | :return: list containing dicts or fully built out roles 128 | """ 129 | 130 | roles = [] 131 | account_roles = get_account_authorization_details('Role', **conn) 132 | 133 | for role in account_roles: 134 | roles.append( 135 | { 136 | 'Arn': role['Arn'], 137 | 'AssumeRolePolicyDocument': role['AssumeRolePolicyDocument'], 138 | 'CreateDate': get_iso_string(role['CreateDate']), 139 | 'InlinePolicies': role['RolePolicyList'], 140 | 'InstanceProfiles': [{ 141 | 'path': ip['Path'], 142 | 'instance_profile_name': ip['InstanceProfileName'], 143 | 'create_date': get_iso_string(ip['CreateDate']), 144 | 'instance_profile_id': ip['InstanceProfileId'], 145 | 'arn': ip['Arn'] 146 | } for ip in role['InstanceProfileList']], 147 | 'ManagedPolicies': [ 148 | { 149 | "name": x['PolicyName'], 150 | "arn": x['PolicyArn'] 151 | } for x in role['AttachedManagedPolicies'] 152 | ], 153 | 'Path': role['Path'], 154 | 'RoleId': role['RoleId'], 155 | 'RoleName': role['RoleName'] 156 | } 157 | ) 158 | 159 | return roles 160 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/iam/saml_provider.py: -------------------------------------------------------------------------------- 1 | from cloudaux.aws.iam import get_saml_provider as boto_get_saml_provider 2 | from cloudaux.decorators import modify_output 3 | from flagpole import FlagRegistry, Flags 4 | import defusedxml.ElementTree as ET 5 | from six import string_types 6 | 7 | 8 | registry = FlagRegistry() 9 | FLAGS = Flags('BASE') 10 | 11 | 12 | @registry.register(flag=FLAGS.BASE) 13 | def get_base(provider, **conn): 14 | # Get the SAML Provider information 15 | saml_provider = boto_get_saml_provider(provider['Arn'], **conn) 16 | 17 | # Parse the SAML Metadata XML Document 18 | root = ET.fromstring(saml_provider['SAMLMetadataDocument']) 19 | 20 | saml_x509 = '' 21 | company = '' 22 | given_name = '' 23 | email_address = '' 24 | 25 | for parent in root.iter(): 26 | for child in parent: 27 | if 'X509Certificate' in child.tag: 28 | saml_x509 = child.text 29 | if 'Company' in child.tag: 30 | company = child.text 31 | if 'GivenName' in child.tag: 32 | given_name = child.text 33 | if 'EmailAddress' in child.tag: 34 | email_address = child.text 35 | 36 | return { 37 | 'Name': root.attrib['entityID'], 38 | 'CreateDate': str(saml_provider['CreateDate']), 39 | 'ValidUntil': str(saml_provider['ValidUntil']), 40 | 'X509': saml_x509, 41 | 'Company': company, 42 | 'GivenName': given_name, 43 | 'Email': email_address 44 | } 45 | 46 | 47 | @modify_output 48 | def get_saml_provider(provider, flags=FLAGS.ALL, **conn): 49 | 50 | # If provided an ARN, cast to a dict 51 | if isinstance(provider, string_types): 52 | provider = dict(Arn=provider) 53 | 54 | return registry.build_out(flags, start_with=provider, pass_datastructure=True, **conn) 55 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/iam/server_certificate.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.orchestration.aws.iam.server_certificate 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Mike Grima 7 | """ 8 | from cloudaux import get_iso_string 9 | from cloudaux.aws.iam import get_server_certificate as get_server_certificate_api 10 | from cloudaux.decorators import modify_output 11 | from flagpole import FlagRegistry, Flags 12 | 13 | from cloudaux.orchestration import modify 14 | from cloudaux.orchestration.aws import _conn_from_args 15 | from cloudaux.orchestration.aws.iam import MissingFieldException 16 | 17 | registry = FlagRegistry() 18 | FLAGS = Flags('BASE') 19 | 20 | 21 | @registry.register(flag=FLAGS.BASE) 22 | def _get_base(server_certificate, **conn): 23 | """Fetch the base IAM Server Certificate.""" 24 | server_certificate['_version'] = 1 25 | 26 | # Get the initial cert details: 27 | cert_details = get_server_certificate_api(server_certificate['ServerCertificateName'], **conn) 28 | 29 | if cert_details: 30 | server_certificate.update(cert_details['ServerCertificateMetadata']) 31 | server_certificate['CertificateBody'] = cert_details['CertificateBody'] 32 | server_certificate['CertificateChain'] = cert_details.get('CertificateChain', None) 33 | 34 | # Cast dates from a datetime to something JSON serializable. 35 | server_certificate['UploadDate'] = get_iso_string(server_certificate['UploadDate']) 36 | server_certificate['Expiration'] = get_iso_string(server_certificate['Expiration']) 37 | 38 | return server_certificate 39 | 40 | 41 | @modify_output 42 | def get_server_certificate(server_certificate, flags=FLAGS.BASE, **conn): 43 | """ 44 | Orchestrates all the calls required to fully build out an IAM User in the following format: 45 | 46 | { 47 | "Arn": ..., 48 | "ServerCertificateName": ..., 49 | "Path": ..., 50 | "ServerCertificateId": ..., 51 | "UploadDate": ..., # str 52 | "Expiration": ..., # str 53 | "CertificateBody": ..., 54 | "CertificateChain": ..., 55 | "_version": 1 56 | } 57 | 58 | :param flags: By default, Users is disabled. This is somewhat expensive as it has to call the 59 | `get_server_certificate` call multiple times. 60 | :param server_certificate: dict MUST contain the ServerCertificateName and also a combination of 61 | either the ARN or the account_number. 62 | :param output: Determines whether keys should be returned camelized or underscored. 63 | :param conn: dict containing enough information to make a connection to the desired account. 64 | Must at least have 'assume_role' key. 65 | :return: dict containing fully built out Server Certificate. 66 | """ 67 | if not server_certificate.get('ServerCertificateName'): 68 | raise MissingFieldException('Must include ServerCertificateName.') 69 | 70 | server_certificate = modify(server_certificate, output='camelized') 71 | _conn_from_args(server_certificate, conn) 72 | return registry.build_out(flags, start_with=server_certificate, pass_datastructure=True, **conn) 73 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/iam/user.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.orchestration.aws.iam.user 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Patrick Kelley @monkeysecurity 7 | .. moduleauthor:: Will Bengtson 8 | """ 9 | from cloudaux import CloudAux, get_iso_string 10 | from cloudaux.aws.iam import get_account_authorization_details 11 | from cloudaux.aws.iam import get_user_inline_policies 12 | from cloudaux.aws.iam import get_user_access_keys 13 | from cloudaux.aws.iam import get_user_login_profile 14 | from cloudaux.aws.iam import get_user_managed_policies 15 | from cloudaux.aws.iam import get_user_mfa_devices 16 | from cloudaux.aws.iam import get_user_signing_certificates 17 | from cloudaux.orchestration.aws import _get_name_from_structure, _conn_from_args 18 | from cloudaux.orchestration import modify 19 | from cloudaux.decorators import modify_output 20 | from flagpole import FlagRegistry, Flags 21 | 22 | 23 | registry = FlagRegistry() 24 | FLAGS = Flags('BASE', 'ACCESS_KEYS', 'INLINE_POLICIES', 'MANAGED_POLICIES', 'MFA_DEVICES', 'LOGIN_PROFILE', 25 | 'SIGNING_CERTIFICATES') 26 | 27 | 28 | @registry.register(flag=FLAGS.ACCESS_KEYS, depends_on=FLAGS.BASE, key='access_keys') 29 | def get_access_keys(user, **conn): 30 | return get_user_access_keys(user, **conn) 31 | 32 | 33 | @registry.register(flag=FLAGS.INLINE_POLICIES, depends_on=FLAGS.BASE, key='inline_policies') 34 | def get_inline_policies(user, **conn): 35 | return get_user_inline_policies(user, **conn) 36 | 37 | 38 | @registry.register(flag=FLAGS.MANAGED_POLICIES, depends_on=FLAGS.BASE, key='managed_policies') 39 | def get_managed_policies(user, **conn): 40 | return get_user_managed_policies(user, **conn) 41 | 42 | 43 | @registry.register(flag=FLAGS.MFA_DEVICES, depends_on=FLAGS.BASE, key='mfa_devices') 44 | def get_mfa_devices(user, **conn): 45 | return get_user_mfa_devices(user, **conn) 46 | 47 | 48 | @registry.register(flag=FLAGS.LOGIN_PROFILE, depends_on=FLAGS.BASE, key='login_profile') 49 | def get_login_profile(user, **conn): 50 | return get_user_login_profile(user, **conn) 51 | 52 | 53 | @registry.register(flag=FLAGS.SIGNING_CERTIFICATES, depends_on=FLAGS.BASE, key='signing_certificates') 54 | def get_signing_certificates(user, **conn): 55 | return get_user_signing_certificates(user, **conn) 56 | 57 | 58 | @registry.register(flag=FLAGS.BASE) 59 | def _get_base(user, **conn): 60 | base_fields = frozenset(['Arn', 'CreateDate', 'Path', 'UserId', 'UserName']) 61 | needs_base = False 62 | for field in base_fields: 63 | if field not in user: 64 | needs_base = True 65 | break 66 | 67 | if needs_base: 68 | user_name = _get_name_from_structure(user, 'UserName') 69 | user = CloudAux.go('iam.client.get_user', UserName=user_name, **conn) 70 | user = user['User'] 71 | 72 | # cast CreateDate from a datetime to something JSON serializable. 73 | user.update(dict(CreateDate=get_iso_string(user['CreateDate']))) 74 | if 'PasswordLastUsed' in user: 75 | user.update(dict(PasswordLastUsed=get_iso_string(user['PasswordLastUsed']))) 76 | 77 | user['_version'] = 2 78 | return user 79 | 80 | 81 | @modify_output 82 | def get_user(user, flags=FLAGS.ALL, **conn): 83 | """ 84 | Orchestrates all the calls required to fully build out an IAM User in the following format: 85 | 86 | { 87 | "Arn": ..., 88 | "AccessKeys": ..., 89 | "CreateDate": ..., # str 90 | "InlinePolicies": ..., 91 | "ManagedPolicies": ..., 92 | "MFADevices": ..., 93 | "Path": ..., 94 | "UserId": ..., 95 | "UserName": ..., 96 | "SigningCerts": ... 97 | } 98 | 99 | :param user: dict MUST contain the UserName and also a combination of either the ARN or the account_number 100 | :param output: Determines whether keys should be returned camelized or underscored. 101 | :param conn: dict containing enough information to make a connection to the desired account. 102 | Must at least have 'assume_role' key. 103 | :return: dict containing fully built out user. 104 | """ 105 | user = modify(user, output='camelized') 106 | _conn_from_args(user, conn) 107 | return registry.build_out(flags, start_with=user, pass_datastructure=True, **conn) 108 | 109 | 110 | def get_all_users(flags=FLAGS.ACCESS_KEYS | FLAGS.MFA_DEVICES | FLAGS.LOGIN_PROFILE | FLAGS.SIGNING_CERTIFICATES, 111 | **conn): 112 | """ 113 | Returns a list of Users represented as dictionary below: 114 | 115 | { 116 | "Arn": ..., 117 | "AccessKeys": ..., 118 | "CreateDate": ..., # str 119 | "InlinePolicies": ..., 120 | "ManagedPolicies": ..., 121 | "MFADevices": ..., 122 | "Path": ..., 123 | "UserId": ..., 124 | "UserName": ..., 125 | "SigningCerts": ... 126 | } 127 | 128 | :param flags: 129 | :param conn: dict containing enough information to make a connection to the desired account. 130 | :return: list of dicts containing fully built out user. 131 | """ 132 | 133 | users = [] 134 | account_users = get_account_authorization_details('User', **conn) 135 | 136 | for user in account_users: 137 | temp_user = { 138 | 'Arn': user['Arn'], 139 | 'CreateDate': get_iso_string(user['CreateDate']), 140 | 'GroupList': user['GroupList'], 141 | 'InlinePolicies': user['UserPolicyList'], 142 | 'ManagedPolicies': [ 143 | { 144 | "name": x['PolicyName'], 145 | "arn": x['PolicyArn'] 146 | } for x in user['AttachedManagedPolicies'] 147 | ], 148 | 'Path': user['Path'], 149 | 'UserId': user['UserId'], 150 | 'UserName': user['UserName'] 151 | } 152 | 153 | user = modify(temp_user, output='camelized') 154 | _conn_from_args(user, conn) 155 | users.append(registry.build_out(flags, start_with=user, pass_datastructure=True, **conn)) 156 | 157 | return users 158 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/image.md: -------------------------------------------------------------------------------- 1 | # CloudAux AWS Image 2 | 3 | CloudAux can build out a JSON object describing an EC2 Image. 4 | 5 | ## Example 6 | 7 | from cloudaux.orchestration.aws.image import get_image, FLAGS 8 | from cloudaux.aws.ec2 import describe_images 9 | 10 | conn = { 11 | 'account_number': '111111111111', 12 | 'assume_role': 'MyRole', 13 | 'session_name': 'MySession', 14 | 'region': 'us-east-1' 15 | } 16 | 17 | # List images in Region 18 | # NOTE: Filter by owner-id or this will take forever to return. 19 | images = describe_images(**conn) 20 | 21 | # List images in region owned by 111111111111 22 | images = describe_images(Owners=['111111111111']) 23 | 24 | # List public images in region owned by 111111111111 25 | describe_images(Filters=[{'Name': 'is-public', 'Values': ['true']}, {'Name': 'owner-id', 'Values': ['111111111111']}]) 26 | 27 | # By Image ID 28 | image = get_image('ami-11111111', flags=FLAGS.ALL, **conn) 29 | 30 | all_images = list() 31 | for image im all_images: 32 | all_images.append(get_image(image['ImageId'], **conn)) 33 | 34 | 35 | ## Flags 36 | 37 | The `get_image` command accepts flags describing what parts of the structure to build out. 38 | 39 | from cloudaux.orchestration.aws.image import FLAGS 40 | 41 | desired_flags = FLAGS.BASE | FLAGS.LAUNCHPERMISSION 42 | image = get_image('ami-11111', flags=desired_flags, **conn) 43 | 44 | If not provided, `get_image` assumes `FLAGS.ALL`. 45 | 46 | - [BASE](#flagsbase) 47 | - [KERNEL](#flagskernel) 48 | - [RAMDISK](#flagsramdisk) 49 | - [LAUNCHPERMISSION](#flagslaunchpermission) 50 | - [PRODUCTCODES](#flagsproductcodes) 51 | 52 | ### FLAGS.BASE 53 | 54 | Call boto3's [`client.describe_images`](http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Client.describe_images). 55 | 56 | { 57 | "Architecture": "x86_64", 58 | "Arn": "arn:aws:ec2:us-east-1::image/ami-11111111", 59 | "BlockDeviceMappings": [ 60 | { 61 | "DeviceName": "/dev/sda1", 62 | "Ebs": { 63 | "DeleteOnTermination": true, 64 | "Encrypted": false, 65 | "SnapshotId": "snap-11111111", 66 | "VolumeSize": 8, 67 | "VolumeType": "standard" 68 | } 69 | }, 70 | { 71 | "DeviceName": "/dev/sdb", 72 | "VirtualName": "ephemeral0" 73 | } 74 | ], 75 | "CreationDate": "2013-07-11T16:04:06.000Z", 76 | "Description": "...", 77 | "Hypervisor": "xen", 78 | "ImageId": "ami-11111111", 79 | "ImageLocation": "111111111111/...", 80 | "ImageType": "machine", 81 | "KernelId": "aki-11111111", 82 | "Name": "...", 83 | "OwnerId": "111111111111", 84 | "Public": false, 85 | "RootDeviceName": "/dev/sda1", 86 | "RootDeviceType": "ebs", 87 | "State": "available", 88 | "Tags": [], 89 | "VirtualizationType": "paravirtual", 90 | "_version": 1 91 | } 92 | 93 | 94 | ## FLAGS.KERNEL 95 | 96 | { 97 | "KernelId": { 98 | "Value": "aki-11111111" 99 | } 100 | } 101 | 102 | 103 | ## FLAGS.RAMDISK 104 | 105 | { 106 | "RamdiskId": {} 107 | } 108 | 109 | 110 | ## FLAGS.LAUNCHPERMISSION 111 | 112 | { 113 | "LaunchPermissions": [ 114 | { 115 | "UserId": "111111111111" 116 | }, 117 | { 118 | "UserId": "222222222222" 119 | } 120 | ] 121 | } 122 | 123 | 124 | ## FLAGS.PRODUCTCODES 125 | 126 | { 127 | "ProductCodes": [] 128 | } -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/image.py: -------------------------------------------------------------------------------- 1 | from cloudaux.aws.ec2 import describe_images 2 | from cloudaux.aws.ec2 import describe_image_attribute 3 | from cloudaux.decorators import modify_output 4 | from flagpole import FlagRegistry, Flags 5 | 6 | registry = FlagRegistry() 7 | FLAGS = Flags('BASE', 'KERNEL', 'RAMDISK', 'LAUNCHPERMISSION', 'PRODUCTCODES') 8 | 9 | 10 | @registry.register(flag=FLAGS.KERNEL) 11 | def get_kernel(image, **conn): 12 | attribute = describe_image_attribute( 13 | Attribute='kernel', ImageId=image['ImageId'], **conn) 14 | return dict(KernelId=attribute['KernelId']) 15 | 16 | 17 | @registry.register(flag=FLAGS.RAMDISK) 18 | def get_ramdisk(image, **conn): 19 | attribute = describe_image_attribute( 20 | Attribute='ramdisk', ImageId=image['ImageId'], **conn) 21 | return dict(RamdiskId=attribute['RamdiskId']) 22 | 23 | 24 | @registry.register(flag=FLAGS.LAUNCHPERMISSION) 25 | def get_launch_permission(image, **conn): 26 | attribute = describe_image_attribute( 27 | Attribute='launchPermission', ImageId=image['ImageId'], **conn) 28 | return dict(LaunchPermissions=attribute['LaunchPermissions']) 29 | 30 | 31 | @registry.register(flag=FLAGS.PRODUCTCODES) 32 | def get_product_codes(image, **conn): 33 | attribute = describe_image_attribute( 34 | Attribute='productCodes', ImageId=image['ImageId'], **conn) 35 | return dict(ProductCodes=attribute['ProductCodes']) 36 | 37 | 38 | @registry.register(flag=FLAGS.BASE) 39 | def get_base(image, **conn): 40 | image = describe_images(ImageIds=[image['ImageId']], **conn) 41 | image = image[0] 42 | arn = 'arn:aws:ec2:{region}::image/{imageid}'.format( 43 | region=conn['region'], 44 | imageid=image['ImageId']) 45 | image.update({'Arn': arn, 'Region': conn['region'], '_version': 1}) 46 | return image 47 | 48 | 49 | @modify_output 50 | def get_image(image_id, flags=FLAGS.ALL, **conn): 51 | """ 52 | Orchestrates all the calls required to fully build out an EC2 Image (AMI, AKI, ARI) 53 | 54 | { 55 | "Architecture": "x86_64", 56 | "Arn": "arn:aws:ec2:us-east-1::image/ami-11111111", 57 | "BlockDeviceMappings": [], 58 | "CreationDate": "2013-07-11T16:04:06.000Z", 59 | "Description": "...", 60 | "Hypervisor": "xen", 61 | "ImageId": "ami-11111111", 62 | "ImageLocation": "111111111111/...", 63 | "ImageType": "machine", 64 | "KernelId": "aki-88888888", 65 | "LaunchPermissions": [], 66 | "Name": "...", 67 | "OwnerId": "111111111111", 68 | "ProductCodes": [], 69 | "Public": false, 70 | "RamdiskId": {}, 71 | "RootDeviceName": "/dev/sda1", 72 | "RootDeviceType": "ebs", 73 | "SriovNetSupport": "simple", 74 | "State": "available", 75 | "Tags": [], 76 | "VirtualizationType": "hvm", 77 | "_version": 1 78 | } 79 | 80 | :param image_id: str ami id 81 | :param flags: By default, set to ALL fields 82 | :param conn: dict containing enough information to make a connection to the desired account. 83 | Must at least have 'assume_role' key. 84 | :return: dict containing a fully built out image. 85 | """ 86 | image = dict(ImageId=image_id) 87 | conn['region'] = conn.get('region', 'us-east-1') 88 | return registry.build_out(flags, image, **conn) 89 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/lambda_function.py: -------------------------------------------------------------------------------- 1 | from cloudaux.aws.lambda_function import * 2 | from cloudaux.decorators import modify_output 3 | from flagpole import FlagRegistry, Flags 4 | import json 5 | 6 | from cloudaux.orchestration.aws import ARN 7 | 8 | registry = FlagRegistry() 9 | FLAGS = Flags('BASE', 'ALIASES', 'EVENT_SOURCE_MAPPINGS', 'VERSIONS', 'TAGS', 'POLICY') 10 | 11 | 12 | @registry.register(flag=FLAGS.POLICY, depends_on=FLAGS.VERSIONS, key='policy') 13 | def _get_policy(lambda_function, **conn): 14 | """Get LambdaFunction Policies. (there can be many of these!) 15 | 16 | Lambda Function Policies are overly complicated. They can be attached to a label, 17 | a version, and there is also a default policy. 18 | 19 | This method attempts to gather all three types. 20 | 21 | AWS returns an exception if the policy requested does not exist. We catch and ignore these exceptions. 22 | """ 23 | policies = dict(Versions=dict(), Aliases=dict(), DEFAULT=dict()) 24 | 25 | for version in [v['Version'] for v in lambda_function['versions']]: 26 | try: 27 | policies['Versions'][version] = get_policy(FunctionName=lambda_function['FunctionName'], Qualifier=version, **conn) 28 | policies['Versions'][version] = json.loads(policies['Versions'][version]) 29 | except Exception as e: 30 | pass 31 | 32 | for alias in [v['Name'] for v in lambda_function['aliases']]: 33 | try: 34 | policies['Aliases'][alias] = get_policy(FunctionName=lambda_function['FunctionName'], Qualifier=alias, **conn) 35 | policies['Aliases'][alias] = json.loads(policies['Aliases'][alias]) 36 | except Exception as e: 37 | pass 38 | 39 | try: 40 | policies['DEFAULT'] = get_policy(FunctionName=lambda_function['FunctionName'], **conn) 41 | policies['DEFAULT'] = json.loads(policies['DEFAULT']) 42 | except Exception as e: 43 | pass 44 | 45 | return policies 46 | 47 | 48 | @registry.register(flag=FLAGS.ALIASES, key='aliases') 49 | def _get_aliases(lambda_function, **conn): 50 | return list_aliases(FunctionName=lambda_function['FunctionName'], **conn) 51 | 52 | 53 | @registry.register(flag=FLAGS.TAGS, depends_on=FLAGS.BASE, key='tags') 54 | def _get_tags(lambda_function, **conn): 55 | return list_tags(Resource=lambda_function['FunctionArn'], **conn) 56 | 57 | 58 | @registry.register(flag=FLAGS.VERSIONS, depends_on=FLAGS.ALIASES, key='versions') 59 | def _get_versions(lambda_function, **conn): 60 | return list_versions_by_function(FunctionName=lambda_function['FunctionName'], **conn) 61 | 62 | 63 | @registry.register(flag=FLAGS.EVENT_SOURCE_MAPPINGS, key='event_source_mappings') 64 | def _get_event_source_mappings(lambda_function, **conn): 65 | mappings = list_event_source_mappings(FunctionName=lambda_function['FunctionName'], **conn) 66 | for mapping in mappings: 67 | if 'LastModified' in mapping: 68 | mapping['LastModified'] = str(mapping['LastModified']) 69 | return mappings 70 | 71 | 72 | @registry.register(flag=FLAGS.BASE) 73 | def get_base(lambda_function, **conn): 74 | base_fields = frozenset( 75 | ['FunctionName', 'FunctionArn', 'Runtime', 'Role', 'Handler', 76 | 'CodeSize', 'Description', 'Timeout', 'MemorySize', 'LastModified', 77 | 'CodeSha256', 'Version', 'VpcConfig', 'DeadLetterConfig', 'Environment', 78 | 'KMSKeyArn', 'TracingConfig', 'MasterArn']) 79 | needs_base = False 80 | 81 | for field in base_fields: 82 | if field not in lambda_function: 83 | needs_base = True 84 | break 85 | 86 | if needs_base: 87 | lambda_function = get_function_configuration(FunctionName=lambda_function['FunctionName'], **conn) 88 | lambda_function.pop('ResponseMetadata', None) 89 | 90 | # Copy FunctionArn to just Arn 91 | lambda_function.update({ 92 | 'Arn': lambda_function.get('FunctionArn'), 93 | '_version': 1 94 | }) 95 | return lambda_function 96 | 97 | 98 | @modify_output 99 | def get_lambda_function(lambda_function, flags=FLAGS.ALL, **conn): 100 | """Fully describes a lambda function. 101 | 102 | Args: 103 | lambda_function: Name, ARN, or dictionary of lambda function. If dictionary, should likely be the return value from list_functions. At a minimum, must contain a key titled 'FunctionName'. 104 | flags: Flags describing which sections should be included in the return value. Default ALL 105 | 106 | Returns: 107 | dictionary describing the requested lambda function. 108 | """ 109 | # Python 2 and 3 support: 110 | try: 111 | basestring 112 | except NameError as _: 113 | basestring = str 114 | 115 | # If STR is passed in, determine if it's a name or ARN and built a dict. 116 | if isinstance(lambda_function, basestring): 117 | lambda_function_arn = ARN(lambda_function) 118 | if lambda_function_arn.error: 119 | lambda_function = dict(FunctionName=lambda_function) 120 | else: 121 | lambda_function = dict(FunctionName=lambda_function_arn.name, FunctionArn=lambda_function) 122 | 123 | # If an ARN is available, override the account_number/region from the conn dict. 124 | if 'FunctionArn' in lambda_function: 125 | lambda_function_arn = ARN(lambda_function['FunctionArn']) 126 | if not lambda_function_arn.error: 127 | if lambda_function_arn.account_number: 128 | conn['account_number'] = lambda_function_arn.account_number 129 | if lambda_function_arn.region: 130 | conn['region'] = lambda_function_arn.region 131 | 132 | return registry.build_out(flags, start_with=lambda_function, pass_datastructure=True, **conn) 133 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/s3.md: -------------------------------------------------------------------------------- 1 | # CloudAux AWS S3 2 | 3 | CloudAux can build out a JSON object describing an S3 Bucket. 4 | 5 | ## Example 6 | 7 | from cloudaux.orchestration.aws.s3 import get_bucket, FLAGS 8 | 9 | conn = dict( 10 | account_number='000000000000', 11 | assume_role='SecurityMonkey') 12 | 13 | bucket = get_bucket('MyS3Bucket', flags=FLAGS.ALL, **conn) 14 | 15 | # The flags parameter is optional but allows the user to indicate that 16 | # only a subset of the full item description is required. 17 | # S3 Flag Options are: 18 | # BASE, GRANTS, GRANT_REFERENCES, OWNER, LIFECYCLE, LOGGING, POLICY, TAGS 19 | # VERSIONING, WEBSITE, CORS, NOTIFICATIONS, ACCELERATION, REPLICATION 20 | # ANALYTICS, METRICS, INVENTORY, CREATED_DATE, ALL (default) 21 | # For instance: flags=FLAGS.WEBSITE | FLAGS.CORS | FLAGS.POLICY 22 | 23 | print(json.dumps(bucket, indent=2, sort_keys=True)) 24 | 25 | { 26 | "Arn": "arn:aws:s3:::MyS3Bucket", 27 | "Grants": { 28 | "SomeIdStringHere": [ 29 | "FULL_CONTROL" 30 | ] 31 | }, 32 | "GrantReferences": { 33 | "SomeIdStringHere": "S3CanonicalNameHere" 34 | }, 35 | "Owner": { 36 | "ID": "SomeIdStringHere" 37 | }, 38 | "LifecycleRules": [ 39 | { 40 | "Expiration": { 41 | "Days": 365 42 | }, 43 | "ID": "deleteoldstuff", 44 | "Prefix": "/doesntactuallyexist", 45 | "Status": "Enabled" 46 | } 47 | ], 48 | "Logging": { 49 | "Enabled": true, 50 | "Grants": [], 51 | "Prefix": "logs/", 52 | "Target": "MyS3LoggingBucket" 53 | }, 54 | "Policy": { 55 | "Statement": [ 56 | { 57 | "Action": "s3:GetObject", 58 | "Effect": "Allow", 59 | "Principal": { 60 | "AWS": "*" 61 | }, 62 | "Resource": "arn:aws:s3:::MyS3Bucket/*", 63 | "Sid": "AddPerm" 64 | } 65 | ], 66 | "Version": "2008-10-17" 67 | }, 68 | "Region": "us-east-1", 69 | "Tags": { 70 | "tagkey": "tagvalue" 71 | }, 72 | "Versioning": { 73 | "Status": "Enabled" 74 | }, 75 | "Website": { 76 | "IndexDocument": { 77 | "Suffix": "index.html" 78 | } 79 | }, 80 | "Cors": { 81 | "AllowedMethods": [ 82 | "GET" 83 | ], 84 | "MaxAgeSeconds": 3000, 85 | "AllowedHeaders": [ 86 | "Authorization" 87 | ], 88 | "AllowedOrigins": [ 89 | "*", 90 | ] 91 | }, 92 | "Notifications": { 93 | "LambdaFunctionConfigurations": [ 94 | { 95 | "LambdaFunctionArn": "arn:aws:lambda:us-east-1:ACCNTNUM:function:LAMBDAFUNC", 96 | "Id": "1234-34534-12-5-123-4213-4123-41235612423", 97 | "Filter": { 98 | "Key": { 99 | "FilterRules": [ 100 | { 101 | "Name": "Prefix", 102 | "Value": "somepath/" 103 | } 104 | ] 105 | }, 106 | "Events": [ 107 | "s3:ObjectCreated:Put" 108 | ] 109 | } 110 | } 111 | ] 112 | }, 113 | "Acceleration": "Enabled", 114 | "Replication": { 115 | "Rules": [ 116 | { 117 | "Prefix": "", 118 | "ID": "MyS3Bucket", 119 | "Destination": { 120 | "Bucket": "arn:aws:s3:::MyOtherS3Bucket" 121 | }, 122 | "Status": "Enabled" 123 | } 124 | ], 125 | "Role": "arn:aws:iam::ACCOUNTNUM:role/MYREPLICATIONROLE" 126 | }, 127 | "AnalyticsConfigurations": [ 128 | "Filter": { 129 | "Prefix": "someprefix" 130 | }, 131 | "StorageClassAnalysis": { 132 | "DataExport": { 133 | "Destination": { 134 | "S3BucketDestination": { 135 | "Prefix": "someother/prefix", 136 | "Format": "CSV", 137 | "Bucket": "arn:aws:s3:::SOMEBUCKETDESTINATION" 138 | } 139 | "OutputSchemaVersion": "V_1" 140 | } 141 | } 142 | "Id": "s3analytics" 143 | } 144 | ], 145 | "MetricsConfigurations": [ 146 | { 147 | "Id": "SomeWholeBucketMetricsConfig" 148 | }, 149 | { 150 | "Filter": { 151 | "Prefix": "some/prefix" 152 | }, 153 | "Id": "SomeOtherMetricsConfig" 154 | } 155 | ], 156 | "InventoryConfigurations": [ 157 | { 158 | "Destination": { 159 | "S3BucketDestination": { 160 | "Prefix": "someother/prefix", 161 | "Format": "CSV", 162 | "Bucket": "arn:aws:s3:::SOMEBUCKETDESTINATION" 163 | }, 164 | "Filter": { 165 | "Prefix": "someprefix/" 166 | }, 167 | "IsEnabled": true, 168 | "OptionalFields": [ 169 | "Size", 170 | "LastModifiedDate", 171 | "StorageClass", 172 | "ETag", 173 | "ReplicationStatus" 174 | ], 175 | "IncludedObjectVersions": "All", 176 | "Schedule": { 177 | "Frequency": "Weekly" 178 | }, 179 | "Id": "inventoryconfig" 180 | } 181 | } 182 | ], 183 | "CreationDate": "2017-09-07T22:28:01Z", 184 | "_version": 9 185 | } 186 | 187 | **NOTE: "GrantReferences" is an ephemeral field -- it is not guaranteed to be consistent - do not base logic off of it** 188 | 189 | ## Flags 190 | 191 | The `get_bucket` command accepts flags describing what parts of the structure to build out. 192 | 193 | If not provided, `get_bucket` assumes `FLAGS.ALL` sans `FLAGS.CREATED_DATE` because that is an expensive operation. 194 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/sg.md: -------------------------------------------------------------------------------- 1 | # CloudAux AWS Lambda Function 2 | 3 | CloudAux can build out a JSON object describing an AWS Security Group. 4 | 5 | ## Example 6 | 7 | from cloudaux.orchestration.aws.sg import get_security_group 8 | 9 | conn = dict( 10 | account_number='111111111111', 11 | assume_role='SecurityMonkey') 12 | 13 | sg = get_security_group('sg-12345678', **conn) 14 | 15 | print(json.dumps(provider, indent=2, sort_keys=True)) 16 | 17 | { 18 | "Description": ..., 19 | "GroupName": ..., 20 | "IpPermissions" ..., 21 | "OwnerId" ..., 22 | "GroupId" ..., 23 | "IpPermissionsEgress" ..., 24 | "VpcId" ... 25 | "_version" ... 26 | } 27 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/sg.py: -------------------------------------------------------------------------------- 1 | from cloudaux.aws.sg import describe_security_group 2 | from cloudaux.decorators import modify_output 3 | from cloudaux.orchestration.aws.arn import ARN 4 | from flagpole import FlagRegistry, Flags 5 | from six import string_types 6 | 7 | registry = FlagRegistry() 8 | FLAGS = Flags('BASE') 9 | 10 | 11 | @registry.register(flag=FLAGS.BASE) 12 | def _get_base(sg_obj, **conn): 13 | base_fields = ['Description', 'GroupName', 'IpPermissions', 'OwnerId', 'GroupId', 'IpPermissionsEgress', 'VpcId'] 14 | 15 | if not all(field in sg_obj for field in base_fields): 16 | sg_obj = describe_security_group(sg_obj['GroupId'], **conn) 17 | 18 | sg_obj['_version'] = 1 19 | return sg_obj 20 | 21 | 22 | @modify_output 23 | def get_security_group(sg_obj, flags=FLAGS.ALL, **conn): 24 | """ 25 | Orchestrates calls to build a Security Group in the following format: 26 | 27 | { 28 | "Description": ..., 29 | "GroupName": ..., 30 | "IpPermissions" ..., 31 | "OwnerId" ..., 32 | "GroupId" ..., 33 | "IpPermissionsEgress" ..., 34 | "VpcId" ... 35 | } 36 | Args: 37 | sg_obj: name, ARN, or dict of Security Group 38 | flags: Flags describing which sections should be included in the return value. Default ALL 39 | 40 | Returns: 41 | dictionary describing the requested Security Group 42 | """ 43 | if isinstance(sg_obj, string_types): 44 | group_arn = ARN(sg_obj) 45 | if group_arn.error: 46 | sg_obj = {'GroupId': sg_obj} 47 | else: 48 | sg_obj = {'GroupId': group_arn.parsed_name} 49 | 50 | return registry.build_out(flags, sg_obj, **conn) 51 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/sqs.md: -------------------------------------------------------------------------------- 1 | # CloudAux AWS SQS 2 | 3 | CloudAux can build out a JSON object describing an SQS Queue. 4 | 5 | ## Example 6 | 7 | from cloudaux.orchestration.aws.sqs import get_queue, FLAGS 8 | 9 | conn = dict( 10 | account_number='012345678910', 11 | assume_role='SecurityMonkey', 12 | region="us-west-2") 13 | 14 | queue = get_queue('MyQueue', flags=FLAGS.ALL, **conn) 15 | # Or the queue URL: 16 | # queue = get_queue('https://sqs.us-west-2.amazonaws.com/012345678910/MyQueue', flags=FLAGS.ALL, **conn) 17 | 18 | # The flags parameter is optional but allows the user to indicate that 19 | # only a subset of the full item description is required. 20 | # SQS Flag Options are: 21 | # BASE, TAGS, DEAD_LETTER_SOURCE_QUEUES, ALL (default) 22 | # For instance: flags=FLAGS.TAGS | FLAGS.DEAD_LETTER_SOURCE_QUEUES 23 | 24 | print(json.dumps(queue, indent=2, sort_keys=True)) 25 | 26 | { 27 | "_version": 1, 28 | "Region": "us-west-2", 29 | "Name": "MyQueue", 30 | "Arn": "arn:aws:sqs:us-west-2:012345678910:MyQueue", 31 | "Url": "https://sqs.us-west-2.amazonaws.com/012345678910/MyQueue" 32 | "DeadLetterSourceQueues": [ 33 | "https://sqs.us-west-2.amazonaws.com/012345678910/MyDeadLetterQueue" 34 | ], 35 | "Tags": { 36 | "TagKey": "TagValue" 37 | }, 38 | "Attributes": { 39 | "MessageRetentionPeriod": "345600", 40 | "CreatedTimestamp": "1516728270", 41 | "ApproximateNumberOfMessagesNotVisible": "0", 42 | "ApproximateNumberOfMessages": "0", 43 | "ApproximateNumberOfMessagesDelayed": "0", 44 | "DelaySeconds": "0", 45 | "QueueArn": "arn:aws:sqs:us-west-2:012345678910:MyQueue", 46 | "MaximumMessageSize": "262144", 47 | "VisibilityTimeout": "30", 48 | "LastModifiedTimestamp": "1516746742", 49 | "ReceiveMessageWaitTimeSeconds": "0" 50 | } 51 | } 52 | 53 | # NOTE: `Attributes` is defaulted to "All", so it will return all attributes boto currently supports. 54 | 55 | 56 | ## Flags 57 | 58 | The `get_queue` command accepts flags describing what parts of the structure to build out. 59 | 60 | If not provided, `get_queue` assumes `FLAGS.ALL`. 61 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/sqs.py: -------------------------------------------------------------------------------- 1 | from cloudaux.aws.sqs import get_queue_url, get_queue_attributes, list_queue_tags, list_dead_letter_source_queues 2 | from cloudaux.decorators import modify_output 3 | from flagpole import FlagRegistry, Flags 4 | 5 | import logging 6 | 7 | from cloudaux.orchestration.aws import ARN 8 | 9 | logger = logging.getLogger('cloudaux') 10 | 11 | registry = FlagRegistry() 12 | FLAGS = Flags('BASE', 'TAGS', 'DEAD_LETTER_SOURCE_QUEUES') 13 | 14 | 15 | @registry.register(flag=FLAGS.TAGS, key='tags') 16 | def get_sqs_tags(sqs_queue, **conn): 17 | return list_queue_tags(QueueUrl=sqs_queue["QueueUrl"], **conn) 18 | 19 | 20 | @registry.register(flag=FLAGS.DEAD_LETTER_SOURCE_QUEUES, key='dead_letter_source_queues') 21 | def get_dead_letter_queues(sqs_queue, **conn): 22 | return list_dead_letter_source_queues(QueueUrl=sqs_queue["QueueUrl"], **conn) 23 | 24 | 25 | @registry.register(flag=FLAGS.BASE) 26 | def get_base(sqs_queue, **conn): 27 | sqs_queue["Attributes"] = get_queue_attributes(QueueUrl=sqs_queue["QueueUrl"], AttributeNames=["All"], **conn) 28 | 29 | # Get the Queue name: 30 | name = ARN(sqs_queue["Attributes"]["QueueArn"]).parsed_name 31 | 32 | return { 33 | 'arn': sqs_queue["Attributes"]["QueueArn"], 34 | 'url': sqs_queue["QueueUrl"], 35 | 'name': name, 36 | 'region': conn['region'], 37 | 'attributes': sqs_queue["Attributes"], 38 | '_version': 1 39 | } 40 | 41 | 42 | @modify_output 43 | def get_queue(queue, flags=FLAGS.ALL, **conn): 44 | """ 45 | Orchestrates all the calls required to fully fetch details about an SQS Queue: 46 | 47 | { 48 | "Arn": ..., 49 | "Region": ..., 50 | "Name": ..., 51 | "Url": ..., 52 | "Attributes": ..., 53 | "Tags": ..., 54 | "DeadLetterSourceQueues": ..., 55 | "_version": 1 56 | } 57 | 58 | :param queue: Either the queue name OR the queue url 59 | :param flags: By default, set to ALL fields. 60 | :param conn: dict containing enough information to make a connection to the desired account. Must at least have 61 | 'assume_role' key. 62 | :return: dict containing a fully built out SQS queue. 63 | """ 64 | # Check if this is a Queue URL or a queue name: 65 | if queue.startswith("https://") or queue.startswith("http://"): 66 | queue_name = queue 67 | else: 68 | queue_name = get_queue_url(QueueName=queue, **conn) 69 | 70 | sqs_queue = {"QueueUrl": queue_name} 71 | 72 | return registry.build_out(flags, sqs_queue, **conn) 73 | -------------------------------------------------------------------------------- /cloudaux/orchestration/aws/vpc.md: -------------------------------------------------------------------------------- 1 | # CloudAux AWS VPC 2 | 3 | CloudAux can build out a JSON object describing a VPC. 4 | 5 | ## Example 6 | 7 | from cloudaux.orchestration.aws.vpc import get_vpc, FLAGS 8 | 9 | conn = dict( 10 | account_number='012345678910', 11 | assume_role='SecurityMonkey', 12 | region="us-west-2") 13 | 14 | queue = get_vpc('vpc-xxxxxxxx', flags=FLAGS.ALL, **conn) 15 | 16 | # The flags parameter is optional but allows the user to indicate that 17 | # only a subset of the full item description is required. 18 | # VPC Flag Options are: 19 | # BASE, INTERNET_GATEWAY, CLASSIC_LINK, VPC_PEERING_CONNECTIONS, SUBNETS, ROUTE_TABLES, 20 | # NETWORK_ACLS, FLOW_LOGS, ALL (default) 21 | # For instance: flags=FLAGS.INTERNET_GATEWAY | FLAGS.VPC_PEERING_CONNECTIONS 22 | 23 | print(json.dumps(vpc, indent=2, sort_keys=True)) 24 | 25 | { 26 | "Arn": "arn:aws:ec2:us-east-1:012345678910:vpc/vpc-xxxxxxxx", 27 | "Id": "vpc-xxxxxxxx", 28 | "Name": "MyVPC", 29 | "Region": "us-east-1", 30 | "Tags": [ 31 | { 32 | "Key": "Name", 33 | "Value": "MyVPC" 34 | } 35 | ], 36 | "IsDefault": false, 37 | "InstanceTenancy": "default", 38 | "DhcpOptionsId": "dopt-xxxxxxxx", 39 | "CidrBlock": "10.0.0.0/16", 40 | "CidrBlockAssociationSet": [ 41 | { 42 | "AssociationId": "vpc-cidr-assoc-xxxxxxxx", 43 | "CidrBlock": "10.0.0.0/16", 44 | "CidrBlockState": { 45 | "State": "associated" 46 | } 47 | }, 48 | { 49 | "AssociationId": "vpc-cidr-assoc-xxxxxxxy", 50 | "CidrBlock": "10.1.0.0/16", 51 | "CidrBlockState": { 52 | "State": "associated" 53 | } 54 | } 55 | ], 56 | "Ipv6CidrBlockAssociationSet": [ 57 | { 58 | "AssociationId": "vpc-cidr-assoc-xxxxxxxz", 59 | "Ipv6CidrBlock": "fde8:2d1a:7900:ea96::/64", 60 | "Ipv6CidrBlockState": { 61 | "State": "associated" 62 | } 63 | } 64 | ], 65 | "Attributes": { 66 | "EnableDnsHostnames": { 67 | "Value": true 68 | }, 69 | "EnableDnsSupport": { 70 | "Value": true 71 | } 72 | }, 73 | "_version": 1, 74 | "FlowLogs": [ 75 | "fl-xxxxxxxx", 76 | "fl-xxxxxxxy" 77 | ], 78 | "ClassicLink": { 79 | "Enabled": false, 80 | "DnsEnabled": false 81 | }, 82 | "InternetGateway": { 83 | "State": "available", 84 | "Id": "igw-xxxxxxxx", 85 | "Tags": [ 86 | { 87 | "Key": "Name", 88 | "Value": "MyInternetGateway" 89 | } 90 | ] 91 | }, 92 | "VpcPeeringConnections": [ 93 | "pcx-xxxxxxxx", 94 | "pcx-xxxxxxxy" 95 | ], 96 | "Subnets": [ 97 | "subnet-xxxxxxxx", 98 | "subnet-xxxxxxxy" 99 | ], 100 | "RouteTables": [ 101 | "rtb-xxxxxxxx", 102 | "rtb-xxxxxxxy" 103 | ], 104 | "NetworkAcls": [ 105 | "acl-xxxxxxxx", 106 | "acl-xxxxxxxy" 107 | ] 108 | } 109 | 110 | ## Flags 111 | 112 | The `get_vpc` command accepts flags describing what parts of the structure to build out. 113 | 114 | If not provided, `get_vpc` assumes `FLAGS.ALL`. 115 | -------------------------------------------------------------------------------- /cloudaux/orchestration/gcp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/cloudaux/39d1dc11ccd794f21d95d3d1458abd94425b0927/cloudaux/orchestration/gcp/__init__.py -------------------------------------------------------------------------------- /cloudaux/orchestration/gcp/gce/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/cloudaux/39d1dc11ccd794f21d95d3d1458abd94425b0927/cloudaux/orchestration/gcp/gce/__init__.py -------------------------------------------------------------------------------- /cloudaux/orchestration/gcp/gce/network.py: -------------------------------------------------------------------------------- 1 | from cloudaux.gcp.gce.network import get_network, list_subnetworks 2 | from cloudaux.decorators import modify_output 3 | from flagpole import FlagRegistry, Flags 4 | 5 | 6 | registry = FlagRegistry() 7 | FLAGS = Flags('BASE') 8 | 9 | 10 | @registry.register(flag=FLAGS.BASE) 11 | def _get_base(network, **conn): 12 | result = get_network(Network=network, **conn) 13 | if 'subnetworks' in result: 14 | result['subnetworks'] = list_subnetworks(filter='network eq %s' % result['selfLink'], **conn) 15 | 16 | result['_version'] = 1 17 | return result 18 | 19 | @modify_output 20 | def get_network_and_subnetworks(network, flags=FLAGS.ALL, **conn): 21 | return registry.build_out(flags, network, **conn) 22 | -------------------------------------------------------------------------------- /cloudaux/orchestration/gcp/gcs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/cloudaux/39d1dc11ccd794f21d95d3d1458abd94425b0927/cloudaux/orchestration/gcp/gcs/__init__.py -------------------------------------------------------------------------------- /cloudaux/orchestration/gcp/gcs/bucket.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.orchestration.gcp.gcs.bucket 3 | :platform: Unix 4 | :copyright: (c) 2016 by Google Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Tom Melendez (@supertom) 7 | """ 8 | from cloudaux.gcp.gcs import get_bucket as fetch_bucket 9 | from cloudaux.gcp.utils import strdate 10 | from cloudaux.decorators import modify_output 11 | from flagpole import FlagRegistry, Flags 12 | 13 | 14 | registry = FlagRegistry() 15 | FLAGS = Flags('BASE') 16 | 17 | 18 | @registry.register(flag=FLAGS.BASE) 19 | def _get_base(bucket_name, **conn): 20 | bucket = fetch_bucket(Bucket=bucket_name, **conn) 21 | if not bucket: 22 | return dict(Error='Unauthorized') 23 | 24 | result = dict() 25 | result['acl'] = list(bucket.acl) 26 | result['default_object_acl'] = list(bucket.default_object_acl) 27 | result['cors'] = bucket.cors 28 | result['etag'] = bucket.etag 29 | result['id'] = bucket.id 30 | result['location'] = bucket.location 31 | result['metageneration'] = bucket.metageneration 32 | result['owner'] = bucket.owner 33 | result['path'] = bucket.path 34 | result['project_number'] = bucket.project_number 35 | result['self_link'] = bucket.self_link 36 | result['storage_class'] = bucket.storage_class 37 | result['versioning_enabled'] = bucket.versioning_enabled 38 | result['time_created'] = strdate(bucket.time_created) 39 | result['_version'] = 1 40 | return result 41 | 42 | 43 | @modify_output 44 | def get_bucket(bucket_name, flags=FLAGS.ALL, **conn): 45 | return registry.build_out(flags, bucket_name, **conn) 46 | -------------------------------------------------------------------------------- /cloudaux/orchestration/gcp/iam/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/cloudaux/39d1dc11ccd794f21d95d3d1458abd94425b0927/cloudaux/orchestration/gcp/iam/__init__.py -------------------------------------------------------------------------------- /cloudaux/orchestration/gcp/iam/serviceaccount.py: -------------------------------------------------------------------------------- 1 | from cloudaux.gcp.iam import get_iam_policy, get_serviceaccount, get_serviceaccount_keys 2 | from cloudaux.decorators import modify_output 3 | from flagpole import FlagRegistry, Flags 4 | 5 | 6 | registry = FlagRegistry() 7 | FLAGS = Flags('BASE', 'KEYS', 'POLICY') 8 | 9 | 10 | @registry.register(flag=FLAGS.KEYS, key='keys') 11 | def get_keys(service_account, **conn): 12 | return get_serviceaccount_keys(service_account=service_account, **conn) 13 | 14 | 15 | @registry.register(flag=FLAGS.POLICY, key='policy') 16 | def get_policy(service_account, **conn): 17 | return get_iam_policy(service_account=service_account, **conn) 18 | 19 | 20 | @registry.register(flag=FLAGS.BASE) 21 | def _get_base(service_account, **conn): 22 | sa = get_serviceaccount(service_account=service_account, **conn) 23 | sa['_version'] = 1 24 | return sa 25 | 26 | 27 | @modify_output 28 | def get_serviceaccount_complete(service_account, flags=FLAGS.ALL, **conn): 29 | return registry.build_out(flags, service_account, **conn) 30 | -------------------------------------------------------------------------------- /cloudaux/orchestration/openstack/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/cloudaux/39d1dc11ccd794f21d95d3d1458abd94425b0927/cloudaux/orchestration/openstack/__init__.py -------------------------------------------------------------------------------- /cloudaux/orchestration/openstack/security_group.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.openstack.orchestration.security_group 3 | :platform: Unix 4 | :copyright: Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. See AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Michael Stair 7 | """ 8 | from cloudaux.openstack.utils import list_items 9 | from flagpole import FlagRegistry, Flags 10 | 11 | registry = FlagRegistry() 12 | FLAGS = Flags('RULES','INSTANCES') 13 | 14 | 15 | @registry.register(flag=FLAGS.INSTANCES, depends_on=FLAGS.RULES, key='assigned_to') 16 | def get_instances(security_group, **kwargs): 17 | detail = kwargs.pop('instance_detail', 'FULL') 18 | 19 | kwargs['service'] = 'compute' 20 | kwargs['generator'] = 'servers' 21 | instances = list_items(**kwargs) 22 | 23 | sg_instances = {} 24 | for instance in instances: 25 | for group in instance.security_groups: 26 | if group['name'] not in sg_instances: 27 | sg_instances[group['name']] = [instance] 28 | else: 29 | sg_instances[group['name']].append(instance) 30 | if detail == 'SUMMARY': 31 | if security_group['name'] in sg_instances: 32 | assigned_to = "{} instances".format(len(sg_instances[security_group['name']])) 33 | else: 34 | assigned_to = "0 instances" 35 | elif detail == 'FULL': 36 | assigned_to = [] 37 | if security_group['name'] in sg_instances: 38 | for instance in sg_instances[security_group['name']]: 39 | tagdict = {"instance_id": instance.id} 40 | assigned_to.append(tagdict) 41 | return assigned_to 42 | 43 | @registry.register(flag=FLAGS.RULES) 44 | def get_rules(security_group, **kwargs): 45 | """ format the rule fields to match AWS to support auditor reuse, 46 | will need to remap back if we want to orchestrate from our stored items """ 47 | rules = security_group.pop('security_group_rules',[]) 48 | for rule in rules: 49 | rule['ip_protocol'] = rule.pop('protocol') 50 | rule['from_port'] = rule.pop('port_range_max') 51 | rule['to_port'] = rule.pop('port_range_min') 52 | rule['cidr_ip'] = rule.pop('remote_ip_prefix') 53 | rule['rule_type'] = rule.pop('direction') 54 | security_group['rules'] = sorted(rules) 55 | return security_group 56 | 57 | def get_security_group(security_group, flags=FLAGS.ALL, **kwargs): 58 | result = registry.build_out(flags, start_with=security_group, pass_datastructure=True, **kwargs) 59 | """ just store the AWS formatted rules """ 60 | result.pop('security_group_rules', []) 61 | return result 62 | -------------------------------------------------------------------------------- /cloudaux/orchestration/openstack/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.openstack.orchestration.utils 3 | :platform: Unix 4 | :copyright: Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. See AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Michael Stair 7 | """ 8 | import inspect 9 | 10 | from six import text_type 11 | 12 | """ global ignore_list for OpenStack SDK service object members """ 13 | ignore_list = [ 'allow_create', 'allow_delete', 'allow_get', 'allow_head', 14 | 'allow_list', 'allow_update', 'patch_update', 'put_create', 15 | 'service', 'base_path', 'resource_key', 'resources_key' ] 16 | 17 | def get_item(item, **kwargs): 18 | """ 19 | API versioning for each OpenStack service is independent. Generically capture 20 | the public members (non-routine and non-private) of the OpenStack SDK objects. 21 | 22 | Note the lack of the modify_output decorator. Preserving the field naming allows 23 | us to reconstruct objects and orchestrate from stored items. 24 | """ 25 | _item = {} 26 | for k,v in inspect.getmembers(item, lambda a:not(inspect.isroutine(a))): 27 | if not k.startswith('_') and not k in ignore_list: 28 | _item[k] = v 29 | 30 | return sub_dict(_item) 31 | 32 | 33 | """ from security_monkey.common.utils. Need to convert any embedded OpenStack classes 34 | to their string/JSON representation """ 35 | 36 | prims = [int, str, text_type, bool, float, type(None)] 37 | 38 | def sub_list(l): 39 | """ 40 | Recursively walk a data-structure sorting any lists along the way. 41 | Any unknown types get mapped to string representation 42 | 43 | :param l: list 44 | :return: sorted list, where any child lists are also sorted. 45 | """ 46 | r = [] 47 | 48 | for i in l: 49 | if type(i) in prims: 50 | r.append(i) 51 | elif type(i) is list: 52 | r.append(sub_list(i)) 53 | elif type(i) is dict: 54 | r.append(sub_dict(i)) 55 | else: 56 | r.append(str(i)) 57 | r = sorted(r) 58 | return r 59 | 60 | 61 | def sub_dict(d): 62 | """ 63 | Recursively walk a data-structure sorting any lists along the way. 64 | Any unknown types get mapped to string representation 65 | 66 | :param d: dict 67 | :return: dict where any lists, even those buried deep in the structure, have been sorted. 68 | """ 69 | r = {} 70 | for k in d: 71 | if type(d[k]) in prims: 72 | r[k] = d[k] 73 | elif type(d[k]) is list: 74 | r[k] = sub_list(d[k]) 75 | elif type(d[k]) is dict: 76 | r[k] = sub_dict(d[k]) 77 | else: 78 | r[k] = str(d[k]) 79 | return r 80 | -------------------------------------------------------------------------------- /cloudaux/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/cloudaux/39d1dc11ccd794f21d95d3d1458abd94425b0927/cloudaux/tests/__init__.py -------------------------------------------------------------------------------- /cloudaux/tests/aws/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/cloudaux/39d1dc11ccd794f21d95d3d1458abd94425b0927/cloudaux/tests/aws/__init__.py -------------------------------------------------------------------------------- /cloudaux/tests/aws/test_arn.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.tests.aws.test_ec2 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Mike Grima 7 | """ 8 | 9 | from cloudaux.orchestration.aws.arn import ARN 10 | 11 | 12 | def test_arn(): 13 | 14 | test_arn = 'arn:aws:iam::123456789123:role/testRole' 15 | 16 | arn = ARN(test_arn) 17 | 18 | assert arn.partition == 'aws' 19 | assert arn.tech == 'iam' 20 | assert arn.region == '' 21 | assert arn.account_number == '123456789123' 22 | assert arn.name == 'role/testRole' 23 | assert arn.resource_type == 'role' 24 | assert arn.resource == 'testRole' 25 | 26 | test_arn2 = 'arn:aws:iam::123456789123:role/service-role/DynamoDBAutoscaleRole' 27 | 28 | arn = ARN(test_arn2) 29 | 30 | assert arn.partition == 'aws' 31 | assert arn.tech == 'iam' 32 | assert arn.region == '' 33 | assert arn.account_number == '123456789123' 34 | assert arn.name == 'role/service-role/DynamoDBAutoscaleRole' 35 | assert arn.resource_type == 'role' 36 | assert arn.resource == 'service-role/DynamoDBAutoscaleRole' 37 | 38 | # Test for GovCloud Partition 39 | test_arn3 = 'arn:aws-us-gov:iam::123456789123:role/service-role/DynamoDBAutoscaleRole' 40 | 41 | arn = ARN(test_arn3) 42 | 43 | assert arn.partition == 'aws-us-gov' 44 | assert arn.tech == 'iam' 45 | assert arn.region == '' 46 | assert arn.account_number == '123456789123' 47 | assert arn.name == 'role/service-role/DynamoDBAutoscaleRole' 48 | assert arn.resource_type == 'role' 49 | assert arn.resource == 'service-role/DynamoDBAutoscaleRole' 50 | -------------------------------------------------------------------------------- /cloudaux/tests/aws/test_decorators.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.tests.aws.test_decorators 3 | :platform: Unix 4 | :copyright: (c) 2021 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Patrick Sanders 7 | """ 8 | 9 | from mock import MagicMock, call 10 | 11 | from cloudaux.aws.decorators import paginated 12 | 13 | 14 | def test_paginated_single_page(): 15 | mock_responder = MagicMock() 16 | mock_responder.side_effect = [ 17 | { 18 | "NextToken": None, 19 | "Data": ["a", "b"] 20 | }, 21 | ] 22 | 23 | @paginated("Data", request_pagination_marker="NextToken", response_pagination_marker="NextToken") 24 | def retrieve_letters(**kwargs): 25 | return mock_responder(**kwargs) 26 | 27 | result = retrieve_letters() 28 | assert result == ["a", "b"] 29 | assert mock_responder.call_count == 1 30 | mock_responder.assert_has_calls([call()]) 31 | 32 | 33 | def test_paginated_multiple_pages(): 34 | mock_responder = MagicMock() 35 | mock_responder.side_effect = [ 36 | { 37 | "NextToken": "1", 38 | "Data": ["a", "b"] 39 | }, 40 | { 41 | "NextToken": "2", 42 | "Data": ["c", "d"] 43 | }, 44 | { 45 | "NextToken": None, 46 | "Data": ["e", "f"] 47 | }, 48 | ] 49 | 50 | @paginated("Data", request_pagination_marker="NextToken", response_pagination_marker="NextToken") 51 | def retrieve_letters(**kwargs): 52 | return mock_responder(**kwargs) 53 | 54 | result = retrieve_letters() 55 | assert result == ["a", "b", "c", "d", "e", "f"] 56 | assert mock_responder.call_count == 3 57 | mock_responder.assert_has_calls([call(), call(NextToken="1"), call(NextToken="2")]) 58 | 59 | 60 | def test_paginated_multiple_pages_empty_results(): 61 | mock_responder = MagicMock() 62 | mock_responder.side_effect = [ 63 | { 64 | "NextToken": "1", 65 | "Data": [] 66 | }, 67 | { 68 | "NextToken": "2", 69 | "Data": [] 70 | }, 71 | { 72 | "NextToken": "3", 73 | "Data": ["e", "f"] 74 | }, 75 | { 76 | "NextToken": None, 77 | "Data": [] 78 | }, 79 | ] 80 | 81 | @paginated("Data", request_pagination_marker="NextToken", response_pagination_marker="NextToken") 82 | def retrieve_letters(**kwargs): 83 | return mock_responder(**kwargs) 84 | 85 | result = retrieve_letters() 86 | assert result == ["e", "f"] 87 | assert mock_responder.call_count == 4 88 | mock_responder.assert_has_calls([call(), call(NextToken="1"), call(NextToken="2"), call(NextToken="3")]) 89 | -------------------------------------------------------------------------------- /cloudaux/tests/aws/test_ec2.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.tests.aws.test_ec2 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Mike Grima 7 | """ 8 | 9 | from cloudaux.aws.ec2 import describe_vpcs, describe_dhcp_options, describe_internet_gateways, \ 10 | describe_vpc_peering_connections, describe_subnets, describe_route_tables, describe_network_acls, \ 11 | describe_vpc_attribute 12 | 13 | 14 | def test_describe_vpcs(test_vpc): 15 | result = describe_vpcs() 16 | 17 | assert len(result) == 2 18 | 19 | # Find the non-default VPC and do stuff: 20 | for v in result: 21 | if v["CidrBlock"] == '172.31.0.0/16': 22 | assert v["IsDefault"] 23 | 24 | else: 25 | vpc_cidrs = ["10.0.0.0/16", "10.1.0.0/16"] 26 | assert v["CidrBlock"] == "10.0.0.0/16" 27 | assert v["InstanceTenancy"] == "default" 28 | assert len(v["Tags"]) == 2 29 | assert not v["IsDefault"] 30 | assert len(v["CidrBlockAssociationSet"]) == 2 31 | vpc_cidrs.remove(v["CidrBlockAssociationSet"][0]["CidrBlock"]) 32 | vpc_cidrs.remove(v["CidrBlockAssociationSet"][1]["CidrBlock"]) 33 | assert not vpc_cidrs 34 | assert len(v["Ipv6CidrBlockAssociationSet"]) == 1 35 | assert v["Ipv6CidrBlockAssociationSet"][0]["Ipv6CidrBlock"] == "fde8:2d1a:7900:ea96::/64" 36 | 37 | 38 | def test_describe_dhcp_options(test_vpc, dhcp_options): 39 | result = describe_dhcp_options(DhcpOptionsIds=[dhcp_options["DhcpOptionsId"]]) 40 | 41 | assert len(result) == 1 42 | assert result[0]["DhcpConfigurations"][0]["Key"] == 'domain-name-servers' 43 | assert len(result[0]["DhcpConfigurations"][0]["Values"]) == 2 44 | 45 | test_vpc = describe_vpcs(VpcIds=[test_vpc["VpcId"]]) 46 | assert test_vpc[0]["DhcpOptionsId"] == result[0]["DhcpOptionsId"] 47 | 48 | 49 | def test_describe_internet_gateways(test_vpc, internet_gateway_vpc): 50 | result = describe_internet_gateways(InternetGatewayIds=[internet_gateway_vpc["InternetGatewayId"]]) 51 | 52 | assert len(result) == 1 53 | assert len(result[0]["Attachments"]) == 1 54 | assert result[0]["Attachments"][0]["State"] == 'available' 55 | assert result[0]["Attachments"][0]["VpcId"] == test_vpc["VpcId"] 56 | assert result[0]["InternetGatewayId"] == internet_gateway_vpc["InternetGatewayId"] 57 | 58 | # Try with a filter: 59 | result = describe_internet_gateways(Filters=[{"Name": "attachment.vpc-id", "Values": [test_vpc["VpcId"]]}]) 60 | assert len(result) == 1 61 | assert result[0]["Attachments"][0]["VpcId"] == test_vpc["VpcId"] 62 | 63 | 64 | def test_describe_vpc_peering_connections(test_vpc, vpc_peering_connection): 65 | result = describe_vpc_peering_connections(VpcPeeringConnectionIds= 66 | [vpc_peering_connection["VpcPeeringConnectionId"]]) 67 | 68 | assert len(result) == 1 69 | assert result[0]["RequesterVpcInfo"]["CidrBlock"] == "10.0.0.0/16" 70 | assert result[0]["RequesterVpcInfo"]["VpcId"] == test_vpc["VpcId"] 71 | 72 | 73 | def test_describe_subnets(test_vpc, subnet): 74 | result = describe_subnets(SubnetIds=[subnet["SubnetId"]]) 75 | 76 | assert len(result) == 1 77 | assert result[0]["CidrBlock"] == "10.0.0.0/16" 78 | assert result[0]["SubnetId"] == subnet["SubnetId"] 79 | assert result[0]["VpcId"] == test_vpc["VpcId"] 80 | 81 | # Try with a filter: 82 | result = describe_subnets(Filters=[{"Name": "vpc-id", "Values": [test_vpc["VpcId"]]}]) 83 | assert len(result) == 1 84 | assert result[0]["VpcId"] == test_vpc["VpcId"] 85 | assert result[0]["SubnetId"] == subnet["SubnetId"] 86 | 87 | 88 | def test_describe_route_tables(test_vpc, route_table): 89 | result = describe_route_tables(RouteTableIds=[route_table["RouteTableId"]]) 90 | 91 | assert len(result) == 1 92 | assert result[0]["RouteTableId"] == route_table["RouteTableId"] 93 | assert result[0]["VpcId"] == test_vpc["VpcId"] 94 | 95 | # Try with a filter: 96 | result = describe_route_tables(Filters=[{"Name": "vpc-id", "Values": [test_vpc["VpcId"]]}]) 97 | # Moto returns 2: 98 | found = False 99 | for r in result: 100 | assert r["VpcId"] == test_vpc["VpcId"] 101 | if r["RouteTableId"] == route_table["RouteTableId"]: 102 | found = True 103 | 104 | assert found 105 | 106 | 107 | def test_describe_network_acls(test_vpc, network_acl): 108 | result = describe_network_acls(NetworkAclIds=[network_acl["NetworkAclId"]]) 109 | 110 | assert len(result) == 1 111 | assert result[0]["NetworkAclId"] == network_acl["NetworkAclId"] 112 | assert result[0]["VpcId"] == test_vpc["VpcId"] 113 | 114 | # Try with a filter: 115 | result = describe_network_acls(Filters=[{"Name": "vpc-id", "Values": [test_vpc["VpcId"]]}]) 116 | # Moto returns 2: 117 | found = False 118 | for r in result: 119 | assert r["VpcId"] == test_vpc["VpcId"] 120 | if r["NetworkAclId"] == network_acl["NetworkAclId"]: 121 | found = True 122 | 123 | assert found 124 | 125 | 126 | def test_describe_vpc_attribute(test_vpc): 127 | result = describe_vpc_attribute(VpcId=test_vpc["VpcId"], Attribute="enableDnsSupport") 128 | assert result["EnableDnsSupport"] 129 | result = describe_vpc_attribute(VpcId=test_vpc["VpcId"], Attribute="enableDnsHostnames") 130 | assert result["EnableDnsHostnames"] 131 | -------------------------------------------------------------------------------- /cloudaux/tests/aws/test_sts.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.tests.aws.test_sts 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Josafat Gonzalez 7 | """ 8 | from botocore.client import Config 9 | from cloudaux.aws.sts import boto3_cached_conn 10 | from mock import patch 11 | 12 | 13 | def test_boto3_cached_conn_read_only(): 14 | # Arrange 15 | conn_details = { 16 | 'account_number': '111111111111', 17 | 'assume_role': 'role_one', 18 | 'region': 'us-east-1', 19 | 'read_only': True 20 | } 21 | 22 | with patch('boto3.session.Session.client'): 23 | 24 | # Act 25 | conn = boto3_cached_conn('s3', **conn_details) 26 | 27 | # Assert 28 | assert 'PolicyArns' in conn.assume_role.call_args.kwargs 29 | 30 | 31 | def test_boto3_cached_conn_default(): 32 | # Arrange 33 | conn_details = { 34 | 'account_number': '111111111111', 35 | 'assume_role': 'role_one', 36 | 'region': 'us-east-1' 37 | } 38 | 39 | with patch('boto3.session.Session.client'): 40 | 41 | # Act 42 | conn = boto3_cached_conn('s3', **conn_details) 43 | 44 | # Assert 45 | assert 'PolicyArns' not in conn.assume_role.call_args.kwargs 46 | 47 | 48 | def test_boto3_cached_conn_retry_config(sts): 49 | from cloudaux.aws.sts import _client 50 | import cloudaux.aws.sts 51 | 52 | def mock_client(*args, **kwargs): 53 | with patch('boto3.session.Session') as p: 54 | _client(*args, **kwargs) 55 | 56 | return p 57 | 58 | # With the default: 59 | with patch('cloudaux.aws.sts._client', mock_client): 60 | conn = boto3_cached_conn('s3') 61 | assert conn.mock_calls[1].kwargs['config'].retries == {'max_attempts': 10} 62 | cloudaux.aws.sts.CACHE = {} 63 | 64 | # With STS role assumption: 65 | conn_details = { 66 | 'account_number': '111111111111', 67 | 'assume_role': 'role_one', 68 | 'region': 'us-east-1' 69 | } 70 | with patch('cloudaux.aws.sts._client', mock_client): 71 | conn = boto3_cached_conn('s3', **conn_details) 72 | assert conn.mock_calls[1].kwargs['config'].retries == {'max_attempts': 10} 73 | cloudaux.aws.sts.CACHE = {} 74 | 75 | # With a specified retry Config: 76 | with patch('cloudaux.aws.sts._client', mock_client): 77 | conn = boto3_cached_conn('s3', retry_max_attempts=1000) 78 | assert conn.mock_calls[1].kwargs['config'].retries == {'max_attempts': 1000} 79 | cloudaux.aws.sts.CACHE = {} 80 | 81 | # With STS role assumption: 82 | conn_details['retry_max_attempts'] = 1000 83 | with patch('cloudaux.aws.sts._client', mock_client): 84 | conn = boto3_cached_conn('s3', **conn_details) 85 | assert conn.mock_calls[1].kwargs['config'].retries == {'max_attempts': 1000} 86 | cloudaux.aws.sts.CACHE = {} 87 | 88 | def test_boto3_cached_conn_config(sts): 89 | from cloudaux.aws.sts import _client 90 | import cloudaux.aws.sts 91 | 92 | def mock_client(*args, **kwargs): 93 | with patch('boto3.session.Session') as p: 94 | _client(*args, **kwargs) 95 | 96 | return p 97 | 98 | # With the default: 99 | with patch('cloudaux.aws.sts._client', mock_client): 100 | conn = boto3_cached_conn('s3', config=Config(signature_version='s3v4')) 101 | assert conn.mock_calls[1].kwargs['config'].signature_version == 's3v4' 102 | cloudaux.aws.sts.CACHE = {} 103 | 104 | # With STS role assumption: 105 | conn_details = { 106 | 'account_number': '111111111111', 107 | 'assume_role': 'role_one', 108 | 'region': 'us-east-1' 109 | } 110 | with patch('cloudaux.aws.sts._client', mock_client): 111 | conn = boto3_cached_conn('s3', config=Config(signature_version='s3v4'), **conn_details) 112 | assert conn.mock_calls[1].kwargs['config'].signature_version == 's3v4' 113 | cloudaux.aws.sts.CACHE = {} 114 | -------------------------------------------------------------------------------- /cloudaux/tests/cloudaux/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/cloudaux/39d1dc11ccd794f21d95d3d1458abd94425b0927/cloudaux/tests/cloudaux/__init__.py -------------------------------------------------------------------------------- /cloudaux/tests/cloudaux/test_cloudaux.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.tests.cloudaux.test_cloudaux 3 | :platform: Unix 4 | :copyright: (c) 2019 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Patrick Kelley 7 | """ 8 | from cloudaux import CloudAux 9 | 10 | 11 | def test_cloudaux(): 12 | conn_one = { 13 | "account_number": "111111111111", 14 | "assume_role": "role_one", 15 | "region": "us-east-1", 16 | "session_name": "conn_one" 17 | } 18 | 19 | conn_two = { 20 | "account_number": "222222222222", 21 | "assume_role": "role_two", 22 | "region": "us-east-2", 23 | "session_name": "conn_two" 24 | } 25 | 26 | ca_one = CloudAux(**conn_one) 27 | ca_two = CloudAux(**conn_two) 28 | 29 | assert ca_one.conn_details["account_number"] == "111111111111" 30 | assert ca_one.conn_details["assume_role"] == "role_one" 31 | assert ca_one.conn_details["region"] == "us-east-1" 32 | assert ca_one.conn_details["session_name"] == "conn_one" 33 | 34 | assert ca_two.conn_details["account_number"] == "222222222222" 35 | assert ca_two.conn_details["assume_role"] == "role_two" 36 | assert ca_two.conn_details["region"] == "us-east-2" 37 | assert ca_two.conn_details["session_name"] == "conn_two" 38 | -------------------------------------------------------------------------------- /cloudaux/tests/gcp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/cloudaux/39d1dc11ccd794f21d95d3d1458abd94425b0927/cloudaux/tests/gcp/__init__.py -------------------------------------------------------------------------------- /cloudaux/tests/gcp/fixtures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/cloudaux/39d1dc11ccd794f21d95d3d1458abd94425b0927/cloudaux/tests/gcp/fixtures/__init__.py -------------------------------------------------------------------------------- /cloudaux/tests/gcp/fixtures/testkey.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "service_account", 3 | "project_id": "security-monkey-test-project", 4 | "private_key_id": "625f4bbfba9b3e7e26428cbcd1d6911a936f0ce8", 5 | "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCSQJ4WpxwH51v6\nMmBde0oMaQM27S1Uku0eXTY7Xy0283quWjHz8EtYVUx1S1rMl1a/5OY0NbY8ZH4s\nlr5A+WfELsovv9PQBq1YiF5Z1N6ibgkKVsTh6LpjBilRH3tcC2bsI8F7ggcMNWtk\n5PnAfVlctDN6QYdfipfghG3uxIETZbVjUf/1u1/EANlktkjFbEhTCcDdxSZnYUDR\nbbwgR23PH7fQRRQEOusDj1riUZMfiE7JZUcmrN2L/PQmrk+BfJS1sVsttJ/wgbwQ\nkhZqO/EpwJBZE9W7jGniiN4QQZlU3BNjAQmu7Ll9DTJzHqe+ZnDSjrWw4f/hHOSE\nQD6JNF5FAgMBAAECggEAcOfVo6NKrodbqGFvp3tghQgk5Shg0GPFDqXjEh695yg8\nU5Fp8upVIFJpfju4Uy2aWPY7CZ9VmnXyOjX9lmd0Ri3hBM/Qk040UJ6mSC9f2IGe\nFQ87WzOxgHTnPKtTLpHW2QIZgmG1UyWCTRe//2xFqp2rB/zCp211JmzV+XhWNUQq\nE8vehw6+8hly1s/E1uJptlfnBshp3i4eelRwHq3BhC8owYtFATgKPqegjT9g+vdS\nDWt2my6a3TlJgGvGSypfosMVeEsGCNklLsbaEiq2L7GSruywDMQ6VnvTjs5MFypa\n3NMlH1OooXY0MzN9UpmA/7ET47DBYtipK7TV2VLfGQKBgQDQmHv+YUv9LCG+AE4s\nlmmBDpclJ4Imttoaty/j/+ziZgv54zBiJV6Cgp9DMoyl1nZuKmjO2pIZZNZEG1yb\nU/VVGeqCiNQtiKY5I+dou9kfmfDyiWjhRoo6P+xrAlaJZjxOAjlYRp9OO4CSgeBy\nogf2ezGulw7vUJvg9R+mv56ZawKBgQCzfS1T/ZfnmnM3Hnxur9biyhI4dr4iCHgG\nzL1cfMc64Ki7CdS8K4XqzWnl7u/vTsFwtnoGnlobDWVBLGTtgwG2xfPMEX1kkpZa\nS0GPXgfw84W65JargaBiCcgLk5Iina8SbI82/HEfK359OXL9VIT+7K4o/PWopYCU\nI3PueP9jDwKBgAFvezL18nLskdLf56nQ4yr7MbkUu4WOrpOOfSrhgPzR3PU8RgH2\n5d6aDYSF3YfqoOgDLLN3t5erJPpVAd8jxcVvJzj8vhEOJq6v+5Cx8j2QbqMMpQcf\n2CWsePEHlBbf+DxFMaXU3pXIPQtFD4laahmdIWtlKLom4Nu8dBpOqUW/AoGBAJKM\nIOdTxaRG+c1GKeihzzyKKkhJw0G8UmuS3QVXp79N0NYt17DvhD7LCqy+4s4sKkqh\n4o3m2sD9te95lMpLzalyhHTYzt2/xdzhVxz8dFVqL4UBwHc1tkJ1pAy3p3h18IyF\ncIU/pgkCFRufPWDTkmclzakPqG/S+WF6dNx/lF1HAoGAejimfb2ehMx5mkos1cdI\n958yEdw1aernGV2u8pFLPF4WCzGDecHKubtIkzOC2Y9vwK/L03iwemh1pGLdXXCY\nHyG54ueTJawBOzWeposLG+ilhgsNbPiMW8WGBIzCoJdX5BlQj4qIQaLvKs8+wBy7\nN/lIjRGIlvMnEeIg+J8va3k=\n-----END PRIVATE KEY-----\n", 6 | "client_email": "sa-key@security-monkey-test-project.iam.gserviceaccount.com", 7 | "client_id": "915386704809902483491", 8 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 9 | "token_uri": "https://accounts.google.com/o/oauth2/token", 10 | "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", 11 | "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/sa-key%40security-monkey-test-project.iam.gserviceaccount.com" 12 | } 13 | -------------------------------------------------------------------------------- /cloudaux/tests/gcp/test_auth.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.tests.gcp.test-auth 3 | :platform: Unix 4 | :copyright: (c) 2016 by Google Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Tom Melendez (@supertom) 7 | """ 8 | import unittest 9 | import mock 10 | 11 | from apiclient.discovery import build, Resource 12 | from apiclient.http import HttpMock 13 | #from oauth2client.client import GoogleCredentials 14 | import os 15 | 16 | from cloudaux.gcp import auth 17 | 18 | 19 | class TestAuth(unittest.TestCase): 20 | 21 | def setUp(self): 22 | self.base_dir = os.getcwd() 23 | self.fixtures_dir = 'cloudaux/tests/gcp/fixtures' 24 | 25 | def _get_fixture(self, file_name): 26 | return os.path.join(self.base_dir, 27 | self.fixtures_dir, 28 | file_name) 29 | def _make_http(): 30 | import httplib2 31 | return mock.Mock(spec=httplib2.Http) 32 | 33 | def _make_credentials(): 34 | import google.auth.credentials 35 | return mock.Mock(spec=google.auth.credentials.Credentials) 36 | 37 | def test_get_available_clients(self): 38 | actual = auth.get_available_clients('foobaradfadfdfadf332343') 39 | self.assertFalse(actual) 40 | 41 | actual = auth.get_available_clients('gce') 42 | self.assertTrue(isinstance(actual, list)) 43 | self.assertTrue(isinstance(actual[0], dict)) 44 | self.assertTrue('client_type' in actual[0]) 45 | self.assertTrue('module_name' in actual[0]) 46 | 47 | def test_choose_client(self): 48 | actual = auth.choose_client('foobaradfadfdfadf332343') 49 | self.assertFalse(actual) 50 | 51 | actual = auth.choose_client('gce') 52 | self.assertTrue(isinstance(actual, dict)) 53 | self.assertTrue('client_type' in actual) 54 | self.assertTrue('module_name' in actual) 55 | 56 | def test__build_google_client(self): 57 | http_auth = HttpMock(self._get_fixture('compute.json'), {'status': '200'}) 58 | client = auth._build_google_client('compute', 'v1', http_auth=http_auth) 59 | self.assertTrue(hasattr(client, '__class__')) 60 | self.assertTrue(isinstance(client, Resource)) 61 | 62 | def test__googleauth(self): 63 | """ 64 | TODO(supertom): add mocking, make more robust, etc. 65 | This test make a lot of assumptions: 66 | 1. Running on GCE 67 | 3. Doesn't truly verify the Http object is authorized. 68 | However, this function is critical for valid GCP operation 69 | so it is good to have a sanity check that we have an Http object. 70 | """ 71 | from httplib2 import Http 72 | # default creds 73 | http_auth = auth._googleauth() 74 | self.assertTrue(isinstance(http_auth, Http)) 75 | 76 | # service account key 77 | test_key_file = self._get_fixture('testkey.json') 78 | http_auth = auth._googleauth(key_file=test_key_file) 79 | self.assertTrue(isinstance(http_auth, Http)) 80 | 81 | if __name__ == '__main__': 82 | unittest.main() 83 | -------------------------------------------------------------------------------- /cloudaux/tests/gcp/test_gcpcache.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import time 3 | 4 | from cloudaux.gcp.gcpcache import GCPCache 5 | 6 | class TestConnectionCache(unittest.TestCase): 7 | 8 | def test_get(self): 9 | myobj = object() 10 | c = GCPCache() 11 | 12 | key = 'strkey' 13 | self.assertTrue(c.insert(key, myobj)) 14 | 15 | actualobj = c.get(key) 16 | self.assertEqual(myobj, actualobj) 17 | 18 | def test_get_expired(self): 19 | myobj = object() 20 | c = GCPCache() 21 | # Check that an object is expired. Should return none 22 | key = 'strkey' 23 | exp_minutes = .000001 24 | self.assertTrue(c.insert(key, myobj, 25 | future_expiration_minutes=exp_minutes)) 26 | 27 | # sleep for two seconds to ensure the client is expired. 28 | time.sleep(2) 29 | self.assertFalse(c.get(key)) 30 | 31 | # delete_if_expired is True, so it should have been removed. 32 | self.assertTrue(key not in c._CACHE) 33 | 34 | def test_delete(self): 35 | myobj = object() 36 | c = GCPCache() 37 | key = 'strkey' 38 | self.assertTrue(c.insert(key, myobj)) 39 | 40 | self.assertTrue(c.delete(key)) 41 | 42 | # Verify that it is not present 43 | self.assertTrue(key not in c._CACHE) 44 | 45 | if __name__ == '__main__': 46 | unittest.main() 47 | -------------------------------------------------------------------------------- /cloudaux/tests/gcp/test_integration.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | 4 | from cloudaux.gcp.gcs import ( 5 | list_buckets, 6 | list_objects_in_bucket, 7 | ) 8 | from cloudaux.gcp.iam import get_project_iam_policy 9 | from cloudaux.gcp.gce.project import get_project 10 | from cloudaux.gcp.crm import get_iam_policy 11 | from cloudaux.gcp.gce.address import ( 12 | list_addresses, 13 | list_global_addresses, 14 | ) 15 | from cloudaux.gcp.gce.disk import ( 16 | list_disks, 17 | ) 18 | from cloudaux.gcp.gce.forwarding_rule import ( 19 | list_forwarding_rules, 20 | list_global_forwarding_rules, 21 | ) 22 | from cloudaux.gcp.gce.instance import ( 23 | list_instances 24 | ) 25 | from cloudaux.gcp.gce.zone import ( 26 | list_zones 27 | ) 28 | 29 | @pytest.fixture 30 | def project(): 31 | return os.getenv('CLOUDAUX_GCP_TEST_PROJECT') 32 | 33 | @pytest.mark.skipif( 34 | os.getenv('CLOUDAUX_GCP_TEST_PROJECT') is None, 35 | reason="Cannot run integration tests unless GCP project configured" 36 | ) 37 | @pytest.mark.parametrize('function,p_param', [ 38 | (list_addresses, 'project'), 39 | (list_forwarding_rules, 'project'), 40 | (list_global_addresses, 'project'), 41 | (list_global_forwarding_rules, 'project'), 42 | (get_iam_policy, 'resource'), 43 | (get_project, 'project'), 44 | (get_project_iam_policy, 'resource'), 45 | ]) 46 | def test_cloudaux_gcp_global_integration(function, p_param, project): 47 | result = function(**{p_param: project}) 48 | assert result is not None 49 | 50 | @pytest.mark.skipif( 51 | os.getenv('CLOUDAUX_GCP_TEST_PROJECT') is None, 52 | reason="Cannot run integration tests unless GCP project configured" 53 | ) 54 | @pytest.mark.parametrize('function,p_param,z_param', [ 55 | (list_disks, 'project', 'zone'), 56 | (list_instances, 'project', 'zone'), 57 | ]) 58 | def test_cloudaux_gcp_zoned_integration(function, p_param, z_param, project): 59 | for zone in list_zones(project=project): 60 | result = function(**{p_param: project, z_param: zone['name']}) 61 | assert result is not None 62 | 63 | @pytest.mark.skipif( 64 | os.getenv('CLOUDAUX_GCP_TEST_PROJECT') is None, 65 | reason="Cannot run integration tests unless GCP project configured" 66 | ) 67 | def test_cloudaux_gcs(project): 68 | for bucket in list_buckets(project=project): 69 | for bucket_object in list_objects_in_bucket(Bucket=bucket['name']): 70 | assert bucket_object is not None 71 | -------------------------------------------------------------------------------- /cloudaux/tests/gcp/test_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.tests.gcp.test-utils 3 | :platform: Unix 4 | :copyright: (c) 2016 by Google Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Tom Melendez (@supertom) 7 | """ 8 | import unittest 9 | 10 | from cloudaux.gcp import utils 11 | 12 | 13 | class TestUtils(unittest.TestCase): 14 | def test_get_creds_from_kwargs(self): 15 | data = {'project': 'my-project', 16 | 'key_file': '/path/to/myfile.json', 17 | 'foo': 'bar', 18 | 'user_agent': 'cloudaux'} 19 | expected_creds = { 20 | 'api_version': 'v1', 21 | 'project': 'my-project', 22 | 'key_file': '/path/to/myfile.json', 23 | 'http_auth': None, 24 | 'user_agent': 'cloudaux' 25 | } 26 | expected_kwargs = {'project': 'my-project', 'foo': 'bar'} 27 | actual_creds, actual_kwargs = utils.get_creds_from_kwargs(data) 28 | self.assertEqual(expected_creds, actual_creds) 29 | self.assertEqual(expected_kwargs, actual_kwargs) 30 | 31 | def test_rewrite_kwargs(self): 32 | data = {'project': 'my-project', 'foo': 'bar'} 33 | expected_general = {'name': 'projects/my-project', 'foo': 'bar'} 34 | actual_general = utils.rewrite_kwargs('general', data) 35 | self.assertEqual(expected_general, actual_general) 36 | 37 | data = {'project': 'my-project', 'foo': 'bar'} 38 | expected_cloud_storage = {'foo': 'bar'} 39 | actual_cloud_storage = utils.rewrite_kwargs('cloud', data, 40 | module_name='storage') 41 | self.assertEqual(expected_cloud_storage, actual_cloud_storage) 42 | 43 | data = {'foo': 'bar'} 44 | expected_no_change = {'foo': 'bar'} 45 | actual_no_change = utils.rewrite_kwargs('cloud', data, 46 | module_name='storage') 47 | self.assertEqual(expected_no_change, actual_no_change) 48 | 49 | data = {'foo': 'bar'} 50 | expected_no_change = {'foo': 'bar'} 51 | actual_no_change = utils.rewrite_kwargs('general', data) 52 | self.assertEqual(expected_no_change, actual_no_change) 53 | 54 | 55 | if __name__ == '__main__': 56 | unittest.main() 57 | -------------------------------------------------------------------------------- /cloudaux/tests/openstack/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/cloudaux/39d1dc11ccd794f21d95d3d1458abd94425b0927/cloudaux/tests/openstack/__init__.py -------------------------------------------------------------------------------- /cloudaux/tests/openstack/fixtures/test-clouds.yaml: -------------------------------------------------------------------------------- 1 | clouds: 2 | mycloud: 3 | auth: 4 | auth_url: 'https://localhost/v2.0/' 5 | project_name: test 6 | username: secmonkey 7 | password: foo 8 | regions: 9 | - RegionOne 10 | - RegionTwo 11 | -------------------------------------------------------------------------------- /cloudaux/tests/openstack/mock_decorators.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.tests.openstack.mock_decorators 3 | :platform: Unix 4 | :copyright: Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. See AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Michael Stair 7 | """ 8 | from functools import wraps 9 | 10 | 11 | def mock_get_regions(cloud_name, yaml_file): 12 | return [ {'name':'RegionOne'} ] 13 | 14 | def mock_openstack_conn(): 15 | def decorator(f): 16 | @wraps(f) 17 | def decorated_function(*args, **kwargs): 18 | kwargs['conn'] = None 19 | return f(*args, **kwargs) 20 | return decorated_function 21 | return decorator 22 | 23 | 24 | def mock_iter_account_region(account_regions): 25 | def decorator(func): 26 | @wraps(func) 27 | def decorated_function(*args, **kwargs): 28 | results = [] 29 | kwargs['account_name'] = 'TEST_ACCOUNT' 30 | kwargs['cloud_name'] = 'foo' 31 | kwargs['yaml_file'] = 'bar' 32 | kwargs['region'] = 'RegionOne' 33 | results.append(func(*args, **kwargs)) 34 | return results 35 | return decorated_function 36 | return decorator 37 | -------------------------------------------------------------------------------- /cloudaux/tests/openstack/mock_object_container.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.tests.openstack.mock_object_container 3 | :platform: Unix 4 | :copyright: Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. See AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Michael Stair 7 | """ 8 | def mock_get_container_metadata(conn=None, **kwargs): 9 | from openstack.object_store.v1.container import Container 10 | from cloudaux.tests.openstack.mock_utils import container_body, container_headers 11 | 12 | body_plus_headers = dict(container_body, **container_headers) 13 | 14 | container = Container(body_plus_headers) 15 | return container 16 | -------------------------------------------------------------------------------- /cloudaux/tests/openstack/mock_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.tests.openstack.mock_utils 3 | :platform: Unix 4 | :copyright: Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. See AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Michael Stair 7 | """ 8 | from mock import patch 9 | from openstack import connection 10 | 11 | 12 | """ Utilizes the OpenStack SDK unit test data. Avoids duplication, but does couple """ 13 | 14 | def mock_list_items(conn=None, **kwargs): 15 | with patch('openstack.connection.Connection'): 16 | service = kwargs['service'] 17 | generator = kwargs['generator'] 18 | f = 'mock_list_%s'%generator 19 | 20 | return globals()[f](conn, **kwargs) 21 | 22 | def mock_list_networks(conn=None, **kwargs): 23 | with patch('openstack.connection.Connection'): 24 | from openstack.network.v2.network import Network 25 | from openstack.tests.unit.network.v2.test_network import EXAMPLE 26 | 27 | conn = connection.Connection() 28 | conn.network.networks.side_effect = [ [Network(**EXAMPLE)] ] 29 | return [x for x in conn.network.networks()] 30 | 31 | def mock_list_ports(conn=None, **kwargs): 32 | with patch('openstack.connection.Connection'): 33 | from openstack.network.v2.port import Port 34 | from openstack.tests.unit.network.v2.test_port import EXAMPLE 35 | 36 | conn = connection.Connection() 37 | conn.network.ports.side_effect = [ [Port(**EXAMPLE)] ] 38 | return [x for x in conn.network.ports()] 39 | 40 | def mock_list_subnets(conn=None, **kwargs): 41 | with patch('openstack.connection.Connection'): 42 | from openstack.network.v2.subnet import Subnet 43 | from openstack.tests.unit.network.v2.test_subnet import EXAMPLE 44 | 45 | conn = connection.Connection() 46 | conn.network.subnets.side_effect = [ [Subnet(**EXAMPLE)] ] 47 | return [x for x in conn.network.subnets()] 48 | 49 | def mock_list_routers(conn=None, **kwargs): 50 | with patch('openstack.connection.Connection'): 51 | from openstack.network.v2.router import Router 52 | from openstack.tests.unit.network.v2.test_router import EXAMPLE 53 | 54 | conn = connection.Connection() 55 | conn.network.routers.side_effect = [ [Router(**EXAMPLE)] ] 56 | return [x for x in conn.network.routers()] 57 | 58 | def mock_list_ips(conn=None, **kwargs): 59 | with patch('openstack.connection.Connection'): 60 | from openstack.network.v2.floating_ip import FloatingIP 61 | from openstack.tests.unit.network.v2.test_floating_ip import EXAMPLE 62 | 63 | conn = connection.Connection() 64 | conn.network.ips.side_effect = [ [FloatingIP(**EXAMPLE)] ] 65 | return [x for x in conn.network.ips()] 66 | 67 | def mock_list_security_groups(conn=None, **kwargs): 68 | with patch('openstack.connection.Connection'): 69 | from openstack.network.v2.security_group import SecurityGroup 70 | from openstack.tests.unit.network.v2.test_security_group import EXAMPLE 71 | 72 | conn = connection.Connection() 73 | conn.network.security_groups.side_effect = [ [SecurityGroup(**EXAMPLE)] ] 74 | return [x for x in conn.network.security_groups()] 75 | 76 | def mock_list_servers(conn=None, **kwargs): 77 | with patch('openstack.connection.Connection'): 78 | from openstack.compute.v2.server import Server 79 | from openstack.tests.unit.compute.v2.test_server import EXAMPLE 80 | 81 | # The server example has an invalid security_group list 82 | server = Server(**EXAMPLE) 83 | server.security_groups=[{u'name': u'default'}] 84 | 85 | conn = connection.Connection() 86 | conn.compute.servers.side_effect = [ [server] ] 87 | return [x for x in conn.compute.servers()] 88 | 89 | def mock_list_images(conn=None, **kwargs): 90 | with patch('openstack.connection.Connection'): 91 | from openstack.compute.v2.image import Image 92 | from openstack.tests.unit.compute.v2.test_image import DETAIL_EXAMPLE 93 | 94 | conn = connection.Connection() 95 | conn.compute.images.side_effect = [ [Image(**DETAIL_EXAMPLE)] ] 96 | return [x for x in conn.compute.images()] 97 | 98 | def mock_list_users(conn=None, **kwargs): 99 | with patch('openstack.connection.Connection'): 100 | from openstack.identity.v3.user import User 101 | from openstack.tests.unit.identity.v3.test_user import EXAMPLE 102 | 103 | conn = connection.Connection() 104 | conn.identity.users.side_effect = [ [User(**EXAMPLE)] ] 105 | return [x for x in conn.identity.users()] 106 | 107 | def mock_list_load_balancers(conn=None, **kwargs): 108 | with patch('openstack.connection.Connection'): 109 | from openstack.load_balancer.v2.load_balancer import LoadBalancer 110 | from openstack.tests.unit.load_balancer.test_load_balancer import EXAMPLE 111 | 112 | conn = connection.Connection() 113 | conn.load_balancer.load_balancers.side_effect = [ [LoadBalancer(**EXAMPLE)] ] 114 | return [x for x in conn.load_balancer.load_balancers()] 115 | 116 | container_body = { 117 | 'count': 2, 118 | 'bytes': 630666, 119 | 'name': 'test', 120 | } 121 | 122 | container_headers = { 123 | 'x-container-object-count': '2', 124 | 'x-container-read': 'read-settings', 125 | 'x-container-write': 'write-settings', 126 | 'x-container-sync-to': 'sync-to', 127 | 'x-container-sync-key': 'sync-key', 128 | 'x-container-bytes-used': '630666', 129 | 'x-versions-location': 'versions-location', 130 | 'content-type': 'application/json; charset=utf-8', 131 | 'x-timestamp': '1453414055.48672' 132 | } 133 | 134 | def mock_list_containers(conn=None, **kwargs): 135 | with patch('openstack.connection.Connection'): 136 | from openstack.object_store.v1.container import Container 137 | 138 | body_plus_headers = dict(container_body, **container_headers) 139 | 140 | conn = connection.Connection() 141 | conn.object_store.containers.side_effect = [ [Container(body_plus_headers)] ] 142 | return [x for x in conn.object_store.containers()] 143 | -------------------------------------------------------------------------------- /cloudaux/tests/openstack/test_conn.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: cloudaux.tests.openstack.test-conn 3 | :platform: Unix 4 | :copyright: Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. See AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Michael Stair 7 | """ 8 | import os 9 | import unittest 10 | import mock 11 | 12 | from cloudaux.openstack.decorators import _connect, get_regions 13 | 14 | class TestConn(unittest.TestCase): 15 | 16 | def setUp(self): 17 | self.base_dir = os.getcwd() 18 | self.fixtures_dir = 'cloudaux/tests/openstack/fixtures' 19 | 20 | def _get_fixture(self, file_name): 21 | return os.path.join(self.base_dir, 22 | self.fixtures_dir, 23 | file_name) 24 | def test_connect(self): 25 | conn = _connect("mycloud", "RegionOne", self._get_fixture("test-clouds.yaml") ) 26 | self.assertEqual(conn.name, "mycloud") 27 | 28 | def test_get_regions(self): 29 | regions = [{'name': u'RegionOne', 'values': {}}, {'name': u'RegionTwo', 'values': {}}] 30 | self.assertEqual(regions, get_regions("mycloud", self._get_fixture("test-clouds.yaml") ) ) 31 | 32 | if __name__ == '__main__': 33 | unittest.main() 34 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | 4 | [wheel] 5 | universal = 0 6 | 7 | [egg_info] 8 | tag_build = 9 | tag_date = 0 10 | tag_svn_revision = 0 11 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | cloudaux 3 | ===== 4 | 5 | Cloud Auxiliary is a python wrapper and orchestration module for interacting with cloud providers 6 | 7 | :copyright: (c) 2016 by Netflix, see AUTHORS for more 8 | :license: Apache, see LICENSE for more details. 9 | """ 10 | import sys 11 | import os.path 12 | 13 | from setuptools import setup, find_packages 14 | 15 | ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__))) 16 | 17 | # When executing the setup.py, we need to be able to import ourselves. This 18 | # means that we need to add the src/ directory to the sys.path 19 | 20 | sys.path.insert(0, ROOT) 21 | 22 | about = {} 23 | with open(os.path.join(ROOT, "cloudaux", "__about__.py")) as f: 24 | exec(f.read(), about) 25 | 26 | classifiers = [ 27 | 'Programming Language :: Python :: 3', 28 | 'Programming Language :: Python :: 3.6', 29 | 'Programming Language :: Python :: 3.7', 30 | 'Programming Language :: Python :: 3.8', 31 | ] 32 | 33 | install_requires = [ 34 | 'boto3', 35 | 'botocore', 36 | 'boto>=2.41.0', 37 | 'joblib>=0.9.4', 38 | 'inflection', 39 | 'flagpole>=1.0.1', 40 | 'defusedxml', 41 | 'six>=1.11.0', 42 | ] 43 | 44 | gcp_require = [ 45 | 'google-api-python-client>=1.6.1', 46 | 'google-cloud-storage==0.22.0', 47 | 'oauth2client>=4.1.2' 48 | ] 49 | 50 | openstack_require = [ 51 | 'openstacksdk>=0.13.0' 52 | ] 53 | 54 | tests_require = [ 55 | 'pytest', 56 | 'pytest-cov', 57 | 'moto', 58 | 'mock', 59 | 'coveralls', 60 | 'tox', 61 | 'flake8' 62 | ] 63 | 64 | docs_require = [] 65 | 66 | dev_require = [] 67 | 68 | setup( 69 | name=about["__title__"], 70 | version=about["__version__"], 71 | author=about["__author__"], 72 | author_email=about["__email__"], 73 | url=about["__uri__"], 74 | description=about["__summary__"], 75 | long_description=open(os.path.join(ROOT, 'README.md')).read(), 76 | long_description_content_type="text/markdown", 77 | packages=find_packages(), 78 | include_package_data=True, 79 | zip_safe=False, 80 | install_requires=install_requires, 81 | classifiers=classifiers, 82 | extras_require={ 83 | 'gcp': gcp_require, 84 | 'openstack': openstack_require, 85 | 'tests': tests_require, 86 | 'docs': docs_require, 87 | 'dev': dev_require 88 | } 89 | ) 90 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,py37 3 | 4 | [testenv] 5 | usedevelop = True 6 | passenv = TRAVIS TRAVIS_* GOOGLE_APPLICATION_CREDENTIALS 7 | 8 | setenv = 9 | PYTEST_ADDOPTS = --junitxml=test-reports/{envname}/junit.xml -vv 10 | BOTO_CONFIG = /dev/null 11 | AWS_SECRET_ACCESS_KEY = foobar_secret 12 | AWS_ACCESS_KEY_ID = foobar_key 13 | 14 | deps = 15 | ; This is needed because managed policy mocks aren't in the release just yet: 16 | git+https://github.com/mikegrima/moto.git@roletags#egg=moto 17 | 18 | mock 19 | pytest 20 | coveralls 21 | .[tests] 22 | .[gcp] 23 | .[openstack] 24 | 25 | commands = 26 | pytest {posargs} --cov=cloudaux cloudaux/tests/cloudaux cloudaux/tests/aws cloudaux/tests/openstack cloudaux/tests/gcp 27 | ; flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics 28 | ; flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 29 | 30 | 31 | ;[testenv:linters] 32 | ;basepython = python3 33 | ;usedevelop = true 34 | ;deps = 35 | ; {[testenv:flake8]deps} 36 | ; {[testenv:pylint]deps} 37 | ; {[testenv:setuppy]deps} 38 | ; {[testenv:bandit]deps} 39 | ;commands = 40 | ; {[testenv:flake8]commands} 41 | ; {[testenv:pylint]commands} 42 | ; {[testenv:setuppy]commands} 43 | ; {[testenv:bandit]commands} 44 | 45 | ;[testenv:flake8] 46 | ;basepython = python3 47 | ;skip_install = true 48 | ;deps = 49 | ; flake8 50 | ; flake8-docstrings>=0.2.7 51 | ; flake8-import-order>=0.9 52 | ;commands = 53 | ; flake8 cloudaux setup.py test 54 | 55 | ;[testenv:pylint] 56 | ;basepython = python3 57 | ;skip_install = false 58 | ;deps = 59 | ; pyflakes 60 | ; pylint 61 | ;commands = 62 | ; pylint --rcfile={toxinidir}/.pylintrc cloudaux 63 | 64 | [testenv:setuppy] 65 | basepython = python3 66 | skip_install = true 67 | deps = 68 | commands = 69 | python setup.py check -m -s 70 | 71 | ;[testenv:bandit] 72 | ;basepython = python3 73 | ;skip_install = true 74 | ;deps = 75 | ; bandit 76 | ;commands = 77 | ; bandit --ini tox.ini -r cloudaux 78 | 79 | ;[bandit] 80 | ;skips = B101 81 | ; 82 | ;[flake8] 83 | ;ignore = E501,I100,D205,D400,D401,I202,R0913,C901 84 | ;exclude = 85 | ; *.egg-info, 86 | ; *.pyc, 87 | ; .cache, 88 | ; .coverage.*, 89 | ; .gradle, 90 | ; .tox, 91 | ; build, 92 | ; dist, 93 | ; htmlcov.* 94 | ; *-cookiecutter 95 | ;max-complexity = 10 96 | ;import-order-style = google 97 | ;application-import-names = flake8 98 | 99 | [pytest] 100 | norecursedirs=.* 101 | --------------------------------------------------------------------------------