├── LICENSE ├── README.md ├── api.py ├── api_signer.py ├── aws-api-definition.json ├── aws_api_shapeshifter.py ├── operation_obj.py ├── protocol_formatter.py └── requirements.txt /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Nick Frichette 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # aws_api_shapeshifter 2 | A small library to alter AWS API requests; Used for fuzzing research 3 | 4 | ## NOTE 5 | This tool was used to identify two XSS vulnerabilities in the [AWS Console](https://frichetten.com/blog/xss_in_aws_console/) and is based off previous work in [silently enumerating API permissions](https://frichetten.com/blog/aws-api-enum-vuln/). It is also what almost got me stuck with a $3,000/month bill (thank you to AWS for getting me out of that). 6 | 7 | **This software is VERY hacky and should be considered alpha-quality at best**. It was originally designed with a very different intention and grew from there. If you are intendeding to use it I would advise you to intercept all traffic (https_proxy) with something like [Burp Suite](https://frichetten.com/blog/aws-api-enum-vuln/) so that you can inspect it. 8 | 9 | This library is **NOT** maintained and is provided as is. I realized it was marked as private and didn't see a reason not to make it public. 10 | 11 | **DO NOT BLINDLY FUZZ EVERY API ACTION. YOU WILL END UP WITH A HEFTY BILL. I DO NOT TAKE RESPONSIBILITY FOR ANY COSTS ASSOCIATED WITH USING THIS SOFTWARE.** 12 | 13 | ## How to Use 14 | There are some examples in [this](https://twitter.com/Frichette_n/status/1492707114250383368?s=20&t=7h1yvuHaKwVu60q9e2Y_uQ) Twitter thread. 15 | 16 | ## Currently Supported Parameter Options 17 | * content_type (String) 18 | * noparams (Boolean) 19 | * creds (Credentials object) 20 | * host (String) 21 | * param (dictionary for params) 22 | * protocol (json | ec2 | query | rest-json | rest-xml) 23 | -------------------------------------------------------------------------------- /api.py: -------------------------------------------------------------------------------- 1 | import api_signer 2 | import protocol_formatter 3 | import operation_obj 4 | 5 | 6 | SAFE_REGIONS = [ 7 | "af-south-1", 8 | "ap-east-1", 9 | "ap-northeast-1", 10 | "ap-northeast-2", 11 | "ap-northeast-3", 12 | "ap-south-1", 13 | "ap-southeast-1", 14 | "ap-southeast-2", 15 | "ca-central-1", 16 | "eu-central-1", 17 | "eu-north-1", 18 | "eu-south-1", 19 | "eu-west-1", 20 | "eu-west-2", 21 | "eu-west-3", 22 | "me-south-1", 23 | "sa-east-1", 24 | "us-east-1", 25 | "us-east-2", 26 | "us-west-1", 27 | "us-west-2", 28 | ] 29 | 30 | 31 | class API: 32 | 33 | def __init__(self, key, service_definition): 34 | self._definition = service_definition 35 | self._latest_version = self.latest_version() 36 | self.protocol = self.latest()['metadata']['protocol'] 37 | self.endpoints = self.latest()['endpoints'] 38 | self.operations = self._make_operations_list( 39 | self.latest()['metadata'], 40 | self.latest()['endpoints'], 41 | self.latest()['shapes'], 42 | self.latest()['operations'] 43 | ) 44 | 45 | 46 | # Returns the definiton of the lastest api version 47 | def latest(self): 48 | # Find the most recent api version 49 | # We can do this by sorting keys 50 | options = list(self._definition.keys()) 51 | options.sort() 52 | return self._definition[list(reversed(options))[0]] 53 | 54 | 55 | # Returns the latest api version 56 | def latest_version(self): 57 | return list(self._definition.keys())[0] 58 | 59 | 60 | # Returns a list of the available API versions 61 | def api_versions(self): 62 | return list(self._definition.keys()) 63 | 64 | 65 | # Returns a list of all operations 66 | def list_operations(self): 67 | return list(self.operations.keys()) 68 | 69 | 70 | # Returns the protocol 71 | def get_protocol(self): 72 | return self.protocol 73 | 74 | 75 | # Returns a dictionary with all the available operations 76 | def _make_operations_list(self, metadata, endpoints, shapes, operations): 77 | to_return = {} 78 | for operation_name, operation in operations.items(): 79 | to_return[operation_name] = operation_obj.Operation(metadata, endpoints, shapes, operation) 80 | 81 | return to_return 82 | -------------------------------------------------------------------------------- /api_signer.py: -------------------------------------------------------------------------------- 1 | import sys, os, base64, datetime, hashlib, hmac 2 | import requests 3 | 4 | from urllib3.exceptions import InsecureRequestWarning 5 | # Suppress only the single warning from urllib3 needed. 6 | requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) 7 | 8 | def _sign(key, msg): 9 | return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest() 10 | 11 | 12 | def _getSignatureKey(key, date_stamp, regionName, serviceName): 13 | kDate = _sign(('AWS4' + key).encode('utf-8'), date_stamp) 14 | kRegion = _sign(kDate, regionName) 15 | kService = _sign(kRegion, serviceName) 16 | kSigning = _sign(kService, 'aws4_request') 17 | return kSigning 18 | 19 | 20 | def query_signer(credentials, method, endpoint_prefix, 21 | host, region, endpoint, 22 | request_uri, formatted_request): 23 | 24 | request_parameters = formatted_request['body'] 25 | 26 | t = datetime.datetime.utcnow() 27 | date_stamp = t.strftime('%Y%m%d') 28 | 29 | canonical_uri = request_uri 30 | 31 | ## Step 3: Create the canonical query string. In this example, request 32 | # parameters are passed in the body of the request and the query string 33 | # is blank. 34 | canonical_querystring = '' 35 | 36 | canonical_headers = _build_canonical_headers(formatted_request['headers']) 37 | 38 | signed_headers = _build_signed_headers(formatted_request['headers']) 39 | 40 | payload_hash = hashlib.sha256(request_parameters.encode('utf-8')).hexdigest() 41 | 42 | canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash 43 | 44 | algorithm = 'AWS4-HMAC-SHA256' 45 | credential_scope = date_stamp + '/' + region + '/' + endpoint_prefix + '/' + 'aws4_request' 46 | string_to_sign = algorithm + '\n' + formatted_request['headers']['X-Amz-Date'] + '\n' + credential_scope + '\n' + hashlib.sha256(canonical_request.encode('utf-8')).hexdigest() 47 | 48 | signing_key = _getSignatureKey(credentials.secret_key, date_stamp, region, endpoint_prefix) 49 | 50 | signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), hashlib.sha256).hexdigest() 51 | 52 | authorization_header = algorithm + ' ' + 'Credential=' + credentials.access_key + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature 53 | 54 | headers = formatted_request['headers'] 55 | headers['Authorization'] = authorization_header 56 | 57 | r = requests.request(method, endpoint, data=request_parameters, headers=headers) 58 | headers.pop("Authorization") 59 | 60 | return r 61 | 62 | 63 | def json_signer(credentials, method, endpoint_prefix, 64 | host, region, endpoint, signing_name, 65 | request_uri, formatted_request): 66 | 67 | request_parameters = formatted_request['body'] 68 | 69 | t = datetime.datetime.utcnow() 70 | date_stamp = t.strftime('%Y%m%d') 71 | 72 | canonical_uri = request_uri 73 | 74 | ## Step 3: Create the canonical query string. In this example, request 75 | # parameters are passed in the body of the request and the query string 76 | # is blank. THIS BREAKS SIGNING. SIGN THIS STRING TO FIND THAT WEIRD THING 77 | canonical_querystring = '' 78 | 79 | canonical_headers = _build_canonical_headers(formatted_request['headers']) 80 | 81 | signed_headers = _build_signed_headers(formatted_request['headers']) 82 | 83 | payload_hash = hashlib.sha256(request_parameters.encode('utf-8')).hexdigest() 84 | 85 | canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash 86 | 87 | algorithm = 'AWS4-HMAC-SHA256' 88 | credential_scope = date_stamp + '/' + region + '/' + signing_name + '/' + 'aws4_request' 89 | string_to_sign = algorithm + '\n' + formatted_request['headers']['X-Amz-Date'] + '\n' + credential_scope + '\n' + hashlib.sha256(canonical_request.encode('utf-8')).hexdigest() 90 | 91 | signing_key = _getSignatureKey(credentials.secret_key, date_stamp, region, signing_name) 92 | 93 | signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), hashlib.sha256).hexdigest() 94 | 95 | authorization_header = algorithm + ' ' + 'Credential=' + credentials.access_key + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature 96 | 97 | headers = formatted_request['headers'] 98 | headers['Authorization'] = authorization_header 99 | 100 | r = requests.request(method, endpoint+canonical_uri, data=request_parameters, headers=headers) 101 | headers.pop("Authorization") 102 | 103 | return r 104 | 105 | def rest_json_signer(credentials, method, endpoint_prefix, 106 | host, region, endpoint, signing_name, 107 | request_uri, formatted_request): 108 | 109 | # For json we supply {} 110 | # For rest-json we do NOT provide {} 111 | request_parameters = formatted_request['body'] 112 | if request_parameters == "{}": 113 | request_parameters = "" 114 | 115 | t = datetime.datetime.utcnow() 116 | date_stamp = t.strftime('%Y%m%d') 117 | 118 | # TODO: UGLY HACK PLEASE FIX 119 | if "prod" in host: 120 | canonical_uri_in_request = "/prod" + request_uri 121 | else: 122 | canonical_uri_in_request = request_uri 123 | 124 | canonical_uri = request_uri 125 | 126 | ## Step 3: Create the canonical query string. In this example, request 127 | # parameters are passed in the body of the request and the query string 128 | # is blank. THIS BREAKS SIGNING. SIGN THIS STRING TO FIND THAT WEIRD THING 129 | canonical_querystring = '' 130 | 131 | canonical_headers = _build_canonical_headers(formatted_request['headers']) 132 | 133 | signed_headers = _build_signed_headers(formatted_request['headers']) 134 | 135 | payload_hash = hashlib.sha256(request_parameters.encode('utf-8')).hexdigest() 136 | 137 | canonical_request = method + '\n' + canonical_uri_in_request + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash 138 | 139 | algorithm = 'AWS4-HMAC-SHA256' 140 | credential_scope = date_stamp + '/' + region + '/' + signing_name + '/' + 'aws4_request' 141 | string_to_sign = algorithm + '\n' + formatted_request['headers']['X-Amz-Date'] + '\n' + credential_scope + '\n' + hashlib.sha256(canonical_request.encode('utf-8')).hexdigest() 142 | 143 | signing_key = _getSignatureKey(credentials.secret_key, date_stamp, region, signing_name) 144 | 145 | signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), hashlib.sha256).hexdigest() 146 | 147 | authorization_header = algorithm + ' ' + 'Credential=' + credentials.access_key + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature 148 | 149 | headers = formatted_request['headers'] 150 | headers['Authorization'] = authorization_header 151 | 152 | r = requests.request(method, endpoint+canonical_uri, data=request_parameters, headers=headers, verify=False) 153 | headers.pop("Authorization") 154 | 155 | return r 156 | 157 | # FUZZ IDEA: Canonical headers must be sorted alphabetically 158 | # What if we didn't do that 159 | def _build_canonical_headers(headers): 160 | headers_string = "" 161 | header_list = list(headers.keys()) 162 | header_list.sort() 163 | for header in header_list: 164 | headers_string += header.lower() + ":" + headers[header] + "\n" 165 | 166 | return headers_string 167 | 168 | 169 | # FUZZ IDEA: Signed Headers must be sorted alphabetically 170 | # What if we didn't do that 171 | def _build_signed_headers(headers): 172 | headers_string = "" 173 | header_list = list(headers.keys()) 174 | header_list.sort() 175 | for header in header_list: 176 | headers_string += header.lower() + ";" 177 | # Remove trailing ';' 178 | headers_string = headers_string[:-1] 179 | 180 | return headers_string 181 | -------------------------------------------------------------------------------- /aws_api_shapeshifter.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import api 4 | 5 | DEFAULT_DEF_LOCATION = "./aws-api-definition.json" 6 | 7 | 8 | def new(definition=DEFAULT_DEF_LOCATION): 9 | data = __read_api_definition(definition) 10 | to_return = {} 11 | for key, service in data.items(): 12 | # Cludgy, but let's make sure it's a v4 service 13 | if service[list(service.keys())[0]]['metadata']['signatureVersion'] != "v4": 14 | continue 15 | 16 | to_return[key] = api.API(key, service) 17 | 18 | return to_return 19 | 20 | 21 | def convert_sts_to_cred_object(credentials): 22 | return Credentials( 23 | credentials['AccessKeyId'], 24 | credentials['SecretAccessKey'], 25 | credentials['SessionToken']) 26 | 27 | 28 | def __read_api_definition(definition): 29 | with open(definition, 'r') as r: 30 | return json.loads(''.join(r.read())) 31 | 32 | 33 | class Credentials(): 34 | 35 | def __init__(self, access_key, secret_key, token): 36 | self.access_key = access_key 37 | self.secret_key = secret_key 38 | self.token = token 39 | 40 | 41 | -------------------------------------------------------------------------------- /operation_obj.py: -------------------------------------------------------------------------------- 1 | from xeger import Xeger 2 | 3 | import api_signer 4 | import protocol_formatter 5 | import aws_api_shapeshifter 6 | 7 | """ The operation_obj will be the primary location where parameters are altered and configured. 8 | Every user generated modification will pass throug here. As a result this code is ugly and I'm 9 | only partially sorry. """ 10 | class Operation: 11 | 12 | def __init__(self, metadata, endpoints, shapes, operation): 13 | self.name = operation['name'] 14 | self.method = operation['http']['method'] 15 | self.request_uri = operation['http']['requestUri'] 16 | self.operation = operation 17 | self.endpoints = endpoints['endpoints'] 18 | self.metadata = metadata 19 | self.shapes = shapes 20 | self.endpoint_prefix = metadata['endpointPrefix'] 21 | self.target_prefix = self._resolve_target_prefix(self.metadata) 22 | self.shape_input = self._resolve_shape_input(operation) 23 | 24 | 25 | # make_request will take the requested modifications (if any) and make the request to the AWS API 26 | def make_request(self, **kwargs): 27 | name = self.name 28 | self.input_format = self._parse_input_shape(self.metadata['endpointPrefix'], self.shapes, self.operation, kwargs) 29 | version = self.metadata['apiVersion'] 30 | credentials = _resolve_credentials(kwargs) 31 | endpoint_prefix = self._resolve_endpoint_prefix(kwargs) 32 | method = self._resolve_method(kwargs) 33 | region = self._resolve_region(kwargs) 34 | host = self._resolve_host(region, endpoint_prefix, kwargs) 35 | endpoint = self._resolve_endpoint(host, kwargs) 36 | request_uri = self._resolve_request_uri(kwargs) 37 | 38 | # TODO: Clean this up 39 | if 'noparam' in kwargs.keys() or 'noparams' in kwargs.keys(): 40 | # TODO: Should be {} 41 | self.input_format = {} 42 | 43 | if 'protocol' in kwargs.keys(): 44 | protocol = kwargs['protocol'] 45 | else: 46 | protocol = self.metadata['protocol'] 47 | 48 | 49 | # Depending on the protocol we need to format inputs differently 50 | if protocol == "query": 51 | formatted_request = protocol_formatter.query_protocol_formatter( 52 | host, 53 | credentials.token, 54 | name, 55 | version, 56 | kwargs, 57 | self.input_format 58 | ) 59 | response = api_signer.query_signer( 60 | credentials, 61 | method, 62 | endpoint_prefix, 63 | host, 64 | region, 65 | endpoint, 66 | request_uri, 67 | formatted_request 68 | ) 69 | return response 70 | 71 | if protocol == "json": 72 | json_version = self._resolve_json_version(self.metadata) 73 | amz_target = self._resolve_target_prefix(self.metadata) 74 | amz_target += "." + self.name 75 | signing_name = self._resolve_signing_name(self.metadata, kwargs) 76 | 77 | formatted_request = protocol_formatter.json_protocol_formatter( 78 | host, 79 | credentials.token, 80 | json_version, 81 | amz_target, 82 | kwargs, 83 | self.input_format 84 | ) 85 | response = api_signer.json_signer( 86 | credentials, 87 | method, 88 | endpoint_prefix, 89 | host, 90 | region, 91 | endpoint, 92 | signing_name, 93 | request_uri, 94 | formatted_request 95 | ) 96 | return response 97 | 98 | if protocol == "rest-json": 99 | json_version = self._resolve_json_version(self.metadata) 100 | signing_name = self._resolve_signing_name(self.metadata, kwargs) 101 | formatted_request = protocol_formatter.rest_json_protocol_formatter( 102 | host, 103 | credentials.token, 104 | json_version, 105 | request_uri, 106 | kwargs, 107 | self.input_format 108 | ) 109 | response = api_signer.rest_json_signer( 110 | credentials, 111 | method, 112 | endpoint_prefix, 113 | host, 114 | region, 115 | endpoint, 116 | signing_name, 117 | formatted_request['request_uri'], 118 | formatted_request 119 | ) 120 | return response 121 | 122 | return None 123 | 124 | 125 | def _has_safe_regions(self, regions): 126 | # return true if it has safe regions 127 | # false if not 128 | for region in regions: 129 | if region in SAFE_REGIONS: 130 | return True 131 | return False 132 | 133 | 134 | def _get_safe_region(self, regions): 135 | for region in regions: 136 | if region in SAFE_REGIONS: 137 | return region 138 | 139 | 140 | def _test_hostname(self, hostname): 141 | try: 142 | socket.gethostbyname(hostname) 143 | return True 144 | except socket.error: 145 | return False 146 | 147 | 148 | def _resolve_endpoint_prefix(self, kwargs): 149 | if 'endpoint_prefix' in kwargs.keys(): 150 | return kwargs['endpoint_prefix'] 151 | 152 | return self.endpoint_prefix 153 | 154 | 155 | def _resolve_method(self, kwargs): 156 | if 'method' in kwargs.keys(): 157 | return kwargs['method'] 158 | 159 | return self.method 160 | 161 | def _resolve_signing_name(self, metadata, kwargs): 162 | if 'signing_name' in kwargs.keys(): 163 | return kwargs['signing_name'] 164 | 165 | # if we have a signing name 166 | if 'signingName' in metadata.keys(): 167 | return metadata['signingName'] 168 | 169 | # Give up and go with endpointPrefix 170 | return metadata['endpointPrefix'] 171 | 172 | def _resolve_region(self, kwargs): 173 | if 'region' in kwargs.keys(): 174 | return kwargs['region'] 175 | 176 | # ngl, gonna prefer us-east-1 177 | if 'us-east-1' in self.endpoints.keys(): 178 | return 'us-east-1' 179 | 180 | # Otherwise, pick a random region that they support 181 | # Need to check for a credential scope region 182 | potential = list(self.endpoints.keys())[0] 183 | if "credentialScope" in self.endpoints[potential].keys(): 184 | return self.endpoints[potential]['credentialScope']['region'] 185 | 186 | return potential 187 | 188 | 189 | def _resolve_host(self, region, endpoint_prefix, kwargs): 190 | if 'host' in kwargs.keys(): 191 | return kwargs['host'] 192 | 193 | # iam is an example of this - Need to check for a hostname for a region 194 | # not in the keys (aws-global) 195 | potential = list(self.endpoints.keys())[0] 196 | if 'credentialScope' in self.endpoints[potential].keys() and region == self.endpoints[potential]['credentialScope']['region']: 197 | return self.endpoints[potential]['hostname'] 198 | 199 | if 'hostname' in self.endpoints[region].keys(): 200 | return self.endpoints[region]['hostname'] 201 | 202 | # TODO: Check ,but I don't think we ever get here. 203 | # I know this is broken but will wait to fix until I have 204 | # more examples 205 | return endpoint_prefix + "." + region + ".amazonaws.com" 206 | 207 | 208 | def _resolve_endpoint(self, host, kwargs): 209 | if 'endpoint' in kwargs.keys(): 210 | return kwargs['endpoint'] 211 | 212 | return "https://" + host 213 | 214 | 215 | def _resolve_request_uri(self, kwargs): 216 | if 'request_uri' in kwargs.keys(): 217 | return kwargs['request_uri'] 218 | 219 | return self.request_uri 220 | 221 | 222 | def _resolve_target_prefix(self, metadata): 223 | if 'targetPrefix' in metadata.keys(): 224 | return metadata['targetPrefix'] 225 | else: 226 | return metadata['endpointPrefix'] 227 | 228 | 229 | def _resolve_json_version(self, metadata): 230 | if 'jsonVersion' in metadata.keys(): 231 | return metadata['jsonVersion'] 232 | 233 | # Otherwise you likely know what you want to do 234 | # In fact it's likely what you're testing 235 | # I'll give you a 1.0 so you don't complain 236 | return "1.0" 237 | 238 | 239 | def _resolve_shape_input(self, operation): 240 | if 'input' in operation.keys(): 241 | return operation['input']['shape'] 242 | return "" 243 | 244 | 245 | # GIVE THIS THE SHAPE, NOT THE NAME 246 | def _resolve_unknown_shape(self, shapes, unknown_shape): 247 | unknown_shape_type = unknown_shape['type'] 248 | if unknown_shape_type == 'string': 249 | return self._gen_string_shape(unknown_shape) 250 | if unknown_shape_type == 'integer': 251 | return 1 252 | if unknown_shape_type == 'boolean': 253 | return "false" 254 | if unknown_shape_type == 'structure': 255 | return self._resolve_structure(shapes, unknown_shape) 256 | if unknown_shape_type == 'list': 257 | return self._resolve_list(shapes, unknown_shape) 258 | if unknown_shape_type == 'timestamp': 259 | return self._resolve_timestamp(shapes, unknown_shape) 260 | if unknown_shape_type == 'blob': 261 | return self._resolve_blob(shapes, unknown_shape) 262 | if unknown_shape_type == 'long': 263 | return 1 264 | if unknown_shape_type == 'map': 265 | return "map" 266 | if unknown_shape_type == 'double' or unknown_shape_type == 'float': 267 | return 2.0 268 | # Map not implemented -Xray 269 | print(unknown_shape_type) 270 | 271 | 272 | def _resolve_structure(self, shapes, structure): 273 | to_return = {} 274 | for member in structure['members']: 275 | if 'required' in structure.keys() and member in structure['required']: 276 | shape_name = structure['members'][member]['shape'] 277 | to_return[member] = self._resolve_unknown_shape(shapes, shapes[shape_name]) 278 | return to_return 279 | 280 | 281 | def _resolve_list(self, shapes, list_shape): 282 | # This is an interesting problem. We should return this in a list 283 | # The reason being that NORMAL operations may have multiple items. 284 | # In our current form, we only give one. 285 | member_shape = list_shape['member']['shape'] 286 | #if 'locationName' in list_shape['member'].keys(): 287 | # location_name = list_shape['member']['locationName'] 288 | #else: 289 | # # Learned from elasticache RemoveTagsFromResource 290 | # location_name = 'member' 291 | result = self._resolve_unknown_shape(shapes, shapes[member_shape]) 292 | return [result] 293 | 294 | 295 | def _resolve_timestamp(self, shapes, timestamp): 296 | return "1615593755.796672" 297 | 298 | 299 | def _resolve_blob(self, shapes, blob): 300 | return "bbbbbbbbebfbebebbebebb" 301 | 302 | 303 | def _parse_input_shape(self, name, shapes, operation, kwargs): 304 | to_return = {} 305 | 306 | # We may have params in our kwargs and we need to process them 307 | # We are going to support a "Append" format 308 | # Not sure what that means right now. Going to work on it 309 | 310 | # Not every operation has an input 311 | if 'input' in operation.keys(): 312 | input_shape_name = operation['input']['shape'] 313 | shape = shapes[input_shape_name] 314 | 315 | if "required" in shape.keys(): 316 | # This is actual torture 317 | # TODO: Refactor 318 | for required in shape['required']: 319 | shape_name = shape['members'][required]['shape'] 320 | result = self._resolve_unknown_shape(shapes, shapes[shape_name]) 321 | to_return[required] = result 322 | 323 | # Parsing params for kwargs 324 | if 'params' in kwargs.keys(): 325 | passed_params = kwargs['params'] 326 | for key in passed_params.keys(): 327 | to_return[key] = passed_params[key] 328 | 329 | return to_return 330 | # Leaving this code here for now. Going to need to 331 | # improve this. Query protocol is an edge case 332 | # because it uses a weird naming format for 333 | # lists. Maybe move this into the query protocol 334 | # formatter? 335 | i = 2 336 | for item in values: 337 | if "." in item[0]: 338 | if "Key" in item[0]: 339 | new_name = item[0] + "." + str(i//2) 340 | new_name = new_name.replace(".Key.",".") + ".Key" 341 | elif "Value" in item[0]: 342 | new_name = item[0] + "." + str(i//2) 343 | new_name = item[0].replace(".Value.", ".") + ".Value" 344 | else: 345 | new_name = item[0] + "." + str(i//2) 346 | i += 1 347 | to_return[new_name] = item[1] 348 | else: 349 | to_return[item[0]] = item[1] 350 | 351 | return to_return 352 | 353 | return {} 354 | 355 | 356 | def _flatten_list(self, list_in): 357 | if isinstance(list_in, list): 358 | for l in list_in: 359 | for y in self._flatten_list(l): 360 | yield y 361 | 362 | else: 363 | yield list_in 364 | 365 | 366 | def _gen_string_shape(self, member_shape): 367 | # min, max, pattern, enum 368 | if "pattern" in member_shape.keys(): 369 | return self._gen_regex_pattern(member_shape['pattern']) 370 | if "enum" in member_shape.keys(): 371 | return member_shape['enum'][0] 372 | if "min" in member_shape.keys(): 373 | return 'a'*member_shape['min'] 374 | return 'aareturngen' 375 | 376 | 377 | def _gen_regex_pattern(self, pattern): 378 | # Some patterns break 379 | x = Xeger() 380 | try: 381 | result = x.xeger(pattern) 382 | return result 383 | except: 384 | return "aregex" 385 | 386 | 387 | def _resolve_credentials(kwargs): 388 | """ A user can send a few types of creds at us, and we 389 | have to be able to resolve them. Output should 390 | always be a credential object with .access_key, 391 | .secret_key, and .token accessible """ 392 | kwargs_keys = kwargs.keys() 393 | # Assume that access key modification is intentional and 394 | # is a higher priority 395 | if 'access_key' in kwargs_keys: 396 | return aws_api_shapeshifter.Credentials( 397 | kwargs['access_key'], 398 | kwargs['secret_key'], 399 | kwargs['token'] 400 | ) 401 | 402 | elif 'creds' in kwargs_keys: 403 | return kwargs['creds'] 404 | 405 | elif 'credentials' in kwargs_keys: 406 | return kwargs['credentials'] 407 | 408 | return aws_api_shapeshifter.Credentials("", "", "") 409 | 410 | 411 | def _resolve_region_hostname(endpoints, preferred=''): 412 | # If there is a hostname return in 413 | # otherwise just return the region 414 | None -------------------------------------------------------------------------------- /protocol_formatter.py: -------------------------------------------------------------------------------- 1 | import json 2 | import datetime 3 | import re 4 | 5 | from urllib.parse import urlencode 6 | 7 | # TODO: If we are using temp creds, need to include X-Amz-Security-Token header 8 | # Otherwise, don't include 9 | ALL_DEFAULT_HEADERS = { 10 | 'Host': '', 11 | 'X-Amz-Date': '', 12 | } 13 | 14 | QUERY_DEFAULT_HEADERS = { 15 | 'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8', 16 | } 17 | 18 | JSON_DEFAULT_HEADERS = { 19 | 'Content-Type': 'application/x-amz-json-' 20 | } 21 | # json version fills in the rest of the content type 22 | 23 | # Intentionally left blank 24 | REST_JSON_DEFAULT_HEADERS = {} 25 | 26 | 27 | def query_protocol_formatter(host, token, name, version, kwargs, input_format): 28 | headers = {} 29 | gathered_headers = _resolve_headers(headers, QUERY_DEFAULT_HEADERS) 30 | complete_headers = _complete_headers(gathered_headers, host, token) 31 | to_return = { 'headers' : complete_headers } 32 | if "content_type" in kwargs.keys(): 33 | to_return['headers']['Content-Type'] = kwargs['content_type'] 34 | 35 | body_map = { 'Action': name } 36 | body_map['Version'] = version 37 | body_map.update(input_format) 38 | 39 | to_return['body'] = urlencode(body_map) 40 | 41 | return to_return 42 | 43 | 44 | def json_protocol_formatter(host, token, json_version, amz_target, kwargs, input_format): 45 | # Need to fill in the Content-Type first 46 | headers = {} 47 | to_return = {} 48 | headers['Content-Type'] = JSON_DEFAULT_HEADERS['Content-Type'] + json_version 49 | # TODO: Make the kwargs header modifications better 50 | if "content_type" in kwargs.keys(): 51 | headers['Content-Type'] = kwargs['content_type'] 52 | 53 | headers['X-Amz-Target'] = amz_target 54 | gathered_headers = _resolve_headers(headers, JSON_DEFAULT_HEADERS) 55 | complete_headers = _complete_headers(gathered_headers, host, token) 56 | to_return['headers'] = complete_headers 57 | 58 | to_return['body'] = json.dumps(input_format) 59 | 60 | return to_return 61 | 62 | def rest_json_protocol_formatter(host, token, json_version, request_uri, kwargs, input_format): 63 | # Need to fill in the Content-Type first 64 | headers = {} 65 | to_return = {} 66 | 67 | if "content_type" in kwargs.keys(): 68 | headers['Content-Type'] = kwargs['content_type'] 69 | 70 | gathered_headers = _resolve_headers(headers, REST_JSON_DEFAULT_HEADERS) 71 | complete_headers = _complete_headers(gathered_headers, host, token) 72 | to_return['headers'] = complete_headers 73 | 74 | # Need to act on the parameters in the URI 75 | # TODO replace params with type appropriate ones 76 | to_return['request_uri'] = re.sub("\{(.*?)\}", "", request_uri) 77 | 78 | to_return['body'] = json.dumps(input_format) 79 | 80 | return to_return 81 | 82 | 83 | 84 | def _resolve_headers(custom_headers, default_headers): 85 | if custom_headers == '': 86 | default_headers.update(ALL_DEFAULT_HEADERS) 87 | return default_headers 88 | else: 89 | return _apply_custom_headers(custom_headers, default_headers) 90 | 91 | 92 | def _apply_custom_headers(custom_headers, default_headers): 93 | """ A user may apply a custom header for their request. These will be 94 | appended to the default headers. """ 95 | to_return = default_headers.copy() 96 | to_return.update(custom_headers) 97 | to_return.update(ALL_DEFAULT_HEADERS) 98 | 99 | return to_return 100 | 101 | 102 | def _complete_headers(headers, host, token): 103 | """ We need to fill in those headers with the info we have """ 104 | for header in headers.keys(): 105 | if header == 'X-Amz-Date': 106 | headers[header] = _get_date_string() 107 | if header == 'Host': 108 | headers[header] = host.split("/")[0] 109 | if header == 'X-Amz-Security-Token': 110 | headers[header] = token 111 | 112 | return headers 113 | 114 | 115 | def _get_date_string(): 116 | t = datetime.datetime.utcnow() 117 | amz_date = t.strftime('%Y%m%dT%H%M%SZ') 118 | return amz_date -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | xeger 3 | boto3 --------------------------------------------------------------------------------