├── tea ├── __init__.py └── gen │ ├── __init__.py │ ├── get_acls.py │ ├── get_groups.py │ ├── utils.py │ ├── get_collections.py │ └── create_tea_config.py ├── test ├── __init__.py ├── tea │ ├── __init__.py │ └── gen │ │ ├── __init__.py │ │ ├── get_groups_test.py │ │ ├── get_acls_test.py │ │ ├── get_collections_test.py │ │ └── utils_test.py └── handler_test.py ├── requirements.txt ├── .gitignore ├── .editorconfig ├── Dockerfile ├── serverless-local.yml ├── harbor.sh ├── serverless-sandbox.yml ├── public └── api.md ├── serverless.yml ├── test.sh ├── main.py ├── capabilities.py ├── README.md ├── run.sh └── handler.py /tea/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tea/gen/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/tea/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/tea/gen/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # pip3 install -r requirements.txt 2 | 3 | boto3 4 | requests 5 | pytest 6 | coverage -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | local-bucket 2 | build 3 | node_modules 4 | *.pyc 5 | *.log 6 | *.swp 7 | public/*.html 8 | package*.json -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | indent_style = space 3 | indent_size = 2 4 | 5 | [*.md] 6 | indent_style = space 7 | indent_size = 4 #this is important, the markdown for API docs will not see 2 spaces 8 | 9 | [*.py] 10 | indent_style = space 11 | indent_size = 4 12 | insert_final_newline = true 13 | trim_trailing_whitespace = true 14 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:18 2 | 3 | RUN apt-get update && \ 4 | apt install -y \ 5 | python3 \ 6 | python3-pip \ 7 | pylint \ 8 | && rm -rf /var/lib/apt/lists/* 9 | 10 | WORKDIR /build 11 | 12 | RUN npm install -g serverless@3 13 | RUN pip3 install --break-system-packages pylint 14 | RUN pip3 install --break-system-packages coverage 15 | RUN pip3 install --break-system-packages pytest 16 | 17 | RUN echo "# Aliases" >> /etc/bash.bashrc 18 | RUN echo "alias ls='ls --color'" >> /etc/bash.bashrc 19 | RUN echo "alias ll='ls -l'" >> /etc/bash.bashrc 20 | 21 | ENV PATH "$PATH:/build/node_modules/.bin" 22 | -------------------------------------------------------------------------------- /serverless-local.yml: -------------------------------------------------------------------------------- 1 | service: tea-config-generator 2 | 3 | frameworkVersion: '2 || 3' 4 | 5 | provider: 6 | name: aws 7 | runtime: python3.10 8 | memorySize: 128 # default was 1024, 512 would also be accepted 9 | timeout: 15 # 7 seconds is the average run time currently 10 | region: us-east-1 11 | environment: 12 | AWS_TEA_CONFIG_CMR: ${env:AWS_TEA_CONFIG_CMR, 'https://cmr.uat.earthdata.nasa.gov'} 13 | AWS_TEA_CONFIG_LOG_LEVEL: ${env:AWS_TEA_CONFIG_CMR, 'INFO'} 14 | 15 | package: 16 | # individually: true 17 | exclude: 18 | - node_modules/** 19 | 20 | functions: 21 | capabilities: 22 | handler: capabilities.capabilities 23 | events: 24 | - http: GET ${self:custom.endPoint}/ 25 | - http: GET ${self:custom.endPoint}/capabilities 26 | debug: 27 | handler: handler.debug 28 | events: 29 | - http: GET ${self:custom.endPoint}/debug 30 | status: 31 | handler: handler.health 32 | events: 33 | - http: GET ${self:custom.endPoint}/status 34 | provider: 35 | handler: handler.generate_tea_config 36 | events: 37 | - http: GET ${self:custom.endPoint}/provider/{id} 38 | 39 | plugins: 40 | - serverless-python-requirements 41 | - serverless-offline 42 | - serverless-s3-local 43 | custom: 44 | env: ${env:AWS_TEA_CONFIG_ENV, 'sit'} 45 | endPoint: /configuration/tea 46 | pythonRequirements: 47 | pythonBin: /usr/bin/python3 48 | s3: 49 | port: 7000 50 | host: localhost 51 | directory: ./build 52 | 53 | resources: 54 | Resources: 55 | LocalBucket: 56 | Type: AWS::S3::Bucket 57 | Properties: 58 | BucketName: local-bucket 59 | -------------------------------------------------------------------------------- /tea/gen/get_acls.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module contains functions to get ACLs 3 | """ 4 | import logging 5 | import requests 6 | import tea.gen.utils as util 7 | 8 | def get_acls(env,provider,token): 9 | """ Method used to get all ACLs for given provider """ 10 | 11 | logger = util.get_logger(env) 12 | logger.debug('TEA configuraton ACL') 13 | cmr_url = util.get_env(env) 14 | headers = util.standard_headers({'Authorization': token, 'Content-Type': 'application/json'}) 15 | url = (f'{cmr_url}/access-control/acls' 16 | f'?provider={provider}' 17 | f'&identity_type=catalog_item' 18 | f'&page_size=2000') 19 | try: 20 | response = requests.get(url, headers=headers) 21 | json_data = response.json() 22 | logging.debug('get_acls: response=%s', json_data) 23 | if response.status_code == 200: 24 | if 'items' in json_data: 25 | items = json_data['items'] 26 | return items 27 | except requests.exceptions.RequestException as error: 28 | logging.error('Error occurred in get_acl: %s', error) 29 | return [] 30 | 31 | def get_acl(env,acl_url, token): 32 | """ Method retrieves ACL for given ACL URL """ 33 | logger = util.get_logger(env) 34 | headers = util.standard_headers({'Authorization': token, 'Content-Type': 'application/json'}) 35 | try: 36 | response = requests.get(acl_url, headers=headers) 37 | json_data = response.json() 38 | logger.debug("get_acl: response=%s", json_data) 39 | if response.status_code == 200: 40 | return json_data 41 | except requests.exceptions.RequestException as error: 42 | logging.error('Error occurred in get_acl: %s', error) 43 | return {} 44 | -------------------------------------------------------------------------------- /harbor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script is made to be called from a CI/CD system and manages all the 4 | # docker commands the build process needs to perform. 5 | 6 | RED='\033[0;31m' 7 | GREEN='\033[0;32m' 8 | YELLOW='\033[0;33m' 9 | BLUE='\033[0;34m' 10 | WHITE='\033[0;37m' 11 | BOLD='\033[1m' 12 | UNDERLINE='\033[4m' 13 | NC='\033[0m' # No Color 14 | color_mode='yes' 15 | 16 | cprintf() { 17 | color=$1 18 | content=$2 19 | if [ "$color_mode" == "no" ] 20 | then 21 | printf "${content}\n" 22 | else 23 | printf "${color}%s${NC}\n" "${content}" 24 | fi 25 | } 26 | 27 | docker_options='' 28 | 29 | help_doc() 30 | { 31 | echo 'Script to manage the life cycle of the Tea Configuration code' 32 | echo 'Usage:' 33 | echo ' ./run.sh -[c|C] -t -[hulre]' 34 | echo ' ./run.sh -[o|I]' 35 | echo 36 | 37 | format="%4s %-8s %10s | %s\n" 38 | printf "${format}" 'flag' 'value' 'name' 'Description' 39 | printf "${format}" '----' '--------' '------' '-------------------------' 40 | printf "${format}" '-h' '' 'Help' 'Print out a help document' 41 | printf "${format}" '-c' '' 'Color' 'Turn color on (default)' 42 | printf "${format}" '-C' '' 'no color' 'Turn color off' 43 | printf "${format}" '-b' '' 'build' 'Build Docker Image' 44 | printf "${format}" '-d' '' 'run' 'Deploy serverless tasks' 45 | printf "${format}" '-r' '' 'run' 'Run Docker Image tasks' 46 | printf "${format}" '-R' '' 'run' 'Run bash in Docker Image' 47 | } 48 | 49 | while getopts 'hcCbrRd:' opt; do 50 | case ${opt} in 51 | h) help_doc ;; 52 | c) color_mode='yes' ;; 53 | C) color_mode='no' ; docker_options='--progress plain' ;; 54 | b) docker build $docker_options --rm --tag=tea-config-gen . ;; 55 | r) 56 | # -I install libraries 57 | # -U unit test, write file 58 | # -j junit output of tests 59 | # -l lint report 60 | docker run --volume $(pwd):/build tea-config-gen \ 61 | sh -c "./run.sh -I -U -j -l" 62 | ;; 63 | R) docker run --volume $(pwd):/build -it tea-config-gen bash ;; 64 | d) #deploy 65 | docker run --rm \ 66 | --volume $(pwd):/build \ 67 | tea-config-gen \ 68 | sh -c "./run.sh -d" 69 | ;; 70 | *) cprintf $RED "option required" ;; 71 | esac 72 | done 73 | -------------------------------------------------------------------------------- /serverless-sandbox.yml: -------------------------------------------------------------------------------- 1 | service: tea-config-generator 2 | 3 | frameworkVersion: '2 || 3' 4 | 5 | variablesResolutionMode: 20210326 6 | 7 | provider: 8 | name: aws 9 | runtime: python3.10 10 | memorySize: 128 # default was 1024, 512 would also be accepted 11 | timeout: 15 # 7 seconds is the average run time currently 12 | region: us-east-1 13 | role: IamRoleTeaLambdaExecution 14 | environment: 15 | AWS_TEA_CONFIG_CMR: ${env:AWS_TEA_CONFIG_CMR, 'https://cmr.uat.earthdata.nasa.gov'} 16 | AWS_TEA_CONFIG_LOG_LEVEL: ${env:AWS_TEA_CONFIG_LOG, 'INFO'} 17 | 18 | package: 19 | # individually: true 20 | exclude: 21 | - node_modules/** 22 | 23 | functions: 24 | capabilities: 25 | handler: capabilities.capabilities 26 | events: 27 | - http: GET ${self:custom.endPoint}/ 28 | - http: GET ${self:custom.endPoint}/capabilities 29 | status: 30 | handler: handler.health 31 | events: 32 | - http: GET ${self:custom.endPoint}/status 33 | provider: 34 | handler: handler.generate_tea_config 35 | events: 36 | - http: GET ${self:custom.endPoint}/provider/{id} 37 | 38 | resources: 39 | Resources: 40 | # this property will not work locally till the following is fixed: 41 | # https://github.com/dherault/serverless-offline/issues/1278 42 | IamRoleTeaLambdaExecution: 43 | Type: AWS::IAM::Role 44 | Properties: 45 | RoleName: tea-config-generator-role-${self:custom.env}-${self:provider.region} 46 | PermissionsBoundary: arn:aws:iam::${aws:accountId}:policy/NGAPShRoleBoundary 47 | ManagedPolicyArns: 48 | - arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole 49 | AssumeRolePolicyDocument: 50 | Version: '2012-10-17' 51 | Statement: 52 | - Effect: 'Allow' 53 | Principal: 54 | Service: 55 | - 'lambda.amazonaws.com' 56 | Action: 57 | - 'sts:AssumeRole' 58 | 59 | plugins: 60 | - serverless-python-requirements 61 | - serverless-offline 62 | - serverless-s3-local 63 | custom: 64 | loadBalancer: ${env:AWS_TEA_CONFIG_LOADBALANCER, ''} 65 | env: ${env:AWS_TEA_CONFIG_ENV, 'sit'} 66 | endPoint: /configuration/tea 67 | pythonRequirements: 68 | pythonBin: /usr/bin/python3 69 | -------------------------------------------------------------------------------- /tea/gen/get_groups.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module contains functions to get Groups 3 | """ 4 | import requests 5 | import tea.gen.utils as util 6 | 7 | #logging.basicConfig(filename='script.log', format='%(asctime)s %(message)s', 8 | #encoding='utf-8', level=logging.INFO) 9 | 10 | def get_groups(env:dict, acl_url, token): 11 | """ Method gets groups for given ACL URL """ 12 | logger = util.get_logger(env) 13 | headers = util.standard_headers({'Authorization': token, 'Content-Type': 'application/json'}) 14 | try: 15 | response = requests.get(acl_url, headers=headers) 16 | json_data = response.json() 17 | logger.debug('get_groups: response=%s', json_data) 18 | if response.status_code == 200: 19 | if 'group_permissions' in json_data: 20 | items = json_data['group_permissions'] 21 | return items 22 | except requests.exceptions.RequestException as error: 23 | logger.error('Error occurred in get_groups: %s', error) 24 | return [] 25 | 26 | def get_group(env:dict, group_id, token): 27 | """ Method used to get group for given group ID """ 28 | logger = util.get_logger({}) 29 | cmr_base = util.get_env(env) 30 | url = f'{cmr_base}/access-control/groups/{group_id}' 31 | headers = util.standard_headers({'Authorization': token, 'Content-Type': 'application/json'}) 32 | try: 33 | response = requests.get(url, headers=headers) 34 | json_data = response.json() 35 | logger.debug('get_group: response=%s', json_data) 36 | if response.status_code == 200: 37 | return json_data 38 | except requests.exceptions.RequestException as error: 39 | logger.error('Error occurred in get_group: %s', error) 40 | return {} 41 | 42 | def get_group_names(env:dict, acl_json, token): 43 | """ Method used to get group names for given ACL json """ 44 | logger = util.get_logger({}) 45 | group_names = [] 46 | if 'group_permissions' in acl_json: 47 | all_groups = acl_json['group_permissions'] 48 | for group in all_groups: 49 | if ('group_id' in group) and (not group['group_id'].endswith('-CMR')): 50 | group_json = get_group(env, group['group_id'], token) 51 | if 'name' in group_json: 52 | group_name = group_json['name'] 53 | group_names.append(group_name) 54 | logger.info('Found non-CMR group: %s', group_name) 55 | return group_names 56 | -------------------------------------------------------------------------------- /tea/gen/utils.py: -------------------------------------------------------------------------------- 1 | """ Utility methods """ 2 | 3 | import logging 4 | 5 | def standard_headers(base:dict = None): 6 | """ 7 | Return a dictionary containing the standard headers which should always be 8 | used when communicating to CMR from this app. Append to an existing dictionary 9 | if one exists. 10 | """ 11 | if base is None: 12 | base = {} 13 | base['User-Agent'] = 'ESDIS TEA Config Generator' 14 | return base 15 | 16 | def get_env(env: dict): 17 | """ Returns CMR server URL, uses 'https://cmr.earthdata.nasa.gov' as default """ 18 | return env.get('cmr-url', 'https://cmr.earthdata.nasa.gov') 19 | 20 | def get_s3_prefixes(collection): 21 | """ Returns array of S3 prefixes for given collection """ 22 | if 'DirectDistributionInformation' in collection: 23 | direct_dist = collection['DirectDistributionInformation'] 24 | if 'S3BucketAndObjectPrefixNames' in direct_dist: 25 | return direct_dist['S3BucketAndObjectPrefixNames'] 26 | return [] 27 | 28 | def add_to_dict(all_s3_prefix_groups_dict, s3_prefixes_set, group_names_set): 29 | """ Adds new elements to S3 prefixes groups dictionary """ 30 | for s3_prefix in s3_prefixes_set: 31 | if s3_prefix in all_s3_prefix_groups_dict: 32 | existing_groups_set = all_s3_prefix_groups_dict[s3_prefix] 33 | existing_groups_set.update(group_names_set) 34 | else: 35 | all_s3_prefix_groups_dict[s3_prefix] = group_names_set 36 | 37 | def create_tea_config(all_s3_prefix_groups_dict): 38 | """ For given S3 prefixes groups dicionary creates the result string""" 39 | result_string = 'PRIVATE_BUCKETS:\n' 40 | for key, value in all_s3_prefix_groups_dict.items(): 41 | result_string += ' '*2 42 | result_string += key.strip() 43 | result_string += ':\n' 44 | for group in value: 45 | result_string += ' '*4 + '- '# 4x space dash space 46 | result_string += group.strip() 47 | result_string += '\n' 48 | return result_string 49 | 50 | def get_logger(envirnment): 51 | """ 52 | Create a logger using the logging info from the calling environment, configure 53 | that logger and return it for use by the code. 54 | """ 55 | level = envirnment.get('logging-level', 'INFO') 56 | logging.basicConfig(format="%(name)s - %(module)s - %(message)s",level=level) 57 | logger = logging.getLogger() 58 | logger.setLevel(level) 59 | return logger 60 | -------------------------------------------------------------------------------- /public/api.md: -------------------------------------------------------------------------------- 1 | # TEA Configuration Generator API 2 | 3 | ## General 4 | Calls can be made with curl as such: 5 | 6 | curl -is -H 'Authorization: Bearer abcd1234' http://localhost:3000/dev/configuration/tea/provider/POCLOUD 7 | 8 | ## Capabilities 9 | To get a programatic output of the supported urls with descriptions, call the capabilities end point which is availible as either the root of the service or named: 10 | 11 | * Request 12 | * GET /configuration/tea/ 13 | * GET /configuration/tea/capabilities 14 | * Headers: none 15 | * Response 16 | * Content Returns a JSON dictionary containing a list of urls 17 | * Headers: `cmr-took` - number of seconds of execution 18 | * Status Codes: 200 - success 19 | 20 | Example response: 21 | 22 | { 23 | "urls": [ 24 | { 25 | "name": "Root", 26 | "path": "/configuration/tea", 27 | "description": "Alias for capabilities" 28 | }, 29 | { 30 | "name": "Capabilities", 31 | "path": "/configuration/tea/capabilities", 32 | "description": "Show which endpoints exist on this service" 33 | }, 34 | { 35 | "name": "Status", 36 | "path": "/configuration/tea/status", 37 | "description": "Returns service status" 38 | }, 39 | { 40 | "name": "Generate", 41 | "path": "/configuration/tea/provider/", 42 | "description": "Generate a TEA config for provider" 43 | } 44 | ] 45 | } 46 | 47 | URL dictionary contains: 48 | 49 | * name : human readable name 50 | * description : human readable description of the parameter 51 | * path : URL fragment to be appended to the host name and end point 52 | 53 | ## Generate Configuration 54 | Generate a TEA configuration file based on the supplied CMR provider id and [Lanchpad][lpad]/[EDL][edl] user token. 55 | 56 | * Request 57 | * GET /configuration/tea/provider 58 | * Headers: `Authorization`: [CMR token] 59 | * Response 60 | * Content: Tea configuration file on success, 400 on error 61 | * Headers: `cmr-took` - number of seconds of execution 62 | * status codes: 63 | * 200 - success 64 | * 400 - Token is required 65 | * 404 - No S3 prefixes returned 66 | 67 | ## Status 68 | Return a simple response indicating that the service is running 69 | 70 | * Request 71 | * GET /configuration/tea/status 72 | * Headers: none 73 | * Response 74 | * Content: `I'm a teapot` 75 | * Headers: `cmr-took` - number of seconds of execution 76 | * Status Codes: 418 - active 77 | 78 | ## License 79 | Copyright © 2022-2022 United States Government as represented by the Administrator 80 | of the National Aeronautics and Space Administration. All Rights Reserved. 81 | 82 | [lpad]: https://launchpad.nasa.gov/ "NASA LaunchPad" 83 | [edl]: https://urs.earthdata.nasa.gov/ "EarthData Login" -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | service: tea-config-generator 2 | 3 | frameworkVersion: '2 || 3' 4 | 5 | variablesResolutionMode: 20210326 6 | 7 | provider: 8 | name: aws 9 | runtime: python3.10 10 | memorySize: 128 # default was 1024, 512 would also be accepted 11 | region: us-east-1 12 | role: IamRoleTeaLambdaExecution 13 | environment: 14 | AWS_TEA_CONFIG_CMR: ${env:AWS_TEA_CONFIG_CMR, 'https://cmr.earthdata.nasa.gov'} 15 | AWS_TEA_CONFIG_LOG_LEVEL: ${env:AWS_TEA_CONFIG_LOG, 'INFO'} 16 | stage: ${opt:stage, 'prod'} 17 | 18 | package: 19 | exclude: 20 | - node_modules/** 21 | 22 | functions: 23 | capabilities: 24 | handler: capabilities.capabilities 25 | description: Return a summary of the calls that can be made to this application 26 | timeout: 5 27 | events: 28 | - alb: 29 | listenerArn: ${cf:${self:provider.stage}.servicesLbListenerArn} 30 | priority: 999 31 | conditions: 32 | path: 33 | - ${self:custom.endPoint}/capabilities 34 | - ${self:custom.endPoint}/ 35 | vpc: ~ 36 | status: 37 | handler: handler.health 38 | description: Return a nominal response for confirming the status of this app 39 | timeout: 5 40 | events: 41 | - alb: 42 | listenerArn: ${cf:${self:provider.stage}.servicesLbListenerArn} 43 | priority: 998 44 | conditions: 45 | path: ${self:custom.endPoint}/status 46 | vpc: ~ 47 | provider: 48 | handler: handler.generate_tea_config 49 | description: Generate a TEA Config YAML file 50 | timeout: 30 51 | events: 52 | - alb: 53 | listenerArn: ${cf:${self:provider.stage}.servicesLbListenerArn} 54 | priority: 997 55 | conditions: 56 | path: ${self:custom.endPoint}/provider* 57 | vpc: 58 | securityGroupIds: 59 | - ${cf:${self:provider.stage}.servicesSecurityGroupId} 60 | subnetIds: !Split [',', '${cf:${self:provider.stage}.subnetIds}'] 61 | 62 | resources: 63 | Resources: 64 | # this property will not work locally till the following is fixed: 65 | # https://github.com/dherault/serverless-offline/issues/1278 66 | IamRoleTeaLambdaExecution: 67 | Type: AWS::IAM::Role 68 | Properties: 69 | RoleName: tea-config-generator-role-${self:provider.stage}-${self:provider.region} 70 | PermissionsBoundary: arn:aws:iam::${aws:accountId}:policy/NGAPShRoleBoundary 71 | ManagedPolicyArns: 72 | - arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole 73 | AssumeRolePolicyDocument: 74 | Version: '2012-10-17' 75 | Statement: 76 | - Effect: 'Allow' 77 | Principal: 78 | Service: 79 | - 'lambda.amazonaws.com' 80 | Action: 81 | - 'sts:AssumeRole' 82 | plugins: 83 | - serverless-python-requirements 84 | - serverless-offline 85 | - serverless-s3-local 86 | custom: 87 | endPoint: /configuration/tea 88 | pythonRequirements: 89 | pythonBin: /usr/bin/python3 90 | -------------------------------------------------------------------------------- /test/tea/gen/get_groups_test.py: -------------------------------------------------------------------------------- 1 | """ Test module """ 2 | from unittest import TestCase, mock 3 | from tea.gen.get_groups import get_group, get_groups 4 | 5 | #python -m unittest discover -s ./ -p '*_test.py' 6 | class GetGroupsTest(TestCase): 7 | """ Test class to test GetGroups """ 8 | @mock.patch('tea.gen.get_groups.requests.get') 9 | def test_get_groups(self, mock_get): 10 | """ Tests get_groups """ 11 | my_mock_response = mock.Mock(status_code=200) 12 | my_mock_response.json.return_value = { 13 | 'group_permissions' : [ { 14 | 'permissions' : [ 'read' ], 15 | 'user_type' : 'guest' 16 | }, { 17 | 'permissions' : [ 'read' ], 18 | 'user_type' : 'registered' 19 | }, { 20 | 'permissions' : [ 'read' ], 21 | 'group_id' : 'AG1222486916-CMR' 22 | }, { 23 | 'permissions' : [ 'read' ], 24 | 'group_id' : 'AG1236456866-CMR' 25 | }, { 26 | 'permissions' : [ 'read', 'order' ], 27 | 'group_id' : 'AG1216375421-SCIOPS' 28 | }, { 29 | 'permissions' : [ 'read', 'order' ], 30 | 'group_id' : 'AG1216375422-SCIOPS' 31 | }, { 32 | 'permissions' : [ 'read', 'order' ], 33 | 'group_id' : 'AG1215550981-CMR' 34 | } ], 35 | 'catalog_item_identity' : { 36 | 'name' : 'All Collections', 37 | 'provider_id' : 'SCIOPS', 38 | 'granule_applicable' : False, 39 | 'collection_applicable' : True, 40 | 'collection_identifier' : { 41 | 'concept_ids' : [ 'C1240032460-SCIOPS'], 42 | 'entry_titles' : [ '2000 Pilot Environmental Sustainability Index (ESI)' ] 43 | } 44 | }, 45 | 'legacy_guid' : 'F4E6573E-B97E-8BBC-A553-37DCE8F28D9D' 46 | } 47 | mock_get.return_value = my_mock_response 48 | 49 | acl_url = 'XXX' 50 | token = 'EDL-XXX' 51 | 52 | response = get_groups({}, acl_url, token) 53 | self.assertEqual(response[2]['group_id'], 'AG1222486916-CMR') 54 | self.assertEqual(response[4]['group_id'], 'AG1216375421-SCIOPS') 55 | 56 | @mock.patch('tea.gen.get_groups.requests.get') 57 | def test_get_group(self, mock_get): 58 | """ Tests get_group """ 59 | my_mock_response = mock.Mock(status_code=200) 60 | my_mock_response.json.return_value = { 61 | 'name' : 'Science Coordinators', 62 | 'description' : 'List of science coordinators for metadata curation \ 63 | in docBUILDER and CMR API', 64 | 'legacy_guid' : 'E825D79F-A110-A251-7110-97208B5C2987', 65 | 'provider_id' : 'SCIOPS', 66 | 'num_members' : 4 67 | } 68 | mock_get.return_value = my_mock_response 69 | 70 | env = {'cmr-url': 'XXX'} 71 | group_id = 'XXX' 72 | token = 'EDL-XXX' 73 | 74 | response = get_group(env, group_id, token) 75 | self.assertEqual(response['name'], 'Science Coordinators') 76 | self.assertEqual(response['legacy_guid'], 'E825D79F-A110-A251-7110-97208B5C2987') 77 | -------------------------------------------------------------------------------- /tea/gen/get_collections.py: -------------------------------------------------------------------------------- 1 | """ Module used to get Collections """ 2 | 3 | import requests 4 | import tea.gen.utils as util 5 | 6 | def get_collections_s3_prefixes_dict(env: dict, token, provider, page_num, page_size): 7 | """ Method returns a dictionary with concept_ids as keys and S3 prefixes array as values """ 8 | all_collections_s3_prefixes_dict = {} 9 | json_data = get_collections(env, token, provider, page_num, page_size) 10 | if 'hits' not in json_data or json_data['hits'] == '0': 11 | return {} 12 | hits = json_data['hits'] 13 | logger = util.get_logger(env) 14 | logger.debug('Hits=%d',hits) 15 | for item in json_data['items']: 16 | if 's3-links' in item['meta']: 17 | all_collections_s3_prefixes_dict[item['meta']['concept-id']] = item['meta']['s3-links'] 18 | if page_size < hits: 19 | remainder = hits % page_size 20 | pages = (hits - remainder) / page_size 21 | pages = int(pages) 22 | index = 1 23 | while index < pages + 1: 24 | index += 1 25 | json_data = get_collections(env, token, provider, index, page_size) 26 | collections_s3_prefixes_dict = {} 27 | items = json_data['items'] 28 | for item in items: 29 | if 's3-links' in item['meta']: 30 | collections_s3_prefixes_dict[item['meta']['concept-id']] = \ 31 | item['meta']['s3-links'] 32 | all_collections_s3_prefixes_dict.update(collections_s3_prefixes_dict) 33 | 34 | return all_collections_s3_prefixes_dict 35 | 36 | def get_collections(env:dict, token, provider, page_num, page_size): 37 | """ Method returns collections for given provider """ 38 | logger = util.get_logger(env) 39 | headers = util.standard_headers({'Authorization': token}) 40 | cmr_base = util.get_env(env) 41 | url = (f'{cmr_base}/search/collections.umm-json' 42 | f'?provider={provider}' 43 | f'&sort_key=entry_title' 44 | f'&page_num={page_num}' 45 | f'&page_size={page_size}') 46 | try: 47 | logger.debug('request url: %s', url) 48 | response = requests.get(url, headers=headers) 49 | json_data = response.json() 50 | logger.debug('get_collections: response=%s', json_data) 51 | if response.status_code == 200: 52 | return json_data 53 | except requests.exceptions.RequestException as error: 54 | logger.error('Error occurred in get_collections from calling %s:\n%s', url, error) 55 | return {} 56 | 57 | def get_collection(env:dict, token, concept_id): 58 | """ Method returns collection for given concept_id """ 59 | logger = util.get_logger(env) 60 | headers = util.standard_headers({'Authorization': token, 'Content-Type': 'application/json'}) 61 | cmr_base = util.get_env(env) 62 | url = f'{cmr_base}/search/concepts/{concept_id}.umm_json' 63 | try: 64 | response = requests.get(env, url, headers=headers) 65 | json_data = response.json() 66 | logger.debug('get_collection: response=%s', json_data) 67 | if response.status_code == 200: 68 | return json_data 69 | except requests.exceptions.RequestException as error: 70 | logger.error('Error occurred in get_collection: %s', error) 71 | return {} 72 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Test the TEA Config Generator in AWS. A CMR token and the AWS API Gateway 4 | # Server instance must be provided 5 | 6 | # ############################################################################## 7 | # Values 8 | 9 | aws_instance="" 10 | base_url="https://${aws_instance}.execute-api.us-east-1.amazonaws.com/dev" 11 | 12 | # ############################################################################## 13 | # Color Util 14 | 15 | RED='\033[0;31m' 16 | GREEN='\033[0;32m' 17 | YELLOW='\033[0;33m' 18 | BLUE='\033[0;34m' 19 | WHITE='\033[0;37m' 20 | BOLD='\033[1m' 21 | UNDERLINE='\033[4m' 22 | NC='\033[0m' # No Color 23 | color_mode='yes' 24 | 25 | cprintf() { 26 | color=$1 27 | content=$2 28 | if [ "$color_mode" == "no" ] 29 | then 30 | printf "${content}\n" 31 | else 32 | printf "${color}%s${NC}\n" "${content}" 33 | fi 34 | } 35 | 36 | # ############################################################################## 37 | # Functions 38 | 39 | function capabilities 40 | { 41 | url="$base_url/configuration/tea/capabilities" 42 | cprintf $GREEN "calling $url" 43 | curl -s -H 'Cmr-Pretty: true' "$url" 44 | } 45 | 46 | function status 47 | { 48 | url="$base_url/configuration/tea/status" 49 | cprintf $GREEN "calling $url" 50 | curl -is -H 'Cmr-Pretty: true' "$url" 51 | } 52 | 53 | function debug 54 | { 55 | url="$base_url/configuration/tea/debug" 56 | cprintf $GREEN "calling $url" 57 | curl -is -H 'Cmr-Pretty: true' \ 58 | -H "authorization: Bearer ${token}" \ 59 | "$url" 60 | } 61 | 62 | function generate 63 | { 64 | url="$base_url/configuration/tea/provider/POCLOUD" 65 | cprintf $GREEN "calling $url" 66 | curl -si \ 67 | -H "Authorization: Bearer ${token}" \ 68 | -H 'Cmr-Pretty: true' \ 69 | "$url" 70 | } 71 | 72 | help_doc() 73 | { 74 | echo 'Script to call interfaces on AWS' 75 | echo 'Usage:' 76 | echo ' ./test.sh -a -t [-c | -d | -g | -s ]' 77 | echo 78 | echo ' ./test.sh -a yruab01 -t /Users/MacUser/.hidden/token-in-file.txt -g' 79 | echo 80 | 81 | format="%4s %-6s %12s | %s\n" 82 | printf "${format}" 'Flag' 'Value' 'Name' 'Description' 83 | printf "${format}" '----' '------' '------------' '-------------------------' 84 | printf "${format}" '-h' '' 'Help' 'Print out a help document.' 85 | printf "${format}" '-a' '' 'AWS' 'REQUIRED: API Gateway host name added in front of execute-api.us-east-1.amazonaws.com.' 86 | printf "${format}" '-c' '' 'Capabilities' 'Show capabilities' 87 | printf "${format}" '-d' '' 'Debug' 'display debug info' 88 | printf "${format}" '-g' '' 'Generate' 'Generate the tea config YAML file.' 89 | printf "${format}" '-s' '' 'Status' 'check service status.' 90 | printf "${format}" '-t' '' 'Token' 'REQUIRED: File with a CMR token, first line without a pound is used.' 91 | } 92 | 93 | while getopts 'ha:cdgst:' opt; do 94 | case ${opt} in 95 | h) help_doc ;; 96 | a) aws_instance="${OPTARG}" 97 | base_url="https://${aws_instance}.execute-api.us-east-1.amazonaws.com/dev" 98 | ;; 99 | c) capabilities ;; 100 | d) debug ;; 101 | g) generate ;; 102 | s) status ;; 103 | t) token=$(grep -v '#' ${OPTARG} | head -n 1) ;; 104 | *) cprintf $RED "option required" ; help_doc ;; 105 | esac 106 | done 107 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | """ Main method to run the Lambda functions in handler.py.""" 2 | 3 | import logging 4 | import os 5 | import argparse 6 | 7 | import handler 8 | 9 | # ****************************************************************************** 10 | #mark - utility functions 11 | 12 | def handle_arguments(): 13 | """ 14 | Setup the application parameteres and return a parsed args object allowing 15 | the caller to get the param values 16 | """ 17 | parser = argparse.ArgumentParser(description='Test AWS handler functions') 18 | 19 | aws_group = parser.add_argument_group() 20 | aws_group.add_argument('-l', '--log-level', default='debug', 21 | help='Set the python log level') 22 | aws_group.add_argument('-p', '--provider', default='POCLOUD', 23 | help='CMR Provider name, such as POCLOUD') 24 | aws_group.add_argument('-e', '--env', default='uat', help='sit, uat, ops') 25 | 26 | token_ops = parser.add_mutually_exclusive_group() 27 | token_ops.add_argument('-t', '--token', default='', help='EDL Token') 28 | token_ops.add_argument('-tf', '--token-file', default='', 29 | help='Path to a file holding an EDL Token') 30 | 31 | args = parser.parse_args() 32 | return args 33 | 34 | def read_token(args): 35 | """ Read a token file or token parameter and return the content """ 36 | if args.token_file: 37 | with open(args.token_file, encoding='utf8') as file_obj: 38 | token = file_obj.readline().strip() 39 | elif args.token: 40 | token = args.token 41 | else: 42 | token = None 43 | return token 44 | 45 | def setup_logging(log_level): 46 | """ Used to test functions locally """ 47 | if len(logging.getLogger().handlers)>0: 48 | logging.getLogger().setLevel(log_level) 49 | else: 50 | logging.basicConfig(filename='script.log', 51 | format="%(name)s - %(module)s - %(message)s", 52 | level=log_level) 53 | #format='%(asctime)s %(message)s', 54 | 55 | def cmr_url_for_env(which_cmr): 56 | """ Return the environment specific URL for CMR """ 57 | cmrs = {'sit':'https://cmr.sit.earthdata.nasa.gov', 58 | 'uat':'https://cmr.uat.earthdata.nasa.gov', 59 | 'ops':'https://cmr.earthdata.nasa.gov', 60 | 'prod':'https://cmr.earthdata.nasa.gov'} 61 | return cmrs.get(which_cmr, 'https://cmr.earthdata.nasa.gov') 62 | 63 | # ****************************************************************************** 64 | #mark - command line function 65 | 66 | def main(): 67 | """ 68 | This method is just to test the lambda functions by passing along the defined 69 | input values in a way as to emulate AWS. Currently the generate_tea_config() 70 | is run. 71 | """ 72 | args = handle_arguments() 73 | 74 | provider = args.provider 75 | which_cmr = args.env 76 | token = read_token(args) 77 | log_level = args.log_level.upper() 78 | 79 | setup_logging(log_level) 80 | cmr_url = cmr_url_for_env(which_cmr) 81 | 82 | # Setup environment and parameters 83 | os.environ['AWS_TEA_CONFIG_LOG_LEVEL'] = log_level 84 | os.environ['AWS_TEA_CONFIG_CMR'] = cmr_url 85 | event = {'headers': {'authorization': token}, 86 | 'pathParameters': {'id': provider}, 87 | 'path': f'/configuration/tea/provider/{provider}'} 88 | context = {} 89 | 90 | print(handler.generate_tea_config(event, context)) 91 | 92 | if __name__ == "__main__": 93 | main() 94 | -------------------------------------------------------------------------------- /capabilities.py: -------------------------------------------------------------------------------- 1 | """ 2 | AWS lambda function for generating a TEA configuration capabilities file 3 | """ 4 | 5 | import datetime 6 | import handler 7 | 8 | #pylint: disable=W0613 # AWS requires event and context, but these are not always used 9 | 10 | # ****************************************************************************** 11 | #mark - Utility functions 12 | 13 | def capability(event, endpoint, name, description, others=None): 14 | """ 15 | A helper function to generate one capability line for the capabilities() 16 | """ 17 | path = event['path'] 18 | named_path_element = path.rfind('/capabilities') 19 | if named_path_element>0: 20 | prefix = path[0:named_path_element] 21 | else: 22 | prefix = path 23 | ret = {'name': name, 'path': prefix+endpoint, 'description': description} 24 | 25 | if others is not None: 26 | for item in others: 27 | if others[item] is not None: 28 | ret[item] = others[item] 29 | return ret 30 | 31 | def header_line(name, description, head_type=None, values=None): 32 | """ 33 | A helper function to generate one header line for the capabilities() 34 | """ 35 | ret = {'name': name, 'description': description} 36 | if head_type is None: 37 | ret['type'] = 'string' 38 | else: 39 | ret['type'] = head_type 40 | if values is not None: 41 | ret['values'] = values 42 | return ret 43 | 44 | # ****************************************************************************** 45 | #mark - AWS Lambda functions 46 | 47 | def capabilities(event, context): 48 | """ Return a static output stating the calls supported by this package """ 49 | logger = handler.init_logging() 50 | logger.debug("capabilities have been loaded") 51 | 52 | start = datetime.datetime.now() 53 | 54 | h_pretty = header_line('Cmr-Pretty', 'format output with new lines', 55 | 'string', ['true', 'false']) 56 | h_took = header_line('Cmr-Took', 'number of seconds used to process request', 'real') 57 | h_token = header_line('Authorization', 'CMR Compatable Bearer token') 58 | h_type_json = header_line('content-type', 'content mime-type', None, 'application/json') 59 | h_type_text = header_line('content-type', 'content mime-type', None, 'text/plain') 60 | h_type_yaml = header_line('content-type', 'content mime-type', None, 'text/yaml') 61 | 62 | # optional return values 63 | optionals = lambda r,i,o : {'headers-in':i,'headers-out':o,'response':r} 64 | 65 | body = {} 66 | handler.append_version(body) 67 | body['urls'] = [ 68 | capability(event, 69 | '/', 70 | 'Root', 71 | 'Alias for the Capabilities'), 72 | capability(event, 73 | '/capabilities', 74 | 'Capabilities', 75 | 'Show which endpoints exist on this service', 76 | optionals('JSON', 77 | [h_pretty], 78 | [h_took, h_type_json])), 79 | capability(event, 80 | '/status', 81 | 'Status', 82 | 'Returns service status', 83 | optionals('418 I\'m a Teapot', 84 | None, 85 | [h_took, h_type_text])), 86 | capability(event, 87 | '/provider/', 88 | 'Generate', 89 | 'Generate a TEA config for provider', 90 | optionals('YAML', 91 | [h_pretty] + [h_token], 92 | [h_took, h_type_yaml])), 93 | ] 94 | 95 | return handler.aws_return_message(event, 200, body, start=start) 96 | -------------------------------------------------------------------------------- /test/tea/gen/get_acls_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | ACL tests 3 | """ 4 | 5 | from unittest import TestCase, mock 6 | from tea.gen.get_acls import get_acl, get_acls 7 | 8 | #python -m unittest discover -s ./ -p '*_test.py' 9 | class GetAclsTest(TestCase): 10 | "Do all the ACL test" 11 | 12 | @mock.patch('tea.gen.get_acls.requests.get') 13 | def test_get_acls(self, mock_get): 14 | "Get ACL check" 15 | my_mock_response = mock.Mock(status_code=200) 16 | my_mock_response.json.return_value = { 17 | 'hits' : 2, 18 | 'took' : 81, 19 | 'items' : [ { 20 | 'concept_id' : 'ACL1218667606555-CMR', 21 | 'revision_id' : 9, 22 | 'identity_type' : 'Catalog Item', 23 | 'name' : 'All Collections', 24 | 'location' : 'https://cmr.uat.earthdata.nasa.gov:443/' + 25 | 'access-control/acls/ACL1218667606555-CMR' 26 | }, { 27 | 'concept_id' : 'ACL1218667506777-CMR', 28 | 'revision_id' : 9, 29 | 'identity_type' : 'Catalog Item', 30 | 'name' : 'All Granules', 31 | 'location' : 'https://cmr.uat.earthdata.nasa.gov:443/' + 32 | 'access-control/acls/ACL1218667506777-CMR' 33 | } ]} 34 | mock_get.return_value = my_mock_response 35 | 36 | provider = 'XXX' 37 | env = {'cmr-url': 'XXX'} 38 | token = 'EDL-XXX' 39 | response = get_acls(env,provider,token) 40 | 41 | self.assertEqual(response[0]['location'], 42 | 'https://cmr.uat.earthdata.nasa.gov:443/access-control/acls/ACL1218667606555-CMR') 43 | self.assertEqual(response[1]['location'], 44 | 'https://cmr.uat.earthdata.nasa.gov:443/access-control/acls/ACL1218667506777-CMR') 45 | 46 | @mock.patch('tea.gen.get_acls.requests.get') 47 | def test_get_acl(self, mock_get): 48 | "test getting one acl" 49 | my_mock_response = mock.Mock(status_code=200) 50 | my_mock_response.json.return_value = { 51 | 'group_permissions' : [ { 52 | 'permissions' : [ 'read' ], 53 | 'user_type' : 'guest' 54 | }, { 55 | 'permissions' : [ 'read' ], 56 | 'user_type' : 'registered' 57 | }, { 58 | 'permissions' : [ 'read' ], 59 | 'group_id' : 'AG1222486916-CMR' 60 | }, { 61 | 'permissions' : [ 'read' ], 62 | 'group_id' : 'AG1236456866-CMR' 63 | }, { 64 | 'permissions' : [ 'read', 'order' ], 65 | 'group_id' : 'AG1216375421-SCIOPS' 66 | }, { 67 | 'permissions' : [ 'read', 'order' ], 68 | 'group_id' : 'AG1216375422-SCIOPS' 69 | }, { 70 | 'permissions' : [ 'read', 'order' ], 71 | 'group_id' : 'AG1215550981-CMR' 72 | } ], 73 | 'catalog_item_identity' : { 74 | 'name' : 'All Collections', 75 | 'provider_id' : 'SCIOPS', 76 | 'granule_applicable' : 'false', 77 | 'collection_applicable' : 'true', 78 | 'collection_identifier' : { 79 | 'concept_ids' : [ 'C1240032460-SCIOPS'], 80 | 'entry_titles' : [ '2000 Pilot Environmental Sustainability Index (ESI)' ] 81 | } 82 | }, 83 | 'legacy_guid' : 'F4E6573E-B97E-8BBC-A553-37DCE8F28D9D' 84 | } 85 | mock_get.return_value = my_mock_response 86 | 87 | acl_url = 'XXX' 88 | token = 'EDL-XXX' 89 | response = get_acl({}, acl_url, token) 90 | 91 | self.assertEqual(response['group_permissions'][3]['group_id'], 'AG1236456866-CMR') 92 | self.assertEqual(response['group_permissions'][5]['group_id'], 'AG1216375422-SCIOPS') 93 | -------------------------------------------------------------------------------- /test/handler_test.py: -------------------------------------------------------------------------------- 1 | """ Test module """ 2 | from unittest import TestCase, mock 3 | 4 | import handler as handler 5 | 6 | 7 | #python3 -m unittest discover -p '*_test.py' 8 | #python -m unittest discover -s ./ -p '*_test.py' 9 | class HandlerTest(TestCase): 10 | """ Test class to test Utils """ 11 | 12 | def test_lowercase_dictionary(self): 13 | """ Make sure that dictionaries are standardized correctly """ 14 | 15 | expected = {'y':"why", "r":"are", 'u':'you', 'a':'a', 'b':'bee'} 16 | 17 | test = lambda e, g, m : self.assertEqual(e, handler.lowercase_dictionary(g), m) 18 | 19 | test(expected, expected, "All lower case check, nothing should change") 20 | test(expected, 21 | {'Y':"why", "R":"are", 'U':'you', 'A':'a', 'B':'bee'}, 22 | "All upper case check, all keys should change") 23 | test(expected, 24 | {'Y':"why", "r":"are", 'U':'you', 'A':'a', 'b':'bee'}, 25 | "Some upper, some lower, upper should change") 26 | test(expected, 27 | {'Y': 'drop-this', 'y':"why", "r":"are", 'u':'you', 'a':'a', 'b':'bee'}, 28 | "double keys, only one should exist") 29 | 30 | def test_pretty_indent(self): 31 | """ Test pretty_indent """ 32 | 33 | # perform test: expected, given, message 34 | tester = lambda e, g, m : self.assertEqual(e, handler.pretty_indent(g), m) 35 | 36 | tester(None, None, "No envirnment") 37 | tester(None, {}, "Blank") 38 | tester(1, {'headers':{'cmr-pretty': 'true'}}, "lowercase test") 39 | tester(None, {'headers':{'Cmr-Pretty': 'false'}}, "false test") 40 | tester(1, {'headers':{'Cmr-Pretty': 'true'}}, "true test") 41 | 42 | tester(None, {'headers':{}}, "empty header") 43 | tester(None, {'headers':{'Cmr-Pretty':''}}, "empty header") 44 | #tester(None, {'headers':{'Cmr-Pretty':None}}, "empty header") 45 | 46 | # create dictionary: header value, query value 47 | env = lambda h,q : {'headers':{'Cmr-Pretty':h},'queryStringParameters':{'pretty':q}} 48 | 49 | tester(None, env('false', 'false'), "00") 50 | tester(1, env('false', 'true'), "01") 51 | tester(1, env('true', 'false'), "10") 52 | tester(1, env('true', 'true'), "11") 53 | 54 | tester(None, env('False', 'False'), "Upper case False check") 55 | tester(1, env('True', 'True'), "Upper case True check") 56 | 57 | tester(None, env('', 'false'), "empty header") 58 | tester(None, env('false', ''), "empty header") 59 | 60 | tester(1, env('', 'true'), "blank header") 61 | #tester(1, env('true', ''), "blank param") 62 | 63 | 64 | @mock.patch('handler.read_file') 65 | def test_get_group(self, mock_read): 66 | """ test that the version can be read and parsesd """ 67 | mock_read.return_value = None 68 | self.assertEqual(None, handler.load_version(), "file does not exist") 69 | 70 | mock_read.return_value = '{"version":"1.2.3","release":"1.2.3","when":"2022-03-01"}' 71 | self.assertEqual({"version":"1.2.3","release":"1.2.3","when":"2022-03-01"}, 72 | handler.load_version(), 73 | "can read version and parse") 74 | 75 | @mock.patch('handler.read_file') 76 | def test_get_group(self, mock_read): 77 | """ test that the version is appended when it needs to be """ 78 | mock_read.return_value = None 79 | 80 | data = None 81 | handler.append_version(data) 82 | self.assertEqual(data, handler.append_version(data), "data does not exist") 83 | 84 | data = {'fish':'food'} 85 | handler.append_version(data) 86 | self.assertEqual({'fish':'food'}, data, "have data, no file") 87 | 88 | # ############################## 89 | 90 | mock_read.return_value = '{"version":"1.2.3","release":"1.2.3","when":"2022-03-01"}' 91 | 92 | data = None 93 | handler.append_version(data) 94 | self.assertEqual(None, data, "data and file") 95 | 96 | data = {'fish':'food'} 97 | handler.append_version(data) 98 | expected = {'fish':'food', 99 | 'application':{"version":"1.2.3","release":"1.2.3","when":"2022-03-01"}} 100 | self.assertEqual(expected, data, "data and file") 101 | 102 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CMR TEA Configuration Generator 2 | A lambda project to generate [Thin Egress App][teacode] (TEA) configuration files using [Serverless][sls] 3 | 4 | ## Overrview 5 | A set of lambda functions to run under the CMR domain name which generate a TEA configuration. 6 | 7 | See: [Cumulus thin egress app][tea] 8 | 9 | ## Usage 10 | 11 | The [run.sh](run.sh) command will execute many of the stages of the software needed 12 | get the application started. To see the options supported, run `./run.sh -h`. To 13 | start the application, run in one terminal window: `./run.sh -o`. This will install 14 | the lambda functions and start the offline server. In another terminal window try 15 | the following commands: 16 | 17 | curl 'http://localhost:3000/dev/configuration/tea/capabilities' 18 | curl -I 'http://localhost:3000/dev/configuration/tea/status' 19 | curl -H 'Authorization: Bearer uat-token-value' 'http://localhost:3000/dev/configuration/tea/provider/POCLOUD' 20 | 21 | | Action | URL | Description | 22 | | ------------ | ----------------------------------------- | ----------- | 23 | | Capabilities | /configuration/tea/ | Describe the service | 24 | | Capabilities | /configuration/tea/capabilities | Describe the service | 25 | | Status | /configuration/tea/status | Returns the service status | 26 | | Generate | /configuration/tea/provider/{provider-id} | Generate the TEA configuration file | 27 | 28 | ## Building 29 | 30 | 1. Install serverless using one of these commands: 31 | * `npm install -g serverless` 32 | * `curl -o- -L https://slss.io/install | bash` 33 | 2. Plugin: `serverless plugin install -n serverless-python-requirements` 34 | 3. Plugin: `serverless plugin install -n serverless-s3-local` 35 | 4. Python dependencies: `pip3 install -r requirements.txt` 36 | 5. Install locally: `serverless offline` 37 | 6. Run some example URLS: `./run.sh -e` 38 | 39 | ### Docker 40 | 41 | The Dockerfile defines a node image which includes python3 and serverless for 42 | use as a build and deployment environment when running under a CI/CD system. 43 | 44 | ## Deploying to AWS 45 | 46 | Several methods are provided to publish this application to specific CMR AWS 47 | environments. These processes can be reused for other similar environments based 48 | on need. 49 | 50 | ### Deployment 51 | 52 | The [serverless.yml](serverless.yml) file will create an IAM role for the lambda 53 | functions and install them under an Application Load Balancer. The ARN is to be 54 | provided. The IAM role will use the following Role name: 55 | `tea-config-generator-role-${self:custom.env}-${self:provider.region}` and will 56 | use `arn:aws:iam::${aws:accountId}:policy/NGAPShRoleBoundary` for a Permissions 57 | Boundary. This setup is suitable for situations where a public URL is applied at 58 | the Loadbalancer level. The CMR URL must be set if production CMR is not to be used. See example below. 59 | 60 | * **NOTE**: This is the file used for deployment to SIT, UAT, and OPS 61 | 62 | Example usage for UAT: 63 | 64 | AWS_PROFILE= \ 65 | AWS_TEA_CONFIG_CMR='https://cmr.uat.earthdata.nasa.gov' \ 66 | serverless deploy --stage uat 67 | 68 | ### Sandbox 69 | The [serverless-sandbox.yml](serverless-sandbox.yml) file will also create an IAM 70 | role and be bound to the same Permissions Boundary, however it will use an API 71 | Gateway to publish the lambda functions. The URL for the API Gateway can be used 72 | for internal testing or for use within an AWS account. 73 | 74 | * **NOTE**: This is the file used for testing on AWS but not SIT, UAT, or OPS. 75 | 76 | 77 | ### Local Use 78 | The [serverless-local.yml](serverless-local.yml) file is meant for use locally by 79 | developers of the application. It will locally publish an API Gateway and allow 80 | access to the lambda functions through that. It also includes an S3 bucket for 81 | displaying the HTML API documentation. 82 | 83 | * **NOTE**: for localhost use only. 84 | 85 | ## Testing 86 | Read the details in the ./run.sh script. There are functions which have all the 87 | predefined parameters for running pylint and unit testing: 88 | 89 | * Unit Testing: `./run.sh -u` 90 | * Lint: `./run -l` 91 | 92 | ## License 93 | Copyright © 2022-2022 United States Government as represented by the Administrator 94 | of the National Aeronautics and Space Administration. All Rights Reserved. 95 | 96 | ---- 97 | 98 | [tea]: https://nasa.github.io/cumulus/docs/deployment/thin_egress_app "Thin Egress App" 99 | [teacode]: https://github.com/asfadmin/thin-egress-app "TEA @ GitHub" 100 | [sls]: https://serverless.com "Serverless" 101 | -------------------------------------------------------------------------------- /test/tea/gen/get_collections_test.py: -------------------------------------------------------------------------------- 1 | """ Test module """ 2 | from unittest import TestCase, mock 3 | from tea.gen.get_collections import get_collection 4 | 5 | #python -m unittest discover -s ./ -p '*_test.py' 6 | class GetCollectionsTest(TestCase): 7 | """ Test class to test GetCollections """ 8 | @mock.patch('tea.gen.get_collections.requests.get') 9 | def test_get_collections(self, mock_get): 10 | """ Tests get_collection """ 11 | my_mock_response = mock.Mock(status_code=200) 12 | my_mock_response.json.return_value = { 13 | 'DataLanguage' : 'eng', 14 | 'AncillaryKeywords' : [ 'GHRSST', 'sea surface temperature', 'Level 4', \ 15 | 'SST', 'surface temperature', ' MUR', ' foundation SST', ' SST anomaly', ' anomaly' ], 16 | 'CollectionCitations' : [ { 17 | 'Creator' : 'JPL MUR MEaSUREs Project', 18 | 'OnlineResource' : { 19 | 'Linkage' : 'https://podaac.jpl.nasa.gov/MEaSUREs-MUR' 20 | }, 21 | 'Publisher' : 'JPL NASA', 22 | 'Title' : 'GHRSST Level 4 MUR Global Foundation Sea Surface Temperature Analysis', 23 | 'SeriesName' : 'GHRSST Level 4 MUR Global Foundation Sea Surface Temperature Analysis', 24 | 'OtherCitationDetails' : 'JPL MUR MEaSUREs Project, JPL NASA, 2015-03-11, \ 25 | GHRSST Level 4 MUR Global Foundation Sea Surface Temperature Analysis (v4.1)', 26 | 'ReleaseDate' : '2015-03-11T00:00:00.000Z', 27 | 'Version' : '4.1', 28 | 'ReleasePlace' : 'Jet Propulsion Laboratory' 29 | } ], 30 | 'AdditionalAttributes' : [ { 31 | 'Name' : 'earliest_granule_start_time', 32 | 'Description' : 'Earliest Granule Start Time for dataset.', 33 | 'Value' : '2002-06-01T09:00:00.000Z', 34 | 'DataType' : 'DATETIME' 35 | }, { 36 | 'Name' : 'latest_granule_end_time', 37 | 'Description' : 'Latest Granule Stop/End Time for dataset.', 38 | 'Value' : '2021-01-25T09:00:00.000Z', 39 | 'DataType' : 'DATETIME' 40 | }, { 41 | 'Name' : 'Series Name', 42 | 'Description' : 'Dataset citation series name', 43 | 'Value' : 'GHRSST Level 4 MUR Global Foundation Sea Surface Temperature Analysis', 44 | 'DataType' : 'STRING' 45 | }, { 46 | 'Name' : 'Persistent ID', 47 | 'Description' : 'Dataset Persistent ID', 48 | 'Value' : 'PODAAC-GHGMR-4FJ04', 49 | 'DataType' : 'STRING' 50 | } ], 51 | 'SpatialExtent' : { 52 | 'SpatialCoverageType' : 'HORIZONTAL', 53 | 'HorizontalSpatialDomain' : { 54 | 'Geometry' : { 55 | 'CoordinateSystem' : 'CARTESIAN', 56 | 'BoundingRectangles' : [ { 57 | 'NorthBoundingCoordinate' : 90.0, 58 | 'WestBoundingCoordinate' : -180.0, 59 | 'EastBoundingCoordinate' : 180.0, 60 | 'SouthBoundingCoordinate' : -90.0 61 | } ] 62 | }, 63 | 'ResolutionAndCoordinateSystem' : { 64 | 'Description' : 'Projection Type: Cylindrical Lat-Lon, Projection Detail: \ 65 | Geolocation information included for each pixel', 66 | 'GeodeticModel' : { 67 | 'HorizontalDatumName' : 'World Geodetic System 1984', 68 | 'EllipsoidName' : 'WGS 84', 69 | 'SemiMajorAxis' : 6378137.0, 70 | 'DenominatorOfFlatteningRatio' : 298.2572236 71 | }, 72 | 'HorizontalDataResolution' : { 73 | 'GenericResolutions' : [ { 74 | 'XDimension' : 0.01, 75 | 'YDimension' : 0.01, 76 | 'Unit' : 'Decimal Degrees' 77 | } ] 78 | } 79 | } 80 | }, 81 | 'GranuleSpatialRepresentation' : 'CARTESIAN' 82 | }, 83 | 'DirectDistributionInformation' : { 84 | 'Region' : 'us-west-2', 85 | 'S3BucketAndObjectPrefixNames' : [ 'podaac-ops-cumulus-public/MUR-JPL-L4-GLOB-v4.1/', \ 86 | 'podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/' ], 87 | 'S3CredentialsAPIEndpoint' : 'https://archive.podaac.earthdata.nasa.gov/s3credentials', 88 | 'S3CredentialsAPIDocumentationURL' : \ 89 | 'https://archive.podaac.earthdata.nasa.gov/s3credentialsREADME' 90 | } 91 | } 92 | mock_get.return_value = my_mock_response 93 | 94 | env = {'cmr-url': 'XXX'} 95 | token = 'EDL-XXX' 96 | concept_id = 'XXX' 97 | response = get_collection(env, token, concept_id) 98 | 99 | self.assertEqual(response['DataLanguage'], 'eng') 100 | self.assertEqual(response['DirectDistributionInformation']['Region'], 'us-west-2') 101 | -------------------------------------------------------------------------------- /tea/gen/create_tea_config.py: -------------------------------------------------------------------------------- 1 | """ Main module to create TEA config """ 2 | import logging 3 | import sys 4 | import tea.gen.utils as util 5 | from tea.gen.utils import add_to_dict, create_tea_config 6 | from tea.gen.get_acls import get_acl, get_acls 7 | from tea.gen.get_groups import get_group_names 8 | from tea.gen.get_collections import get_collections_s3_prefixes_dict 9 | 10 | #pylint: disable=E1101 #No issue, just want number of handlers 11 | #pylint: disable=R0914 #We want parameters separated 12 | #pylint: disable=R0201 #We want it as class method 13 | #pylint: disable=R0903 #No point to add more public methods 14 | class CreateTeaConfig: 15 | """ Main class to create TEA config """ 16 | def __init__(self, env): 17 | self.logger = util.get_logger(env) 18 | self.logger.debug('Creating TEA configuraton') 19 | 20 | def create_tea_config(self, env:dict, provider:str, token:str): 21 | """ Main method to retrieve data and create TEA config """ 22 | all_s3_prefix_groups_dict = {} 23 | all_collections_s3_prefixes_dict = \ 24 | get_collections_s3_prefixes_dict(env, token, provider, 1, 2000) 25 | if not all_collections_s3_prefixes_dict: 26 | return {'statusCode': 404, 'body': 'No S3 prefixes returned'} 27 | acls = get_acls(env, provider, token) 28 | for acl in acls: 29 | acl_url = acl['location'] 30 | self.logger.debug('---------------------') 31 | self.logger.info('Found ACL %s', acl_url) 32 | acl_json = get_acl(env, acl_url, token) 33 | catalog_item_identity = acl_json['catalog_item_identity'] 34 | if 'collection_identifier' in catalog_item_identity: 35 | self.logger.info('Getting group names for ACL') 36 | group_names_set = set() 37 | group_names = get_group_names(env, acl_json, token) 38 | group_names_set.update(group_names) 39 | concept_ids = catalog_item_identity['collection_identifier']['concept_ids'] 40 | s3_prefixes_set = set() 41 | for concept_id in concept_ids: 42 | self.logger.info('Found concept id in ACL: %s', concept_id) 43 | if concept_id in all_collections_s3_prefixes_dict: 44 | col_s3_prefixes = all_collections_s3_prefixes_dict[concept_id] 45 | self.logger.info('Found S3 prefixes: %s', col_s3_prefixes) 46 | s3_prefixes_set.update(col_s3_prefixes) 47 | else: 48 | self.logger.info('No S3 prefixes found for concept id %s', concept_id) 49 | if s3_prefixes_set: 50 | self.logger.info('Number of S3 prefixes for ACL: %d', len(s3_prefixes_set)) 51 | self.logger.info('Found Group Name Set for ACL: %s', group_names_set) 52 | if group_names_set: 53 | add_to_dict(all_s3_prefix_groups_dict, s3_prefixes_set, group_names_set) 54 | else: 55 | self.logger.info('No S3 prefixes found for ACL') 56 | else: 57 | self.logger.info('ACL does not have concept ids assigned') 58 | if all_s3_prefix_groups_dict: 59 | tea_config_text = create_tea_config(all_s3_prefix_groups_dict) 60 | self.logger.info('result mapping:\n%s', tea_config_text) 61 | return {'statusCode': 200, 'body': tea_config_text} 62 | 63 | self.logger.info('No S3 prefixes found') 64 | return {'statusCode': 404, 'body': 'No S3 prefixes found'} 65 | 66 | def main(): 67 | """ Main method - a direct unit test """ 68 | if len(logging.getLogger().handlers) > 0: 69 | logging.getLogger().setLevel(logging.INFO) 70 | else: 71 | logging.basicConfig(filename='script.log', \ 72 | format='%(asctime)s %(message)s', \ 73 | level=logging.INFO) 74 | 75 | provider = input('Enter provider (POCLOUD is default): ') 76 | if provider is None or len(provider)<1: 77 | provider = 'POCLOUD' 78 | print(f"Using {provider}.") 79 | 80 | cmr_env = input('Enter env (sit, uat or prod; uat is default): ') 81 | if cmr_env is None or len(cmr_env)<1: 82 | cmr_env = 'uat' 83 | print (f"Using {cmr_env}.") 84 | 85 | token = input('Enter EDL token: ') 86 | if token is None or len(token)<1: 87 | print ('A CMR compatable token must be provided') 88 | sys.exit() 89 | 90 | cmrs = {'sit':'https://cmr.sit.earthdata.nasa.gov', 91 | 'uat':'https://cmr.uat.earthdata.nasa.gov', 92 | 'ops':'https://cmr.earthdata.nasa.gov', 93 | 'prod':'https://cmr.earthdata.nasa.gov'} 94 | 95 | env = {'cmr-url': cmrs[cmr_env.lower()]} 96 | env['logging-level'] = 'INFO' 97 | env['pretty-print'] = True 98 | 99 | processor = CreateTeaConfig(env) 100 | print(processor.create_tea_config(env, provider, token)) 101 | 102 | if __name__ == "__main__": 103 | main() 104 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | RED='\033[0;31m' 4 | GREEN='\033[0;32m' 5 | YELLOW='\033[0;33m' 6 | BLUE='\033[0;34m' 7 | WHITE='\033[0;37m' 8 | BOLD='\033[1m' 9 | UNDERLINE='\033[4m' 10 | NC='\033[0m' # No Color 11 | color_mode='yes' 12 | 13 | cprintf() { 14 | color=$1 15 | content=$2 16 | if [ "$color_mode" == "no" ] 17 | then 18 | printf "${content}\n" 19 | else 20 | printf "${color}%s${NC}\n" "${content}" 21 | fi 22 | } 23 | 24 | # Check the syntax of the code for PIP8 violations 25 | lint() 26 | { 27 | printf '*****************************************************************\n' 28 | printf 'Run pylint to check for common code convention warnings\n' 29 | pylint *.py tea \ 30 | --disable=duplicate-code \ 31 | --extension-pkg-allow-list=math \ 32 | --ignore-patterns=".*\.md,.*\.sh,.*\.html,pylintrc,LICENSE,build,dist,tags,eo_metadata_tools_cmr.egg-info" \ 33 | > lint.results.txt 34 | cat lint.results.txt 35 | } 36 | 37 | # Run all the Unit Tests 38 | report_code_coverage() 39 | { 40 | # https://docs.codecov.com/docs/codecov-uploader 41 | printf '*****************************************************************\n' 42 | printf 'Run the unit tests for all subdirectories\n' 43 | # coverage should have already been installed with pip3 44 | coverage run --source=cmr -m unittest discover 45 | coverage html 46 | } 47 | 48 | # Generate documentation and copy it into a local S3 bucket for viewing 49 | documentation() 50 | { 51 | if command -v markdown &> /dev/null ; then 52 | markdown -T public/api.md > public/api.html 53 | else 54 | echo "could not found markdown, try running the following" 55 | cprint $GREEN "brew install discount" 56 | fi 57 | if command -v aws &> /dev/null ; then 58 | aws --endpoint http://localhost:7000 \ 59 | s3 cp public/api.html s3://local-bucket/api.html \ 60 | --profile s3local 61 | else 62 | echo "aws command not found" 63 | fi 64 | } 65 | 66 | # Have serverless deploy, called from within the docker container 67 | deploy() 68 | { 69 | if [ -e ./credentials ] ; then 70 | # CI/CD creates this file, only install credentials if they exist 71 | if [ -d ~/.aws ] ; then 72 | echo 'skipping, do not override' 73 | else 74 | mkdir ~/.aws/ 75 | cp ./credentials ~/.aws/. 76 | fi 77 | fi 78 | serverless deploy --stage "${deploy_env}" 79 | } 80 | 81 | help_doc() 82 | { 83 | echo 'Script to manage the life cycle of the Tea Configuration code' 84 | echo 'Usage:' 85 | echo ' ./run.sh -[c|C] -t -[hulre]' 86 | echo ' ./run.sh -[o|I]' 87 | echo 88 | 89 | format="%4s %-8s %10s | %s\n" 90 | printf "${format}" 'flag' 'value' 'name' 'Description' 91 | printf "${format}" '----' '--------' '------' '-------------------------' 92 | printf "${format}" '-h' '' 'Help' 'Print out a help document' 93 | printf "${format}" '-c' '' 'Color on' 'Turn color on (default)' 94 | printf "${format}" '-C' '' 'Color off' 'Turn color off' 95 | printf "${format}" '-d' '' 'Documentation' 'Generate Documentation for AWS' 96 | printf "${format}" '-D' '' 'Deploy' 'Run deployment tasks' 97 | printf "${format}" '-u' '' 'Unittest' 'Run python unit test' 98 | printf "${format}" '-U' '' 'Unittest' 'Run python unit test and save results' 99 | printf "${format}" '-j' '' 'JUnittest' 'Run python unit test and save junit results' 100 | printf "${format}" '-l' '' 'Lint' 'Run pylint over all files' 101 | printf "${format}" '-L' '' 'Lint' 'Run pylint over all files and save results' 102 | printf "${format}" '-t' '' 'Token' 'Set token' 103 | printf "${format}" '-r' '' 'Report' 'Generate code coverage report' 104 | printf "${format}" '-e' '' 'Example' 'Do curl examples' 105 | printf "${format}" '-o' '' 'Offline' 'Run serverless offline then exit' 106 | printf "${format}" '-I' '' 'Install' 'Install dependent libraries' 107 | } 108 | 109 | while getopts 'hcCdDe:uUlLjt:orSeIx' opt; do 110 | case ${opt} in 111 | h) help_doc ;; 112 | c) color_mode='yes';; 113 | C) color_mode='no' ;; 114 | d) documentation ;; 115 | e) deploy_env=${OPTARG} ;; 116 | D) deploy ; exit $? ;; 117 | u) python3 -m unittest discover -s ./ -p '*test.py' ;; 118 | U) python3 -m unittest discover -s ./ -p '*test.py' &> test.results.txt ;; 119 | l) lint ;; 120 | L) lint &> list.results ;; 121 | j) pip3 install --break-system-packages pytest ; py.test --junitxml junit.xml ;; 122 | t) token=${OPTARG} ;; 123 | o) serverless offline ; exit ;; 124 | r) report_code_coverage ;; 125 | S) serverless doctor &> doctor.txt ;; 126 | 127 | x) rm -rf build script.log ;; 128 | e) curl -H "Authorization: ${token}" "${baseEndPoint}/configuration/tea/provider/POCLOUD" ;; 129 | I) 130 | #alternet ways to install serverless, enable as needed 131 | #npm install -g serverless 132 | #curl --silent -o- --location https://slss.io/install | bash 133 | pip3 install --break-system-packages -r requirements.txt 134 | serverless plugin install --name serverless-offline 135 | serverless plugin install --name serverless-python-requirements 136 | serverless plugin install --name serverless-s3-local 137 | ;; 138 | *) cprintf $RED "option required" ; exit 42 ;; 139 | esac 140 | done 141 | -------------------------------------------------------------------------------- /test/tea/gen/utils_test.py: -------------------------------------------------------------------------------- 1 | """ Test module """ 2 | from unittest import TestCase 3 | import tea.gen.utils as util 4 | from tea.gen.utils import add_to_dict, get_s3_prefixes, get_env 5 | 6 | #python -m unittest discover -s ./ -p '*_test.py' 7 | class UtilsTest(TestCase): 8 | """ Test class to test Utils """ 9 | 10 | def test_standard_headers(self): 11 | """ 12 | Test that the standard header function always returns the correct user agent 13 | """ 14 | test = lambda e,p,m : self.assertEqual(e, p['User-Agent'], m) 15 | 16 | expected='ESDIS TEA Config Generator' 17 | 18 | test(expected, util.standard_headers(), 'dictionary not provided') 19 | test(expected, util.standard_headers(None), 'none dictionary') 20 | test(expected, util.standard_headers({'u':'you'}), 'dictionary provided') 21 | test(expected, util.standard_headers({'User-Agent':'Wrong'}), 'overwrite check') 22 | 23 | actual=util.standard_headers({'Other-Header':'Keep'}) 24 | self.assertEqual('Keep', actual['Other-Header'], 'other values') 25 | 26 | def test_add_to_dict(self): 27 | """ Test add_to_dict """ 28 | dict_a = {} 29 | set_b = set() 30 | set_b.update([1,2]) 31 | set_c = set() 32 | set_c.update(['a','b','c']) 33 | add_to_dict(dict_a,set_b,set_c) 34 | self.assertEqual(len(dict_a[1]), 3) 35 | self.assertEqual(len(dict_a[2]), 3) 36 | 37 | def test_get_env(self): 38 | """ 39 | Test that the get_env variable is always able to get a URL out of the env 40 | """ 41 | tester = lambda data,exp,desc : self.assertEqual(get_env(data), exp,desc) 42 | same = lambda url,reason : tester({'cmr-url':url}, url, reason) 43 | 44 | same('https://cmr.sit.earthdata.nasa.gov', 'sit') 45 | same('https://cmr.uat.earthdata.nasa.gov', 'uat') 46 | same('https://cmr.earthdata.nasa.gov', 'ops') 47 | tester({'bad-key':'value'}, 'https://cmr.earthdata.nasa.gov', 'wrong key') 48 | tester({}, 'https://cmr.earthdata.nasa.gov', 'empty') 49 | tester({'cmr-url':None}, None, 'None value') 50 | 51 | def test_get_s3_prefixes(self): 52 | """ Test get_s3_prefixes """ 53 | collection = { 54 | 'DataLanguage' : 'eng', 55 | 'AncillaryKeywords' : [ 'GHRSST', 'sea surface temperature', 'Level 4', \ 56 | 'SST', 'surface temperature', ' MUR', ' foundation SST', ' SST anomaly', ' anomaly' ], 57 | 'CollectionCitations' : [ { 58 | 'Creator' : 'JPL MUR MEaSUREs Project', 59 | 'OnlineResource' : { 60 | 'Linkage' : 'https://podaac.jpl.nasa.gov/MEaSUREs-MUR' 61 | }, 62 | 'Publisher' : 'JPL NASA', 63 | 'Title' : 'GHRSST Level 4 MUR Global Foundation Sea Surface Temperature Analysis', 64 | 'SeriesName' : 'GHRSST Level 4 MUR Global Foundation Sea Surface Temperature Analysis', 65 | 'OtherCitationDetails' : 'JPL MUR MEaSUREs Project, JPL NASA, 2015-03-11, \ 66 | GHRSST Level 4 MUR Global Foundation Sea Surface Temperature Analysis (v4.1)', 67 | 'ReleaseDate' : '2015-03-11T00:00:00.000Z', 68 | 'Version' : '4.1', 69 | 'ReleasePlace' : 'Jet Propulsion Laboratory' 70 | } ], 71 | 'AdditionalAttributes' : [ { 72 | 'Name' : 'earliest_granule_start_time', 73 | 'Description' : 'Earliest Granule Start Time for dataset.', 74 | 'Value' : '2002-06-01T09:00:00.000Z', 75 | 'DataType' : 'DATETIME' 76 | }, { 77 | 'Name' : 'latest_granule_end_time', 78 | 'Description' : 'Latest Granule Stop/End Time for dataset.', 79 | 'Value' : '2021-01-25T09:00:00.000Z', 80 | 'DataType' : 'DATETIME' 81 | }, { 82 | 'Name' : 'Series Name', 83 | 'Description' : 'Dataset citation series name', 84 | 'Value' : 'GHRSST Level 4 MUR Global Foundation Sea Surface Temperature Analysis', 85 | 'DataType' : 'STRING' 86 | }, { 87 | 'Name' : 'Persistent ID', 88 | 'Description' : 'Dataset Persistent ID', 89 | 'Value' : 'PODAAC-GHGMR-4FJ04', 90 | 'DataType' : 'STRING' 91 | } ], 92 | 'SpatialExtent' : { 93 | 'SpatialCoverageType' : 'HORIZONTAL', 94 | 'HorizontalSpatialDomain' : { 95 | 'Geometry' : { 96 | 'CoordinateSystem' : 'CARTESIAN', 97 | 'BoundingRectangles' : [ { 98 | 'NorthBoundingCoordinate' : 90.0, 99 | 'WestBoundingCoordinate' : -180.0, 100 | 'EastBoundingCoordinate' : 180.0, 101 | 'SouthBoundingCoordinate' : -90.0 102 | } ] 103 | }, 104 | 'ResolutionAndCoordinateSystem' : { 105 | 'Description' : 'Projection Type: Cylindrical Lat-Lon, Projection Detail: \ 106 | Geolocation information included for each pixel', 107 | 'GeodeticModel' : { 108 | 'HorizontalDatumName' : 'World Geodetic System 1984', 109 | 'EllipsoidName' : 'WGS 84', 110 | 'SemiMajorAxis' : 6378137.0, 111 | 'DenominatorOfFlatteningRatio' : 298.2572236 112 | }, 113 | 'HorizontalDataResolution' : { 114 | 'GenericResolutions' : [ { 115 | 'XDimension' : 0.01, 116 | 'YDimension' : 0.01, 117 | 'Unit' : 'Decimal Degrees' 118 | } ] 119 | } 120 | } 121 | }, 122 | 'GranuleSpatialRepresentation' : 'CARTESIAN' 123 | }, 124 | 'DirectDistributionInformation' : { 125 | 'Region' : 'us-west-2', 126 | 'S3BucketAndObjectPrefixNames' : [ 'podaac-ops-cumulus-public/MUR-JPL-L4-GLOB-v4.1/', \ 127 | 'podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/' ], 128 | 'S3CredentialsAPIEndpoint' : 'https://archive.podaac.earthdata.nasa.gov/s3credentials', 129 | 'S3CredentialsAPIDocumentationURL' : \ 130 | 'https://archive.podaac.earthdata.nasa.gov/s3credentialsREADME' 131 | } 132 | } 133 | 134 | s3_prefixes = get_s3_prefixes(collection) 135 | self.assertEqual(s3_prefixes[0], 'podaac-ops-cumulus-public/MUR-JPL-L4-GLOB-v4.1/') 136 | self.assertEqual(s3_prefixes[1], 'podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/') 137 | -------------------------------------------------------------------------------- /handler.py: -------------------------------------------------------------------------------- 1 | """ 2 | AWS lambda functions for generating a TEA configuration. 3 | """ 4 | 5 | import os 6 | import json 7 | import logging 8 | import datetime 9 | import boto3 10 | 11 | import tea.gen.create_tea_config as tea 12 | 13 | #pylint: disable=W0613 # AWS requires event and context, but these are not always used 14 | 15 | # ****************************************************************************** 16 | #mark - Utility functions 17 | 18 | def lowercase_dictionary(dirty_dictionary): 19 | """ 20 | Standardize the keys in a header to lower case to reduce the number of lookup 21 | cases that need to be supported. Assumes that there are no duplicates, if 22 | there are, the last one is saved. 23 | """ 24 | return dict((key.lower(), value) for key,value in dirty_dictionary.items()) 25 | 26 | def finish_timer(start): 27 | """ Calculate and format the number of seconds from start to 'now'. """ 28 | delta = datetime.datetime.now() - start 29 | sec = delta.total_seconds() 30 | return f"{sec:.6f}" 31 | 32 | def pretty_indent(event): 33 | """ Look for pretty in both the header and query parameters """ 34 | indent = None # none means do not pretty print, a number is the tab count 35 | if event: 36 | pretty = 'false' 37 | 38 | # look first at query 39 | query_params = event.get('queryStringParameters') 40 | if query_params: 41 | pretty = query_params.get('pretty', "false") 42 | 43 | # next try the header 44 | if pretty=='false' and 'headers' in event: 45 | # standardize the keys to lowercase 46 | headers = lowercase_dictionary(event['headers']) 47 | pretty = headers.get('cmr-pretty', 'false') 48 | 49 | # set the output 50 | if pretty.lower()=="true": 51 | indent=1 52 | return indent 53 | 54 | def read_file(file_name): 55 | """ 56 | Read version file content written by the CI/CD system, return None if it 57 | does not exist 58 | """ 59 | if os.path.exists(file_name): 60 | with open(file_name, encoding='utf-8') as out_file: 61 | return out_file.read() 62 | return None 63 | 64 | def load_version(): 65 | """ Load version information from a file. This file is written by CI/CD """ 66 | ver = read_file('ver.txt') 67 | if ver is not None: 68 | return json.loads(ver) 69 | return None 70 | 71 | def append_version(data:dict=None): 72 | """ Append CI/CD version information to a dictionary if it exists. """ 73 | if data is not None: 74 | ver = load_version() 75 | if ver is not None: 76 | data['application'] = ver 77 | 78 | def aws_return_message(event, status, body, headers=None, start=None): 79 | """ build a dictionary which AWS Lambda will accept as a return value """ 80 | indent = pretty_indent(event) 81 | 82 | ret = {"statusCode": status, 83 | "body": json.dumps(body, indent=indent), 84 | 'headers': {}} 85 | if start is not None: 86 | ret['headers']['cmr-took'] = finish_timer(start) 87 | if headers is not None: 88 | for header in headers: 89 | ret['headers'][header] = headers[header] 90 | return ret 91 | 92 | def parameter_read(pname, default_value=''): 93 | """ 94 | Try to read a parameter first from SSM, if not there, then try the os environment 95 | """ 96 | #pylint: disable=W0703 # no, fall back to os in all cases 97 | try: 98 | ssm = boto3.client('ssm') 99 | param = ssm.get_parameter(Name=pname.upper(), WithDecryption=True) 100 | return param['Parameter']['Value'] 101 | except Exception: 102 | return os.environ.get(pname.upper(), default_value) 103 | return default_value 104 | 105 | def init_logging(): 106 | """ 107 | Initialize the logging system using the logging level provided by the calling 108 | 'environment' and return a logger 109 | """ 110 | level = parameter_read('AWS_TEA_CONFIG_LOG_LEVEL', default_value='INFO') 111 | logging.basicConfig(format="%(name)s - %(module)s - %(message)s",level=level) 112 | logger = logging.getLogger() 113 | logger.setLevel(level) 114 | return logger 115 | 116 | # ****************************************************************************** 117 | #mark - AWS Lambda functions 118 | 119 | def debug(event, context): 120 | """ Return debug information about AWS in general """ 121 | 122 | logger = init_logging() 123 | logger.info("Debug have been loaded") 124 | 125 | start = datetime.datetime.now() 126 | body = { 127 | 'context': context.get_remaining_time_in_millis(), 128 | 'event': event, 129 | 'clean-header': lowercase_dictionary(event['headers']), 130 | 'tea_config_cmr': parameter_read('AWS_TEA_CONFIG_CMR', 131 | default_value='https://cmr.earthdata.nasa.gov'), 132 | 'tea_config_log_level': parameter_read('AWS_TEA_CONFIG_LOG_LEVEL', 133 | default_value='INFO'), 134 | } 135 | 136 | for env, value in os.environ.items(): 137 | if env.startswith('AWS_'): 138 | if env in ['AWS_SESSION_TOKEN', 'AWS_SECRET_ACCESS_KEY', 'AWS_ACCESS_KEY_ID']: 139 | body[env] = '~redacted~' 140 | else: 141 | body[env] = value 142 | append_version(body) 143 | 144 | return aws_return_message(event, 200, body, start=start) 145 | 146 | def health(event, context): 147 | """ 148 | Provide an endpoint for testing service availability and for complicance with 149 | RFC 7168 150 | """ 151 | logger = init_logging() 152 | logger.debug("health check has been requested") 153 | 154 | return aws_return_message(event, 155 | 418, 156 | "I'm a teapot", 157 | headers={'content-type': 'text/plain'}, 158 | start=datetime.datetime.now()) 159 | 160 | def generate_tea_config(event, context): 161 | """ 162 | Lambda function to return a TEA configuration. Requires that event contain 163 | the following: 164 | * CMR must be configured with an env variable 165 | * Path Parameter named 'id' with CMR provider name 166 | * HTTP Header named 'Authorization' with a CMR compatible token 167 | """ 168 | logger = init_logging() 169 | logger.debug("generate tea config started") 170 | 171 | start = datetime.datetime.now() 172 | 173 | provider = None 174 | if 'pathParameters' in event: 175 | provider = event['pathParameters'].get('id') 176 | elif 'path' in event: 177 | parts = event['path'].split('/', -1) 178 | if 1