├── .gitignore ├── AWS_Glue_Athena_Quicksight ├── age-groupings.sql ├── create-records-program.py ├── gender-percentages.sql ├── most-common-ages.sql └── start-glue-job-lamba.js ├── Amazon_Simple_Storage_Service ├── Getting_Data_Into_S3 │ ├── boto3 │ │ ├── boto3actions.py │ │ └── boto3upload.txt │ ├── cli │ │ ├── cli.txt │ │ ├── movemeCLI.txt │ │ ├── syncmeCLI.txt │ │ └── webconsole.txt │ └── web │ │ └── webconsole.txt ├── Lifecycle_Policies │ ├── ArchivePolicy.json │ └── addlifecycle.txt └── Multipart_Upload │ ├── WheretheBuffaloRoam-URI │ └── multipartupload.py ├── Data_Analytics_with_Spark_and_EMR ├── cloudformation │ └── SetupEnv.yml ├── pyspark-script │ ├── emr-pyspark-code-local.py │ └── emr-pyspark-code.py ├── results │ ├── ._SUCCESS.crc │ ├── .part-00000-71cf5249-7476-4501-b86e-04ed053f4a99-c000.csv.crc │ └── _SUCCESS └── user-data-acg.zip ├── Elasticsearch ├── query_utility │ └── query_es.py └── reddit_loader │ ├── scrape.py │ └── scrape_stream.py ├── Lab_Assets ├── Brocks_Lab_Assets │ ├── groceries.csv │ ├── sandbox-notebook.ipynb │ └── simulate-customer-orders.py ├── advanced_s3_security_configuration │ ├── bucket_policy.json │ ├── group_policy.json │ └── user_policy.json ├── implementing_an_elasticsearch_backed_search_microservice │ └── function_solved.py ├── manually_migrating_data_between_redshift_clusters │ └── solution.sql ├── programmatically_utilizing_data_from_s3 │ ├── lambda │ │ ├── function.py │ │ └── function_solved.py │ └── web │ │ ├── icon.png │ │ ├── index.html │ │ ├── randomusers.css │ │ └── randomusers.js ├── programmatically_utilizing_s3_select │ ├── lambda │ │ ├── function.py │ │ └── function_solved.py │ └── web │ │ ├── icon.png │ │ ├── index.html │ │ ├── randomusers.css │ │ └── randomusers.js └── querying_data_from_multiple_redshift_spectrum_tables │ └── solution.sql ├── Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis ├── cloudformation │ └── SetupKinesisHelper_CloudCraft.yml ├── create-users-dynamodb-lambda │ ├── function │ │ ├── create-users-dynamodb-lambda.zip │ │ ├── index.js │ │ ├── package-lock.json │ │ ├── package.json │ │ └── user_json.json │ └── simple-event.json ├── dockerized-angular-app │ ├── .dockerignore │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ ├── ansible.cfg │ ├── hosts │ ├── playbooks │ │ └── setup-angular-app.yml │ └── site │ │ ├── angular.json │ │ ├── e2e │ │ ├── protractor.conf.js │ │ ├── src │ │ │ ├── app.e2e-spec.ts │ │ │ └── app.po.ts │ │ └── tsconfig.e2e.json │ │ ├── package-lock.json │ │ ├── package.json │ │ ├── src │ │ ├── app │ │ │ ├── app-routing.module.ts │ │ │ ├── app.component.css │ │ │ ├── app.component.html │ │ │ ├── app.component.ts │ │ │ ├── app.module.ts │ │ │ ├── home │ │ │ │ ├── home.component.css │ │ │ │ ├── home.component.html │ │ │ │ └── home.component.ts │ │ │ └── kinesis-helper │ │ │ │ ├── kinesis-helper.component.css │ │ │ │ ├── kinesis-helper.component.html │ │ │ │ ├── kinesis-helper.component.ts │ │ │ │ └── kinesis-helper.service.ts │ │ ├── assets │ │ │ ├── .gitkeep │ │ │ ├── acloudguru_long_logo.png │ │ │ ├── angular_logo.png │ │ │ ├── ansible-setup-diagram.png │ │ │ ├── ansible_logo.png │ │ │ ├── docker_logo.png │ │ │ └── middle_img.png │ │ ├── browserslist │ │ ├── environments │ │ │ ├── environment.prod.ts │ │ │ └── environment.ts │ │ ├── index.html │ │ ├── karma.conf.js │ │ ├── main.ts │ │ ├── polyfills.ts │ │ ├── styles.css │ │ ├── test.ts │ │ ├── tsconfig.app.json │ │ ├── tsconfig.spec.json │ │ └── tslint.json │ │ ├── tsconfig.json │ │ └── tslint.json └── other-scripts │ ├── enrich-data-lambda-function.py │ ├── filter-top-orders.sql │ └── new-line-function.py ├── README.md ├── Redshift_Maintenance_And_Operations └── Launching_A_Redshift_Cluster │ ├── createcluster.py │ └── createcluster.sh └── Using_Redshift └── Redshift_Spectrum └── Spectrum_demo_queries.md /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # System Files 132 | .DS_Store 133 | 134 | */node_modules 135 | */dist 136 | -------------------------------------------------------------------------------- /AWS_Glue_Athena_Quicksight/age-groupings.sql: -------------------------------------------------------------------------------- 1 | SELECT SUM(CASE WHEN "dob.age" BETWEEN 21 AND 29 THEN 1 ELSE 0 END) AS "21-29", 2 | SUM(CASE WHEN "dob.age" BETWEEN 30 AND 39 THEN 1 ELSE 0 END) AS "30-39", 3 | SUM(CASE WHEN "dob.age" BETWEEN 40 AND 49 THEN 1 ELSE 0 END) AS "40-49", 4 | SUM(CASE WHEN "dob.age" BETWEEN 50 AND 59 THEN 1 ELSE 0 END) AS "50-59", 5 | SUM(CASE WHEN "dob.age" BETWEEN 60 AND 69 THEN 1 ELSE 0 END) AS "60-69", 6 | SUM(CASE WHEN "dob.age" BETWEEN 70 AND 79 THEN 1 ELSE 0 END) AS "70-79" 7 | FROM transformed_data; -------------------------------------------------------------------------------- /AWS_Glue_Athena_Quicksight/create-records-program.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import uuid 3 | import requests 4 | import json 5 | import random 6 | import time 7 | import csv 8 | 9 | # Initialize interfaces 10 | s3Client = boto3.client('s3', 11 | region_name='us-east-1', 12 | aws_access_key_id='XXX', 13 | aws_secret_access_key='XXX') 14 | s3Resource = boto3.resource('s3') 15 | number_of_results = 500 16 | 17 | 18 | def createRecords(format, bucket_name, key, number_of_files): 19 | 20 | if(format == "csv"): 21 | createCSVRecords(bucket_name, key, number_of_files) 22 | else: 23 | createJSONRecords(bucket_name, number_of_files) 24 | 25 | def createJSONRecords(bucket_name, number_of_files): 26 | count = 1 27 | r = requests.get('https://randomuser.me/api/?exc=login&results=' + str(number_of_results)) 28 | data = r.json()["results"] 29 | 30 | for _ in range(number_of_files): 31 | uuid_str = str(uuid.uuid4()) 32 | json_data = '' 33 | 34 | for x in data: 35 | random_user_index = int(random.uniform(0, (number_of_results - 1))) 36 | json_data += json.dumps(data[random_user_index]) + '\n' 37 | 38 | json_data = bytes(json_data) 39 | response = s3Client.put_object( 40 | Body = json_data, 41 | Bucket = bucket_name, 42 | Key = 'user-data-' + uuid_str + '.json', 43 | ) 44 | print('Added {} file...').format(count) 45 | count = count + 1 46 | 47 | def createCSVRecords(bucket_name, key, number_of_files): 48 | count = 1 49 | 50 | for _ in range(number_of_files): 51 | # csv_data = 'gender,name.title,name.first,name.last,location.street.number,location.street.name,location.city,location.state,location.country,location.postcode,location.coordinates.latitude,location.coordinates.longitude,location.timezone.offset,location.timezone.description,email,login.uuid,login.username,login.password,login.salt,login.md5,login.sha1,login.sha256,dob.date,dob.age,registered.date,registered.age,phone,cell,id.name,id.value,picture.large,picture.medium,picture.thumbnail,nat' 52 | csv_data = '' 53 | r = requests.get('https://randomuser.me/api/?exc=login&nat=us&results=' + str(number_of_results) + '&format=csv') 54 | text = r.iter_lines() 55 | data = csv.reader(text, delimiter=',') 56 | 57 | 58 | for row in data: 59 | error = False 60 | for index in range(27): 61 | try: 62 | row[index].encode(encoding='UTF-8',errors='strict') 63 | if(',' in row[index]): 64 | error = True 65 | break 66 | except UnicodeDecodeError: 67 | error = True 68 | break 69 | if(error == False): 70 | csv_data += '{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}\n'.format(row[0], row[1], row[2], row[3], row[4],row[5], row[6], row[7], row[8], row[9],row[10], row[11], row[12], row[13], row[14],row[15], row[16], row[17], row[18], row[19],row[20], row[21], row[22], row[23], row[24],row[25], row[26]) 71 | 72 | uuid_str = str(uuid.uuid4()) 73 | 74 | csv_data = bytes(csv_data) 75 | response = s3Client.put_object( 76 | Body = csv_data, 77 | Bucket = bucket_name, 78 | Key = key + 'user-data-' + uuid_str + '.csv', 79 | ) 80 | print('Added {} file...').format(count) 81 | count = count + 1 82 | time.sleep(5) 83 | 84 | createRecords("json", "user-data-acg", '', 100) 85 | 86 | # first job - 2:36:31 -------------------------------------------------------------------------------- /AWS_Glue_Athena_Quicksight/gender-percentages.sql: -------------------------------------------------------------------------------- 1 | SELECT gender, (COUNT(gender) * 100.0 / (SELECT COUNT(*) FROM transformed_data)) AS percent 2 | FROM transformed_data 3 | GROUP BY gender; -------------------------------------------------------------------------------- /AWS_Glue_Athena_Quicksight/most-common-ages.sql: -------------------------------------------------------------------------------- 1 | SELECT "dob.age", COUNT("dob.age") AS occurances 2 | FROM transformed_data 3 | GROUP BY "dob.age" 4 | ORDER BY occurances DESC 5 | LIMIT 5; -------------------------------------------------------------------------------- /AWS_Glue_Athena_Quicksight/start-glue-job-lamba.js: -------------------------------------------------------------------------------- 1 | var AWS = require('aws-sdk'); 2 | var glue = new AWS.Glue(); 3 | 4 | exports.handler = async (event, callback, context) => { 5 | 6 | // Lets check to see if any jobs are currently running 7 | var params = { 8 | JobName: 'new-job', 9 | }; 10 | return glue.getJobRuns(params).promise().then((data) => { 11 | var isJobRunning = false; 12 | var noJobs = false; 13 | if(data["JobRuns"].length === 0) { 14 | noJobs = true; 15 | } 16 | console.log(noJobs) 17 | data["JobRuns"].forEach(job => { 18 | 19 | if(job["JobRunState"] !== 'SUCCEEDED') { 20 | isJobRunning = true; 21 | } 22 | }); 23 | 24 | if(!isJobRunning || noJobs) { 25 | var params = { 26 | JobName: 'new-job', 27 | }; 28 | console.log('Starting a new job....'); 29 | return glue.startJobRun(params).promise(); 30 | } else { 31 | return console.log('Not starting a new job...'); 32 | } 33 | 34 | 35 | }); 36 | }; 37 | -------------------------------------------------------------------------------- /Amazon_Simple_Storage_Service/Getting_Data_Into_S3/boto3/boto3actions.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | 3 | # Initialize interfaces 4 | s3Client = boto3.client('s3') 5 | s3Resource = boto3.resource('s3') 6 | 7 | # Create byte string to send to our bucket 8 | putMessage = b'Hi! I came from Boto3!' 9 | 10 | # put_object 11 | response = s3Client.put_object( 12 | Body = putMessage, 13 | Bucket = 'das-demos', 14 | Key = 'boto3put.txt' 15 | ) 16 | 17 | print(response) 18 | 19 | # copy 20 | toCopy = { 21 | 'Bucket': 'das-demos', 22 | 'Key': 'boto3put.txt' 23 | } 24 | 25 | s3Resource.meta.client.copy(toCopy, 'das-demos', 'boto3copy.txt') 26 | 27 | # copy_object 28 | response = s3Client.copy_object( 29 | Bucket = 'das-demos', 30 | CopySource = '/das-demos/boto3put.txt', 31 | Key = 'boto3copyobject.txt' 32 | ) 33 | 34 | print(response) 35 | 36 | # upload_file 37 | boto3Upload = 'boto3upload.txt' 38 | 39 | s3Resource.meta.client.upload_file(boto3Upload, 'das-demos', boto3Upload) 40 | 41 | # upload_fileobj 42 | with open(boto3Upload, 'rb') as fileObj: 43 | response = s3Client.upload_fileobj(fileObj, 'das-demos', 'boto3uploadobj.txt') 44 | print(response) 45 | 46 | ''' 47 | response = s3Client.put_bucket_accelerate_configuration( 48 | Bucket='das-demos', 49 | AccelerateConfiguration={ 50 | 'Status': 'Enabled' 51 | } 52 | ) 53 | 54 | print(response) 55 | ''' -------------------------------------------------------------------------------- /Amazon_Simple_Storage_Service/Getting_Data_Into_S3/boto3/boto3upload.txt: -------------------------------------------------------------------------------- 1 | Hi I was uploaded with boto3! -------------------------------------------------------------------------------- /Amazon_Simple_Storage_Service/Getting_Data_Into_S3/cli/cli.txt: -------------------------------------------------------------------------------- 1 | Hi! I came from the CLI! -------------------------------------------------------------------------------- /Amazon_Simple_Storage_Service/Getting_Data_Into_S3/cli/movemeCLI.txt: -------------------------------------------------------------------------------- 1 | Hi I got moved here from the AWS CLI! -------------------------------------------------------------------------------- /Amazon_Simple_Storage_Service/Getting_Data_Into_S3/cli/syncmeCLI.txt: -------------------------------------------------------------------------------- 1 | I got here from a sync! Update me and sync again! -------------------------------------------------------------------------------- /Amazon_Simple_Storage_Service/Getting_Data_Into_S3/cli/webconsole.txt: -------------------------------------------------------------------------------- 1 | Hi! I came from the web console -------------------------------------------------------------------------------- /Amazon_Simple_Storage_Service/Getting_Data_Into_S3/web/webconsole.txt: -------------------------------------------------------------------------------- 1 | Hi! I came from the web console -------------------------------------------------------------------------------- /Amazon_Simple_Storage_Service/Lifecycle_Policies/ArchivePolicy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Rules": [ 3 | { 4 | "Filter": { 5 | "Prefix": "logs/" 6 | }, 7 | "Status": "Enabled", 8 | "Transitions": [ 9 | { 10 | "Days": 30, 11 | "StorageClass": "STANDARD_IA" 12 | }, 13 | { 14 | "Days": 120, 15 | "StorageClass": "DEEP_ARCHIVE" 16 | } 17 | ], 18 | "Expiration": { 19 | "Days": 2555 20 | }, 21 | "ID": "ApplicationLogArchiving" 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /Amazon_Simple_Storage_Service/Lifecycle_Policies/addlifecycle.txt: -------------------------------------------------------------------------------- 1 | aws s3api put-bucket-lifecycle-configuration --bucket das-demos --lifecycle-configuration file://ArchivePolicy.json -------------------------------------------------------------------------------- /Amazon_Simple_Storage_Service/Multipart_Upload/WheretheBuffaloRoam-URI: -------------------------------------------------------------------------------- 1 | https://das-c01-data-analytics-specialty.s3.amazonaws.com/amazon-simple-storage-service/multipart-upload/WheretheBuffaloRoam.mp4 -------------------------------------------------------------------------------- /Amazon_Simple_Storage_Service/Multipart_Upload/multipartupload.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import boto3 3 | import json 4 | import multiprocessing 5 | 6 | # Starts Multipart Upload 7 | def start_upload(bucket, key): 8 | s3_client = boto3.client('s3') 9 | 10 | response = s3_client.create_multipart_upload( 11 | Bucket = bucket, 12 | Key = key 13 | ) 14 | 15 | return response['UploadId'] 16 | 17 | # Add upload part 18 | def add_part(proc_queue, body, bucket, key, part_number, upload_id): 19 | s3_client = boto3.client('s3') 20 | 21 | response = s3_client.upload_part( 22 | Body = body, 23 | Bucket = bucket, 24 | Key = key, 25 | PartNumber = part_number, 26 | UploadId = upload_id 27 | ) 28 | 29 | print(f"Finished Part: {part_number}, ETag: {response['ETag']}") 30 | proc_queue.put({'PartNumber': part_number, 'ETag': response['ETag']}) 31 | return 32 | 33 | # End Multipart Upload 34 | def end_upload(bucket, key, upload_id, finished_parts): 35 | s3_client = boto3.client('s3') 36 | 37 | response = s3_client.complete_multipart_upload( 38 | Bucket = bucket, 39 | Key = key, 40 | MultipartUpload={ 41 | 'Parts': finished_parts 42 | }, 43 | UploadId = upload_id 44 | ) 45 | 46 | return response 47 | 48 | # Primary logic 49 | def main(): 50 | ap = argparse.ArgumentParser() 51 | ap.add_argument('-f', '--file', required = True, help = "File to be chunked and uploaded") 52 | ap.add_argument('-k', '--key', help = "Key for destination object") 53 | ap.add_argument('-b', '--bucket', required = True, help = "Destination bucket") 54 | ap.add_argument('-cs', '--chunk_size', required = True, type = int, choices = range(5,101), metavar = '[5-100]', help = "Chunk size in MB, must be > 5MiB") 55 | ap.add_argument('-p', '--processes', type = int, choices = range(1,256), metavar = '[1-256]', default = 10, help = "Number of upload processes to run simultaneously") 56 | args = vars(ap.parse_args()) 57 | 58 | if args['key'] in [None, '']: 59 | args['key'] = args['file'] 60 | 61 | file = args['file'] 62 | key = args['key'] 63 | bucket = args['bucket'] 64 | sim_proc = args['processes'] 65 | upload_id = start_upload(bucket, key) 66 | print(f'Starting upload: {upload_id}') 67 | 68 | file_upload = open(file, 'rb') 69 | part_procs = [] 70 | proc_queue = multiprocessing.Queue() 71 | queue_returns = [] 72 | chunk_size = (args['chunk_size'] * 1024) * 1024 73 | part_num = 1 74 | chunk = file_upload.read(chunk_size) 75 | 76 | while len(chunk) > 0: 77 | proc = multiprocessing.Process(target=add_part, args=(proc_queue, chunk, bucket, key, part_num, upload_id)) 78 | part_procs.append(proc) 79 | part_num += 1 80 | chunk = file_upload.read(chunk_size) 81 | 82 | part_procs = [part_procs[i * sim_proc:(i +1) * sim_proc] for i in range((len(part_procs) + (sim_proc - 1)) // sim_proc)] 83 | 84 | for i in range(len(part_procs)): 85 | for p in part_procs[i]: 86 | p.start() 87 | 88 | for p in part_procs[i]: 89 | p.join() 90 | 91 | for p in part_procs[i]: 92 | queue_returns.append(proc_queue.get()) 93 | 94 | queue_returns = sorted(queue_returns, key = lambda i: i['PartNumber']) 95 | response = end_upload(bucket, key, upload_id, queue_returns) 96 | print(json.dumps(response, sort_keys=True, indent=4)) 97 | 98 | if __name__ == '__main__': 99 | main() -------------------------------------------------------------------------------- /Data_Analytics_with_Spark_and_EMR/cloudformation/SetupEnv.yml: -------------------------------------------------------------------------------- 1 | Description: 2 | This is a template that launches a simple AWS account. 3 | Resources: 4 | AccessKey: 5 | Type: AWS::IAM::AccessKey 6 | Properties: 7 | UserName: cloud_user 8 | LambdaRole: 9 | Type: AWS::IAM::Role 10 | Properties: 11 | AssumeRolePolicyDocument: 12 | Version: '2012-10-17' 13 | Statement: 14 | - Effect: Allow 15 | Principal: 16 | Service: 17 | - lambda.amazonaws.com 18 | Action: 19 | - sts:AssumeRole 20 | Path: "/" 21 | Policies: 22 | - PolicyName: EC2AccessRole 23 | PolicyDocument: 24 | Version: '2012-10-17' 25 | Statement: 26 | - Effect: Allow 27 | Action: 28 | - ec2:* 29 | Resource: "*" 30 | - Effect: Allow 31 | Action: 32 | - logs:* 33 | Resource: "*" 34 | 35 | InitFunction: 36 | Type: AWS::Lambda::Function 37 | Properties: 38 | Code: 39 | ZipFile: | 40 | import json 41 | import boto3 42 | import threading 43 | import urllib3 44 | SUCCESS = "SUCCESS" 45 | FAILED = "FAILED" 46 | http = urllib3.PoolManager() 47 | def send(event, context, responseStatus, responseData, physicalResourceId=None, noEcho=False, reason=None): 48 | responseUrl = event['ResponseURL'] 49 | print(responseUrl) 50 | responseBody = {} 51 | responseBody['Status'] = responseStatus 52 | responseBody['Reason'] = reason or "See the details in CloudWatch Log Stream: {}".format(context.log_stream_name) 53 | responseBody['PhysicalResourceId'] = physicalResourceId or context.log_stream_name 54 | responseBody['StackId'] = event['StackId'] 55 | responseBody['RequestId'] = event['RequestId'] 56 | responseBody['LogicalResourceId'] = event['LogicalResourceId'] 57 | responseBody['NoEcho'] = noEcho 58 | responseBody['Data'] = responseData 59 | json_responseBody = json.dumps(responseBody) 60 | print("Response body:\n" + json_responseBody) 61 | headers = { 62 | 'content-type' : '', 63 | 'content-length' : str(len(json_responseBody)) 64 | } 65 | try: 66 | response = http.request('PUT',responseUrl,headers=headers,body=json_responseBody) 67 | print("Status code: {}".format(str(response.status))) 68 | except Exception as e: 69 | print("send(..) failed executing requests.put(..): " + str(e)) 70 | def createDefault(): 71 | print("Creating default VPC") 72 | ec2 = boto3.client('ec2') 73 | response = ec2.create_default_vpc() 74 | return response 75 | def deleteDefault(): 76 | return "" 77 | def timeout(event, context): 78 | print('Timing out, sending failure response to CFN') 79 | send(event, context, FAILED, {}, None) 80 | def lambda_handler(event, context): 81 | print(f'Received event: {json.dumps(event)}') 82 | timer = threading.Timer((context.get_remaining_time_in_millis() / 1000.00) - 0.5, timeout, args=[event, context]) 83 | timer.start() 84 | status = SUCCESS 85 | responseData = {} 86 | try: 87 | if event['RequestType'] == 'Delete': 88 | deleteDefault() 89 | else: 90 | response = createDefault() 91 | print(response) 92 | responseData['Data'] = response 93 | except Exception as e: 94 | print(e) 95 | status = FAILED 96 | finally: 97 | timer.cancel() 98 | send(event, context, status, responseData, None) 99 | Handler: index.lambda_handler 100 | Role: !GetAtt LambdaRole.Arn 101 | Runtime: python3.7 102 | Timeout: 60 103 | 104 | InitializeVPC: 105 | Type: Custom::InitFunction 106 | Properties: 107 | ServiceToken: !GetAtt InitFunction.Arn 108 | Outputs: 109 | pubIpAddress1: 110 | Description: cloud_user Access Key 111 | Value: !Ref AccessKey 112 | pubIpAddress2: 113 | Description: cloud_user Secret Access Key 114 | Value: !GetAtt AccessKey.SecretAccessKey 115 | 116 | -------------------------------------------------------------------------------- /Data_Analytics_with_Spark_and_EMR/pyspark-script/emr-pyspark-code-local.py: -------------------------------------------------------------------------------- 1 | from pyspark import SparkConf, SparkContext 2 | from pyspark.sql import SparkSession 3 | 4 | spark = SparkSession.builder.master("local").config(conf=SparkConf()).getOrCreate() 5 | 6 | df = spark.read.format("csv").option("header", "true").load("../user-data-acg/user-data-*.csv") 7 | results = df.groupBy("`dob.age`","`gender`").count().orderBy('count', ascending=False) 8 | results.show() 9 | results.coalesce(1).write.csv("../results", sep=',', header='true') -------------------------------------------------------------------------------- /Data_Analytics_with_Spark_and_EMR/pyspark-script/emr-pyspark-code.py: -------------------------------------------------------------------------------- 1 | from pyspark import SparkConf, SparkContext 2 | from pyspark.sql import SparkSession 3 | 4 | spark = SparkSession.builder.master("local") 5 | .config(conf=SparkConf()).getOrCreate() 6 | 7 | df = spark.read.format("csv") 8 | .option("header", "true") 9 | .load("hdfs:///user-data-acg/user-data-*.csv") 10 | 11 | results = df.groupBy("`dob.age`","`gender`") 12 | .count() 13 | .orderBy("count", ascending=False) 14 | 15 | results.show() 16 | 17 | results.coalesce(1).write.csv("hdfs:///results", sep=",", header="true") -------------------------------------------------------------------------------- /Data_Analytics_with_Spark_and_EMR/results/._SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /Data_Analytics_with_Spark_and_EMR/results/.part-00000-71cf5249-7476-4501-b86e-04ed053f4a99-c000.csv.crc: -------------------------------------------------------------------------------- 1 | crcIKr; -------------------------------------------------------------------------------- /Data_Analytics_with_Spark_and_EMR/results/_SUCCESS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality/881ac05f8b7e4218f3bdebddb82a91be8ec1e156/Data_Analytics_with_Spark_and_EMR/results/_SUCCESS -------------------------------------------------------------------------------- /Data_Analytics_with_Spark_and_EMR/user-data-acg.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality/881ac05f8b7e4218f3bdebddb82a91be8ec1e156/Data_Analytics_with_Spark_and_EMR/user-data-acg.zip -------------------------------------------------------------------------------- /Elasticsearch/query_utility/query_es.py: -------------------------------------------------------------------------------- 1 | from elasticsearch import Elasticsearch, RequestsHttpConnection 2 | from requests_aws4auth import AWS4Auth 3 | import boto3 4 | import json 5 | import argparse 6 | 7 | 8 | def query(term): 9 | host = 'search-r-aws-scrape-ipb6jt4hdfgwwbkedxmg2gr2uy.us-west-2.es.amazonaws.com' 10 | region = 'us-west-2' 11 | 12 | service = 'es' 13 | credentials = boto3.Session().get_credentials() 14 | awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service, session_token=credentials.token) 15 | 16 | es = Elasticsearch( 17 | hosts = [{'host': host, 'port': 443}], 18 | http_auth = awsauth, 19 | use_ssl = True, 20 | verify_certs = True, 21 | connection_class = RequestsHttpConnection 22 | ) 23 | 24 | query = { 25 | 'query': { 26 | 'query_string': { 27 | 'query': term, 28 | 'default_field': 'body' 29 | } 30 | } 31 | } 32 | response = es.search(index="r-aws-scrape", body=json.dumps(query)) 33 | 34 | return response 35 | 36 | def main(): 37 | ap = argparse.ArgumentParser() 38 | ap.add_argument('-qt', '--query_term', required = True, type = str, help = 'term to query for') 39 | query_term = vars(ap.parse_args())['query_term'] 40 | 41 | print(json.dumps(query(query_term), indent=4, separators=(',', ': '), sort_keys=True)) 42 | 43 | if __name__ == '__main__': 44 | main() -------------------------------------------------------------------------------- /Elasticsearch/reddit_loader/scrape.py: -------------------------------------------------------------------------------- 1 | import praw 2 | import boto3 3 | import datetime 4 | 5 | client_ident = '9LjBYEs4MIv0Tw' 6 | client_skey = '2IqklztP5SJCZVcc8wn72hxUMa0' 7 | client_agent = 'scrapetest' 8 | 9 | def handler(event, context): 10 | dynamodb = boto3.client('dynamodb', region_name='us-west-2') 11 | 12 | try: 13 | latest_submission = dynamodb.query( 14 | TableName = 'r-aws-scrape', 15 | IndexName = 'created_unix-index', 16 | ExpressionAttributeValues = { 17 | ':sr': { 18 | 'S': 'aws' 19 | } 20 | }, 21 | KeyConditionExpression = 'subreddit = :sr', 22 | ProjectionExpression = 'created_unix', 23 | Limit = 1, 24 | ScanIndexForward = False 25 | )['Items'][0]['created_unix']['N'] 26 | except Exception: 27 | latest_submission = '0.0' 28 | 29 | print(f'Latest Post: {latest_submission}') 30 | 31 | reddit = praw.Reddit( 32 | client_id = client_ident, 33 | client_secret = client_skey, 34 | user_agent = client_agent 35 | ) 36 | 37 | aws_subreddit = reddit.subreddit('aws') 38 | submissions = aws_subreddit.new(limit=None) 39 | 40 | for submission in submissions: 41 | if submission.created_utc >= float(latest_submission): 42 | submission_data = { 43 | 'created_unix': {'N': str(submission.created_utc)}, 44 | 'created_utc': {'S': str(datetime.datetime.fromtimestamp(submission.created_utc))}, 45 | 'id': {'S': submission.id}, 46 | 'num_comments': {'N': str(submission.num_comments)}, 47 | 'score': {'N': str(submission.score)}, 48 | 'title': {'S': submission.title}, 49 | 'link': {'S': submission.url}, 50 | 'subreddit': {'S': 'aws'} 51 | } 52 | 53 | try: 54 | submission_data['author'] = {'S': submission.author.name} 55 | except Exception: 56 | submission_data['author'] = {'S': '[deleted]'} 57 | 58 | if len(submission.selftext) > 0: 59 | submission_data['body'] = {'S': submission.selftext} 60 | 61 | try: 62 | dynamodb.put_item( 63 | TableName = 'r-aws-scrape', 64 | Item = submission_data, 65 | ) 66 | except Exception as e: 67 | print(e) 68 | 69 | return -------------------------------------------------------------------------------- /Elasticsearch/reddit_loader/scrape_stream.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import requests 3 | from requests_aws4auth import AWS4Auth 4 | 5 | region = 'us-west-2' 6 | service = 'es' 7 | credentials = boto3.Session().get_credentials() 8 | awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service, session_token=credentials.token) 9 | 10 | host = 'https://search-r-aws-scrape-ipb6jt4hdfgwwbkedxmg2gr2uy.us-west-2.es.amazonaws.com' 11 | index = 'r-aws-scrape' 12 | type = 'reddit-submission-type' 13 | url = f'{host}/{index}/{type}/' 14 | 15 | headers = { "Content-Type": "application/json" } 16 | 17 | def handler(event, context): 18 | count = 0 19 | 20 | for record in event['Records']: 21 | # Get the primary key for use as the Elasticsearch ID 22 | id = record['dynamodb']['Keys']['id']['S'] 23 | 24 | if record['eventName'] == 'REMOVE': 25 | r = requests.delete(url + id, auth=awsauth) 26 | print(r.status_code) 27 | else: 28 | document = deserialize(record['dynamodb']['NewImage']) 29 | r = requests.put(url + id, auth=awsauth, json=document, headers=headers) 30 | print(r.status_code) 31 | print(r.content.decode()) 32 | count += 1 33 | 34 | return f'{count} records processed.' 35 | 36 | def deserialize(item): 37 | record = {} 38 | for k,v in item.items(): 39 | for value in v.values(): 40 | if isinstance(value, list): 41 | record[k] = [] 42 | for li in value: 43 | for s in li.values(): 44 | if '.' in value: 45 | try: 46 | record[k].append(float(s)) 47 | except ValueError: 48 | record[k].append(s) 49 | else: 50 | try: 51 | record[k].append(int(s)) 52 | except ValueError: 53 | record[k].append(s) 54 | elif '.' in value: 55 | try: 56 | record[k] = float(value) 57 | except ValueError: 58 | record[k] = value 59 | else: 60 | try: 61 | record[k] = int(value) 62 | except ValueError: 63 | record[k] = value 64 | 65 | return record -------------------------------------------------------------------------------- /Lab_Assets/Brocks_Lab_Assets/simulate-customer-orders.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import uuid 3 | import requests 4 | import json 5 | import random 6 | import time 7 | 8 | client = boto3.client('kinesis', 9 | region_name='us-east-1', 10 | aws_access_key_id='XXX', 11 | aws_secret_access_key='XXX') 12 | # partition_key = str(uuid.uuid4()) 13 | 14 | number_of_results = 500 15 | r = requests.get('https://randomuser.me/api/?results=' + str(number_of_results)) 16 | data = r.json()["results"] 17 | 18 | # while True: 19 | # # The following chooses a random user from the 500 random users pulled from the API in a single API call. 20 | # random_user_index = int(random.uniform(0, (number_of_results - 1))) 21 | # random_user = data[random_user_index] 22 | # random_user = json.dumps(data[random_user_index]) 23 | # client.put_record( 24 | # StreamName='', 25 | # Data=random_user, 26 | # PartitionKey=partition_key) 27 | # time.sleep(random.uniform(0, 1)) 28 | 29 | random_user_index = int(random.uniform(0, (number_of_results - 1))) 30 | random_user = data[random_user_index] 31 | # random_user = json.dumps(data[random_user_index]) 32 | # partition_key = 33 | print(random_user["email"]) -------------------------------------------------------------------------------- /Lab_Assets/advanced_s3_security_configuration/bucket_policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version":"2012-10-17", 3 | "Statement":[ 4 | { 5 | "Sid":"PublicRead", 6 | "Effect":"Allow", 7 | "Principal": "*", 8 | "Action":[ 9 | "s3:GetObject", 10 | "s3:GetObjectVersion" 11 | ], 12 | "Resource":[ 13 | "arn:aws:s3:::/public/*" 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /Lab_Assets/advanced_s3_security_configuration/group_policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement":[ 4 | { 5 | "Sid":"AllowList", 6 | "Action":[ 7 | "s3:ListBucket" 8 | ], 9 | "Effect":"Allow", 10 | "Resource":[ 11 | "arn:aws:s3:::" 12 | ], 13 | "Condition":{ 14 | "StringLike":{ 15 | "s3:prefix":[ 16 | "team/*" 17 | ] 18 | } 19 | } 20 | }, 21 | { 22 | "Sid":"AllowGroupToReadWrite", 23 | "Action":[ 24 | "s3:GetObject", 25 | "s3:PutObject" 26 | ], 27 | "Effect":"Allow", 28 | "Resource":[ 29 | "arn:aws:s3:::/team/*" 30 | ] 31 | } 32 | ] 33 | } -------------------------------------------------------------------------------- /Lab_Assets/advanced_s3_security_configuration/user_policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement":[ 4 | { 5 | "Sid":"AllowList", 6 | "Action":[ 7 | "s3:ListBucket" 8 | ], 9 | "Effect":"Allow", 10 | "Resource":[ 11 | "arn:aws:s3:::" 12 | ], 13 | "Condition":{ 14 | "StringLike":{ 15 | "s3:prefix":[ 16 | "${aws:username}/*" 17 | ] 18 | } 19 | } 20 | }, 21 | { 22 | "Sid":"AllowUserToReadWrite", 23 | "Action":[ 24 | "s3:GetObject", 25 | "s3:PutObject" 26 | ], 27 | "Effect":"Allow", 28 | "Resource":[ 29 | "arn:aws:s3:::/${aws:username}/*" 30 | ] 31 | } 32 | ] 33 | } -------------------------------------------------------------------------------- /Lab_Assets/implementing_an_elasticsearch_backed_search_microservice/function_solved.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | from elasticsearch import Elasticsearch 4 | 5 | def query_es(query_term): 6 | es = Elasticsearch( 7 | [{'host': '', 'port': 443, 'use_ssl': True}], 8 | http_auth = ('cloud_user', 'Strongpass1!') 9 | ) 10 | 11 | es_query = { 12 | 'query': { 13 | 'simple_query_string': { 14 | 'query': query_term, 15 | 'default_operator': 'and' 16 | } 17 | } 18 | } 19 | 20 | ''' 21 | es_query = { 22 | 'query': { 23 | 'match': { 24 | 'Text': { 25 | 'query': query_term, 26 | 'fuzziness': 'AUTO' 27 | } 28 | } 29 | } 30 | } 31 | ''' 32 | es_query = { 33 | 'query': { 34 | 'simple_query_string': { 35 | 'query': query_term, 36 | 'default_operator': 'and' 37 | } 38 | } 39 | } 40 | 41 | es_response = json.loads(json.dumps(es.search(index = 'frankenstein', body = json.dumps(es_query)))) 42 | print(es_response) 43 | 44 | response = { 45 | 'total_hits': es_response["hits"]["total"]["value"], 46 | 'chapter_count': 0, 47 | 'total_score': 0.0, 48 | 'hits': [] 49 | } 50 | 51 | for hit in es_response['hits']['hits']: 52 | response['chapter_count'] += 1 53 | for key, value in hit['_source'].items(): 54 | if key != 'Text': 55 | location = f'{key} {value}' 56 | score = hit['_score'] 57 | response['hits'].append( 58 | { 59 | 'location': location, 60 | 'score': score 61 | 62 | } 63 | ) 64 | 65 | response['total_score'] += hit['_score'] 66 | 67 | response['total_score'] = round(response['total_score'], 2) 68 | 69 | # print(json.dumps(response, indent = 4) ) 70 | return response 71 | 72 | def handler(event, context): 73 | return { 74 | 'isBase64Encoded': False, 75 | 'statusCode': 200, 76 | 'body': json.dumps(query_es(event['queryStringParameters']['query'])), 77 | 'headers': {"Access-Control-Allow-Origin": "*"} 78 | } -------------------------------------------------------------------------------- /Lab_Assets/manually_migrating_data_between_redshift_clusters/solution.sql: -------------------------------------------------------------------------------- 1 | UNLOAD ('select * from users_data') 2 | TO '' 3 | IAM_ROLE '' 4 | FORMAT AS PARQUET; 5 | 6 | create table users_data( 7 | id_value varchar(64), 8 | name_first varchar(64), 9 | name_last varchar(64), 10 | location_country varchar(32), 11 | dob_age int, 12 | picture_large varchar(64), 13 | primary key(id_value) 14 | ) 15 | distkey(location_country) 16 | compound sortkey(id_value); 17 | 18 | COPY users_data 19 | FROM '' 20 | IAM_ROLE '' 21 | FORMAT AS PARQUET; -------------------------------------------------------------------------------- /Lab_Assets/programmatically_utilizing_data_from_s3/lambda/function.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | 4 | def get_data(): 5 | data = [] 6 | 7 | return data 8 | 9 | def handler(event, context): 10 | # Call the "get_data" function and return appropriately formatted results. 11 | return {'isBase64Encoded': False,'statusCode': 200,'body': json.dumps(get_data()), 'headers': {"Access-Control-Allow-Origin": "*"}} -------------------------------------------------------------------------------- /Lab_Assets/programmatically_utilizing_data_from_s3/lambda/function_solved.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | 4 | def get_data(): 5 | # S3 bucket we'll be interacting with 6 | s3_bucket = '' 7 | # Because we need to combine data from multiple S3 objects, initialize a list to hold this data before returning it. 8 | data = [] 9 | # Initialize an boto3 S3 client, and list the objects in our bucket. The data about the contents of our bucket will be stored in a list called s3_keys. 10 | s3 = boto3.client('s3') 11 | objects = s3.list_objects_v2( 12 | Bucket = s3_bucket 13 | )['Contents'] 14 | 15 | s3_keys = [] 16 | for object in objects: 17 | if object['Key'].startswith('users_'): 18 | s3_keys.append(object['Key']) 19 | 20 | # After collecting the appropriate keys that begin with "users_" gather each object, and combine the returned data with the existing "data" list. 21 | for key in s3_keys: 22 | object = s3.get_object( 23 | Bucket = s3_bucket, 24 | Key = key 25 | ) 26 | 27 | object_data = json.loads(object['Body'].read()) 28 | data += object_data 29 | 30 | # Return our combined data from all "users_" objects. 31 | return data 32 | 33 | def handler(event, context): 34 | # Call the "get_data" function and return appropriately formatted results. 35 | return {'isBase64Encoded': False,'statusCode': 200,'body': json.dumps(get_data()), 'headers': {"Access-Control-Allow-Origin": "*"}} 36 | -------------------------------------------------------------------------------- /Lab_Assets/programmatically_utilizing_data_from_s3/web/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality/881ac05f8b7e4218f3bdebddb82a91be8ec1e156/Lab_Assets/programmatically_utilizing_data_from_s3/web/icon.png -------------------------------------------------------------------------------- /Lab_Assets/programmatically_utilizing_data_from_s3/web/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Random Users! 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 17 |
18 |

User Count:

19 |
20 |
21 |
22 |
23 | 24 | 25 |
26 |
27 | 28 | 29 |
30 |
31 | 32 | 33 |
34 |
35 | 36 | 37 |
38 |
39 | 40 | 41 |
42 | 43 |
44 |
45 |
46 |
47 |
48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 |
PictureFirst NameLast NameAgeCountryState
68 |
69 |
70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /Lab_Assets/programmatically_utilizing_data_from_s3/web/randomusers.css: -------------------------------------------------------------------------------- 1 | body { 2 | background-image: linear-gradient(#70e7f7a2, #dddddd); 3 | } 4 | 5 | body, 6 | html { 7 | height: 100%; 8 | } 9 | 10 | table { 11 | table-layout: fixed; 12 | word-wrap: break-word; 13 | } 14 | 15 | th { 16 | width: 5%; 17 | } 18 | 19 | .custombg { 20 | background-color: #a4f0fa; 21 | } 22 | 23 | .statusbox { 24 | padding-top: 10px; 25 | padding-right: 10px; 26 | } 27 | 28 | .welcome { 29 | position: absolute; 30 | padding-top: 40px; 31 | padding-left: 10px; 32 | } 33 | 34 | .toppad { 35 | padding-top: 10px; 36 | } 37 | 38 | .contentbox { 39 | max-height: 95vh; 40 | padding: 0px; 41 | background-image: linear-gradient(#dddddd, #70e7f7a2); 42 | } 43 | 44 | .countbox { 45 | position: absolute; 46 | top: 65px; 47 | left: 30px; 48 | } 49 | 50 | .filterbox { 51 | position: absolute; 52 | bottom: 65px; 53 | left: 30px; 54 | } 55 | 56 | .loadbox { 57 | position: absolute; 58 | top: 45vh; 59 | left: 45vw; 60 | } 61 | 62 | .circle { 63 | margin: 0 auto; 64 | border-radius: 50%; 65 | border: 10px solid rgba(0, 0, 0, 0); 66 | border-top-color: #347fc3; 67 | } 68 | 69 | .circle1-1, .circle1-2, .circle1-3, .circle1-4, .circle1-5, .circle1-6 { 70 | height: 100px; 71 | width: 100px; 72 | margin-top: 15px; 73 | } 74 | 75 | .circle2-1, .circle2-2, .circle2-3, .circle2-4, .circle2-5, .circle2-6 { 76 | height: 70px; 77 | width: 70px; 78 | margin-top: 5px; 79 | } 80 | 81 | .circle3-1, .circle3-2, .circle3-3, .circle3-4, .circle3-5, .circle3-6 { 82 | height: 40px; 83 | width: 40px; 84 | margin-top: 5px; 85 | } 86 | 87 | .circle1 { 88 | -webkit-animation: load1 4s infinite; 89 | } 90 | 91 | @-webkit-keyframes load1 { 92 | 0% { 93 | } 94 | 100% { 95 | -webkit-transform: rotate(360deg); 96 | } 97 | } -------------------------------------------------------------------------------- /Lab_Assets/programmatically_utilizing_data_from_s3/web/randomusers.js: -------------------------------------------------------------------------------- 1 | var ViewModel = function () { 2 | var self = this; 3 | self.url = window.location.href 4 | self.users = ko.observableArray([]); 5 | self.userCount = ko.observable(0); 6 | self.tableClass = ko.observable("d-none"); 7 | self.filterFirst = ko.observable(); 8 | self.filterLast = ko.observable(); 9 | self.filterAge = ko.observable(); 10 | self.filterCountry = ko.observable(); 11 | self.filterUState = ko.observable(); 12 | var tableVis = "container overflow-auto contentbox w-75" 13 | 14 | var displayValue = function(element, valueAccessor){ 15 | var value = ko.utils.unwrapObservable(valueAccessor()); 16 | var isCurrentlyVisible = !(element.style.display == "none"); 17 | if(value && !isCurrentlyVisible) 18 | element.style.display = ""; 19 | else if((!value) && isCurrentlyVisible) 20 | element.style.display = "none"; 21 | }; 22 | 23 | 24 | ko.bindingHandlers['loading-animation'] = { 25 | 26 | 'init' : function(element, valueAccessor){ 27 | $(element) 28 | .append( 29 | '
'); 30 | 31 | displayValue(element, valueAccessor); 32 | }, 33 | 'update' : function(element, valueAccessor){ 34 | displayValue(element, valueAccessor); 35 | } 36 | }; 37 | 38 | IsLoading = ko.observable(true); 39 | 40 | self.filterUsers = function() { 41 | var filterData = { 42 | "name.\"first\"": self.filterFirst(), 43 | "name.\"last\"": self.filterLast(), 44 | "dob.age": self.filterAge(), 45 | "location.country": self.filterCountry(), 46 | "location.state": self.filterUState(), 47 | }; 48 | fetch('https://abwq7mqaeg.execute-api.us-west-2.amazonaws.com/users-test?filters='+JSON.stringify(filterData)) 49 | .then(response => response.json()) 50 | .then(data => self.users(data)); 51 | }; 52 | 53 | fetch('https://abwq7mqaeg.execute-api.us-west-2.amazonaws.com/users-test') 54 | .then(response => response.json()) 55 | .then(data => self.users(data)); 56 | 57 | 58 | setInterval(function() { 59 | self.userCount(self.users().length) 60 | if (self.users().length > 0) { 61 | IsLoading(false); 62 | self.tableClass(tableVis); 63 | } 64 | }, 3000); 65 | 66 | }; 67 | 68 | ko.applyBindings(new ViewModel()); -------------------------------------------------------------------------------- /Lab_Assets/programmatically_utilizing_s3_select/lambda/function.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | 4 | def filter_data(filters): 5 | data = [] 6 | 7 | return data 8 | 9 | def get_data(): 10 | # S3 bucket we'll be interacting with 11 | s3_bucket = '' 12 | # Because we need to combine data from multiple S3 objects, initialize a list to hold this data before returning it. 13 | data = [] 14 | # Initialize an boto3 S3 client, and list the objects in our bucket. The data about the contents of our bucket will be stored in a list called s3_keys. 15 | s3 = boto3.client('s3') 16 | objects = s3.list_objects_v2( 17 | Bucket = s3_bucket 18 | )['Contents'] 19 | 20 | s3_keys = [] 21 | for object in objects: 22 | if object['Key'].startswith('users_'): 23 | s3_keys.append(object['Key']) 24 | 25 | # After collecting the appropriate keys that begin with "users_" gather each object, and combine the returned data with the existing "data" list. 26 | for key in s3_keys: 27 | object = s3.get_object( 28 | Bucket = s3_bucket, 29 | Key = key 30 | ) 31 | 32 | object_data = json.loads(object['Body'].read()) 33 | data += object_data 34 | 35 | # Return our combined data from all "users_" objects. 36 | return data 37 | 38 | def handler(event, context): 39 | # If the events object contains query string parameters call the "filter_data" function and return the results in the appropriate format. 40 | if 'queryStringParameters' in event.keys(): 41 | return {'isBase64Encoded': False,'statusCode': 200,'body': json.dumps(filter_data(json.loads(event['queryStringParameters']['filters']))), 'headers': {"Access-Control-Allow-Origin": "*"}} 42 | # Otherwise call the "get_data" function and return appropriately formatted results. 43 | return {'isBase64Encoded': False,'statusCode': 200,'body': json.dumps(get_data()), 'headers': {"Access-Control-Allow-Origin": "*"}} -------------------------------------------------------------------------------- /Lab_Assets/programmatically_utilizing_s3_select/lambda/function_solved.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | 4 | def filter_data(filters): 5 | # S3 bucket we'll be interacting with 6 | s3_bucket = '' 7 | # Check our filters for any empty strings to ensure our S3 Select queries don't have empty where clauses. 8 | empty_filters = [] 9 | for key, value in filters.items(): 10 | if len(value) <= 0: 11 | empty_filters.append(key) 12 | 13 | for key in empty_filters: 14 | del filters[key] 15 | 16 | # If all filters are removed by the above, call the get_data() function to just return the entire dataset. 17 | if len(filters) <= 0: 18 | return get_data() 19 | 20 | # We have to run our query against each object in the bucket. We know that all of our data is in objects with the filename prefix "users_" so we'll list the contents of the bucket and add any objects that start with "users_" to a s3_keys list. 21 | s3 = boto3.client('s3') 22 | objects = s3.list_objects_v2( 23 | Bucket = s3_bucket 24 | )['Contents'] 25 | 26 | s3_keys = [] 27 | for object in objects: 28 | if object['Key'].startswith('users_'): 29 | s3_keys.append(object['Key']) 30 | 31 | # Process our filters into a single WHERE clause for our S3 Select query. 32 | filter_string = '' 33 | for key, value in filters.items(): 34 | if key == "dob.age": 35 | if len(filter_string) == 0: 36 | filter_string += f's3o.{key} = {value}' 37 | else: 38 | filter_string += f' AND s3o.{key} = {value}' 39 | else: 40 | if len(filter_string) == 0: 41 | filter_string += f's3o.{key} = \'{value}\'' 42 | else: 43 | filter_string += f' AND s3o.{key} = \'{value}\'' 44 | 45 | # For each of the S3 keys gathered above we're going to make a select_object_content() call to our data storage bucket. We use the query filter_string generated above in our WHERE clause. 46 | data = [] 47 | for key in s3_keys: 48 | response = s3.select_object_content( 49 | Bucket = s3_bucket, 50 | Key = key, 51 | Expression = f'SELECT * FROM S3Object[*][*] as s3o WHERE {filter_string}', 52 | ExpressionType='SQL', 53 | InputSerialization = {'JSON': {'Type': 'Document'}}, 54 | OutputSerialization = {'JSON': {}} 55 | ) 56 | 57 | # S3 Select(select_object_content() in boto3) returns a Payload object that contains an interable stream object. We need to extract the data from this object which returns binary strings. Each object is separated by a \n so we decode the binary string, and then split records on \n. Each set of records ends with a \n so there will be a zero length string at the end of the records list, we need to ommit this as seen below. We then use json.loads() to convert the string into a List/Dict structure and append it to our data list. 58 | for event in response['Payload']: 59 | if 'Records' in event: 60 | records = event['Records']['Payload'].decode('utf-8').split('\n') 61 | for record in records: 62 | if len(record) > 0: 63 | data.append(json.loads(record)) 64 | # Return the data list and we're done! 65 | return data 66 | 67 | def get_data(): 68 | # S3 bucket we'll be interacting with 69 | s3_bucket = '' 70 | # Because we need to combine data from multiple S3 objects, initialize a list to hold this data before returning it. 71 | data = [] 72 | # Initialize an boto3 S3 client, and list the objects in our bucket. The data about the contents of our bucket will be stored in a list called s3_keys. 73 | s3 = boto3.client('s3') 74 | objects = s3.list_objects_v2( 75 | Bucket = s3_bucket 76 | )['Contents'] 77 | 78 | s3_keys = [] 79 | for object in objects: 80 | if object['Key'].startswith('users_'): 81 | s3_keys.append(object['Key']) 82 | 83 | # After collecting the appropriate keys that begin with "users_" gather each object, and combine the returned data with the existing "data" list. 84 | for key in s3_keys: 85 | object = s3.get_object( 86 | Bucket = s3_bucket, 87 | Key = key 88 | ) 89 | 90 | object_data = json.loads(object['Body'].read()) 91 | data += object_data 92 | 93 | # Return our combined data from all "users_" objects. 94 | return data 95 | 96 | def handler(event, context): 97 | # If the events object contains query string parameters call the "filter_data" function and return the results in the appropriate format. 98 | if 'queryStringParameters' in event.keys(): 99 | return {'isBase64Encoded': False,'statusCode': 200,'body': json.dumps(filter_data(json.loads(event['queryStringParameters']['filters']))), 'headers': {"Access-Control-Allow-Origin": "*"}} 100 | # Otherwise call the "get_data" function and return appropriately formatted results. 101 | return {'isBase64Encoded': False,'statusCode': 200,'body': json.dumps(get_data()), 'headers': {"Access-Control-Allow-Origin": "*"}} 102 | -------------------------------------------------------------------------------- /Lab_Assets/programmatically_utilizing_s3_select/web/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality/881ac05f8b7e4218f3bdebddb82a91be8ec1e156/Lab_Assets/programmatically_utilizing_s3_select/web/icon.png -------------------------------------------------------------------------------- /Lab_Assets/programmatically_utilizing_s3_select/web/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Random Users! 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 17 |
18 |

User Count:

19 |
20 |
21 |
22 |
23 | 24 | 25 |
26 |
27 | 28 | 29 |
30 |
31 | 32 | 33 |
34 |
35 | 36 | 37 |
38 |
39 | 40 | 41 |
42 | 43 |
44 |
45 |
46 |
47 |
48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 |
PictureFirst NameLast NameAgeCountryState
68 |
69 |
70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /Lab_Assets/programmatically_utilizing_s3_select/web/randomusers.css: -------------------------------------------------------------------------------- 1 | body { 2 | background-image: linear-gradient(#70e7f7a2, #dddddd); 3 | } 4 | 5 | body, 6 | html { 7 | height: 100%; 8 | } 9 | 10 | table { 11 | table-layout: fixed; 12 | word-wrap: break-word; 13 | } 14 | 15 | th { 16 | width: 5%; 17 | } 18 | 19 | .custombg { 20 | background-color: #a4f0fa; 21 | } 22 | 23 | .statusbox { 24 | padding-top: 10px; 25 | padding-right: 10px; 26 | } 27 | 28 | .welcome { 29 | position: absolute; 30 | padding-top: 40px; 31 | padding-left: 10px; 32 | } 33 | 34 | .toppad { 35 | padding-top: 10px; 36 | } 37 | 38 | .contentbox { 39 | max-height: 95vh; 40 | padding: 0px; 41 | background-image: linear-gradient(#dddddd, #70e7f7a2); 42 | } 43 | 44 | .countbox { 45 | position: absolute; 46 | top: 65px; 47 | left: 30px; 48 | } 49 | 50 | .filterbox { 51 | position: absolute; 52 | bottom: 65px; 53 | left: 30px; 54 | } 55 | 56 | .loadbox { 57 | position: absolute; 58 | top: 45vh; 59 | left: 45vw; 60 | } 61 | 62 | .circle { 63 | margin: 0 auto; 64 | border-radius: 50%; 65 | border: 10px solid rgba(0, 0, 0, 0); 66 | border-top-color: #347fc3; 67 | } 68 | 69 | .circle1-1, .circle1-2, .circle1-3, .circle1-4, .circle1-5, .circle1-6 { 70 | height: 100px; 71 | width: 100px; 72 | margin-top: 15px; 73 | } 74 | 75 | .circle2-1, .circle2-2, .circle2-3, .circle2-4, .circle2-5, .circle2-6 { 76 | height: 70px; 77 | width: 70px; 78 | margin-top: 5px; 79 | } 80 | 81 | .circle3-1, .circle3-2, .circle3-3, .circle3-4, .circle3-5, .circle3-6 { 82 | height: 40px; 83 | width: 40px; 84 | margin-top: 5px; 85 | } 86 | 87 | .circle1 { 88 | -webkit-animation: load1 4s infinite; 89 | } 90 | 91 | @-webkit-keyframes load1 { 92 | 0% { 93 | } 94 | 100% { 95 | -webkit-transform: rotate(360deg); 96 | } 97 | } -------------------------------------------------------------------------------- /Lab_Assets/programmatically_utilizing_s3_select/web/randomusers.js: -------------------------------------------------------------------------------- 1 | var ViewModel = function () { 2 | var self = this; 3 | self.url = window.location.href 4 | self.users = ko.observableArray([]); 5 | self.userCount = ko.observable(0); 6 | self.tableClass = ko.observable("d-none"); 7 | self.filterFirst = ko.observable(); 8 | self.filterLast = ko.observable(); 9 | self.filterAge = ko.observable(); 10 | self.filterCountry = ko.observable(); 11 | self.filterUState = ko.observable(); 12 | var apigateway = "APGATEWAY"; 13 | var tableVis = "container overflow-auto contentbox w-75"; 14 | 15 | var displayValue = function(element, valueAccessor){ 16 | var value = ko.utils.unwrapObservable(valueAccessor()); 17 | var isCurrentlyVisible = !(element.style.display == "none"); 18 | if(value && !isCurrentlyVisible) 19 | element.style.display = ""; 20 | else if((!value) && isCurrentlyVisible) 21 | element.style.display = "none"; 22 | }; 23 | 24 | 25 | ko.bindingHandlers['loading-animation'] = { 26 | 27 | 'init' : function(element, valueAccessor){ 28 | $(element) 29 | .append( 30 | '
'); 31 | 32 | displayValue(element, valueAccessor); 33 | }, 34 | 'update' : function(element, valueAccessor){ 35 | displayValue(element, valueAccessor); 36 | } 37 | }; 38 | 39 | IsLoading = ko.observable(true); 40 | 41 | self.filterUsers = function() { 42 | var filterData = { 43 | "name.\"first\"": self.filterFirst(), 44 | "name.\"last\"": self.filterLast(), 45 | "dob.age": self.filterAge(), 46 | "location.country": self.filterCountry(), 47 | "location.state": self.filterUState(), 48 | }; 49 | fetch(apigateway+'?filters='+JSON.stringify(filterData)) 50 | .then(response => response.json()) 51 | .then(data => self.users(data)); 52 | }; 53 | 54 | fetch(apigateway) 55 | .then(response => response.json()) 56 | .then(data => self.users(data)); 57 | 58 | 59 | setInterval(function() { 60 | self.userCount(self.users().length) 61 | if (self.users().length > 0) { 62 | IsLoading(false); 63 | self.tableClass(tableVis); 64 | } 65 | }, 3000); 66 | 67 | }; 68 | 69 | ko.applyBindings(new ViewModel()); -------------------------------------------------------------------------------- /Lab_Assets/querying_data_from_multiple_redshift_spectrum_tables/solution.sql: -------------------------------------------------------------------------------- 1 | create external schema users_data 2 | from data catalog 3 | database 'users' 4 | iam_role '' 5 | create external database if not exists; 6 | 7 | 8 | create external table users_data.names( 9 | id_name varchar(32), 10 | id_value varchar(64), 11 | gender varchar(16), 12 | name_title varchar(32), 13 | name_first varchar(64), 14 | name_last varchar(64) 15 | ) 16 | ROW FORMAT SERDE 17 | 'org.openx.data.jsonserde.JsonSerDe' 18 | LOCATION ''; 19 | 20 | create external table users_data.location( 21 | id_name varchar(32), 22 | id_value varchar(32), 23 | location_street_number int, 24 | location_street_name varchar(64), 25 | location_city varchar(32), 26 | location_state varchar(32), 27 | location_country varchar(32), 28 | location_postcode varchar(32), 29 | location_coordinates_latitude varchar(64), 30 | location_coordinates_longitude varchar(64), 31 | location_timezone_offset varchar(32), 32 | location_timezone_description varchar(32), 33 | nat varchar(16) 34 | ) 35 | ROW FORMAT SERDE 36 | 'org.openx.data.jsonserde.JsonSerDe' 37 | LOCATION ''; 38 | 39 | create external table users_data.age( 40 | id_name varchar(32), 41 | id_value varchar(32), 42 | dob_date varchar(32), 43 | dob_age int, 44 | registered_date varchar(32), 45 | registered_age int 46 | ) 47 | ROW FORMAT SERDE 48 | 'org.openx.data.jsonserde.JsonSerDe' 49 | LOCATION ''; 50 | 51 | create external table users_data.contact( 52 | id_name varchar(32), 53 | id_value varchar(32), 54 | email varchar(32), 55 | phone varchar(32), 56 | cell varchar(32) 57 | ) 58 | ROW FORMAT SERDE 59 | 'org.openx.data.jsonserde.JsonSerDe' 60 | LOCATION ''; 61 | 62 | create external table users_data.picture( 63 | id_name varchar(32), 64 | id_value varchar(32), 65 | picture_large varchar(64), 66 | picture_medium varchar(64), 67 | picture_thumbnail varchar(64) 68 | ) 69 | ROW FORMAT SERDE 70 | 'org.openx.data.jsonserde.JsonSerDe' 71 | LOCATION ''; 72 | 73 | select 74 | names.name_first as first_name, 75 | names.name_last as last_name, 76 | location.location_state as state, 77 | age.dob_age as age, 78 | contact.cell as cell, 79 | picture.picture_large as picture 80 | from users_data.names 81 | join users_data.location on users_data.names.id_value = users_data.location.id_value 82 | join users_data.age on users_data.names.id_value = users_data.age.id_value 83 | join users_data.contact on users_data.names.id_value = users_data.contact.id_value 84 | join users_data.picture on users_data.names.id_value = users_data.picture.id_value 85 | order by age 86 | limit 10; -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/cloudformation/SetupKinesisHelper_CloudCraft.yml: -------------------------------------------------------------------------------- 1 | Description: 2 | This is a template that launches an EC2 instance and deploys a dockerized angular app 3 | to a public EC2 instance. The EC2 instance is configured using files from 4 | github repository here - https://github.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality 5 | This template also sets up a DynamoDB table and populates it with some data. 6 | Mappings: 7 | RegionMap: 8 | us-east-1: 9 | AMI: %ami-203% 10 | Resources: 11 | AccessKey: 12 | Type: AWS::IAM::AccessKey 13 | Properties: 14 | UserName: cloud_user 15 | LambdaRole: 16 | Type: AWS::IAM::Role 17 | Properties: 18 | AssumeRolePolicyDocument: 19 | Version: '2012-10-17' 20 | Statement: 21 | - Effect: Allow 22 | Principal: 23 | Service: 24 | - lambda.amazonaws.com 25 | Action: 26 | - sts:AssumeRole 27 | Path: "/" 28 | Policies: 29 | - PolicyName: EC2AccessRole 30 | PolicyDocument: 31 | Version: '2012-10-17' 32 | Statement: 33 | - Effect: Allow 34 | Action: 35 | - ec2:* 36 | Resource: "*" 37 | - Effect: Allow 38 | Action: 39 | - logs:* 40 | Resource: "*" 41 | 42 | InitFunction: 43 | Type: AWS::Lambda::Function 44 | Properties: 45 | Code: 46 | ZipFile: | 47 | import json 48 | import boto3 49 | import threading 50 | import urllib3 51 | SUCCESS = "SUCCESS" 52 | FAILED = "FAILED" 53 | http = urllib3.PoolManager() 54 | def send(event, context, responseStatus, responseData, physicalResourceId=None, noEcho=False, reason=None): 55 | responseUrl = event['ResponseURL'] 56 | print(responseUrl) 57 | responseBody = {} 58 | responseBody['Status'] = responseStatus 59 | responseBody['Reason'] = reason or "See the details in CloudWatch Log Stream: {}".format(context.log_stream_name) 60 | responseBody['PhysicalResourceId'] = physicalResourceId or context.log_stream_name 61 | responseBody['StackId'] = event['StackId'] 62 | responseBody['RequestId'] = event['RequestId'] 63 | responseBody['LogicalResourceId'] = event['LogicalResourceId'] 64 | responseBody['NoEcho'] = noEcho 65 | responseBody['Data'] = responseData 66 | json_responseBody = json.dumps(responseBody) 67 | print("Response body:\n" + json_responseBody) 68 | headers = { 69 | 'content-type' : '', 70 | 'content-length' : str(len(json_responseBody)) 71 | } 72 | try: 73 | response = http.request('PUT',responseUrl,headers=headers,body=json_responseBody) 74 | print("Status code: {}".format(str(response.status))) 75 | except Exception as e: 76 | print("send(..) failed executing requests.put(..): " + str(e)) 77 | def createDefault(): 78 | print("Creating default VPC") 79 | ec2 = boto3.client('ec2') 80 | response = ec2.create_default_vpc() 81 | return response 82 | def deleteDefault(): 83 | return "" 84 | def timeout(event, context): 85 | print('Timing out, sending failure response to CFN') 86 | send(event, context, FAILED, {}, None) 87 | def lambda_handler(event, context): 88 | print(f'Received event: {json.dumps(event)}') 89 | timer = threading.Timer((context.get_remaining_time_in_millis() / 1000.00) - 0.5, timeout, args=[event, context]) 90 | timer.start() 91 | status = SUCCESS 92 | responseData = {} 93 | try: 94 | if event['RequestType'] == 'Delete': 95 | deleteDefault() 96 | else: 97 | response = createDefault() 98 | print(response) 99 | responseData['Data'] = response 100 | except Exception as e: 101 | print(e) 102 | status = FAILED 103 | finally: 104 | timer.cancel() 105 | send(event, context, status, responseData, None) 106 | Handler: index.lambda_handler 107 | Role: !GetAtt LambdaRole.Arn 108 | Runtime: python3.7 109 | Timeout: 60 110 | 111 | InitializeVPC: 112 | Type: Custom::InitFunction 113 | Properties: 114 | ServiceToken: !GetAtt InitFunction.Arn 115 | 116 | LnDynamoDBUserTable: 117 | Type: AWS::DynamoDB::Table 118 | Properties: 119 | AttributeDefinitions: 120 | - AttributeName: user_id 121 | AttributeType: S 122 | KeySchema: 123 | - AttributeName: user_id 124 | KeyType: HASH 125 | TableName: users-information 126 | BillingMode: PAY_PER_REQUEST 127 | LnLambdaExecutionAndDynamoRole: 128 | Type: AWS::IAM::Role 129 | DependsOn: 130 | - LnDynamoDBUserTable 131 | Properties: 132 | AssumeRolePolicyDocument: 133 | Version: '2012-10-17' 134 | Statement: 135 | - Effect: Allow 136 | Principal: 137 | Service: 138 | - lambda.amazonaws.com 139 | Action: 140 | - sts:AssumeRole 141 | Path: "/" 142 | Policies: 143 | - PolicyName: root 144 | PolicyDocument: 145 | Version: '2012-10-17' 146 | Statement: 147 | - Effect: Allow 148 | Action: 149 | - logs:* 150 | Resource: arn:aws:logs:*:*:* 151 | - Effect: Allow 152 | Action: 153 | - dynamodb:PutItem 154 | Resource: !GetAtt LnDynamoDBUserTable.Arn 155 | LnPopulateDynamoDBTable: 156 | Type: AWS::Lambda::Function 157 | DependsOn: 158 | - LnDynamoDBUserTable 159 | - LnLambdaExecutionAndDynamoRole 160 | Properties: 161 | Code: 162 | S3Bucket: 'das-c01-data-analytics-specialty' 163 | S3Key: 'Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/create-users-dynamodb-lambda.zip' 164 | Handler: index.handler 165 | Runtime: nodejs12.x 166 | Role: !GetAtt LnLambdaExecutionAndDynamoRole.Arn 167 | Timeout: 60 168 | LnPopulateDynamoDBTableInit: 169 | Type: Custom::LnPopulateDynamoDBTable 170 | Properties: 171 | ServiceToken: !GetAtt LnPopulateDynamoDBTable.Arn 172 | 173 | LnSecurityGroupWebserver: 174 | Type: AWS::EC2::SecurityGroup 175 | DependsOn: 176 | - InitializeVPC 177 | Properties: 178 | GroupDescription: !Sub 'Security Group created with CF template for web servers.' 179 | SecurityGroupIngress: 180 | - IpProtocol: tcp 181 | FromPort: 80 182 | ToPort: 80 183 | CidrIp: 0.0.0.0/0 184 | Tags: 185 | - Key: Name 186 | Value: !Sub '${AWS::StackName}-kinesis-helper-webserver-sg' 187 | - Key: Description 188 | Value: !Sub 'Security Group created for kinesis web server with ${AWS::StackName}.' 189 | 190 | LnCreateKinesisHelperServer: 191 | Type: AWS::EC2::Instance 192 | DependsOn: 193 | - LnSecurityGroupWebserver 194 | Properties: 195 | InstanceType: t3a.medium 196 | ImageId: 197 | Fn::FindInMap: 198 | - RegionMap 199 | - !Ref AWS::Region 200 | - AMI 201 | Tags: 202 | - Key: Name 203 | Value: !Join [ '-', [!Sub '${AWS::StackName}', 'kinesis-helper-server'] ] 204 | SecurityGroupIds: 205 | - !Ref LnSecurityGroupWebserver 206 | UserData: 207 | Fn::Base64: 208 | !Join [ "", [ 209 | "#!/bin/bash -xe\n", 210 | "yum update -y\n", 211 | "/bin/echo '%password%' | /bin/passwd cloud_user --stdin\n", 212 | "/opt/aws/bin/cfn-init -v ", #use cfn-init to install packages in cloudformation init 213 | !Sub "--stack ${AWS::StackName} ", 214 | "--resource LnCreateKinesisHelperServer ", 215 | "--configsets InstallAndConfigure ", 216 | !Sub "--region ${AWS::Region}\n", 217 | !Sub "/opt/aws/bin/cfn-signal -e $? ", 218 | !Sub "--stack ${AWS::StackName} ", 219 | "--resource LnCreateKinesisHelperServer ", 220 | !Sub "--region ${AWS::Region}", 221 | "\n"] ] 222 | Metadata: 223 | AWS::CloudFormation::Init: 224 | configSets: 225 | InstallAndConfigure: 226 | - "install_docker" 227 | - "start_docker" 228 | - "install_git" 229 | - "get_angular_app" 230 | - "build_docker_image" 231 | - "docker_run_app" 232 | install_docker: 233 | commands: 234 | test: 235 | command: amazon-linux-extras install docker 236 | cwd: /home/ec2-user 237 | start_docker: 238 | commands: 239 | test: 240 | command: service docker start && usermod -a -G docker ec2-user 241 | cwd: /home/ec2-user 242 | install_git: 243 | commands: 244 | test: 245 | command: yum install git -y 246 | cwd: /home/ec2-user 247 | get_angular_app: 248 | commands: 249 | test: 250 | command: git clone https://github.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality 251 | cwd: /home/ec2-user 252 | build_docker_image: 253 | commands: 254 | test: 255 | command: docker image build -t ubuntu-angular . 256 | cwd: /home/ec2-user/Content-AWS-Certified-Data-Analytics---Speciality/Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app 257 | docker_run_app: 258 | commands: 259 | test: 260 | command: docker run -d -p 80:80 ubuntu-angular 261 | cwd: /home/ec2-user/Content-AWS-Certified-Data-Analytics---Speciality/Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app 262 | CreationPolicy: 263 | ResourceSignal: 264 | Count: 1 265 | Timeout: PT60M 266 | Outputs: 267 | pubIpAddress1: 268 | Description: cloud_user Access Key 269 | Value: !Ref AccessKey 270 | pubIpAddress2: 271 | Description: cloud_user Secret Access Key 272 | Value: !GetAtt AccessKey.SecretAccessKey 273 | pubIpAddress3: 274 | Description: Public IP address of Kinesis Helper Server 275 | Value: !GetAtt LnCreateKinesisHelperServer.PublicIp 276 | 277 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/create-users-dynamodb-lambda/function/create-users-dynamodb-lambda.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality/881ac05f8b7e4218f3bdebddb82a91be8ec1e156/Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/create-users-dynamodb-lambda/function/create-users-dynamodb-lambda.zip -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/create-users-dynamodb-lambda/function/index.js: -------------------------------------------------------------------------------- 1 | // Load the AWS SDK for Node.js 2 | var response = require('cfn-response') 3 | var AWS = require('aws-sdk'); 4 | var ddb = new AWS.DynamoDB({apiVersion: '2012-08-10'}); 5 | AWS.config.update({region: 'us-east-1'}); 6 | var fs = require('fs'); 7 | var ddb = new AWS.DynamoDB({apiVersion: '2012-08-10'}); 8 | const TABLE_NAME = 'users-information' 9 | 10 | exports.handler = async (event, context, callback) => { 11 | console.log('Received event: %s', event); 12 | 13 | if (event.RequestType == "Delete") { 14 | return response.send(event, context, "SUCCESS") 15 | } 16 | console.log('Populating DynamoDB %s table with users.', TABLE_NAME); 17 | var filePath = './user_json.json'; 18 | var allUsers = JSON.parse(fs.readFileSync(filePath)); 19 | var allPromises = new Array(); 20 | 21 | for (var i = 0; i < allUsers.length; i++) { 22 | var user = allUsers[i]; 23 | var params = { 24 | Item: { 25 | "user_id": { S: user.user_id }, 26 | "first_name": { S: user.first_name }, 27 | "last_name": { S: user.last_name }, 28 | "email": { S: user.email }, 29 | "street_number": { N: user.street_number.toString() }, 30 | "street_name": { S: user.street_name }, 31 | "city": { S: user.city }, 32 | "state": { S: user.state }, 33 | "country": { S: user.country }, 34 | "postcode": { S: user.postcode.toString() }, 35 | "latitude": { N: user.latitude.toString() }, 36 | "longitude": { N: user.longitude.toString() } 37 | }, 38 | TableName: TABLE_NAME 39 | }; 40 | 41 | var promise = ddb.putItem(params).promise(); 42 | allPromises.push(promise); 43 | } 44 | console.log("Creating %s users.", allPromises.length); 45 | 46 | var responseStatus = "FAILED" 47 | var responseData = {} 48 | 49 | if (allPromises.length == 0) { 50 | responseData = {Error: "Invoke call failed - no items where put into DynamoDB table"}; 51 | console.log(responseData.Error); 52 | } 53 | else { 54 | responseStatus = "SUCCESS"; 55 | responseData = {Success: "Users added to " + TABLE_NAME + " DynamoDB table."}; 56 | console.log("Response body: %s", responseData); 57 | } 58 | 59 | return Promise.all(allPromises).then((values) => { 60 | return send(event, context, responseStatus, responseData); 61 | }); 62 | }; 63 | 64 | async function send(event, context, responseStatus, responseData, physicalResourceId, noEcho) { 65 | var responsePromise = new Promise((resolve, reject) => { 66 | console.log("Sending signal back to CloudFormation."); 67 | 68 | var responseBody = JSON.stringify({ 69 | Status: responseStatus, 70 | Reason: "See the details in CloudWatch Log Stream: " + context.logStreamName, 71 | PhysicalResourceId: physicalResourceId || context.logStreamName, 72 | StackId: event.StackId, 73 | RequestId: event.RequestId, 74 | LogicalResourceId: event.LogicalResourceId, 75 | NoEcho: noEcho || false, 76 | Data: responseData 77 | }); 78 | 79 | console.log("Response body:\n", responseBody); 80 | 81 | var https = require("https"); 82 | var url = require("url"); 83 | 84 | var parsedUrl = url.parse(event.ResponseURL); 85 | var options = { 86 | hostname: parsedUrl.hostname, 87 | port: 443, 88 | path: parsedUrl.path, 89 | method: "PUT", 90 | headers: { 91 | "content-type": "", 92 | "content-length": responseBody.length 93 | } 94 | }; 95 | 96 | console.log("SENDING RESPONSE...\n"); 97 | var request = https.request(options, function(response) { 98 | console.log("STATUS: " + response.statusCode); 99 | console.log("HEADERS: " + JSON.stringify(response.headers)); 100 | resolve(JSON.parse(responseBody)); 101 | context.done(); 102 | }); 103 | request.on("error", function(error) { 104 | console.log("sendResponse Error:" + error); 105 | reject(error); 106 | context.done(); 107 | }); 108 | request.write(responseBody); 109 | request.end(); 110 | }); 111 | return await responsePromise; 112 | } 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/create-users-dynamodb-lambda/function/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "create-users-dynamodb-lambda", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "@dabh/diagnostics": { 8 | "version": "2.0.2", 9 | "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz", 10 | "integrity": "sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q==", 11 | "requires": { 12 | "colorspace": "1.1.x", 13 | "enabled": "2.0.x", 14 | "kuler": "^2.0.0" 15 | } 16 | }, 17 | "async": { 18 | "version": "3.2.0", 19 | "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz", 20 | "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==" 21 | }, 22 | "aws-sdk": { 23 | "version": "2.824.0", 24 | "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.824.0.tgz", 25 | "integrity": "sha512-9KNRQBkIMPn+6DWb4gR+RzqTMNyGLEwOgXbE4dDehOIAflfLnv3IFwLnzrhxJnleB4guYrILIsBroJFBzjiekg==", 26 | "requires": { 27 | "buffer": "4.9.2", 28 | "events": "1.1.1", 29 | "ieee754": "1.1.13", 30 | "jmespath": "0.15.0", 31 | "querystring": "0.2.0", 32 | "sax": "1.2.1", 33 | "url": "0.10.3", 34 | "uuid": "3.3.2", 35 | "xml2js": "0.4.19" 36 | }, 37 | "dependencies": { 38 | "uuid": { 39 | "version": "3.3.2", 40 | "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", 41 | "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" 42 | } 43 | } 44 | }, 45 | "base64-js": { 46 | "version": "1.5.1", 47 | "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", 48 | "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" 49 | }, 50 | "buffer": { 51 | "version": "4.9.2", 52 | "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", 53 | "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", 54 | "requires": { 55 | "base64-js": "^1.0.2", 56 | "ieee754": "^1.1.4", 57 | "isarray": "^1.0.0" 58 | } 59 | }, 60 | "cfn-response": { 61 | "version": "1.0.1", 62 | "resolved": "https://registry.npmjs.org/cfn-response/-/cfn-response-1.0.1.tgz", 63 | "integrity": "sha1-qOwDlQwGg8UUlejKaA2dwLiEsTc=" 64 | }, 65 | "color": { 66 | "version": "3.0.0", 67 | "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz", 68 | "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==", 69 | "requires": { 70 | "color-convert": "^1.9.1", 71 | "color-string": "^1.5.2" 72 | } 73 | }, 74 | "color-convert": { 75 | "version": "1.9.3", 76 | "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", 77 | "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", 78 | "requires": { 79 | "color-name": "1.1.3" 80 | } 81 | }, 82 | "color-name": { 83 | "version": "1.1.3", 84 | "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", 85 | "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" 86 | }, 87 | "color-string": { 88 | "version": "1.5.4", 89 | "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.4.tgz", 90 | "integrity": "sha512-57yF5yt8Xa3czSEW1jfQDE79Idk0+AkN/4KWad6tbdxUmAs3MvjxlWSWD4deYytcRfoZ9nhKyFl1kj5tBvidbw==", 91 | "requires": { 92 | "color-name": "^1.0.0", 93 | "simple-swizzle": "^0.2.2" 94 | } 95 | }, 96 | "colors": { 97 | "version": "1.4.0", 98 | "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", 99 | "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==" 100 | }, 101 | "colorspace": { 102 | "version": "1.1.2", 103 | "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.2.tgz", 104 | "integrity": "sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ==", 105 | "requires": { 106 | "color": "3.0.x", 107 | "text-hex": "1.0.x" 108 | } 109 | }, 110 | "commander": { 111 | "version": "6.2.1", 112 | "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz", 113 | "integrity": "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==" 114 | }, 115 | "core-util-is": { 116 | "version": "1.0.2", 117 | "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", 118 | "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" 119 | }, 120 | "dotenv": { 121 | "version": "8.2.0", 122 | "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", 123 | "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==" 124 | }, 125 | "enabled": { 126 | "version": "2.0.0", 127 | "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", 128 | "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" 129 | }, 130 | "events": { 131 | "version": "1.1.1", 132 | "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", 133 | "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" 134 | }, 135 | "fast-safe-stringify": { 136 | "version": "2.0.7", 137 | "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", 138 | "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" 139 | }, 140 | "fecha": { 141 | "version": "4.2.0", 142 | "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.0.tgz", 143 | "integrity": "sha512-aN3pcx/DSmtyoovUudctc8+6Hl4T+hI9GBBHLjA76jdZl7+b1sgh5g4k+u/GL3dTy1/pnYzKp69FpJ0OicE3Wg==" 144 | }, 145 | "fn.name": { 146 | "version": "1.1.0", 147 | "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", 148 | "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" 149 | }, 150 | "fs": { 151 | "version": "0.0.1-security", 152 | "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", 153 | "integrity": "sha1-invTcYa23d84E/I4WLV+yq9eQdQ=" 154 | }, 155 | "ieee754": { 156 | "version": "1.1.13", 157 | "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", 158 | "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" 159 | }, 160 | "inherits": { 161 | "version": "2.0.4", 162 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", 163 | "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" 164 | }, 165 | "is-arrayish": { 166 | "version": "0.3.2", 167 | "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", 168 | "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" 169 | }, 170 | "is-stream": { 171 | "version": "2.0.0", 172 | "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", 173 | "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" 174 | }, 175 | "isarray": { 176 | "version": "1.0.0", 177 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", 178 | "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" 179 | }, 180 | "jmespath": { 181 | "version": "0.15.0", 182 | "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", 183 | "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" 184 | }, 185 | "kuler": { 186 | "version": "2.0.0", 187 | "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", 188 | "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" 189 | }, 190 | "lambda-local": { 191 | "version": "1.7.4", 192 | "resolved": "https://registry.npmjs.org/lambda-local/-/lambda-local-1.7.4.tgz", 193 | "integrity": "sha512-uLrFPGj2//glOgJGLZn8hNTNlhU+eGx0WFRLZxIoC39nfjLRZ1fncHcPK2t5gA2GcvgtGUT2dnw60M8vJAOIkQ==", 194 | "requires": { 195 | "aws-sdk": "^2.689.0", 196 | "commander": "^6.1.0", 197 | "dotenv": "^8.2.0", 198 | "winston": "^3.2.1" 199 | } 200 | }, 201 | "logform": { 202 | "version": "2.2.0", 203 | "resolved": "https://registry.npmjs.org/logform/-/logform-2.2.0.tgz", 204 | "integrity": "sha512-N0qPlqfypFx7UHNn4B3lzS/b0uLqt2hmuoa+PpuXNYgozdJYAyauF5Ky0BWVjrxDlMWiT3qN4zPq3vVAfZy7Yg==", 205 | "requires": { 206 | "colors": "^1.2.1", 207 | "fast-safe-stringify": "^2.0.4", 208 | "fecha": "^4.2.0", 209 | "ms": "^2.1.1", 210 | "triple-beam": "^1.3.0" 211 | } 212 | }, 213 | "ms": { 214 | "version": "2.1.3", 215 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", 216 | "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" 217 | }, 218 | "one-time": { 219 | "version": "1.0.0", 220 | "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", 221 | "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", 222 | "requires": { 223 | "fn.name": "1.x.x" 224 | } 225 | }, 226 | "process-nextick-args": { 227 | "version": "2.0.1", 228 | "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", 229 | "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" 230 | }, 231 | "punycode": { 232 | "version": "1.3.2", 233 | "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", 234 | "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" 235 | }, 236 | "querystring": { 237 | "version": "0.2.0", 238 | "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", 239 | "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" 240 | }, 241 | "readable-stream": { 242 | "version": "3.6.0", 243 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", 244 | "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", 245 | "requires": { 246 | "inherits": "^2.0.3", 247 | "string_decoder": "^1.1.1", 248 | "util-deprecate": "^1.0.1" 249 | } 250 | }, 251 | "safe-buffer": { 252 | "version": "5.2.1", 253 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", 254 | "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" 255 | }, 256 | "sax": { 257 | "version": "1.2.1", 258 | "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", 259 | "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" 260 | }, 261 | "simple-swizzle": { 262 | "version": "0.2.2", 263 | "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", 264 | "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=", 265 | "requires": { 266 | "is-arrayish": "^0.3.1" 267 | } 268 | }, 269 | "stack-trace": { 270 | "version": "0.0.10", 271 | "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", 272 | "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" 273 | }, 274 | "string_decoder": { 275 | "version": "1.3.0", 276 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", 277 | "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", 278 | "requires": { 279 | "safe-buffer": "~5.2.0" 280 | } 281 | }, 282 | "text-hex": { 283 | "version": "1.0.0", 284 | "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", 285 | "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" 286 | }, 287 | "triple-beam": { 288 | "version": "1.3.0", 289 | "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", 290 | "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" 291 | }, 292 | "url": { 293 | "version": "0.10.3", 294 | "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", 295 | "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", 296 | "requires": { 297 | "punycode": "1.3.2", 298 | "querystring": "0.2.0" 299 | } 300 | }, 301 | "util-deprecate": { 302 | "version": "1.0.2", 303 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", 304 | "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" 305 | }, 306 | "winston": { 307 | "version": "3.3.3", 308 | "resolved": "https://registry.npmjs.org/winston/-/winston-3.3.3.tgz", 309 | "integrity": "sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw==", 310 | "requires": { 311 | "@dabh/diagnostics": "^2.0.2", 312 | "async": "^3.1.0", 313 | "is-stream": "^2.0.0", 314 | "logform": "^2.2.0", 315 | "one-time": "^1.0.0", 316 | "readable-stream": "^3.4.0", 317 | "stack-trace": "0.0.x", 318 | "triple-beam": "^1.3.0", 319 | "winston-transport": "^4.4.0" 320 | } 321 | }, 322 | "winston-transport": { 323 | "version": "4.4.0", 324 | "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.4.0.tgz", 325 | "integrity": "sha512-Lc7/p3GtqtqPBYYtS6KCN3c77/2QCev51DvcJKbkFPQNoj1sinkGwLGFDxkXY9J6p9+EPnYs+D90uwbnaiURTw==", 326 | "requires": { 327 | "readable-stream": "^2.3.7", 328 | "triple-beam": "^1.2.0" 329 | }, 330 | "dependencies": { 331 | "readable-stream": { 332 | "version": "2.3.7", 333 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", 334 | "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", 335 | "requires": { 336 | "core-util-is": "~1.0.0", 337 | "inherits": "~2.0.3", 338 | "isarray": "~1.0.0", 339 | "process-nextick-args": "~2.0.0", 340 | "safe-buffer": "~5.1.1", 341 | "string_decoder": "~1.1.1", 342 | "util-deprecate": "~1.0.1" 343 | } 344 | }, 345 | "safe-buffer": { 346 | "version": "5.1.2", 347 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", 348 | "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" 349 | }, 350 | "string_decoder": { 351 | "version": "1.1.1", 352 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", 353 | "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", 354 | "requires": { 355 | "safe-buffer": "~5.1.0" 356 | } 357 | } 358 | } 359 | }, 360 | "xml2js": { 361 | "version": "0.4.19", 362 | "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", 363 | "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", 364 | "requires": { 365 | "sax": ">=0.6.0", 366 | "xmlbuilder": "~9.0.1" 367 | } 368 | }, 369 | "xmlbuilder": { 370 | "version": "9.0.7", 371 | "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", 372 | "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" 373 | } 374 | } 375 | } 376 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/create-users-dynamodb-lambda/function/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "create-users-dynamodb-lambda", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "aws-sdk": "^2.824.0", 13 | "cfn-response": "^1.0.1", 14 | "fs": "0.0.1-security", 15 | "lambda-local": "^1.7.4" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/create-users-dynamodb-lambda/simple-event.json: -------------------------------------------------------------------------------- 1 | { 2 | "RequestType": "Create", 3 | "ResponseURL": "http://pre-signed-S3-url-for-response", 4 | "StackId": "arn:aws:cloudformation:us-east-1:123456789012:stack/MyStack/guid", 5 | "RequestId": "unique id for this create request", 6 | "ResourceType": "Custom::TestResource", 7 | "LogicalResourceId": "MyTestResource", 8 | "ResourceProperties": { 9 | "StackName": "MyStack", 10 | "List": [ 11 | "1", 12 | "2", 13 | "3" 14 | ] 15 | } 16 | } -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/ .dockerignore: -------------------------------------------------------------------------------- 1 | site/node_modules 2 | site/dist -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | RUN apt-get update -y 3 | RUN apt-get install software-properties-common -y --fix-missing 4 | RUN apt-add-repository ppa:ansible/ansible 5 | RUN apt-get update -y 6 | RUN apt-get install ansible -y 7 | RUN apt-get install apache2 -y 8 | RUN echo "ServerName localhost" >> /etc/apache2/apache2.conf 9 | RUN apt-get install curl -y 10 | RUN apt-get install rpm -y 11 | RUN apt-get install gcc g++ make -y 12 | RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - 13 | RUN apt-get install nodejs 14 | 15 | RUN mkdir -p /home/ansible/playbooks 16 | RUN mkdir -p /tmp/site 17 | 18 | COPY ./ansible.cfg ./hosts /home/ansible/ 19 | COPY ./playbooks /home/ansible/playbooks 20 | COPY ./site /tmp/site 21 | WORKDIR /home/ansible 22 | RUN ansible-playbook /home/ansible/playbooks/setup-angular-app.yml 23 | 24 | EXPOSE 80 25 | CMD ["apache2ctl", "-D", "FOREGROUND"] 26 | 27 | ## Below is for development purposes only 28 | # FROM ubuntu:20.04 29 | # RUN apt-get update 30 | # RUN apt-get install curl gcc g++ make git -y 31 | # EXPOSE 4200 32 | 33 | # RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - 34 | # RUN apt-get install nodejs -y 35 | # RUN npm install -g npm 36 | # RUN npm install -g @angular/cli 37 | 38 | # WORKDIR /home/kinesis-live-angular-app 39 | 40 | # docker image build -t kinesis-live-image . 41 | # docker run -p 4200:4200 -v /Users/brocktubre/Desktop/Projects/Content-AWS-Certified-Data-Analytics---Speciality/Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/:/home/kinesis-live-angular-app/site -it kinesis-live-image 42 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/README.md: -------------------------------------------------------------------------------- 1 | # Dockerized Angular Application 2 | 3 | This application is a simple example of how to use Docker + Angular + Ansible to produce a simple user interface to interact with Kinesis Data Streams. This simple application uses Docker to containerize an Angular app and uses Ansible to load and configure the build/distribution of the Angular application. 4 | 5 | 1. Build Docker Image 6 | To build the Docker image use the following: `docker image build -t ubuntu-angular .` 7 | 8 | 2. Run the docker container 9 | Run the docker contianer: `docker run -d -p 80:80 ubuntu-angular` 10 | 11 | 3. Run app locally 12 | To run the Angular application locally, navigate inside the site directory and use: `ng serve` 13 | 14 | 4. Docker Installs 15 | Docker installs the ubuntu container and things like apache, ansible, and nodejs. 16 | 17 | 5. Ansible Configurations 18 | Ansible removes the node_modules, runs npm install, packages the Angular app, and copies the distribution site to the /var/www/html directory. 19 | 20 | 6. Apache Serves App 21 | Apache serves the application on port 80. 22 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | inventory = ./hosts -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/hosts: -------------------------------------------------------------------------------- 1 | [local] 2 | control ansible_connection=local -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/playbooks/setup-angular-app.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: local 3 | become: true 4 | tasks: 5 | - name: Install angular-cli 6 | npm: 7 | name: "@angular/cli" 8 | global: yes 9 | state: present 10 | - name: Remove node_modules 11 | file: 12 | name: /tmp/site/node_modules 13 | state: absent 14 | - name: Install npm dependencies 15 | npm: 16 | name: install 17 | path: /tmp/site 18 | - name: Build distribution 19 | command: ng build --prod 20 | args: 21 | chdir: /tmp/site 22 | - name: Copying site contents to /var/www/html 23 | copy: 24 | src: /tmp/site/dist/site/ 25 | dest: /var/www/html 26 | owner: root 27 | group: root 28 | mode: '0644' 29 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/angular.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "./node_modules/@angular/cli/lib/config/schema.json", 3 | "version": 1, 4 | "newProjectRoot": "projects", 5 | "projects": { 6 | "site": { 7 | "root": "", 8 | "sourceRoot": "src", 9 | "projectType": "application", 10 | "prefix": "app", 11 | "schematics": {}, 12 | "architect": { 13 | "build": { 14 | "builder": "@angular-devkit/build-angular:browser", 15 | "options": { 16 | "outputPath": "dist/site", 17 | "index": "src/index.html", 18 | "main": "src/main.ts", 19 | "polyfills": "src/polyfills.ts", 20 | "tsConfig": "src/tsconfig.app.json", 21 | "assets": [ 22 | "src/favicon.ico", 23 | "src/assets" 24 | ], 25 | "styles": [ 26 | "node_modules/bootstrap/dist/css/bootstrap.min.css", 27 | "node_modules/font-awesome/css/font-awesome.min.css", 28 | "src/styles.css" 29 | ], 30 | "scripts": [ 31 | "node_modules/jquery/dist/jquery.min.js", 32 | "node_modules/bootstrap/dist/js/bootstrap.min.js" 33 | ] 34 | }, 35 | "configurations": { 36 | "production": { 37 | "fileReplacements": [ 38 | { 39 | "replace": "src/environments/environment.ts", 40 | "with": "src/environments/environment.prod.ts" 41 | } 42 | ], 43 | "optimization": true, 44 | "outputHashing": "all", 45 | "sourceMap": false, 46 | "extractCss": true, 47 | "namedChunks": false, 48 | "aot": true, 49 | "extractLicenses": true, 50 | "vendorChunk": false, 51 | "buildOptimizer": true 52 | } 53 | } 54 | }, 55 | "serve": { 56 | "builder": "@angular-devkit/build-angular:dev-server", 57 | "options": { 58 | "browserTarget": "site:build" 59 | }, 60 | "configurations": { 61 | "production": { 62 | "browserTarget": "site:build:production" 63 | } 64 | } 65 | }, 66 | "extract-i18n": { 67 | "builder": "@angular-devkit/build-angular:extract-i18n", 68 | "options": { 69 | "browserTarget": "site:build" 70 | } 71 | }, 72 | "test": { 73 | "builder": "@angular-devkit/build-angular:karma", 74 | "options": { 75 | "main": "src/test.ts", 76 | "polyfills": "src/polyfills.ts", 77 | "tsConfig": "src/tsconfig.spec.json", 78 | "karmaConfig": "src/karma.conf.js", 79 | "styles": [ 80 | "src/styles.css" 81 | ], 82 | "scripts": [], 83 | "assets": [ 84 | "src/favicon.ico", 85 | "src/assets" 86 | ] 87 | } 88 | }, 89 | "lint": { 90 | "builder": "@angular-devkit/build-angular:tslint", 91 | "options": { 92 | "tsConfig": [ 93 | "src/tsconfig.app.json", 94 | "src/tsconfig.spec.json" 95 | ], 96 | "exclude": [ 97 | "**/node_modules/**" 98 | ] 99 | } 100 | } 101 | } 102 | }, 103 | "site-e2e": { 104 | "root": "e2e/", 105 | "projectType": "application", 106 | "architect": { 107 | "e2e": { 108 | "builder": "@angular-devkit/build-angular:protractor", 109 | "options": { 110 | "protractorConfig": "e2e/protractor.conf.js", 111 | "devServerTarget": "site:serve" 112 | }, 113 | "configurations": { 114 | "production": { 115 | "devServerTarget": "site:serve:production" 116 | } 117 | } 118 | }, 119 | "lint": { 120 | "builder": "@angular-devkit/build-angular:tslint", 121 | "options": { 122 | "tsConfig": "e2e/tsconfig.e2e.json", 123 | "exclude": [ 124 | "**/node_modules/**" 125 | ] 126 | } 127 | } 128 | } 129 | } 130 | }, 131 | "defaultProject": "site" 132 | } 133 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/e2e/protractor.conf.js: -------------------------------------------------------------------------------- 1 | // Protractor configuration file, see link for more information 2 | // https://github.com/angular/protractor/blob/master/lib/config.ts 3 | 4 | const { SpecReporter } = require('jasmine-spec-reporter'); 5 | 6 | exports.config = { 7 | allScriptsTimeout: 11000, 8 | specs: [ 9 | './src/**/*.e2e-spec.ts' 10 | ], 11 | capabilities: { 12 | 'browserName': 'chrome' 13 | }, 14 | directConnect: true, 15 | baseUrl: 'http://localhost:4200/', 16 | framework: 'jasmine', 17 | jasmineNodeOpts: { 18 | showColors: true, 19 | defaultTimeoutInterval: 30000, 20 | print: function() {} 21 | }, 22 | onPrepare() { 23 | require('ts-node').register({ 24 | project: require('path').join(__dirname, './tsconfig.e2e.json') 25 | }); 26 | jasmine.getEnv().addReporter(new SpecReporter({ spec: { displayStacktrace: true } })); 27 | } 28 | }; -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/e2e/src/app.e2e-spec.ts: -------------------------------------------------------------------------------- 1 | import { AppPage } from './app.po'; 2 | 3 | describe('workspace-project App', () => { 4 | let page: AppPage; 5 | 6 | beforeEach(() => { 7 | page = new AppPage(); 8 | }); 9 | 10 | it('should display welcome message', () => { 11 | page.navigateTo(); 12 | expect(page.getParagraphText()).toEqual('Welcome to site!'); 13 | }); 14 | }); 15 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/e2e/src/app.po.ts: -------------------------------------------------------------------------------- 1 | import { browser, by, element } from 'protractor'; 2 | 3 | export class AppPage { 4 | navigateTo() { 5 | return browser.get('/'); 6 | } 7 | 8 | getParagraphText() { 9 | return element(by.css('app-root h1')).getText(); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/e2e/tsconfig.e2e.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "../out-tsc/app", 5 | "module": "commonjs", 6 | "target": "es5", 7 | "types": [ 8 | "jasmine", 9 | "jasminewd2", 10 | "node" 11 | ] 12 | } 13 | } -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "site", 3 | "version": "0.0.0", 4 | "scripts": { 5 | "ng": "ng", 6 | "start": "ng serve --host 0.0.0.0 --port 4200", 7 | "build": "ng build", 8 | "test": "ng test", 9 | "lint": "ng lint", 10 | "e2e": "ng e2e" 11 | }, 12 | "private": true, 13 | "dependencies": { 14 | "@angular/animations": "^6.1.0", 15 | "@angular/common": "^6.1.0", 16 | "@angular/compiler": "^6.1.0", 17 | "@angular/core": "^6.1.0", 18 | "@angular/forms": "^6.1.0", 19 | "@angular/http": "^6.1.0", 20 | "@angular/platform-browser": "^6.1.0", 21 | "@angular/platform-browser-dynamic": "^6.1.0", 22 | "@angular/router": "^6.1.0", 23 | "aws-sdk": "^2.824.0", 24 | "bootstrap": "^4.1.3", 25 | "core-js": "^2.5.4", 26 | "font-awesome": "^4.7.0", 27 | "jquery": "^3.5.1", 28 | "rxjs": "~6.2.0", 29 | "zone.js": "~0.8.26" 30 | }, 31 | "devDependencies": { 32 | "@angular-devkit/build-angular": "~0.8.0", 33 | "@angular/cli": "~6.2.3", 34 | "@angular/compiler-cli": "^6.1.0", 35 | "@angular/language-service": "^6.1.0", 36 | "@types/jasmine": "~2.8.8", 37 | "@types/jasminewd2": "~2.0.3", 38 | "@types/node": "^8.9.5", 39 | "codelyzer": "~4.3.0", 40 | "jasmine-core": "~2.99.1", 41 | "jasmine-spec-reporter": "~4.2.1", 42 | "karma": "~3.0.0", 43 | "karma-chrome-launcher": "~2.2.0", 44 | "karma-coverage-istanbul-reporter": "~2.0.1", 45 | "karma-jasmine": "~1.1.2", 46 | "karma-jasmine-html-reporter": "^0.2.2", 47 | "protractor": "~5.4.0", 48 | "ts-node": "~7.0.0", 49 | "tslint": "~5.11.0", 50 | "typescript": "~2.9.2" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/app/app-routing.module.ts: -------------------------------------------------------------------------------- 1 | import { HomeComponent } from './home/home.component'; 2 | import { KinesisHelperComponent } from './kinesis-helper/kinesis-helper.component'; 3 | import { NgModule } from '@angular/core'; 4 | import { Routes, RouterModule } from '@angular/router'; 5 | 6 | const routes: Routes = [ 7 | { path: '', component: KinesisHelperComponent }, 8 | { path: 'kinesis-helper', component: KinesisHelperComponent }, 9 | { path: '**', redirectTo: '' } 10 | 11 | ]; 12 | 13 | @NgModule({ 14 | imports: [RouterModule.forRoot(routes, { relativeLinkResolution: 'legacy' })], 15 | exports: [RouterModule] 16 | }) 17 | export class AppRoutingModule { } 18 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/app/app.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality/881ac05f8b7e4218f3bdebddb82a91be8ec1e156/Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/app/app.component.css -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/app/app.component.html: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/app/app.component.ts: -------------------------------------------------------------------------------- 1 | import { Component } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-root', 5 | templateUrl: './app.component.html', 6 | styleUrls: ['./app.component.css'] 7 | }) 8 | export class AppComponent { } 9 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/app/app.module.ts: -------------------------------------------------------------------------------- 1 | import { AppRoutingModule } from './app-routing.module'; 2 | import { BrowserModule } from '@angular/platform-browser'; 3 | import { NgModule } from '@angular/core'; 4 | 5 | import { AppComponent } from './app.component'; 6 | import { KinesisHelperComponent } from './kinesis-helper/kinesis-helper.component'; 7 | import { HomeComponent } from './home/home.component'; 8 | 9 | @NgModule({ 10 | declarations: [ 11 | AppComponent, 12 | KinesisHelperComponent, 13 | HomeComponent 14 | ], 15 | imports: [ 16 | BrowserModule, 17 | AppRoutingModule 18 | ], 19 | providers: [], 20 | bootstrap: [AppComponent] 21 | }) 22 | export class AppModule { } 23 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/app/home/home.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality/881ac05f8b7e4218f3bdebddb82a91be8ec1e156/Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/app/home/home.component.css -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/app/home/home.component.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | 10 |
11 |
12 |
13 |
14 |
15 |
16 | 17 | 18 | 19 |
20 |

{{title}}

21 |

This application is a simple example of how to use Docker + Angular + Ansible. This simple application uses Docker to containerize an Angular app and uses Ansible to load and configure the build and distribution of the Angular application.

22 |

See source code

23 |
24 | 25 |
26 |
27 |
28 |
29 |
30 |

1. Build Docker Image

31 |

To build the Docker image use the following: docker image build -t ubuntu-angular .

32 |
33 |
34 |

2. Run The Docker Container

35 |

Run the docker container: docker run ubuntu-angular

36 |
37 |
38 |

3. Run App Locally

39 |

Run the angular app (must be inside site folder): ng serve

40 |
41 |
42 |
43 |
44 |

4. Docker Installs

45 |

Docker installs the ubuntu container and things like apache, ansible, and nodejs.

46 |
47 |
48 |

5. Ansible Configurations

49 |

Ansible removes the node_modules, runs npm install, packages the Angular app, and deploys it to /var/www/html directory.

50 |
51 |
52 |

6. Apache Serves App

53 |

Apache serves the application.

54 |
55 |
56 |
57 |
58 |

© Copyright {{date}}

59 |
60 |
61 |
-------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/app/home/home.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-home', 5 | templateUrl: './home.component.html', 6 | styleUrls: ['./home.component.css'] 7 | }) 8 | export class HomeComponent implements OnInit { 9 | 10 | title = 'Docker + Angular + Ansible'; 11 | date = new Date().getFullYear(); 12 | 13 | constructor() { } 14 | 15 | ngOnInit() { 16 | } 17 | 18 | } 19 | 20 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/app/kinesis-helper/kinesis-helper.component.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality/881ac05f8b7e4218f3bdebddb82a91be8ec1e156/Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/app/kinesis-helper/kinesis-helper.component.css -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/app/kinesis-helper/kinesis-helper.component.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | 10 |
11 |
12 | {{someErrorMessage}} 13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 | 21 |
22 |
23 | 24 |
25 |
26 | 27 |
28 |
29 |
30 |
31 | 32 |

{{streamingData}}

33 | 34 |
35 |
36 |
-------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/app/kinesis-helper/kinesis-helper.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, ElementRef, OnInit, ViewChild } from '@angular/core'; 2 | import { KinesisHelperService } from './kinesis-helper.service'; 3 | import * as AWS from 'aws-sdk'; 4 | import * as uuid from 'uuid'; 5 | import { interval } from 'rxjs'; 6 | 7 | @Component({ 8 | selector: 'app-kinesis-helper', 9 | templateUrl: './kinesis-helper.component.html', 10 | styleUrls: ['./kinesis-helper.component.css'] 11 | }) 12 | export class KinesisHelperComponent implements OnInit { 13 | 14 | @ViewChild('secretAccessKey') secretAccessKey: ElementRef; 15 | @ViewChild('streamName') streamName: ElementRef; 16 | @ViewChild('accessKey') accessKey: ElementRef; 17 | @ViewChild('region') region: ElementRef; 18 | 19 | public isLoadingData: boolean 20 | public someError: boolean 21 | public someErrorMessage: String 22 | 23 | public loadingSub; 24 | 25 | public streamingData: String; 26 | 27 | constructor(private kinesisServiceHelper: KinesisHelperService) { 28 | this.isLoadingData = false; 29 | this.someError = false; 30 | } 31 | 32 | ngOnInit() { 33 | } 34 | 35 | public startStream() { 36 | 37 | this.isLoadingData = false; 38 | this.someError = false; 39 | 40 | const secretAccessKey = this.secretAccessKey.nativeElement.value; 41 | const streamName = this.streamName.nativeElement.value; 42 | const accessKey = this.accessKey.nativeElement.value; 43 | const region = this.region.nativeElement.value; 44 | 45 | if(secretAccessKey === "" || streamName === "" || accessKey === "" || region === "") { 46 | this.someErrorMessage = "Please enter in all the fields."; 47 | this.someError = true; 48 | return; 49 | } 50 | 51 | this.loadingSub = interval(1000).subscribe(x => { 52 | this.kinesisServiceHelper.streamData(region, secretAccessKey, accessKey, streamName) 53 | .subscribe( payload => { 54 | this.isLoadingData = true; 55 | this.streamingData = payload; 56 | }, error => { 57 | this.isLoadingData = false; 58 | this.someError = true; 59 | this.someErrorMessage = error; 60 | if(this.someErrorMessage == 'NetworkingError: Network Failure') { 61 | this.someErrorMessage = 'NetworkingError: Stream name and region combination is invalid.' 62 | } 63 | this.loadingSub.unsubscribe(); 64 | }); 65 | }); 66 | } 67 | 68 | public stopStream() { 69 | this.loadingSub.unsubscribe(); 70 | this.isLoadingData = false; 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/app/kinesis-helper/kinesis-helper.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@angular/core'; 2 | import * as AWS from 'aws-sdk'; 3 | import { Observable, Subject } from 'rxjs'; 4 | import * as uuid from 'uuid'; 5 | 6 | @Injectable({ 7 | providedIn: 'root' 8 | }) 9 | export class KinesisHelperService { 10 | 11 | 12 | constructor() { 13 | 14 | } 15 | 16 | public streamData(region: string, secretAccessKey: string, accessKey: string, streamName: string ): Observable{ 17 | const sendResult = new Subject(); 18 | AWS.config.update({region: region, 19 | accessKeyId: accessKey, 20 | secretAccessKey: secretAccessKey}); 21 | const credentials = new AWS.Credentials(AWS.config.credentials); 22 | const kds = new AWS.Kinesis({ region: region, credentials: credentials }); 23 | 24 | const order_id = uuid.v4(); 25 | let order = {}; 26 | let items = []; 27 | let user_ids = []; 28 | for(let i = 0; i < Math.floor(Math.random() * 5) + 1; i++){ 29 | items.push(this.grocery_items[Math.floor(Math.random() * this.grocery_items.length) - 1]) 30 | } 31 | 32 | for(let i = 0; i < 500; i++) { 33 | user_ids.push('uid_'+i); 34 | } 35 | 36 | let user_id = user_ids[Math.floor(Math.random() * (user_ids.length - 1))] 37 | 38 | 39 | order["items"] = items; 40 | order["order_id"] = order_id; 41 | order["total_cost"] = ((Math.random() * 199.99) + 1.00).toFixed(2) 42 | order["user_id"] = user_id; 43 | 44 | var payload = JSON.stringify(order); 45 | 46 | var params = { 47 | Data: payload, 48 | PartitionKey: order_id, 49 | StreamName: streamName, 50 | }; 51 | console.log(payload); 52 | 53 | kds.putRecord(params, function(err, data) { 54 | if (err) { 55 | sendResult.error(err); 56 | }else { 57 | sendResult.next(payload); 58 | } 59 | }); 60 | return sendResult.asObservable(); 61 | } 62 | 63 | private grocery_items = ['tropical fruit', 'whole milk', 'pip fruit', 'other vegetables', 64 | 'rolls/buns', 'pot plants', 'citrus fruit', 'beef', 'frankfurter', 65 | 'chicken', 'butter', 'fruit/vegetable juice', 66 | 'packaged fruit/vegetables', 'chocolate', 'specialty bar', 67 | 'butter milk', 'bottled water', 'yogurt', 'sausage', 'brown bread', 68 | 'hamburger meat', 'root vegetables', 'pork', 'pastry', 69 | 'canned beer', 'berries', 'coffee', 'misc. beverages', 'ham', 70 | 'turkey', 'curd cheese', 'red/blush wine', 71 | 'frozen potato products', 'flour', 'sugar', 'frozen meals', 72 | 'herbs', 'soda', 'detergent', 'grapes', 'processed cheese', 'fish', 73 | 'sparkling wine', 'newspapers', 'curd', 'pasta', 'popcorn', 74 | 'finished products', 'beverages', 'bottled beer', 'dessert', 75 | 'dog food', 'specialty chocolate', 'condensed milk', 'cleaner', 76 | 'white wine', 'meat', 'ice cream', 'hard cheese', 'cream cheese ', 77 | 'liquor', 'pickled vegetables', 'liquor (appetizer)', 'UHT-milk', 78 | 'candy', 'onions', 'hair spray', 'photo/film', 'domestic eggs', 79 | 'margarine', 'shopping bags', 'salt', 'oil', 'whipped/sour cream', 80 | 'frozen vegetables', 'sliced cheese', 'dish cleaner', 81 | 'baking powder', 'specialty cheese', 'salty snack', 82 | 'Instant food products', 'pet care', 'white bread', 83 | 'female sanitary products', 'cling film/bags', 'soap', 84 | 'frozen chicken', 'house keeping products', 'spread cheese', 85 | 'decalcifier', 'frozen dessert', 'vinegar', 'nuts/prunes', 86 | 'potato products', 'frozen fish', 'hygiene articles', 87 | 'artif. sweetener', 'light bulbs', 'canned vegetables', 88 | 'chewing gum', 'canned fish', 'cookware', 'semi-finished bread', 89 | 'cat food', 'bathroom cleaner', 'prosecco', 'liver loaf', 90 | 'zwieback', 'canned fruit', 'frozen fruits', 'brandy', 91 | 'baby cosmetics', 'spices', 'napkins', 'waffles', 'sauces', 'rum', 92 | 'chocolate marshmallow', 'long life bakery product', 'bags', 93 | 'sweet spreads', 'soups', 'mustard', 'specialty fat', 94 | 'instant coffee', 'snack products', 'organic sausage', 95 | 'soft cheese', 'mayonnaise', 'dental care', 'roll products ', 96 | 'kitchen towels', 'flower soil/fertilizer', 'cereals', 97 | 'meat spreads', 'dishes', 'male cosmetics', 'candles', 'whisky', 98 | 'tidbits', 'cooking chocolate', 'seasonal products', 'liqueur', 99 | 'abrasive cleaner', 'syrup', 'ketchup', 'cream', 'skin care', 100 | 'rubbing alcohol', 'nut snack', 'cocoa drinks', 'softener', 101 | 'organic products', 'cake bar', 'honey', 'jam', 'kitchen utensil', 102 | 'flower (seeds)', 'rice', 'tea', 'salad dressing', 103 | 'specialty vegetables', 'pudding powder', 'ready soups', 104 | 'make up remover', 'toilet cleaner', 'preservation products']; 105 | } 106 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/assets/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality/881ac05f8b7e4218f3bdebddb82a91be8ec1e156/Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/assets/.gitkeep -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/assets/acloudguru_long_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality/881ac05f8b7e4218f3bdebddb82a91be8ec1e156/Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/assets/acloudguru_long_logo.png -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/assets/angular_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality/881ac05f8b7e4218f3bdebddb82a91be8ec1e156/Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/assets/angular_logo.png -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/assets/ansible-setup-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality/881ac05f8b7e4218f3bdebddb82a91be8ec1e156/Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/assets/ansible-setup-diagram.png -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/assets/ansible_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality/881ac05f8b7e4218f3bdebddb82a91be8ec1e156/Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/assets/ansible_logo.png -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/assets/docker_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality/881ac05f8b7e4218f3bdebddb82a91be8ec1e156/Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/assets/docker_logo.png -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/assets/middle_img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linuxacademy/Content-AWS-Certified-Data-Analytics---Speciality/881ac05f8b7e4218f3bdebddb82a91be8ec1e156/Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/assets/middle_img.png -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/browserslist: -------------------------------------------------------------------------------- 1 | # This file is currently used by autoprefixer to adjust CSS to support the below specified browsers 2 | # For additional information regarding the format and rule options, please see: 3 | # https://github.com/browserslist/browserslist#queries 4 | # 5 | # For IE 9-11 support, please remove 'not' from the last line of the file and adjust as needed 6 | 7 | > 0.5% 8 | last 2 versions 9 | Firefox ESR 10 | not dead 11 | not IE 9-11 -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/environments/environment.prod.ts: -------------------------------------------------------------------------------- 1 | export const environment = { 2 | production: true 3 | }; 4 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/environments/environment.ts: -------------------------------------------------------------------------------- 1 | // This file can be replaced during build by using the `fileReplacements` array. 2 | // `ng build --prod` replaces `environment.ts` with `environment.prod.ts`. 3 | // The list of file replacements can be found in `angular.json`. 4 | 5 | export const environment = { 6 | production: false 7 | }; 8 | 9 | /* 10 | * For easier debugging in development mode, you can import the following file 11 | * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`. 12 | * 13 | * This import should be commented out in production mode because it will have a negative impact 14 | * on performance if an error is thrown. 15 | */ 16 | // import 'zone.js/dist/zone-error'; // Included with Angular CLI. 17 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Kinesis Live Beta 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/karma.conf.js: -------------------------------------------------------------------------------- 1 | // Karma configuration file, see link for more information 2 | // https://karma-runner.github.io/1.0/config/configuration-file.html 3 | 4 | module.exports = function (config) { 5 | config.set({ 6 | basePath: '', 7 | frameworks: ['jasmine', '@angular-devkit/build-angular'], 8 | plugins: [ 9 | require('karma-jasmine'), 10 | require('karma-chrome-launcher'), 11 | require('karma-jasmine-html-reporter'), 12 | require('karma-coverage-istanbul-reporter'), 13 | require('@angular-devkit/build-angular/plugins/karma') 14 | ], 15 | client: { 16 | clearContext: false // leave Jasmine Spec Runner output visible in browser 17 | }, 18 | coverageIstanbulReporter: { 19 | dir: require('path').join(__dirname, '../coverage'), 20 | reports: ['html', 'lcovonly'], 21 | fixWebpackSourcePaths: true 22 | }, 23 | reporters: ['progress', 'kjhtml'], 24 | port: 9876, 25 | colors: true, 26 | logLevel: config.LOG_INFO, 27 | autoWatch: true, 28 | browsers: ['Chrome'], 29 | singleRun: false 30 | }); 31 | }; -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/main.ts: -------------------------------------------------------------------------------- 1 | import { enableProdMode } from '@angular/core'; 2 | import { platformBrowserDynamic } from '@angular/platform-browser-dynamic'; 3 | 4 | import { AppModule } from './app/app.module'; 5 | import { environment } from './environments/environment'; 6 | 7 | if (environment.production) { 8 | enableProdMode(); 9 | } 10 | 11 | platformBrowserDynamic().bootstrapModule(AppModule) 12 | .catch(err => console.error(err)); 13 | 14 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/polyfills.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This file includes polyfills needed by Angular and is loaded before the app. 3 | * You can add your own extra polyfills to this file. 4 | * 5 | * This file is divided into 2 sections: 6 | * 1. Browser polyfills. These are applied before loading ZoneJS and are sorted by browsers. 7 | * 2. Application imports. Files imported after ZoneJS that should be loaded before your main 8 | * file. 9 | * 10 | * The current setup is for so-called "evergreen" browsers; the last versions of browsers that 11 | * automatically update themselves. This includes Safari >= 10, Chrome >= 55 (including Opera), 12 | * Edge >= 13 on the desktop, and iOS 10 and Chrome on mobile. 13 | * 14 | * Learn more in https://angular.io/docs/ts/latest/guide/browser-support.html 15 | */ 16 | 17 | /*************************************************************************************************** 18 | * BROWSER POLYFILLS 19 | */ 20 | 21 | /** IE9, IE10 and IE11 requires all of the following polyfills. **/ 22 | // import 'core-js/es6/symbol'; 23 | // import 'core-js/es6/object'; 24 | // import 'core-js/es6/function'; 25 | // import 'core-js/es6/parse-int'; 26 | // import 'core-js/es6/parse-float'; 27 | // import 'core-js/es6/number'; 28 | // import 'core-js/es6/math'; 29 | // import 'core-js/es6/string'; 30 | // import 'core-js/es6/date'; 31 | // import 'core-js/es6/array'; 32 | // import 'core-js/es6/regexp'; 33 | // import 'core-js/es6/map'; 34 | // import 'core-js/es6/weak-map'; 35 | // import 'core-js/es6/set'; 36 | 37 | /** IE10 and IE11 requires the following for NgClass support on SVG elements */ 38 | // import 'classlist.js'; // Run `npm install --save classlist.js`. 39 | 40 | /** IE10 and IE11 requires the following for the Reflect API. */ 41 | // import 'core-js/es6/reflect'; 42 | 43 | 44 | /** Evergreen browsers require these. **/ 45 | // Used for reflect-metadata in JIT. If you use AOT (and only Angular decorators), you can remove. 46 | import 'core-js/es7/reflect'; 47 | 48 | 49 | /** 50 | * Web Animations `@angular/platform-browser/animations` 51 | * Only required if AnimationBuilder is used within the application and using IE/Edge or Safari. 52 | * Standard animation support in Angular DOES NOT require any polyfills (as of Angular 6.0). 53 | **/ 54 | // import 'web-animations-js'; // Run `npm install --save web-animations-js`. 55 | 56 | /** 57 | * By default, zone.js will patch all possible macroTask and DomEvents 58 | * user can disable parts of macroTask/DomEvents patch by setting following flags 59 | */ 60 | 61 | // (window as any).__Zone_disable_requestAnimationFrame = true; // disable patch requestAnimationFrame 62 | // (window as any).__Zone_disable_on_property = true; // disable patch onProperty such as onclick 63 | // (window as any).__zone_symbol__BLACK_LISTED_EVENTS = ['scroll', 'mousemove']; // disable patch specified eventNames 64 | 65 | /* 66 | * in IE/Edge developer tools, the addEventListener will also be wrapped by zone.js 67 | * with the following flag, it will bypass `zone.js` patch for IE/Edge 68 | */ 69 | // (window as any).__Zone_enable_cross_context_check = true; 70 | 71 | /*************************************************************************************************** 72 | * Zone JS is required by default for Angular itself. 73 | */ 74 | import 'zone.js/dist/zone'; // Included with Angular CLI. 75 | 76 | 77 | 78 | /*************************************************************************************************** 79 | * APPLICATION IMPORTS 80 | */ 81 | 82 | (window as any).global = window; 83 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/styles.css: -------------------------------------------------------------------------------- 1 | /* You can add global styles to this file, and also import other style files */ 2 | .background { 3 | padding-bottom: 50px; 4 | } 5 | 6 | .img-icon { 7 | max-width: 100px; 8 | } 9 | 10 | .tools-body .row{ 11 | padding-top: 50px; 12 | } 13 | 14 | .footer { 15 | padding-top: 100px; 16 | } 17 | 18 | .brand-img 19 | { 20 | margin-left: 70px; 21 | } 22 | 23 | .ansible-logo { 24 | max-width: 250px; 25 | } 26 | 27 | .info-item { 28 | padding: 15px; 29 | } 30 | 31 | .k-input { 32 | padding-top: 25px; 33 | } -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/test.ts: -------------------------------------------------------------------------------- 1 | // This file is required by karma.conf.js and loads recursively all the .spec and framework files 2 | 3 | import 'zone.js/dist/zone-testing'; 4 | import { getTestBed } from '@angular/core/testing'; 5 | import { 6 | BrowserDynamicTestingModule, 7 | platformBrowserDynamicTesting 8 | } from '@angular/platform-browser-dynamic/testing'; 9 | 10 | declare const require: any; 11 | 12 | // First, initialize the Angular testing environment. 13 | getTestBed().initTestEnvironment( 14 | BrowserDynamicTestingModule, 15 | platformBrowserDynamicTesting() 16 | ); 17 | // Then we find all the tests. 18 | const context = require.context('./', true, /\.spec\.ts$/); 19 | // And load the modules. 20 | context.keys().map(context); 21 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "../out-tsc/app", 5 | "types": ["node"] 6 | }, 7 | "exclude": [ 8 | "test.ts", 9 | "**/*.spec.ts" 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/tsconfig.spec.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "../out-tsc/spec", 5 | "types": [ 6 | "jasmine", 7 | "node" 8 | ] 9 | }, 10 | "files": [ 11 | "test.ts", 12 | "polyfills.ts" 13 | ], 14 | "include": [ 15 | "**/*.spec.ts", 16 | "**/*.d.ts" 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/src/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tslint.json", 3 | "rules": { 4 | "directive-selector": [ 5 | true, 6 | "attribute", 7 | "app", 8 | "camelCase" 9 | ], 10 | "component-selector": [ 11 | true, 12 | "element", 13 | "app", 14 | "kebab-case" 15 | ] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compileOnSave": false, 3 | "compilerOptions": { 4 | "baseUrl": "./", 5 | "outDir": "./dist/out-tsc", 6 | "sourceMap": true, 7 | "declaration": false, 8 | "module": "es2015", 9 | "moduleResolution": "node", 10 | "emitDecoratorMetadata": true, 11 | "experimentalDecorators": true, 12 | "target": "es5", 13 | "typeRoots": [ 14 | "node_modules/@types" 15 | ], 16 | "lib": [ 17 | "es2017", 18 | "dom" 19 | ] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/dockerized-angular-app/site/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "rulesDirectory": [ 3 | "node_modules/codelyzer" 4 | ], 5 | "rules": { 6 | "arrow-return-shorthand": true, 7 | "callable-types": true, 8 | "class-name": true, 9 | "comment-format": [ 10 | true, 11 | "check-space" 12 | ], 13 | "curly": true, 14 | "deprecation": { 15 | "severity": "warn" 16 | }, 17 | "eofline": true, 18 | "forin": true, 19 | "import-blacklist": [ 20 | true, 21 | "rxjs/Rx" 22 | ], 23 | "import-spacing": true, 24 | "indent": [ 25 | true, 26 | "spaces" 27 | ], 28 | "interface-over-type-literal": true, 29 | "label-position": true, 30 | "max-line-length": [ 31 | true, 32 | 140 33 | ], 34 | "member-access": false, 35 | "member-ordering": [ 36 | true, 37 | { 38 | "order": [ 39 | "static-field", 40 | "instance-field", 41 | "static-method", 42 | "instance-method" 43 | ] 44 | } 45 | ], 46 | "no-arg": true, 47 | "no-bitwise": true, 48 | "no-console": [ 49 | true, 50 | "debug", 51 | "info", 52 | "time", 53 | "timeEnd", 54 | "trace" 55 | ], 56 | "no-construct": true, 57 | "no-debugger": true, 58 | "no-duplicate-super": true, 59 | "no-empty": false, 60 | "no-empty-interface": true, 61 | "no-eval": true, 62 | "no-inferrable-types": [ 63 | true, 64 | "ignore-params" 65 | ], 66 | "no-misused-new": true, 67 | "no-non-null-assertion": true, 68 | "no-redundant-jsdoc": true, 69 | "no-shadowed-variable": true, 70 | "no-string-literal": false, 71 | "no-string-throw": true, 72 | "no-switch-case-fall-through": true, 73 | "no-trailing-whitespace": true, 74 | "no-unnecessary-initializer": true, 75 | "no-unused-expression": true, 76 | "no-use-before-declare": true, 77 | "no-var-keyword": true, 78 | "object-literal-sort-keys": false, 79 | "one-line": [ 80 | true, 81 | "check-open-brace", 82 | "check-catch", 83 | "check-else", 84 | "check-whitespace" 85 | ], 86 | "prefer-const": true, 87 | "quotemark": [ 88 | true, 89 | "single" 90 | ], 91 | "radix": true, 92 | "semicolon": [ 93 | true, 94 | "always" 95 | ], 96 | "triple-equals": [ 97 | true, 98 | "allow-null-check" 99 | ], 100 | "typedef-whitespace": [ 101 | true, 102 | { 103 | "call-signature": "nospace", 104 | "index-signature": "nospace", 105 | "parameter": "nospace", 106 | "property-declaration": "nospace", 107 | "variable-declaration": "nospace" 108 | } 109 | ], 110 | "unified-signatures": true, 111 | "variable-name": false, 112 | "whitespace": [ 113 | true, 114 | "check-branch", 115 | "check-decl", 116 | "check-operator", 117 | "check-separator", 118 | "check-type" 119 | ], 120 | "no-output-on-prefix": true, 121 | "use-input-property-decorator": true, 122 | "use-output-property-decorator": true, 123 | "use-host-property-decorator": true, 124 | "no-input-rename": true, 125 | "no-output-rename": true, 126 | "use-life-cycle-interface": true, 127 | "use-pipe-transform-interface": true, 128 | "component-class-suffix": true, 129 | "directive-class-suffix": true 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/other-scripts/enrich-data-lambda-function.py: -------------------------------------------------------------------------------- 1 | import boto3, json, base64 2 | 3 | TABLE_NAME = 'users-information' 4 | OUTPUT_STREAM_NAME = '' 5 | 6 | def lambda_handler(event, context): 7 | 8 | incoming_orders_list = [] # Used to hold the incoming records in JSON format. 9 | user_id_set = set() # Used to hold the ids of the users who placed an order. We'll use this in a mapping function later. 10 | 11 | # Let's loop through the batch records coming in to populate lists 12 | for record in event['Records']: 13 | payload = base64.b64decode(record['kinesis']['data']) 14 | incoming_order = json.loads(payload) 15 | user_id_set.add(incoming_order['user_id']) 16 | incoming_orders_list.append(incoming_order) 17 | 18 | # Let's get the matching batch records from DynamoDB 19 | id_dict = get_records(user_id_set) 20 | 21 | # Holds the records we are going to put onto the output enriched stream 22 | enriched_orders_list = [] 23 | for incoming_order in incoming_orders_list: 24 | enriched_record = incoming_order.copy() 25 | if incoming_order['user_id'] in id_dict: 26 | user_id = incoming_order['user_id'] 27 | enriched_record['first_name'] = id_dict[user_id]['first_name']['S'] 28 | enriched_record['last_name'] = id_dict[user_id]['last_name']['S'] 29 | enriched_record['email'] = id_dict[user_id]['email']['S'] 30 | enriched_orders_list.append(enriched_record) 31 | 32 | # Let's add the matched records to the output kinesis stream 33 | response = put_records_to_stream(enriched_orders_list) 34 | 35 | # If there was some failure, go ahead and print out the details 36 | if response['FailedRecordCount'] > 0: 37 | print('FailedRecordCount = %d' % response['FailedRecordCount']) 38 | print('Received event: ' + json.dumps(event, indent=2)) 39 | print('Records: ' + json.dumps(enriched_orders_list, indent=2)) 40 | raise Exception('FailedRecordCount = %d' % response['FailedRecordCount']) 41 | else: 42 | print('Successfully put %d record(s) onto output stream.' % len(enriched_orders_list)) 43 | print(*enriched_orders_list, sep = ", ") 44 | 45 | def get_records(id_set): 46 | dynamodb_client = boto3.client('dynamodb') 47 | 48 | id_dict = {} 49 | 50 | # Convert the set of user_ids into a format for the batch_get_item API 51 | record_keys = map(lambda i: {'user_id':{'S':i}}, id_set) 52 | 53 | # Retrieve the records in batches using batch_get_item 54 | response = dynamodb_client.batch_get_item( 55 | RequestItems = { 56 | TABLE_NAME: { 57 | 'Keys': list(record_keys), 58 | 'AttributesToGet': [ 59 | 'user_id','first_name','last_name','email' 60 | ], 61 | 'ConsistentRead': True, 62 | } 63 | }, 64 | ReturnConsumedCapacity='TOTAL' 65 | ) 66 | 67 | # Loop through all of ther records returned from DynamoDB 68 | # and populate the id_dict with matching user_ids 69 | for i in response['Responses'][TABLE_NAME]: 70 | id_dict[i['user_id']['S']] = i 71 | 72 | return id_dict 73 | 74 | # This function puts the enriched records onto the output stream 75 | def put_records_to_stream(orders_list = []): 76 | if len(orders_list) > 0: 77 | kinesis_client = boto3.client('kinesis') 78 | response = kinesis_client.put_records( 79 | StreamName = OUTPUT_STREAM_NAME, 80 | Records = list(map(lambda record: { 81 | 'Data': json.dumps(record), 82 | 'PartitionKey':record['user_id'] 83 | }, 84 | orders_list)) 85 | ) 86 | return response 87 | else: 88 | return {'FailedRecordCount':0} -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/other-scripts/filter-top-orders.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE STREAM "DESTINATION_USER_DATA" ( 2 | order_id VARCHAR(64), 3 | user_id VARCHAR(16), 4 | email VARCHAR(16), 5 | first_name VARCHAR(16), 6 | last_name VARCHAR(16), 7 | total_cost FLOAT 8 | ); 9 | CREATE OR REPLACE PUMP "STREAM_PUMP" AS INSERT INTO "DESTINATION_USER_DATA" 10 | 11 | SELECT STREAM "order_id", "user_id", "email", "first_name", "last_name", "total_cost" 12 | FROM "SOURCE_SQL_STREAM_001" 13 | WHERE "total_cost" >= 100; -------------------------------------------------------------------------------- /Lab_Joining_Enriching_Transforming_Streaming_Data_Amazon_Kinesis/other-scripts/new-line-function.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import base64 4 | 5 | print('Loading function') 6 | 7 | 8 | def lambda_handler(event, context): 9 | output = [] 10 | 11 | for record in event['records']: 12 | print(record['recordId']) 13 | payload = base64.b64decode(record['data']) 14 | 15 | # Do custom processing on the payload here 16 | payload = payload + '\n'.encode() 17 | 18 | output_record = { 19 | 'recordId': record['recordId'], 20 | 'result': 'Ok', 21 | 'data': base64.b64encode(payload) 22 | } 23 | output.append(output_record) 24 | 25 | print('Successfully processed {} records.'.format(len(event['records']))) 26 | 27 | return {'records': output} 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AWS CERTIFIED DATA ANALYTICS SPECIALTY 2 | An A Cloud Guru Course - 3 | [AWS Certified Data Analytics Specialty](https://acloud.guru/overview/312375cd-c136-4f1c-81dc-dbdcfff2d06b) 4 | 5 | In late 2019, AWS announced that they would be renaming the AWS Big Data Specialty certification to the AWS Data Analytics specialty certification. With these changes there are some carry over topics but many of the topics are brand new and more in-depth. This course is designed and tailored to teach you the new services and tools that you can use in AWS to build data analytics applications, as well as build and manage the lifecycle of collecting, storing, processing, and visualizing your data. 6 | 7 | Just because the name of the certification is changing doesn’t mean big data is going away. In fact, data is growing for organizations at an alarming rate and that means there is a growing need for engineers who know how to gather insight from that data. That is where the AWS Data Analytics Specialty certification fits into the picture. This certification validates a deep understanding of AWS data analytics services and understands how they integrate with each other. Once you have passed the exam you should be able to explain how AWS data analytics services fit in the data life cycle and answer important questions your organization's data is providing. 8 | 9 | Join us in the course where we cover major tools, topics, and services needed for a successful data analytics pipeline, help you gain hands on experience, as well as help you pass the exam. So when you’re ready, join us in the AWS Data Analytics Specialty course! 10 | 11 | Prepared by [Brock Tubre](https://learn.acloud.guru/profile/brock-tubre) and [John Hanna](https://learn.acloud.guru/profile/john-hanna) 2020 12 | 13 | Files for A Cloud Guru, Course - [AWS Certified Data Analytics Specialty](https://acloud.guru/overview/312375cd-c136-4f1c-81dc-dbdcfff2d06b) 14 | 15 | ## In this course you'll learn: 16 | - The domains of knowledge for the AWS Certified Data Analytics Specialty exam. 17 | - How to define AWS data analytics services and understand how they integrate with each other. 18 | - How to design, build, secure and maintain analytics solutions in AWS. 19 | 20 | ## IMPORTANT 21 | Please note, this is provided as-is, neither I, nor A Cloud Guru support this code. If you do identify any errors, then please identify and we will attempt to fix on a best efforts basis. 22 | 23 | IMPORTANT - We recommend creating a new account or lab specs for this workshop. Using an existing account could cause damage or disruption to the resources in that account. 24 | 25 | These files are distributed on an AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied 26 | 27 | 28 | ## October 2020 29 | Initial Creation. 30 | 31 | -------------------------------------------------------------------------------- /Redshift_Maintenance_And_Operations/Launching_A_Redshift_Cluster/createcluster.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | 3 | redshift = boto3.client('redshift') 4 | 5 | response = redshift.create_cluster( 6 | ClusterIdentifier = 'boto3-cluster', 7 | ClusterType = 'single-node', 8 | NodeType = 'dc2.large', 9 | MasterUsername = 'john', 10 | MasterUserPassword = 'Strongpass1' 11 | ) 12 | 13 | print(response) -------------------------------------------------------------------------------- /Redshift_Maintenance_And_Operations/Launching_A_Redshift_Cluster/createcluster.sh: -------------------------------------------------------------------------------- 1 | aws redshift create-cluster --node-type dc2.large --number-of-nodes 2 --master-username john --master-user-password Strongpass1 --cluster-identifier cli-cluster -------------------------------------------------------------------------------- /Using_Redshift/Redshift_Spectrum/Spectrum_demo_queries.md: -------------------------------------------------------------------------------- 1 | # Spectrum Queries 2 | 3 | ## Number of rows in spectrum.sales 4 | ```SQL 5 | select count(*) from spectrum.sales; 6 | ``` 7 | 8 | ## First 10 rows from spectrum.sales 9 | ```SQL 10 | select * from spectrum.sales limit 10; 11 | ``` 12 | 13 | ## Top 15 events by ticket sales 14 | ```SQL 15 | select top 15 event.eventname as event_name, sum(spectrum.sales.pricepaid) as gross_ticket_sales from spectrum.sales, event 16 | where spectrum.sales.eventid = event.eventid 17 | and spectrum.sales.pricepaid > 30 18 | group by event.eventname 19 | order by 2 desc; 20 | ``` --------------------------------------------------------------------------------