├── .DS_Store ├── LICENSE ├── Lambda Trigger code ├── Lambda └── lambda trigger code ├── README.md ├── lab-deployments ├── README.md ├── v1 │ ├── app.js │ ├── package.json │ └── v1.zip ├── v2 │ ├── app.js │ ├── package.json │ └── v2.zip └── watch_url.sh ├── lab-networking ├── 7508839.jpg ├── Step_1_Setup.yaml └── Step_2_LaFours_Disaster.yaml ├── lab-scaling ├── README.md └── stress-testing-boot.sh └── lab-storage ├── README.md ├── create_air_quality_table.sql └── create_o3_view.sql /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ACloudGuru-Resources/Course_Certified_Solutions_Architect_Professional/bdc3461752e580de25f9f00c6526e4c68467c764/.DS_Store -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 A Cloud Guru 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Lambda Trigger code: -------------------------------------------------------------------------------- 1 | const aws = require("aws-sdk"); 2 | const sns = new aws.SNS({ 3 | region:'us-east-1' 4 | }); 5 | var ses = new aws.SES({ 6 | region: 'us-east-1' 7 | }); 8 | exports.handler = function(event, context, callback) { 9 | console.log("AWS lambda and SNS trigger "); 10 | console.log(event); 11 | const s3message = "Bucket Name:"+event.Records[0].s3.bucket.name+"\nLog details:"+event.Records[0].s3.object.key; 12 | console.log(s3message); 13 | var eParams = { 14 | Destination: { 15 | ToAddresses: ["xxxxxxxxx12@gmail.com"] 16 | }, 17 | Message: { 18 | Body: { 19 | Text: { 20 | Data:s3message 21 | } 22 | }, 23 | Subject: { 24 | Data: "cloudtrail logs" 25 | } 26 | }, 27 | Source: "coxxxxxx@gmail.com" 28 | }; 29 | var email = ses.sendEmail(eParams, function(err, data) { 30 | if (err) console.log(err); 31 | else { 32 | console.log("===EMAIL SENT==="); 33 | console.log("EMAIL CODE END"); 34 | console.log('EMAIL: ', email); 35 | context.succeed(event); 36 | callback(null, "email is send"); 37 | } 38 | }); 39 | }; 40 | ``` 41 | -------------------------------------------------------------------------------- /Lambda/lambda trigger code: -------------------------------------------------------------------------------- 1 | const { SES } = require("@aws-sdk/client-ses"); 2 | const { SNS } = require("@aws-sdk/client-sns"); 3 | 4 | const sns = new SNS({ 5 | region:'us-east-1' 6 | }); 7 | var ses = new SES({ 8 | region: 'us-east-1' 9 | }); 10 | exports.handler = function(event, context, callback) { 11 | console.log("AWS lambda and SNS trigger "); 12 | console.log(event); 13 | const s3message = "Bucket Name:"+event.Records[0].s3.bucket.name+"\nLog details:"+event.Records[0].s3.object.key; 14 | console.log(s3message); 15 | var eParams = { 16 | Destination: { 17 | ToAddresses: ["xxxxxxxxx12@gmail.com"] 18 | }, 19 | Message: { 20 | Body: { 21 | Text: { 22 | Data:s3message 23 | } 24 | }, 25 | Subject: { 26 | Data: "cloudtrail logs" 27 | } 28 | }, 29 | Source: "xxxxxxxxx12@gmail.com" 30 | }; 31 | var email = ses.sendEmail(eParams, function(err, data) { 32 | if (err) console.log(err); 33 | else { 34 | console.log("===EMAIL SENT==="); 35 | console.log("EMAIL CODE END"); 36 | console.log('EMAIL: ', email); 37 | context.succeed(event); 38 | callback(null, "email is send"); 39 | } 40 | }); 41 | }; 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### Certified Solutions Architect Professional 2019 2 | ## Lab Resources 3 | 4 | This repository contains the code samples used during the A Cloud Guru, AWS Certified Solutions Architect Professional course delivered by Scott Pletcher. 5 | -------------------------------------------------------------------------------- /lab-deployments/README.md: -------------------------------------------------------------------------------- 1 | # Lab: Deployments on AWS 2 | 3 | To be used in conjunction with the AWS Certified Solutions Architect Professional course on A Cloud Guru. 4 | -------------------------------------------------------------------------------- /lab-deployments/v1/app.js: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | Shakespeare Insult Flinger 4 | Version 1 5 | 6 | CHANGELOG: 7 | - Initial version 8 | 9 | USAGE: 10 | node app.js 11 | 12 | By default, will listen on port 8080 if no PORT environment variable is set. 13 | */ 14 | 15 | var express = require('express'); 16 | var app = express(); 17 | var insult = require('shakespeare-insult'); 18 | var bodyParser = require('body-parser'); 19 | var os = require('os'); 20 | 21 | app.use(bodyParser.urlencoded({extended:true})); 22 | app.use(bodyParser.json()); 23 | 24 | var port = process.env.PORT || 8080; 25 | 26 | app.get('/', function(req,res){ 27 | res.json( 28 | { 29 | hostname:os.hostname(), 30 | version:1, 31 | insult:insult.random() 32 | } 33 | ); 34 | }) 35 | 36 | app.listen(port,function(){ 37 | console.log('Insult v1 app listening on port',port); 38 | }) 39 | -------------------------------------------------------------------------------- /lab-deployments/v1/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "insults", 3 | "version": "1.0.0", 4 | "description": "Version 1 of the wonderful Shakespeare Insult Flinger", 5 | "main": "app.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "start": "node app.js" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git+https://github.com/scottpletcher/aws-csa-pro.git" 13 | }, 14 | "author": "", 15 | "license": "ISC", 16 | "bugs": { 17 | "url": "https://github.com/scottpletcher/aws-csa-pro/issues" 18 | }, 19 | "homepage": "https://github.com/scottpletcher/aws-csa-pro#readme", 20 | "dependencies": { 21 | "body-parser": "^1.18.3", 22 | "express": "^4.16.4", 23 | "shakespeare-insult": "^1.0.0" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /lab-deployments/v1/v1.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ACloudGuru-Resources/Course_Certified_Solutions_Architect_Professional/bdc3461752e580de25f9f00c6526e4c68467c764/lab-deployments/v1/v1.zip -------------------------------------------------------------------------------- /lab-deployments/v2/app.js: -------------------------------------------------------------------------------- 1 | /* 2 | Shakespeare Insult Flinger 3 | Version 2 4 | 5 | CHANGELOG: 6 | - Now with Extra Mean Mode! 7 | - Version 2 returned in JSON 8 | 9 | USAGE: 10 | node app.js 11 | 12 | By default, will listen on port 8080 if no PORT environment variable is set. 13 | */ 14 | 15 | var express = require('express'); 16 | var app = express(); 17 | var insult = require('shakespeare-insult'); 18 | var bodyParser = require('body-parser'); 19 | var os = require('os'); 20 | 21 | app.use(bodyParser.urlencoded({extended:true})); 22 | app.use(bodyParser.json()); 23 | 24 | var port = process.env.PORT || 8080; 25 | 26 | app.get('/', function(req,res){ 27 | res.json( 28 | { 29 | hostname: os.hostname(), 30 | version: 2, 31 | insult: insult.random(), 32 | extra_mean_mode: true 33 | } 34 | ); 35 | }) 36 | 37 | app.listen(port,function(){ 38 | console.log('Insult v2 app listening on port',port); 39 | }) 40 | -------------------------------------------------------------------------------- /lab-deployments/v2/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "insults", 3 | "version": "2.0.0", 4 | "description": "Version 2 of the wonderful Shakespeare Insult Flinger", 5 | "main": "app.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "start": "node app.js" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git+https://github.com/scottpletcher/aws-csa-pro.git" 13 | }, 14 | "author": "", 15 | "license": "ISC", 16 | "bugs": { 17 | "url": "https://github.com/scottpletcher/aws-csa-pro/issues" 18 | }, 19 | "homepage": "https://github.com/scottpletcher/aws-csa-pro#readme", 20 | "dependencies": { 21 | "body-parser": "^1.18.3", 22 | "express": "^4.16.4", 23 | "shakespeare-insult": "^1.0.0" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /lab-deployments/v2/v2.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ACloudGuru-Resources/Course_Certified_Solutions_Architect_Professional/bdc3461752e580de25f9f00c6526e4c68467c764/lab-deployments/v2/v2.zip -------------------------------------------------------------------------------- /lab-deployments/watch_url.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # USAGE: watch_url 3 | # DESCRIPTION: Repeatedly polls URL and displays the returned result 4 | 5 | watch -n 1 curl -s $1 -------------------------------------------------------------------------------- /lab-networking/7508839.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ACloudGuru-Resources/Course_Certified_Solutions_Architect_Professional/bdc3461752e580de25f9f00c6526e4c68467c764/lab-networking/7508839.jpg -------------------------------------------------------------------------------- /lab-networking/Step_1_Setup.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: '2010-09-09' 3 | Description: ACG CSAP Networking Lab - Initial Setup 4 | Parameters : 5 | LatestAmiId : 6 | Type : 'AWS::SSM::Parameter::Value' 7 | Default: '/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2' 8 | InstanceType: 9 | Description: WebServer EC2 instance type 10 | Type: String 11 | Default: t2.micro 12 | AllowedValues: [t1.micro, t2.micro, t3.micro] 13 | Resources: 14 | VPC: 15 | Type: AWS::EC2::VPC 16 | Properties: 17 | EnableDnsSupport: 'true' 18 | EnableDnsHostnames: 'true' 19 | CidrBlock: 10.0.0.0/16 20 | Tags: 21 | - 22 | Key: 'Name' 23 | Value: 'ACG CSAP Networking Lab' 24 | PublicSubnet: 25 | Type: AWS::EC2::Subnet 26 | Properties: 27 | VpcId: 28 | Ref: VPC 29 | CidrBlock: 10.0.0.0/24 30 | Tags: 31 | - 32 | Key: 'Name' 33 | Value: 'ACG CSAP Networking Lab' 34 | InternetGateway: 35 | Type: AWS::EC2::InternetGateway 36 | Properties: 37 | Tags: 38 | - 39 | Key: 'Name' 40 | Value: 'ACG CSAP Networking Lab' 41 | VPCGatewayAttachment: 42 | Type: AWS::EC2::VPCGatewayAttachment 43 | Properties: 44 | VpcId: 45 | Ref: VPC 46 | InternetGatewayId: 47 | Ref: InternetGateway 48 | PublicRouteTable: 49 | Type: AWS::EC2::RouteTable 50 | Properties: 51 | VpcId: 52 | Ref: VPC 53 | PublicNetworkAcl: 54 | Type: "AWS::EC2::NetworkAcl" 55 | Properties: 56 | VpcId: 57 | Ref: "VPC" 58 | PublicNACLEntryEgress100: 59 | Type: AWS::EC2::NetworkAclEntry 60 | Properties: 61 | CidrBlock: '0.0.0.0/0' 62 | NetworkAclId: 63 | Ref: PublicNetworkAcl 64 | Egress: 'true' 65 | PortRange: 66 | From: '0' 67 | To: '65535' 68 | Protocol: '-1' 69 | RuleAction: Allow 70 | RuleNumber: 100 71 | PublicNACLEntryIngress100: 72 | Type: AWS::EC2::NetworkAclEntry 73 | Properties: 74 | CidrBlock: '0.0.0.0/0' 75 | NetworkAclId: 76 | Ref: PublicNetworkAcl 77 | Egress: 'false' 78 | PortRange: 79 | From: '0' 80 | To: '65535' 81 | Protocol: -1 82 | RuleAction: Allow 83 | RuleNumber: 100 84 | PublicRoute: 85 | Type: AWS::EC2::Route 86 | DependsOn: VPCGatewayAttachment 87 | Properties: 88 | RouteTableId: 89 | Ref: PublicRouteTable 90 | DestinationCidrBlock: 0.0.0.0/0 91 | GatewayId: 92 | Ref: InternetGateway 93 | PublicSubnetRouteTableAssociation: 94 | Type: AWS::EC2::SubnetRouteTableAssociation 95 | Properties: 96 | SubnetId: 97 | Ref: PublicSubnet 98 | RouteTableId: 99 | Ref: PublicRouteTable 100 | PublicSubnetNetworkAclAssociation: 101 | Type: AWS::EC2::SubnetNetworkAclAssociation 102 | Properties: 103 | SubnetId: 104 | Ref: PublicSubnet 105 | NetworkAclId: 106 | Ref: PublicNetworkAcl 107 | WebServerSecurityGroup: 108 | Type: AWS::EC2::SecurityGroup 109 | Properties: 110 | GroupDescription: HTTP ingress 111 | VpcId: 112 | Ref: VPC 113 | SecurityGroupIngress: 114 | - IpProtocol: tcp 115 | FromPort: '80' 116 | ToPort: '80' 117 | CidrIp: 0.0.0.0/0 118 | Tags: 119 | - 120 | Key: 'Name' 121 | Value: 'ACG CSAP Networking Lab' 122 | WebServerInstance: 123 | Type: AWS::EC2::Instance 124 | Properties: 125 | InstanceType: !Ref 'InstanceType' 126 | ImageId: !Ref LatestAmiId 127 | NetworkInterfaces: 128 | - GroupSet: 129 | - Ref: WebServerSecurityGroup 130 | AssociatePublicIpAddress: 'false' 131 | DeviceIndex: '0' 132 | DeleteOnTermination: 'true' 133 | SubnetId: 134 | Ref: PublicSubnet 135 | Tags: 136 | - 137 | Key: 'Name' 138 | Value: 'ACG CSAP Networking Lab' 139 | UserData: 140 | Fn::Base64: !Sub | 141 | #!/bin/bash -xe 142 | sudo yum update -y 143 | sudo yum install httpd -y 144 | sudo systemctl start httpd 145 | sudo systemctl enable httpd 146 | echo "

Keep Being Awesome Cloud Gurus!

" > /var/www/html/index.html 147 | EIP: 148 | Type: AWS::EC2::EIP 149 | Properties: 150 | Domain: vpc 151 | InstanceId: 152 | Ref: WebServerInstance 153 | Outputs: 154 | FQDN: 155 | Value: !GetAtt WebServerInstance.PublicDnsName 156 | 157 | -------------------------------------------------------------------------------- /lab-networking/Step_2_LaFours_Disaster.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: '2010-09-09' 3 | Description: ACG CSAP Networking Lab - LaFours Disaster 4 | Parameters : 5 | LatestAmiId : 6 | Type : 'AWS::SSM::Parameter::Value' 7 | Default: '/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2' 8 | InstanceType: 9 | Description: WebServer EC2 instance type 10 | Type: String 11 | Default: t2.micro 12 | AllowedValues: [t1.micro, t2.micro, t3.micro] 13 | Resources: 14 | VPC: 15 | Type: AWS::EC2::VPC 16 | Properties: 17 | EnableDnsSupport: 'true' 18 | EnableDnsHostnames: 'true' 19 | CidrBlock: 10.0.0.0/16 20 | Tags: 21 | - 22 | Key: 'Name' 23 | Value: 'ACG CSAP Networking Lab' 24 | PublicSubnet: 25 | Type: AWS::EC2::Subnet 26 | Properties: 27 | VpcId: 28 | Ref: VPC 29 | CidrBlock: 10.0.0.0/24 30 | Tags: 31 | - 32 | Key: 'Name' 33 | Value: 'ACG CSAP Networking Lab' 34 | InternetGateway: 35 | Type: AWS::EC2::InternetGateway 36 | Properties: 37 | Tags: 38 | - 39 | Key: 'Name' 40 | Value: 'ACG CSAP Networking Lab' 41 | VPCGatewayAttachment: 42 | Type: AWS::EC2::VPCGatewayAttachment 43 | Properties: 44 | VpcId: 45 | Ref: VPC 46 | InternetGatewayId: 47 | Ref: InternetGateway 48 | PublicRouteTable: 49 | Type: AWS::EC2::RouteTable 50 | Properties: 51 | VpcId: 52 | Ref: VPC 53 | PublicNetworkAcl: 54 | Type: "AWS::EC2::NetworkAcl" 55 | Properties: 56 | VpcId: 57 | Ref: "VPC" 58 | PublicNACLEntryEgress100: 59 | Type: AWS::EC2::NetworkAclEntry 60 | Properties: 61 | CidrBlock: '0.0.0.0/0' 62 | NetworkAclId: 63 | Ref: PublicNetworkAcl 64 | Egress: 'true' 65 | PortRange: 66 | From: '80' 67 | To: '80' 68 | Protocol: '6' 69 | RuleAction: Allow 70 | RuleNumber: 100 71 | PublicNACLEntryIngress110: 72 | Type: AWS::EC2::NetworkAclEntry 73 | Properties: 74 | CidrBlock: '10.0.0.0/0' 75 | NetworkAclId: 76 | Ref: PublicNetworkAcl 77 | Egress: 'false' 78 | PortRange: 79 | From: '80' 80 | To: '80' 81 | Protocol: 17 82 | RuleAction: Allow 83 | RuleNumber: 110 84 | PublicNACLEntryIngress100: 85 | Type: AWS::EC2::NetworkAclEntry 86 | Properties: 87 | CidrBlock: '0.0.0.0/0' 88 | NetworkAclId: 89 | Ref: PublicNetworkAcl 90 | Egress: 'false' 91 | PortRange: 92 | From: '80' 93 | To: '80' 94 | Protocol: -1 95 | RuleAction: Deny 96 | RuleNumber: 100 97 | PublicRoute: 98 | Type: AWS::EC2::Route 99 | DependsOn: VPCGatewayAttachment 100 | Properties: 101 | RouteTableId: 102 | Ref: PublicRouteTable 103 | DestinationCidrBlock: 10.1.0.0/16 104 | GatewayId: 105 | Ref: InternetGateway 106 | PublicSubnetRouteTableAssociation: 107 | Type: AWS::EC2::SubnetRouteTableAssociation 108 | Properties: 109 | SubnetId: 110 | Ref: PublicSubnet 111 | RouteTableId: 112 | Ref: PublicRouteTable 113 | PublicSubnetNetworkAclAssociation: 114 | Type: AWS::EC2::SubnetNetworkAclAssociation 115 | Properties: 116 | SubnetId: 117 | Ref: PublicSubnet 118 | NetworkAclId: 119 | Ref: PublicNetworkAcl 120 | WebServerSecurityGroup: 121 | Type: AWS::EC2::SecurityGroup 122 | Properties: 123 | GroupDescription: HTTP ingress 124 | VpcId: 125 | Ref: VPC 126 | SecurityGroupIngress: 127 | - IpProtocol: tcp 128 | FromPort: '80' 129 | ToPort: '80' 130 | CidrIp: 10.0.0.0/16 131 | SecurityGroupEgress: 132 | - IpProtocol: tcp 133 | FromPort: '80' 134 | ToPort: '80' 135 | CidrIp: 10.0.0.0/16 136 | Tags: 137 | - 138 | Key: 'Name' 139 | Value: 'ACG CSAP Networking Lab' 140 | WebServerInstance: 141 | Type: AWS::EC2::Instance 142 | Properties: 143 | InstanceType: !Ref 'InstanceType' 144 | ImageId: !Ref LatestAmiId 145 | NetworkInterfaces: 146 | - GroupSet: 147 | - Ref: WebServerSecurityGroup 148 | AssociatePublicIpAddress: 'false' 149 | DeviceIndex: '0' 150 | DeleteOnTermination: 'true' 151 | SubnetId: 152 | Ref: PublicSubnet 153 | Tags: 154 | - 155 | Key: 'Name' 156 | Value: 'ACG CSAP Networking Lab' 157 | UserData: 158 | Fn::Base64: !Sub | 159 | #!/bin/bash -xe 160 | sudo yum update -y 161 | sudo yum install httpd -y 162 | sudo systemctl start httpd 163 | sudo systemctl enable httpd 164 | echo "

Keep Being Awesome Cloud Gurus!

" > /var/www/html/index.html 165 | EIP: 166 | Type: AWS::EC2::EIP 167 | Properties: 168 | Domain: vpc 169 | InstanceId: 170 | Ref: WebServerInstance 171 | Outputs: 172 | FQDN: 173 | Value: !GetAtt WebServerInstance.PublicDnsName 174 | Note: 175 | Value: 'LaFOURS WUZ HERE! SNOOGINS!' 176 | -------------------------------------------------------------------------------- /lab-scaling/README.md: -------------------------------------------------------------------------------- 1 | # Lab: Scaling on AWS 2 | 3 | To be used in conjunction with the AWS Certified Solutions Architect Professional course on A Cloud Guru. 4 | -------------------------------------------------------------------------------- /lab-scaling/stress-testing-boot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # USAGE: Place in in the User Data section (under Advanced) when launching an EC2 instance 3 | # DESCRIPTION: After updating from the repo, installs stress, a tool used to create various system load for testing purposes. 4 | yum update -y 5 | wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -P /tmp 6 | yum install -y /tmp/epel-release-latest-7.noarch.rpm 7 | yum install stress -y 8 | /usr/bin/stress --cpu 8 --timeout 10m -------------------------------------------------------------------------------- /lab-storage/README.md: -------------------------------------------------------------------------------- 1 | # Lab: Storage Domain 2 | 3 | To be used in conjunction with the AWS Certified Solutions Architect Professional course on A Cloud Guru. 4 | -------------------------------------------------------------------------------- /lab-storage/create_air_quality_table.sql: -------------------------------------------------------------------------------- 1 | /* Creates a table referencing a directory full of nested JSON files */ 2 | 3 | CREATE EXTERNAL TABLE `air_quality`( 4 | `date` struct COMMENT 'from deserializer', 5 | `parameter` string COMMENT 'from deserializer', 6 | `location` string COMMENT 'from deserializer', 7 | `value` double COMMENT 'from deserializer', 8 | `unit` string COMMENT 'from deserializer', 9 | `city` string COMMENT 'from deserializer', 10 | `attribution` array> COMMENT 'from deserializer', 11 | `coordinates` struct COMMENT 'from deserializer', 12 | `country` string COMMENT 'from deserializer', 13 | `sourcename` string COMMENT 'from deserializer', 14 | `sourcetype` string COMMENT 'from deserializer', 15 | `mobile` boolean COMMENT 'from deserializer', 16 | `averagingperiod` struct COMMENT 'from deserializer') 17 | ROW FORMAT SERDE 18 | 'org.openx.data.jsonserde.JsonSerDe' 19 | WITH SERDEPROPERTIES ( 20 | 'paths'='attribution,averagingPeriod,city,coordinates,country,date,location,mobile,parameter,sourceName,sourceType,unit,value') 21 | STORED AS INPUTFORMAT 22 | 'org.apache.hadoop.mapred.TextInputFormat' 23 | OUTPUTFORMAT 24 | 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' 25 | LOCATION 26 | 's3://openaq-fetches/realtime/2018-10-09/' 27 | TBLPROPERTIES ( 28 | 'CrawlerSchemaDeserializerVersion'='1.0', 29 | 'CrawlerSchemaSerializerVersion'='1.0', 30 | 'UPDATED_BY_CRAWLER'='aq2', 31 | 'averageRecordSize'='1005', 32 | 'classification'='json', 33 | 'compressionType'='none', 34 | 'objectCount'='117', 35 | 'recordCount'='309720', 36 | 'sizeKey'='317032303', 37 | 'typeOfData'='file'); -------------------------------------------------------------------------------- /lab-storage/create_o3_view.sql: -------------------------------------------------------------------------------- 1 | /* Creates a view of just Ozone readings in the US */ 2 | 3 | CREATE OR REPLACE VIEW v_O3_US AS 4 | SELECT 5 | "city" 6 | , "coordinates"."latitude" "latitude" 7 | , "coordinates"."longitude" "longitude" 8 | , CAST("from_iso8601_timestamp"("date"."local") AS timestamp) "timestamp" 9 | , "value" 10 | , "unit" 11 | FROM 12 | "air_quality" 13 | WHERE (("parameter" = 'o3') AND ("country" = 'US')); --------------------------------------------------------------------------------