├── LICENSE ├── README.md ├── course ├── architecting │ ├── as-elb-healthcheck │ │ ├── LICENSE │ │ ├── NOTICE │ │ ├── README.md │ │ ├── web-bootstrap.sh │ │ └── web-examplefiles.zip │ ├── route53_lbr_demo │ │ ├── LICENSE │ │ ├── NOTICE │ │ ├── README.md │ │ ├── cfn-route53-latency-demo-all.json │ │ ├── cfn-route53-latency-demo-vpn.json │ │ ├── cfn-route53-latency-demo-web.json │ │ ├── params.json │ │ ├── vpn-bootstrap.sh │ │ ├── web-bootstrap.sh │ │ └── web-examplefiles.zip │ └── s3_parallel_upload │ │ ├── LICENSE │ │ ├── NOTICE │ │ └── README.md ├── big-data │ ├── process-commoncrawl-with-emr-and-datapipeline │ │ ├── LICENSE │ │ ├── NOTICE │ │ ├── README.md │ │ ├── boto-cfg.sh │ │ ├── data-pipe-line.template │ │ └── emr.py │ └── process-commoncrawl-with-emr │ │ ├── LICENSE │ │ ├── NOTICE │ │ └── README.md ├── developing │ └── sample_code │ │ ├── LICENSE │ │ ├── NOTICE │ │ ├── README.md │ │ ├── nbactions.xml │ │ ├── pom.xml │ │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── awstrainers │ │ │ │ └── devcourse │ │ │ │ └── sdkdemos │ │ │ │ └── AsyncDemo.java │ │ └── resources │ │ │ ├── AwsCredentials.properties │ │ │ └── log4j.properties │ │ └── test │ │ └── java │ │ └── com │ │ └── awstrainers │ │ └── devcourse │ │ └── sdkdemos │ │ ├── DynamoDBTest.java │ │ ├── Patient.java │ │ ├── S3Test.java │ │ └── SMSTest.java └── system_operations │ ├── AMIFinder │ ├── LICENSE │ ├── NOTICE │ ├── README.md │ ├── cfn │ │ ├── amifinder.template.json │ │ └── amifinder_sample.template.json │ ├── findAMI.py │ ├── sh │ │ ├── aws-cfn-resource-bridge-0.1.tar.gz │ │ ├── cfn-resource-bridge.conf │ │ ├── create.sh │ │ ├── delete.sh │ │ └── update.sh │ └── test │ │ └── test_amifinder.py │ ├── SGFinder │ ├── LICENSE │ ├── README.md │ └── findSG.py │ ├── chef │ ├── LICENSE │ ├── NOTICE │ ├── README.md │ ├── admin-instance-resources.tar.gz │ └── chef-solo-admin-instance.json │ ├── cloudtrail-logstash │ ├── LICENSE │ ├── NOTICE │ ├── README.md │ ├── bootstrap.sh │ ├── cf1.png │ ├── cf2.png │ ├── changelog │ └── logstash-cloudtrail.template │ ├── cloudwatch │ ├── LICENSE │ ├── NOTICE │ ├── README.md │ └── stat-set-cw.sh │ ├── sns │ ├── LICENSE │ ├── NOTICE │ ├── README.md │ └── snsDemo.sh │ └── tags │ ├── LICENSE │ ├── NOTICE │ ├── README.md │ ├── tagBased.md │ └── tagBased.sh ├── misc ├── aws-daleks │ ├── LICENSE │ ├── NOTICE │ ├── README.md │ ├── build.sbt │ ├── project │ │ └── plugins.sbt │ └── src │ │ └── main │ │ └── scala │ │ └── aws │ │ └── daleks │ │ ├── eager │ │ ├── EagerAWSDaleks.scala │ │ ├── EagerBeanstalkDalek.scala │ │ ├── EagerCloudFormationDalek.scala │ │ ├── EagerDalek.scala │ │ ├── EagerDynamoDBDalek.scala │ │ ├── EagerEC2Dalek.scala │ │ ├── EagerEMRDalek.scala │ │ ├── EagerElastiCacheDalek.scala │ │ ├── EagerIAMDalek.scala │ │ ├── EagerRDSDalek.scala │ │ ├── EagerRoute53Dalek.scala │ │ ├── EagerS3Dalek.scala │ │ ├── EagerSNSDalek.scala │ │ ├── EagerSQSDalek.scala │ │ └── package.scala │ │ └── util │ │ └── Humid.scala └── video │ └── README.md ├── self-paced-lab └── spl49 │ ├── LICENSE │ ├── NOTICE.txt │ ├── README.md │ ├── lib │ ├── javax.ejb-api.jar │ ├── javax.faces-2.2.1.jar │ └── jstl-1.1.0.jar │ ├── pom.xml │ └── src │ └── main │ ├── java │ └── com │ │ └── amazon │ │ └── aws │ │ └── training │ │ ├── backend │ │ └── GreeterEJBBean.java │ │ └── controler │ │ └── Greeter.java │ └── webapp │ ├── WEB-INF │ ├── beans.xml │ ├── faces-config.xml │ └── web.xml │ ├── css │ └── justified-nav.css │ ├── img │ └── amazon.png │ └── index.xhtml └── service └── lambda ├── Lambda IP Checker.eml ├── cloudtrail ├── LICENSE ├── NOTICE ├── README.md ├── cloudtrail.js ├── filter_config.json ├── infra_create.sh ├── infra_parameters.sh ├── infra_teardown.sh ├── input.json └── main.js └── mobile-lambda ├── LICENSE ├── MobileLambda ├── LICENSE ├── MobileLambda-Bridging-Header.h ├── MobileLambda.xcodeproj │ ├── project.pbxproj │ └── project.xcworkspace │ │ ├── contents.xcworkspacedata │ │ └── xcshareddata │ │ └── MobileLambda.xccheckout ├── MobileLambda │ ├── AppDelegate.swift │ ├── Base.lproj │ │ ├── LaunchScreen.xib │ │ └── Main.storyboard │ ├── Images.xcassets │ │ └── AppIcon.appiconset │ │ │ └── Contents.json │ ├── Info.plist │ └── ViewController.swift ├── MobileLambdaTests │ ├── Info.plist │ └── MobileLambdaTests.swift ├── NOTICE ├── Podfile └── Podfile.lock ├── NOTICE ├── iam_policy_for_cognito_unauth.json ├── iam_policy_for_lambda.json ├── input.json ├── main.js ├── mobile-lambda.js └── upload_function.sh /README.md: -------------------------------------------------------------------------------- 1 | # aws-training-demo 2 | 3 | This repository contains demos managed by the AWS Technical Trainers community. 4 | 5 | Content of the repository is as follow : 6 | 7 | - Course 8 | - Service 9 | - Self-Paced Labs 10 | - Misc 11 | 12 | ## Courses 13 | 14 | This directory contains demos related to [AWS Training](https://aws.amazon.com/training) classes: 15 | 16 | - [Architecting on AWS](course/architecting) 17 | - [Systems Operations on AWS](course/system_operations) 18 | - [Developing on AWS](course/developing/) 19 | - [Big Data in AWS](course/big-data) 20 | 21 | ## Self-Paced Labs 22 | 23 | This directory contains code supporting some [self-paced labs](https://run.qwiklab.com). 24 | 25 | - SPL49 : Sample Java EE Lab to support Lab 49 : "[Deploy a Java EE Application on AWS Elastic Beanstalk Using Docker Containers](https://run.qwiklab.com/focuses/preview/1247?locale=en)" 26 | 27 | ## Service 28 | 29 | This directory contains demos related to AWS services 30 | 31 | - Lambda - CloudTrail 32 | This is the code from the blog post "[How to Receive Alerts When Specific APIs Are Called by Using AWS CloudTrail, Amazon SNS, and AWS Lambda](http://blogs.aws.amazon.com/security/post/Tx2ZTUVN2VGBS85/How-to-Receive-Alerts-When-Specific-APIs-Are-Called-by-Using-AWS-CloudTrail-Amaz)" 33 | 34 | - Lambda - Mobile demo 35 | Simple example of synchronous invocation of a Lambda function from an iOS Swift Application 36 | 37 | ## Misc 38 | 39 | This directory contains miscellaneous code or collections of links. 40 | 41 | - Videos : a collection of video links related to AWS 42 | - Dalek : a tool to delete all resources created in an AWS Account. To be used with extreme caution. 43 | 44 | # License 45 | 46 | Copyright 2015, Amazon Web Services. 47 | 48 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. 49 | You may obtain a copy of the License at 50 | 51 | http://www.apache.org/licenses/LICENSE-2.0 52 | 53 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 54 | See the License for the specific language governing permissions and limitations under the License. 55 | -------------------------------------------------------------------------------- /course/architecting/as-elb-healthcheck/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | -------------------------------------------------------------------------------- /course/architecting/as-elb-healthcheck/README.md: -------------------------------------------------------------------------------- 1 | #AutoScaling ELB HealthCheck Demo 2 | 3 | This scripts demonstrate AutoScaling healthcheck at the ELB level. 4 | In this scenario, the Auto Scaling service will probe the ELB about the health of the instance. It will replace any instance that fails to answer to ELB heatlthcheck during the period configured at ELB level. 5 | 6 | For web application, this scenario is better than traditional AutoScaling healthcheck at instance level since it allows to detect application level failures. 7 | 8 | ## Demo Script 9 | 10 | ### Prerequisites 11 | 12 | This scenario demo script requires that 13 | 14 | - you have installed the AWS CLI package on the machine you will use for the demo 15 | - AWS CLI is correctly configured (AWS_CONFIG_FILE environment variable pointing to a config file) 16 | - the command below are prepared for eu-west-1 region, you will need to adapat it to your region name and availability zones names 17 | 18 | The AWS CLI configuration file must look like this : 19 | 20 | ``` 21 | [default] 22 | aws_access_key_id = AKIA... 23 | aws_secret_access_key = fFVT... 24 | region = eu-west-1 25 | ``` 26 | 27 | ### Demo 28 | 29 | #### Create the demo environment 30 | 31 | Create a load balancer 32 | 33 | ```aws elb create-load-balancer --availability-zones eu-west-1a eu-west-1b --load-balancer-name asdemo --region eu-west-1 --listeners protocol=http,load_balancer_port=80,instance_protocol=http,instance_port=80``` 34 | 35 | Configure ELB's healthcheck 36 | 37 | ```aws elb configure-health-check --load-balancer-name asdemo --health-check target=HTTP:80/index.php,interval=10,timeout=5,unhealthy_threshold=2,healthy_threshold=2``` 38 | 39 | Create an AutoScaling launch configuration (Might need to chang the AMI id) 40 | 41 | ```aws autoscaling create-launch-configuration --image-id ami-c7c0d6b3 --instance-type t1.micro --key-name sst-aws --launch-configuration-name asdemo-lc --region eu-west-1 --security-groups web --user-data file://./web-bootstrap.sh``` 42 | 43 | 44 | Create an AutoScaling group (notice ```--health-check-type ELB ``` parameter) 45 | 46 | ```aws autoscaling create-auto-scaling-group --auto-scaling-group-name asdemo --availability-zones eu-west-1a eu-west-1b --desired-capacity 2 --min-size 2 --max-size 2 --health-check-grace-period 30 --health-check-type ELB --launch-configuration-name asdemo-lc --load-balancer-names asdemo --region eu-west-1``` 47 | 48 | Check AutoScaling is OK 49 | 50 | ```aws autoscaling describe-auto-scaling-instances``` 51 | 52 | 53 | #### SSH connect to one of the machine and stop the web server 54 | 55 | /etc/init.d/httpd stop 56 | 57 | 58 | #### Sit down, relax and watch AS in action 59 | 60 | After a few seconds, you should see one of the two instances marked as ```UNHEALTHY``` 61 | 62 | 63 | ``` 64 | $ aws autoscaling describe-auto-scaling-instances 65 | [ 66 | { 67 | "AvailabilityZone": "eu-west-1b", 68 | "InstanceId": "i-704d4b3d", 69 | "AutoScalingGroupName": "test-as", 70 | "HealthStatus": "HEALTHY", 71 | "LifecycleState": "InService", 72 | "LaunchConfigurationName": "asdemo-lc" 73 | }, 74 | { 75 | "AvailabilityZone": "eu-west-1a", 76 | "InstanceId": "i-764d4b3b", 77 | "AutoScalingGroupName": "test-as", 78 | "HealthStatus": "UNHEALTHY", 79 | "LifecycleState": "Terminating", 80 | "LaunchConfigurationName": "asdemo-lc" 81 | } 82 | ] 83 | ``` 84 | 85 | ### Cleanup 86 | 87 | Delete all resources created : 88 | 89 | ```aws autoscaling update-auto-scaling-group --min-size 0 --max-size 0 --auto-scaling-group-name asdemo``` 90 | 91 | ```aws autoscaling delete-auto-scaling-group --auto-scaling-group-name asdemo``` 92 | 93 | ```aws autoscaling delete-launch-configuration --launch-configuration-name asdemo-lc``` 94 | 95 | ```aws elb delete-load-balancer --load-balancer-name asdemo``` 96 | 97 | ##TODO 98 | 99 | Maybe it would benice to have a CFN template to setup this demo 100 | -------------------------------------------------------------------------------- /course/architecting/as-elb-healthcheck/web-bootstrap.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | # 5 | # http://aws.amazon.com/apache2.0/ 6 | # 7 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | 9 | 10 | #!/bin/bash 11 | yum -y install httpd php 12 | chkconfig httpd on 13 | /etc/init.d/httpd start 14 | cd /var/www/html 15 | wget http://aws-emea.info.s3.amazonaws.com/resources/route53-lbr/web-examplefiles.zip 16 | unzip web-examplefiles.zip 17 | -------------------------------------------------------------------------------- /course/architecting/as-elb-healthcheck/web-examplefiles.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-training-demo/3aa80fd3daf4b761b6425154f3124e167b4c8541/course/architecting/as-elb-healthcheck/web-examplefiles.zip -------------------------------------------------------------------------------- /course/architecting/route53_lbr_demo/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | -------------------------------------------------------------------------------- /course/architecting/route53_lbr_demo/cfn-route53-latency-demo-all.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | 4 | "Parameters" : { 5 | 6 | "SharedSecret": { 7 | "Description" : "The IPSec SharedSecret", 8 | "Type": "String", 9 | "Default" : "SharedSecret" 10 | }, 11 | "Username": { 12 | "Description" : "The VPN username", 13 | "Type": "String", 14 | "Default" : "username" 15 | }, 16 | "Password": { 17 | "Description" : "The VPN password", 18 | "Type": "String", 19 | "Default" : "password" 20 | }, 21 | "HostedDomain": { 22 | "Description" : "The hosted domain where we will add DNS records", 23 | "Type": "String", 24 | "Default" : "example.com" 25 | } 26 | 27 | }, 28 | 29 | "Resources" : { 30 | "WebStack" : { 31 | "Type" : "AWS::CloudFormation::Stack", 32 | "Properties" : { 33 | "TemplateURL" : "http://s3-eu-west-1.amazonaws.com/aws-emea.info/resources/route53-lbr/cfn-route53-latency-demo-web.json", 34 | "TimeoutInMinutes" : "60", 35 | "Parameters" : { 36 | "HostedDomain" : {"Ref" : "HostedDomain"} 37 | } 38 | } 39 | }, 40 | "VPNStack" : { 41 | "Type" : "AWS::CloudFormation::Stack", 42 | "Properties" : { 43 | "TemplateURL" : "http://s3-eu-west-1.amazonaws.com/aws-emea.info/resources/route53-lbr/cfn-route53-latency-demo-vpn.json", 44 | "TimeoutInMinutes" : "60", 45 | "Parameters" : { 46 | "SharedSecret" : {"Ref" : "SharedSecret"}, 47 | "Username" : {"Ref" : "Username"}, 48 | "Password" : {"Ref" : "Password"}, 49 | "HostedDomain" : {"Ref" : "HostedDomain"} 50 | } 51 | } 52 | } 53 | }, 54 | 55 | "Outputs": { 56 | "StackRef": {"Value": { "Ref" : "WebStack"}}, 57 | "WebServerDNSName" : { 58 | "Value" : { "Fn::GetAtt" : [ "WebStack", "Outputs.WebServerDNSName" ] } 59 | } 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /course/architecting/route53_lbr_demo/cfn-route53-latency-demo-vpn.json: -------------------------------------------------------------------------------- 1 | { 2 | "Mappings": { 3 | "AWSRegionToAMI": { 4 | "us-east-1": { "AMI": "ami-05355a6c" }, 5 | "us-west-2": { "AMI": "ami-0358ce33" }, 6 | "us-west-1": { "AMI": "ami-3ffed17a" }, 7 | "eu-west-1": { "AMI": "ami-c7c0d6b3" }, 8 | "ap-southeast-1": { "AMI": "ami-fade91a8" }, 9 | "ap-northeast-1": { "AMI": "ami-39b23d38" }, 10 | "ap-southeast-2": { "AMI": "ami-d16bfbeb" }, 11 | "sa-east-1": { "AMI": "ami-5253894f" } 12 | } 13 | }, 14 | 15 | "Parameters" : { 16 | 17 | "SharedSecret": { 18 | "Description" : "The IPSec SharedSecret", 19 | "Type": "String" 20 | }, 21 | "Username": { 22 | "Description" : "The VPN username", 23 | "Type": "String" 24 | }, 25 | "Password": { 26 | "Description" : "The VPN password", 27 | "Type": "String" 28 | }, 29 | "HostedDomain": { 30 | "Description" : "The hosted domain where we will add DNS records", 31 | "Type": "String" 32 | } 33 | 34 | }, 35 | 36 | "Resources" : { 37 | 38 | "VPNSecurityGroup" : { 39 | "Type" : "AWS::EC2::SecurityGroup", 40 | "Properties" : { 41 | "GroupDescription" : "VPN Access", 42 | "SecurityGroupIngress" : [ 43 | { "IpProtocol" : "tcp", "FromPort" : "22", "ToPort" : "22", "CidrIp" : "0.0.0.0/0"}, 44 | { "IpProtocol" : "tcp", "FromPort" : "500", "ToPort" : "500", "CidrIp" : "0.0.0.0/0"}, 45 | { "IpProtocol" : "udp", "FromPort" : "500", "ToPort" : "500", "CidrIp" : "0.0.0.0/0"}, 46 | { "IpProtocol" : "udp", "FromPort" : "4500", "ToPort" : "4500", "CidrIp" : "0.0.0.0/0"} 47 | ] 48 | } 49 | }, 50 | 51 | "VPNServer" : { 52 | "Type" : "AWS::EC2::Instance", 53 | "Properties" : { 54 | "ImageId" : { "Fn::FindInMap" : [ "AWSRegionToAMI", { "Ref" : "AWS::Region" }, "AMI"]}, 55 | "InstanceType" : "m1.small", 56 | "SecurityGroups" : [{ "Ref" : "VPNSecurityGroup" }], 57 | "UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [ 58 | "#!/bin/bash", "\n", 59 | 60 | "wget -q 'http://aws-emea.info.s3.amazonaws.com/resources/route53-lbr/vpn-bootstrap.sh'", "\n", 61 | "sed -i -e \"2s/^/IPSEC_PSK=", {"Ref" : "SharedSecret"} ,"\\n/\" vpn-bootstrap.sh", "\n", 62 | "sed -i -e \"2s/^/VPN_USER=", {"Ref" : "Username"} ,"\\n/\" vpn-bootstrap.sh", "\n", 63 | "sed -i -e \"2s/^/VPN_PASSWORD=", {"Ref" : "Password"} ,"\\n/\" vpn-bootstrap.sh", "\n", 64 | ". ./vpn-bootstrap.sh", "\n", 65 | "/opt/aws/bin/cfn-signal -e $? ", { "Fn::Base64" : { "Ref" : "WaitForVPNWaitHandle" }}, "\n" 66 | ] ] } }, 67 | "Tags" : [ {"Key" : "Name", "Value" : "VPNServer"}] 68 | } 69 | }, 70 | 71 | "WaitForVPNWaitHandle" : { 72 | "Type" : "AWS::CloudFormation::WaitConditionHandle", 73 | "Properties" : { 74 | } 75 | }, 76 | 77 | "WaitForVPNServer" : { 78 | "Type" : "AWS::CloudFormation::WaitCondition", 79 | "DependsOn" : "VPNServer", 80 | "Properties" : { 81 | "Handle" : {"Ref" : "WaitForVPNWaitHandle"}, 82 | "Timeout" : "300" 83 | } 84 | }, 85 | 86 | "VPNDNSRecord" : { 87 | "Type" : "AWS::Route53::RecordSet", 88 | "Properties" : { 89 | "HostedZoneName" : { "Fn::Join" : ["", [ {"Ref" : "HostedDomain"}, "."] ]}, 90 | "Comment" : "CNAME record for VPN Server", 91 | "Name" : { "Fn::Join" : ["", [ 92 | "vpn-", 93 | { "Ref" : "AWS::Region" }, 94 | ".", 95 | {"Ref" : "HostedDomain"}, 96 | "." 97 | ]]}, 98 | "Type" : "CNAME", 99 | "TTL" : "60", 100 | "ResourceRecords" : [{ "Fn::GetAtt" : [ "VPNServer", "PublicDnsName" ] } ] 101 | } 102 | } 103 | }, 104 | 105 | "Outputs" : { 106 | "VPNServerDNSName" : { 107 | "Description" : "Public DNS name of the newly created VPN Server", 108 | "Value" : { "Fn::GetAtt" : [ "VPNServer", "PublicDnsName" ] } 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /course/architecting/route53_lbr_demo/cfn-route53-latency-demo-web.json: -------------------------------------------------------------------------------- 1 | { 2 | "Mappings": { 3 | "AWSRegionToAMI": { 4 | "us-east-1": { "AMI": "ami-05355a6c" }, 5 | "us-west-2": { "AMI": "ami-0358ce33" }, 6 | "us-west-1": { "AMI": "ami-3ffed17a" }, 7 | "eu-west-1": { "AMI": "ami-c7c0d6b3" }, 8 | "ap-southeast-1": { "AMI": "ami-fade91a8" }, 9 | "ap-northeast-1": { "AMI": "ami-39b23d38" }, 10 | "ap-southeast-2": { "AMI": "ami-d16bfbeb" }, 11 | "sa-east-1": { "AMI": "ami-5253894f" } 12 | } 13 | }, 14 | 15 | "Parameters" : { 16 | 17 | "HostedDomain": { 18 | "Description" : "The hosted domain where we will add DNS records", 19 | "Type": "String" 20 | } 21 | 22 | }, 23 | 24 | "Resources" : { 25 | 26 | "WebSecurityGroup" : { 27 | "Type" : "AWS::EC2::SecurityGroup", 28 | "Properties" : { 29 | "GroupDescription" : "Web Access", 30 | "SecurityGroupIngress" : [ 31 | { "IpProtocol" : "tcp", "FromPort" : "80", "ToPort" : "80", "CidrIp" : "0.0.0.0/0"} 32 | ] 33 | } 34 | }, 35 | 36 | "WebServer" : { 37 | "Type" : "AWS::EC2::Instance", 38 | "Properties" : { 39 | "ImageId" : { "Fn::FindInMap" : [ "AWSRegionToAMI", { "Ref" : "AWS::Region" }, "AMI"]}, 40 | "InstanceType" : "m1.small", 41 | "SecurityGroups" : [{ "Ref" : "WebSecurityGroup" }], 42 | "UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [ 43 | "#!/bin/bash", "\n", 44 | "wget -q -O - 'http://aws-emea.info.s3.amazonaws.com/resources/route53-lbr/web-bootstrap.sh' | sh" , "\n", 45 | "/opt/aws/bin/cfn-signal -e $? ", { "Fn::Base64" : { "Ref" : "WaitForWebServerHandle" }}, "\n" 46 | ] ] } }, 47 | "Tags" : [ {"Key" : "Name", "Value" : "WebServer"}] 48 | } 49 | }, 50 | 51 | "WebDNSRecord" : { 52 | "Type" : "AWS::Route53::RecordSet", 53 | "Properties" : { 54 | "HostedZoneName" : { "Fn::Join" : ["", [ {"Ref" : "HostedDomain"}, "."] ]}, 55 | "Comment" : "CNAME record for Web Server", 56 | "Name" : { "Fn::Join" : ["", [ 57 | "web-", 58 | { "Ref" : "AWS::Region" }, 59 | ".", 60 | {"Ref" : "HostedDomain"}, 61 | "." 62 | ]]}, 63 | "Type" : "CNAME", 64 | "TTL" : "60", 65 | "ResourceRecords" : [ { "Fn::GetAtt" : [ "WebServer", "PublicDnsName" ] } ] 66 | } 67 | }, 68 | 69 | "WaitForWebServerHandle" : { 70 | "Type" : "AWS::CloudFormation::WaitConditionHandle", 71 | "Properties" : { 72 | } 73 | }, 74 | 75 | "WaitForWebServer" : { 76 | "Type" : "AWS::CloudFormation::WaitCondition", 77 | "DependsOn" : "WebServer", 78 | "Properties" : { 79 | "Handle" : {"Ref" : "WaitForWebServerHandle"}, 80 | "Timeout" : "300" 81 | } 82 | } 83 | 84 | }, 85 | 86 | "Outputs" : { 87 | "WebServerDNSName" : { 88 | "Description" : "Public DNS name of the newly created Web Server", 89 | "Value" : { "Fn::GetAtt" : [ "WebServer", "PublicDnsName" ] } 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /course/architecting/route53_lbr_demo/params.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"ParameterKey":"SharedSecret", "ParameterValue":"SharedSecret"}, 3 | {"ParameterKey":"Username", "ParameterValue":"user"}, 4 | {"ParameterKey":"Password", "ParameterValue":"password"}, 5 | {"ParameterKey":"HostedDomain", "ParameterValue":"aws-emea.info"} 6 | ] -------------------------------------------------------------------------------- /course/architecting/route53_lbr_demo/vpn-bootstrap.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | # 5 | # http://aws.amazon.com/apache2.0/ 6 | # 7 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | 9 | 10 | #!/bin/bash -x 11 | 12 | # Please define your own values for those variables 13 | # these will be injected into that script by the CFN template bootstrap script 14 | #IPSEC_PSK=SharedSecret 15 | #VPN_USER=username 16 | #VPN_PASSWORD=password 17 | 18 | # Those two variables will be found automatically 19 | PRIVATE_IP=`wget -q -O - 'http://instance-data/latest/meta-data/local-ipv4'` 20 | 21 | #the following does not work in VPC 22 | #PUBLIC_IP=`wget -q -O - 'http://instance-data/latest/meta-data/public-ipv4'` 23 | # 24 | # use http://169.254.169.254/latest/meta-data/network/interfaces/macs/06:79:3f:b2:49:20/ipv4-associations/ instead but depends on mac address :-( 25 | # 26 | PUBLIC_IP=`wget -q -O - 'checkip.amazonaws.com'` 27 | 28 | 29 | # send std out to log file 30 | exec &>> /home/ec2-user/bootstrap-vpn.log 31 | 32 | function error_exit() { 33 | echo "{\"Reason\": \"$1\"}" 34 | exit $2 35 | } 36 | 37 | RETURN_CODE=$(/usr/bin/yum install -y --enablerepo=epel openswan xl2tpd) 38 | if [ $RETURN_CODE -ne 0 ] 39 | then 40 | error_exit "yum install openswan xl2tpd failed" $RETURN_CODE 41 | fi 42 | 43 | cat > /etc/ipsec.conf < /etc/ipsec.secrets < /etc/xl2tpd/xl2tpd.conf < /etc/ppp/options.xl2tpd < /etc/ppp/chap-secrets < /proc/sys/net/ipv4/ip_forward 126 | 127 | iptables-save > /etc/iptables.rules 128 | 129 | mkdir -p /etc/network/if-pre-up.d 130 | cat > /etc/network/if-pre-up.d/iptablesload < /proc/sys/net/ipv4/ip_forward 134 | exit 0 135 | EOF 136 | 137 | chkconfig ipsec on 138 | chkconfig xl2tpd on 139 | 140 | RETURN_CODE=$(service ipsec start) 141 | if [ $RETURN_CODE -ne 0 ] 142 | then 143 | error_exit "Can not start ipsec" $RETURN_CODE 144 | fi 145 | 146 | RETURN_CODE=$(service xl2tpd start) 147 | if [ $RETURN_CODE -ne 0 ] 148 | then 149 | error_exit "Can not start xl2tpd" $RETURN_CODE 150 | fi 151 | -------------------------------------------------------------------------------- /course/architecting/route53_lbr_demo/web-bootstrap.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | # 5 | # http://aws.amazon.com/apache2.0/ 6 | # 7 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | 9 | 10 | #!/bin/bash -x 11 | 12 | # send std out to log file 13 | exec &>> /home/ec2-user/bootstrap.log 14 | 15 | function error_exit() { 16 | echo "{\"Reason\": \"$1\"}" 17 | exit $2 18 | } 19 | 20 | echo "Installing Apache's HTTPD and PHP" 21 | RETURN_CODE = $(/usr/bin/yum -y install httpd php) 22 | if [ $RETURN_CODE -ne 0 ] 23 | then 24 | error_exit "yum install httpd php failed" $RETURN_CODE 25 | fi 26 | 27 | RETURN_CODE=$(/sbin/chkconfig httpd on) 28 | if [ $RETURN_CODE -ne 0 ] 29 | then 30 | error_exit "chkconfig failed" $RETURN_CODE 31 | fi 32 | 33 | RETURN_CODE=$(/etc/init.d/httpd start) 34 | if [ $RETURN_CODE -ne 0 ] 35 | then 36 | error_exit "httpd start failed" $RETURN_CODE 37 | fi 38 | 39 | cd /var/www/html 40 | RETURN_CODE=$(wget http://aws-emea.info.s3.amazonaws.com/resources/route53-lbr/web-examplefiles.zip) 41 | if [ $RETURN_CODE -ne 0 ] 42 | then 43 | error_exit "download sample web app failed" $RETURN_CODE 44 | fi 45 | 46 | RETURN_CODE=$(unzip web-examplefiles.zip) 47 | if [ $RETURN_CODE -ne 0 ] 48 | then 49 | error_exit "unzip web app failed" $RETURN_CODE 50 | fi 51 | -------------------------------------------------------------------------------- /course/architecting/route53_lbr_demo/web-examplefiles.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-training-demo/3aa80fd3daf4b761b6425154f3124e167b4c8541/course/architecting/route53_lbr_demo/web-examplefiles.zip -------------------------------------------------------------------------------- /course/architecting/s3_parallel_upload/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | -------------------------------------------------------------------------------- /course/architecting/s3_parallel_upload/README.md: -------------------------------------------------------------------------------- 1 | #Parallel vs Sequential Uploads 2 | 3 | **Purpose**: Demonstrate that S3 is massively parallel. 4 | **Summary**: This demonstration uploads 500 x 10MB files (5GB total). When done sequentially, it takes 18 minutes. When done in parallel, it takes 70 seconds. 5 | 6 | Do not confuse parallel uploads with [multipart uploads](http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html) 7 | 8 | ##Step 1: Create a bucket for storing files 9 | The Region for the bucket should match the Region used for EC2 in the next step. I also add a Lifecycle rule to expire files after 0 days, just to keep things clean. 10 | 11 | ``` 12 | export BUCKET= 13 | aws s3 mb s3://$BUCKET 14 | ``` 15 | 16 | ##Step 2: Launch a Linux m2.4XL instance with ephemeral storage in the same Region 17 | - Use ``m2.4XL`` to avoid network bandwidth problems 18 | - Give it a Role with permissions to access your S3 Bucket 19 | - Add one default Instance Store volume to store the 5GB of data. It also makes a great demonstration of ephemeral storage. 20 | - Make sure the instance is in the same Region as the Bucket! 21 | 22 | ``` 23 | Should we add a AWS CLI script for these steps (Role, SG, KP, Instance) ? 24 | Or a cloudformation template ? 25 | ``` 26 | 27 | ##Step 3: Login to EC2 and setup environment 28 | I paste these commands into the Command Line. It downloads GNU Parallel and flicks to the ephemeral storage. 29 | 30 | ``` 31 | # Download Gnu Parallel 32 | wget http://ftp.gnu.org/gnu/parallel/parallel-20101202.tar.bz2 33 | tar -jxvf parallel-20101202.tar.bz2 34 | cd parallel-20101202 35 | ./configure 36 | make && sudo make install 37 | 38 | # Go to instance store 39 | cd /media/ephemeral0/ 40 | sudo chmod 777 . 41 | ``` 42 | ##Step 4: Create 500 x 10MB files 43 | ``` 44 | # Create files 45 | for num in $(seq 1 500); do dd if=/dev/zero of=file-${num} bs=1M count=10; done 46 | ``` 47 | 48 | ##Step 5: Sequential upload demonstration 49 | 50 | ``` 51 | # Sequential upload 52 | time for ob in *; do echo ${ob}; aws s3 $ob s3://$BUCKET/seq-$ob; done 53 | ```` 54 | 55 | This should take approximately 18 minutes. Go back to teaching the course and come back later to see the result. It will look something like this: 56 | 57 | ``` 58 | real 18m20.396s 59 | user 7m25.308s 60 | sys 0m44.327s 61 | ``` 62 | 63 | ##Step 6: Parallel upload demonstration 64 | 65 | ``` 66 | # Parallel upload 67 | time ls * | parallel -j500 aws s3 $ob s3://$BUCKET/par-$ob 68 | ``` 69 | The result will look like: 70 | ``` 71 | real 1m12.777s 72 | user 7m59.842s 73 | sys 1m4.388s 74 | ``` 75 | 76 | **Bottom line**: Parallel uploads on S3 are much faster than sequential uploads 77 | 78 | ## Cleanup 79 | 80 | Delete the bucket 81 | 82 | ``` 83 | aws s3 rb --force s3://$BUCKET 84 | ``` 85 | 86 | ## Credits 87 | 88 | Dean Samuels for the script 89 | John Rotenstein for the wiki page 90 | Sebastien Stormacq to adapt to AWS CLI 1.0 91 | -------------------------------------------------------------------------------- /course/big-data/process-commoncrawl-with-emr-and-datapipeline/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | -------------------------------------------------------------------------------- /course/big-data/process-commoncrawl-with-emr-and-datapipeline/README.md: -------------------------------------------------------------------------------- 1 | # process-commoncrawl-with-emr-and-datapipeline 2 | 3 | A demo that shows how to orchestrate data transformation with the Data Pipeline service. 4 | For a partial manual step by step guide see : 5 | https://github.com/stasov/process-commoncrawl-with-emr 6 | 7 | A predefined Data Pipeline template is used to perform the following steps: 8 | - Launch an EMR cluster 9 | - Copy a part of the CommonCrawl public data set locally to the cluster (HDFS) 10 | - Process the data (Run a regex search with the following pattern : [Bb]ig [Dd]ata is ([a-zA-Z]+) by using the grep utility from hadoop-examples.jar) 11 | - Copy the processed data to a specified dynamoDB table 12 | 13 | To learn more about AWS public data sets or the CommonCrawl project, see: 14 | - AWS Public Data Sets (http://aws.amazon.com/public-data-sets/) 15 | - The commonCrawl project (http://commoncrawl.org/) 16 | 17 | Choose a region where Data Pipeline is supported and use it where \ is specified. 18 | The demo was tested in Ireland (eu-west-1) and Oregon (us-west-2) but should work just 19 | as well in any other region. 20 | 21 | ##Step 1: Create the needed resources 22 | Create a S3 bucket: 23 | ``` 24 | aws s3 mb s3:// --region 25 | ``` 26 | Create a DynamoDB table as follows (The Primary Hash key setup is important): 27 | ``` 28 | aws dynamodb create-table --region --table-name --attribute-definitions AttributeName=phrase,AttributeType=S --key-schema AttributeName=phrase,KeyType=HASH --provisioned-throughput ReadCapacityUnits=1,WriteCapacityUnits=1 29 | ``` 30 | 31 | ##Step 2: Edit the artifacts and upload to your S3 bucket 32 | Download and edit the following files: 33 | emr.py - replace \ and \ with the appropriate values 34 | data-pipe-line.template - no need to edit 35 | Upload the files to your S3 bucket. 36 | 37 | ##Step 3: Create and activate the Data pipeline 38 | In the Data Pipeline console: 39 | - choose a name for the pipeline 40 | - choose "import a definition", then "Load from S3" and browse to the data-pipe-line.template object in your bucket. 41 | In case you are prompt with an "Access denied" error, either set appropriate permissions on the data-pipe-line.template file (for example make it public) or download the file locally and use the "load local file" option. 42 | - Supply a region name, bucket name and key-pair name for the instances of the cluster 43 | - Leave "Path to a part of the common crawl data set" unchanged (It's useful if you'll decide to process a different part of the CommonCrawl project in the future) 44 | - Leave everything else with the default settings and activate the pipeline 45 | If you are prompt with a warning, ignore it and activate the pipeline. 46 | 47 | ##Step 4: Monitor the progress and view results in your dynamoDB table 48 | - Clone the activated pipeline and view the different configurations in the "Architect Editor" 49 | - View the properties of the cluster in the EMR console 50 | - Take a look at the steps that are submitted to the cluster by Data Pipeline 51 | 52 | The process should take around 30 minutes. 53 | It's time for a coffee break. 54 | Take a look at the results in the dynamoDB table. 55 | What is the most common phrase for "Big Data is" ? 56 | 57 | Remember to clean up! 58 | Delete the dynamoDB table, the S3 bucket and the pipeline in Data pipeline. 59 | 60 | The End. 61 | 62 | 63 | 64 | Cloned from https://github.com/stasov/process-commoncrawl-with-emr-and-datapipeline 65 | -------------------------------------------------------------------------------- /course/big-data/process-commoncrawl-with-emr-and-datapipeline/boto-cfg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | var="[Credentials]" 3 | sed "1s/.*/$var/" /home/hadoop/.aws/config > /home/hadoop/.aws/temp 4 | sudo mv /home/hadoop/.aws/temp /etc/boto.cfg 5 | -------------------------------------------------------------------------------- /course/big-data/process-commoncrawl-with-emr-and-datapipeline/data-pipe-line.template: -------------------------------------------------------------------------------- 1 | { 2 | "objects" : [ 3 | { 4 | "id" : "DefaultSchedule", 5 | "name" : "RunOnce", 6 | "occurrences" : "1", 7 | "startAt" : "FIRST_ACTIVATION_DATE_TIME", 8 | "type" : "Schedule", 9 | "period" : "1 Day" 10 | }, 11 | { 12 | "id" : "Default", 13 | "scheduleType" : "cron", 14 | "failureAndRerunMode" : "CASCADE", 15 | "schedule" : { "ref" : "DefaultSchedule" }, 16 | "name" : "Default", 17 | "pipelineLogUri" : "s3://#{myBucketName}/logs-common-crawl-pipe-line", 18 | "role" : "DataPipelineDefaultRole", 19 | "resourceRole" : "DataPipelineDefaultResourceRole" 20 | }, 21 | { 22 | "id" : "ActivityId_01", 23 | "schedule" : { "ref" : "DefaultSchedule" }, 24 | "name" : "copy-public-data-set-to-hdfs", 25 | "runsOn" : { "ref" : "EmrClusterId_01" }, 26 | "type" : "EmrActivity", 27 | "step" : [ "file:///home/hadoop/lib/emr-s3distcp-1.0.jar,--s3Endpoint,s3-external-1.amazonaws.com,--src,#{myPathToDataSet},--dest,hdfs:///grep-data/"] 28 | }, 29 | { 30 | "id" : "ActivityId_02", 31 | "schedule" : { "ref" : "DefaultSchedule" }, 32 | "name" : "grep-results-to-hdfs", 33 | "runsOn" : { "ref" : "EmrClusterId_01" }, 34 | "type" : "EmrActivity", 35 | "step" : ["file:///home/hadoop/hadoop-examples.jar,grep,/grep-data/,/grep-output/,[Bb]ig [Dd]ata is ([a-zA-Z]+)"], 36 | "dependsOn" : { "ref" : "ActivityId_01" } 37 | }, 38 | { 39 | "id" : "ActivityId_03", 40 | "schedule" : { "ref" : "DefaultSchedule" }, 41 | "name" : "write-results-to-dynamo", 42 | "runsOn" : { "ref" : "EmrClusterId_01" }, 43 | "type" : "EmrActivity", 44 | "step" : ["s3://elasticmapreduce/libs/script-runner/script-runner.jar,s3://#{myBucketName}/emr.py"], 45 | "dependsOn" : { "ref" : "ActivityId_02" } 46 | }, 47 | { 48 | "id" : "EmrClusterId_01", 49 | "region" : "#{myClusterResourceRegion}", 50 | "amiVersion" : "3.4.0", 51 | "schedule" : { "ref" : "DefaultSchedule" }, 52 | "keyPair" : "#{myKeyPairName}", 53 | "masterInstanceType" : "m1.medium", 54 | "coreInstanceCount" : "2", 55 | "coreInstanceType" : "m3.xlarge", 56 | "name" : "emr-cluster", 57 | "type" : "EmrCluster", 58 | "terminateAfter" : "55 Minutes", 59 | "resourceRole": "DataPipelineDefaultResourceRole", 60 | "role": "DataPipelineDefaultRole" 61 | } 62 | ], 63 | "parameters": [ 64 | { 65 | "id": "myClusterResourceRegion", 66 | "watermark": "Your region. for example : us-west-2", 67 | "description": "Region to launch cluster in", 68 | "type": "String" 69 | }, 70 | { 71 | "id": "myPathToDataSet", 72 | "watermark": "s3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-2014-23/segments/1404776400583.60/wet/", 73 | "default": "s3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-2014-23/segments/1404776400583.60/wet/", 74 | "description": "Path to a part of the commmon crawl data set", 75 | "type": "String" 76 | }, 77 | { 78 | "id": "myKeyPairName", 79 | "description": "Name of the key pair for the EMR instances. Please make sure it's in the chosen region", 80 | "type": "String" 81 | }, 82 | { 83 | "id": "myBucketName", 84 | "description": "Name of the bucket with the artifacts", 85 | "type": "String" 86 | } 87 | ] 88 | } 89 | -------------------------------------------------------------------------------- /course/big-data/process-commoncrawl-with-emr-and-datapipeline/emr.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | # 5 | # http://aws.amazon.com/apache2.0/ 6 | # 7 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | 9 | 10 | #!/usr/bin/python 11 | 12 | import boto.dynamodb 13 | import subprocess 14 | 15 | conn = boto.dynamodb.connect_to_region('') 16 | table = conn.get_table('') 17 | 18 | cat = subprocess.Popen(["hadoop", "fs", "-cat", "/grep-output/part-r-00000"], stdout=subprocess.PIPE) 19 | 20 | item_phrase = "" 21 | item_count = "" 22 | 23 | tup_bag = [] 24 | for line in cat.stdout: 25 | split_line = line.strip('\n').split('\t') 26 | #In case of empty lines 27 | if split_line[0] == "": 28 | break 29 | item_count = split_line[0] 30 | item_phrase = split_line[1] 31 | item_data = { 'count' : int(item_count)} 32 | item = table.new_item(hash_key=item_phrase,attrs=item_data) 33 | item.save() 34 | -------------------------------------------------------------------------------- /course/big-data/process-commoncrawl-with-emr/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | -------------------------------------------------------------------------------- /course/big-data/process-commoncrawl-with-emr/README.md: -------------------------------------------------------------------------------- 1 | # process-commoncrawl-with-emr 2 | A short demo that shows how to launch an EMR cluster with spot instances using the CLI, copy a part of the commonCrawl AWS public data set using s3distCP and how to use the grep implementation from the Hadoop examples jar to find what Big Data is 3 | 4 | 5 | - AWS Public Data Sets (http://aws.amazon.com/public-data-sets/) 6 | - The commonCrawl project (http://commoncrawl.org/) 7 | 8 | To do all the following with a single command please see "Step 7: Automate" 9 | 10 | ##Step 1: Launch the cluster 11 | The example was tested with the 3.0.4 EMR AMI (With newer versions an error might appear when running the s3DistCP) 12 | When launched in Ireland with a m1.large master node and 4 c3.8xlarge core nodes, 13 | The copy should take around 3 minutes and the grep around 2.5 minutes. 14 | The cost per hour with the bid prices in the example should be up to : 1.43 $ 15 | ``` 16 | aws emr create-cluster --ami-version 3.0.4 --name "" --region --no-auto-terminate\ 17 | --instance-groups InstanceGroupType=MASTER,InstanceCount=1,BidPrice=,InstanceType=""\ 18 | InstanceGroupType=CORE,InstanceCount=,BidPrice=,InstanceType=""\ 19 | --service-role EMR_DefaultRole --ec2-attributes\ InstanceProfile=EMR_EC2_DefaultRole,KeyName=,AvailabilityZone=eu-west-1a\ 20 | --tags project=big-data application=common-crawl --applications Name=Ganglia 21 | ``` 22 | Example: 23 | ``` 24 | aws emr create-cluster --ami-version 3.0.4 --name "My cluster" --region eu-west-1 --no-auto-terminate\ 25 | --instance-groups InstanceGroupType=MASTER,InstanceCount=1,BidPrice=0.03,InstanceType="m1.large"\ 26 | InstanceGroupType=CORE,InstanceCount=4,BidPrice=0.35,InstanceType="c3.8xlarge"\ 27 | --service-role EMR_DefaultRole --ec2-attributes InstanceProfile=EMR_EC2_DefaultRole,KeyName=myKey,AvailabilityZone=eu-west-1a\ 28 | --tags project=big-data application=common-crawl cost-center=1234 --applications Name=Ganglia 29 | ``` 30 | 31 | ##Step 2: Copy part of the commonCrawl data set from s3 to HDFS 32 | The folder size under the specified path is 55.6GB (feel free to include more paths). 33 | At the time of writing s3distCP doesn't support multiple S3 source paths, 34 | one could use distCP instead or use srcPattern with s3distCP. 35 | ``` 36 | hdfs dfs -mkdir /grep-data/ 37 | ``` 38 | ``` 39 | hadoop jar /home/hadoop/lib/emr-s3distcp-1.0.jar --s3Endpoint s3-external-1.amazonaws.com --src\ 40 | "s3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-2014-23/segments/1406510280868.21/wet/" --dest "hdfs:///grep-data/" 41 | ``` 42 | Example for using source pattern to include multiple paths (A larger data set): 43 | ``` 44 | hadoop jar /home/hadoop/lib/emr-s3distcp-1.0.jar --s3Endpoint s3-external-1.amazonaws.com --src\ 45 | "s3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-2014-23/segments/" --dest "hdfs:///grep-data1/" --srcPattern '.*(1404776400583.60|1404776400808.24)\/wet\/.*' 46 | ``` 47 | 48 | ##Step 3: Check the size of the data set copied 49 | ``` 50 | hdfs dfs -du -s -h /grep-data 51 | ``` 52 | ##Step 4: Use the grep utility from the hadoop examples file 53 | The data files provided in the commonCrawl data set are compressed. 54 | the following job will grep through 130GB of uncompressed data. 55 | ``` 56 | hadoop jar hadoop-examples.jar grep /grep-data/ /grep-output/ '[Bb]ig [Dd]ata is ([a-zA-Z]{5,})' 57 | ``` 58 | ##Step 5: View results 59 | ``` 60 | hdfs dfs -cat /grep-output/part-r-00000 | head -10 61 | ``` 62 | ##Step 6: View cluster performance with Ganglia 63 | Edit the "ElasticMapReduce-master" secruity group to allow access to port 80 from your IP 64 | or create an SSH tunnel as described under "View cluster details"/Connections. 65 | Navigate to /ganglia and explore the CPU/Mem/Network metrics for all the instances. 66 | 67 | ##Step 7: Automate 68 | 69 | - Make sure you have an S3 bucket, a preferably newly created folder and that you edit the steps.json to include the correct output path. example : "s3://my-common-crawl/output" 70 | *If the specified folder contains the results of a previously invoked job the new job will fail because the output already exists. Make sure the output folder is empty for each run. 71 | 72 | - If you're interested in things other than Big Data, feel free to change the following line: 73 | "[Bb]ig [Dd]ata is ([a-zA-Z]{5,})" to something like "[Ll]ife is ([a-zA-Z]{5,})" or any valid regular expression. 74 | 75 | - Create a steps.json file : 76 | ``` 77 | [ 78 | { 79 | "Name": "Copy part of the common crawl data set", 80 | "Jar" : "file:///home/hadoop/lib/emr-s3distcp-1.0.jar", 81 | "Args": [ 82 | "--s3Endpoint","s3-external-1.amazonaws.com","--src","s3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-2014-23/segments/1406510280868.21/wet/","--dest","hdfs:///grep-data/" 83 | ] 84 | , 85 | "ActionOnFailure": "CONTINUE", 86 | "Type": "CUSTOM_JAR" 87 | }, 88 | { 89 | "Name": "Grep using the hadoop-examples JAR", 90 | "Jar" : "file:///home/hadoop/hadoop-examples.jar", 91 | "Args": [ 92 | "grep","/grep-data/","s3://","[Bb]ig [Dd]ata is ([a-zA-Z]{5,})" 93 | ], 94 | "ActionOnFailure": "TERMINATE_JOB_FLOW", 95 | "Type": "CUSTOM_JAR" 96 | } 97 | ] 98 | ``` 99 | - Launch the cluster using the steps.json to describe the steps. 100 | *Don't forget to edit the name of the key pair and to provide the path to the steps.json 101 | (Unless you're running the command from the same directory) 102 | Example: 103 | ``` 104 | aws emr create-cluster --ami-version 3.0.4 --name "My cluster" --region eu-west-1 --auto-terminate\ 105 | --instance-groups InstanceGroupType=MASTER,InstanceCount=1,BidPrice=0.03,InstanceType="m1.large"\ 106 | InstanceGroupType=CORE,InstanceCount=4,BidPrice=0.35,InstanceType="c3.8xlarge"\ 107 | --service-role EMR_DefaultRole --ec2-attributes InstanceProfile=EMR_EC2_DefaultRole,\ 108 | KeyName=,AvailabilityZone=eu-west-1a\ 109 | --steps file://./step-prod.json --tags project=big-data application=common-crawl cost-center=1234 --applications Name=Ganglia 110 | ``` 111 | - The "--auto-terminate" arg will make sure the cluster terminates after the steps complete. 112 | - Get the output from the part-r-00000 file that should appear in the S3 path specified (You might have to make the object public in case you're accessing it by using the S3 console) 113 | 114 | The End. 115 | 116 | Cloned from https://github.com/stasov/process-commoncrawl-with-emr 117 | -------------------------------------------------------------------------------- /course/developing/sample_code/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | -------------------------------------------------------------------------------- /course/developing/sample_code/README.md: -------------------------------------------------------------------------------- 1 |

AWS developer course examples

2 | 3 | 4 | Here you can find a few examples that may be useful when teaching the Dev course. They're mostly implemented in the form 5 | of jUnit tests but actually they don't relay heavily into asserting conditions. 6 | 7 |
    8 |
  • DynamoDBTest demostrates how to use the DynamoDB high level API, among other things. 9 |
  • S3Test includes a multiupload demo using the TransferManager. 10 |
  • SESTest will help you to show how to send mail using AWS and the javax.mail API. 11 |
12 | 13 | Feel free to use the code. And of course, all improvements will be appreciated! 14 | 15 | jv 16 | -------------------------------------------------------------------------------- /course/developing/sample_code/nbactions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | build 5 | 6 | * 7 | 8 | 9 | install 10 | 11 | 12 | true 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /course/developing/sample_code/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.javiermoreno.mediaclouder 7 | mediaclouder 8 | 0.1-SNAPSHOT 9 | 10 | com.awstrainers 11 | devcoursedemos 12 | 0.1-SNAPSHOT 13 | devcoursedemos 14 | http://maven.apache.org 15 | 16 | UTF-8 17 | 18 | 19 | 20 | com.amazonaws 21 | aws-java-sdk 22 | 1.6.1 23 | 24 | 25 | junit 26 | junit 27 | 4.11 28 | 29 | 30 | log4j 31 | log4j 32 | 1.2.17 33 | 34 | 35 | commons-logging 36 | commons-logging 37 | 1.1.1 38 | 39 | 40 | javax.mail 41 | mail 42 | 1.4 43 | test 44 | jar 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /course/developing/sample_code/src/main/java/com/awstrainers/devcourse/sdkdemos/AsyncDemo.java: -------------------------------------------------------------------------------- 1 | // Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | // 5 | // http://aws.amazon.com/apache2.0/ 6 | // 7 | // or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | 9 | 10 | 11 | package com.awstrainers.devcourse.sdkdemos; 12 | 13 | import java.util.concurrent.Callable; 14 | import java.util.concurrent.ExecutorService; 15 | import java.util.concurrent.Executors; 16 | import java.util.concurrent.Future; 17 | 18 | /** 19 | * 20 | * @author ciberado 21 | */ 22 | public class AsyncDemo { 23 | 24 | public static void main(String[] args) throws Exception { 25 | ExecutorService executor = Executors.newFixedThreadPool(5); 26 | Future lprA = executor.submit(new LongProcess("A", 1000)); 27 | Future lprB = executor.submit(new LongProcess("B", 500)); 28 | // Uncomment the following line to see how Future.get is a blocking invocation 29 | //System.out.println("**** " + lprA.get()); 30 | Future lprC = executor.submit(new LongProcess("C", 800)); 31 | 32 | } 33 | 34 | } 35 | 36 | class LongProcess implements Callable { 37 | 38 | private String name; 39 | private int pause; 40 | 41 | public LongProcess(String name, int pause) { 42 | this.name = name; 43 | this.pause = pause; 44 | } 45 | 46 | @Override 47 | public String call() throws Exception { 48 | System.out.println("Into call " + name); 49 | Thread.sleep(pause); 50 | System.out.println("Exiting call " + name); 51 | return "Ok from " + name; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /course/developing/sample_code/src/main/resources/AwsCredentials.properties: -------------------------------------------------------------------------------- 1 | #Insert your AWS Credentials from http://aws.amazon.com/security-credentials 2 | #Tue Oct 15 22:58:20 CEST 2013 3 | secretKey= 4 | accessKey= 5 | -------------------------------------------------------------------------------- /course/developing/sample_code/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | 2 | 3 | log4j.rootLogger=debug, stdout 4 | 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Threshold=DEBUG 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - <%m>%n 9 | 10 | log4j.logger.org.apache=INFO 11 | log4j.logger.com.amazonaws=debug 12 | 13 | -------------------------------------------------------------------------------- /course/developing/sample_code/src/test/java/com/awstrainers/devcourse/sdkdemos/Patient.java: -------------------------------------------------------------------------------- 1 | /* 2 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | # 6 | # http://aws.amazon.com/apache2.0/ 7 | # 8 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | 10 | */ 11 | 12 | package com.awstrainers.devcourse.sdkdemos; 13 | 14 | import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAttribute; 15 | import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBHashKey; 16 | import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBRangeKey; 17 | import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTable; 18 | 19 | @DynamoDBTable(tableName = "patients") 20 | public class Patient implements java.io.Serializable { 21 | private static final long serialVersionUID = 7336392782218661341L; 22 | 23 | 24 | private long id; 25 | private long timestamp; 26 | private String payload; 27 | 28 | public Patient(long id, long timestamp, String payload) { 29 | super(); 30 | this.id = id; 31 | this.timestamp = timestamp; 32 | this.payload = payload; 33 | } 34 | 35 | @DynamoDBHashKey 36 | public long getId() { 37 | return id; 38 | } 39 | 40 | public void setId(long id) { 41 | this.id = id; 42 | } 43 | 44 | @DynamoDBRangeKey 45 | public long getTimestamp() { 46 | return timestamp; 47 | } 48 | 49 | public void setTimestamp(long timestamp) { 50 | this.timestamp = timestamp; 51 | } 52 | 53 | @DynamoDBAttribute 54 | public String getPayload() { 55 | return payload; 56 | } 57 | 58 | public void setPayload(String payload) { 59 | this.payload = payload; 60 | } 61 | 62 | @Override 63 | public int hashCode() { 64 | final int prime = 31; 65 | int result = 1; 66 | result = prime * result + (int) (id ^ (id >>> 32)); 67 | result = prime * result + (int) (timestamp ^ (timestamp >>> 32)); 68 | return result; 69 | } 70 | 71 | @Override 72 | public boolean equals(Object obj) { 73 | if (this == obj) 74 | return true; 75 | if (obj == null) 76 | return false; 77 | if (getClass() != obj.getClass()) 78 | return false; 79 | Patient other = (Patient) obj; 80 | if (id != other.id) 81 | return false; 82 | if (timestamp != other.timestamp) 83 | return false; 84 | return true; 85 | } 86 | 87 | 88 | 89 | } 90 | -------------------------------------------------------------------------------- /course/developing/sample_code/src/test/java/com/awstrainers/devcourse/sdkdemos/SMSTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | # 6 | # http://aws.amazon.com/apache2.0/ 7 | # 8 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | 10 | */ 11 | 12 | package com.awstrainers.devcourse.sdkdemos; 13 | 14 | import java.util.Properties; 15 | 16 | import javax.mail.Message; 17 | import javax.mail.Session; 18 | import javax.mail.Transport; 19 | import javax.mail.internet.InternetAddress; 20 | import javax.mail.internet.MimeMessage; 21 | 22 | import org.junit.BeforeClass; 23 | import org.junit.Test; 24 | 25 | import com.amazonaws.auth.AWSCredentials; 26 | import com.amazonaws.auth.ClasspathPropertiesFileCredentialsProvider; 27 | import com.amazonaws.services.simpleemail.AWSJavaMailTransport; 28 | 29 | public class SMSTest { 30 | public static final AWSCredentials cred = 31 | new ClasspathPropertiesFileCredentialsProvider("AwsCredentials.properties").getCredentials(); 32 | 33 | // Check the next url to know more about verifying email addresses in AWS. 34 | // http://docs.aws.amazon.com/ses/latest/DeveloperGuide/verify-email-addresses.html 35 | private static final String FROM_ADDRESS = System.getProperty("sms_test_from_address"); 36 | private static final String TO_ADDRESS = System.getProperty("sms_test_to_address");; 37 | 38 | @BeforeClass 39 | public static void checkSourcePreconditions() throws Exception { 40 | if (FROM_ADDRESS == null || TO_ADDRESS== null) { 41 | throw new /*Precondition*/Exception("Please run this test indicating valid and verifyied email addresses."); 42 | } 43 | } 44 | 45 | @Test 46 | public void checkSendMail() throws Exception { 47 | 48 | Properties props = new Properties(); 49 | props.setProperty("mail.transport.protocol", "aws"); 50 | props.setProperty("mail.aws.user", cred.getAWSAccessKeyId()); 51 | props.setProperty("mail.aws.password", cred.getAWSSecretKey()); 52 | 53 | Session session = Session.getInstance(props); 54 | 55 | Message msg = new MimeMessage(session); 56 | msg.setFrom(new InternetAddress(FROM_ADDRESS)); 57 | msg.addRecipient(Message.RecipientType.TO, new InternetAddress(TO_ADDRESS)); 58 | msg.setSubject("Test"); 59 | msg.setText("Hi!"); 60 | msg.saveChanges(); 61 | 62 | Transport t = new AWSJavaMailTransport(session, null); 63 | t.connect(); 64 | t.sendMessage(msg, null); 65 | t.close(); 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /course/system_operations/AMIFinder/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | -------------------------------------------------------------------------------- /course/system_operations/AMIFinder/README.md: -------------------------------------------------------------------------------- 1 | AMIFinder 2 | ========= 3 | 4 | AMIFinder is a sample [CloudFormation Custom Resource] (http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/crpg-walkthrough.html) environment. 5 | 6 | This Custom Resource uses EC2's DescribeImage API to find an Amazon Windows Base image (64 bits, ebs based) 7 | of the specified version. 8 | 9 | Use this Custom Resource to avoid hard coding AMI ids inside your CFN templates, at the risk of having errors because of deprecated AMI IDs. 10 | 11 | 12 | Usage 13 | ----- 14 | 15 | The ```cfn``` directory contains two CloudFormation templates: 16 | 17 | - ```amifinder.template.json``` setup the complete infrastructure to implement the Custom Resource "AMIFinder". See 18 | below for a list of resources it creates. 19 | 20 | - ```amifinder_sample.template.json``` is a sample template that shows how to use the Custom Resource : 21 | 22 | ``` 23 | { 24 | "AWSTemplateFormatVersion": "2010-09-09", 25 | "Description": "Create Infrastructure required to run finAMI Custom resource", 26 | 27 | "Resources": { 28 | "AMIFinderTest": { 29 | "Type": "Custom::AMIFinder", 30 | "Version": "1.0", 31 | "Properties": { 32 | "ServiceToken": "", 33 | "Version": "2012" 34 | } 35 | } 36 | }, 37 | "Outputs" : { 38 | "WindowAMIID" : { 39 | "Value" : { "Ref" : "AMIFinderTest" } 40 | } 41 | } 42 | } 43 | ``` 44 | 45 | Other resources in the template can use ```{ "Ref" : "AMIFinderTest" }``` to refer to the AMI ID. Typically, you will referer to the AMI from an ```EC2::Instance``` resource. 46 | 47 | This template can not run "as is", you need to insert your Custom Resource's implementation SNS Topic ARN as 48 | ```ServiceToken``` value. 49 | 50 | How does it work ? 51 | ------------------ 52 | 53 | The ```amifinder.template.json``` CFN template creates the following environment : 54 | 55 | - a SNS Topic - to be used by CFN to call the Custom Resource and to be inserted in the ```amifinder_test.template 56 | .json``` 57 | - a SQS Queue subscribed to the topic 58 | - a SQS Policy allowing SNS to post messages to the queue 59 | - an IAM Role to allow an EC2 instance to read from the queue and to call DescribeImage EC2 API 60 | - a Security Group allowing inbound SSH connections (debugging only - can be removed once everything is working) 61 | - an EC2 Instance bootstrapped with [```cfn-resource-bridge```](https://github.com/aws/aws-cfn-resource-bridge) and 62 | ```findAMI```, a custom python helper script 63 | 64 | ```cfn-resource-bridge``` will poll the queue, waiting for CloudFormation messages, and will call appropriate shell 65 | scripts to respond to ```create```, ```update``` and ```delete``` requests. 66 | 67 | In this example, ```update``` and ```delete``` shell scripts are empty. Only ```create``` is implemented. It uses ```findAMI``` to retrieve the correct AMI IDs. 68 | 69 | TODO 70 | ---- 71 | 72 | - improve based on collected feedback 73 | -------------------------------------------------------------------------------- /course/system_operations/AMIFinder/cfn/amifinder_sample.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Create Infrastructure required to run finAMI Custom resource", 4 | 5 | "Parameters" : { 6 | "KeyName" : { 7 | "Description" : "Name of an existing EC2 KeyPair to enable SSH access to the Instance", 8 | "Type" : "String", 9 | "Default" : "sst-aws" 10 | }, 11 | "InstanceType" : { 12 | "Description" : "Instance type", 13 | "Type" : "String", 14 | "Default" : "t1.micro", 15 | "AllowedValues" : [ "t1.micro","m3.medium","m3.large"], 16 | "ConstraintDescription" : "must be a valid EC2 instance type." 17 | } 18 | }, 19 | 20 | "Resources": { 21 | "AMIFinder": { 22 | "Type": "Custom::AMIFinder", 23 | "Version": "1.0", 24 | "Properties": { 25 | "ServiceToken": "", 26 | "Version": "2012" 27 | } 28 | }, 29 | "RDPSecurityGroup" : { 30 | "Type" : "AWS::EC2::SecurityGroup", 31 | "Properties" : { 32 | "GroupDescription" : "RDP to the instance", 33 | "SecurityGroupIngress" : [ 34 | { 35 | "CidrIp" : "0.0.0.0/0", 36 | "FromPort" : "3389", 37 | "ToPort" : "3389", 38 | "IpProtocol" : "tcp" 39 | } 40 | ] 41 | } 42 | }, 43 | "WindowsInstance" : { 44 | "Type" : "AWS::EC2::Instance", 45 | "Properties" : { 46 | "ImageId" : { "Ref" : "AMIFinder" }, 47 | "InstanceType" : { "Ref" : "InstanceType" }, 48 | "KeyName" : { "Ref" : "KeyName" }, 49 | "SecurityGroups" : [{ "Ref" : "RDPSecurityGroup" }] 50 | } 51 | } 52 | }, 53 | "Outputs" : { 54 | "WindowsAMIID" : { 55 | "Value" : { "Ref" : "AMIFinder" } 56 | }, 57 | "WindowsInstanceDNSName" : { 58 | "Value" : { "Fn::GetAtt" : [ "WindowsInstance" , "PublicDnsName" ] } 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /course/system_operations/AMIFinder/findAMI.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | # 5 | # http://aws.amazon.com/apache2.0/ 6 | # 7 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | 9 | #!/usr/bin/python 10 | 11 | import sys, argparse, logging 12 | import requests 13 | import boto.ec2 14 | 15 | 16 | __author__ = 'stormacq' 17 | __version__ = 1.0 18 | 19 | ''' 20 | Usage : findAMI --region 21 | --version 22 | --locale (default: 'en') 23 | 24 | Find an Amazon's Windows Base AMI in the given region, the given version and for the given locale. 25 | This command only search for 64bits EBS based AMI 26 | 27 | Example : findAMI --region eu-west-1 --version '2012' 28 | ''' 29 | 30 | class AMIFinder: 31 | 32 | def __init__(self, logger=None): 33 | 34 | self.logger = logger or logging.getLogger(self.__class__.__name__) 35 | 36 | 37 | self.locales = { 'en' : 'English', 38 | 'fr' : 'French' } 39 | 40 | #shared connection object - but do not cache it as it is region dependant 41 | self.conn = None 42 | 43 | self.DEFAULT_FILTERS = { 'platform' : 'windows', 44 | 'architecture' : 'x86_64', 45 | 'root-device-type' : 'ebs'} 46 | 47 | #cached complete AMI list for DEFAULT_FILTER 48 | self.amiList = None 49 | 50 | def searchDescription(self, image, search, locale): 51 | ''' 52 | Search for matching string values in the image Description attribute 53 | 54 | This search function searches against the Base AMI only 55 | ''' 56 | result = None 57 | 58 | if image.description is not None: 59 | if image.description.find('Base AMI') > -1: 60 | if image.description.find(locale) > -1: 61 | if image.description.find(search) > -1: 62 | result = image 63 | 64 | return result 65 | 66 | 67 | def findWindowsAMIInRegion(self, region, searchCriteria, locale='en'): 68 | ''' 69 | Search for a Amazon's Base AMI Windows, 64 bits, EBS based in the specific region, 70 | the specific searchCriteria and the specific locale 71 | 72 | Typically, searchCriteria is the Windows version number (2012, 2008 SP1 ...) 73 | ''' 74 | 75 | l = self.locales.get(locale); 76 | if l is None: 77 | self.logger.error('Unknown locale : %s' % locale) 78 | return None 79 | 80 | if boto.ec2.get_region(region) is None: 81 | self.logger.error('Invalid region : %s' % region) 82 | return None 83 | 84 | self.conn = boto.ec2.connect_to_region(region) 85 | 86 | if self.conn is None: 87 | self.logger.error('Can not connect to AWS') 88 | return None 89 | 90 | if (self.amiList is None): 91 | self.amiList = self.conn.get_all_images(owners='amazon', filters=self.DEFAULT_FILTERS) 92 | self.logger.debug('Retrieved %d images' % len(self.amiList)) 93 | 94 | if len(self.amiList) < 1: 95 | self.logger.warning('No image retrieved for region "%s" using default filters (Windows, 64 Bits, EBS)' % region) 96 | return None 97 | 98 | result = None 99 | for image in self.amiList: 100 | #print image.description 101 | if self.searchDescription(image, searchCriteria, l) is not None: 102 | #print vars(image) 103 | result = image 104 | 105 | if result is not None: 106 | self.logger.debug('ImageID: %s ImageDescription: %s', result.id, result.description) 107 | else: 108 | self.logger.debug('ImageID: none') 109 | 110 | return result 111 | 112 | def main(finder, **kwargs): 113 | 114 | region = kwargs['region'] 115 | if region is None: 116 | try: 117 | logger.warning('No region name given, trying to find one from EC2 instance meta data service') 118 | f = requests.get("http://169.254.169.254/latest/meta-data/placement/availability-zone/", timeout=1) 119 | region = f.text[:-1] 120 | logger.info('Using %s as region, provided by instance meta-data' % region) 121 | except requests.exceptions.Timeout: 122 | logger.error('Can not find region name (are you running this on EC2 ?). Abording.') 123 | sys.exit(-1) 124 | except: 125 | logger.error('Unknown error while trying to get region name. Abording.') 126 | sys.exit(-1) 127 | 128 | ami = finder.findWindowsAMIInRegion(region, kwargs['amiversion'], kwargs['locale']) 129 | if ami is not None: 130 | print ami.id 131 | sys.exit(0) 132 | else: 133 | sys.exit(-1) 134 | 135 | 136 | if __name__ == '__main__': 137 | 138 | logging.basicConfig() 139 | logger = logging.getLogger('findAMI') 140 | logger.setLevel(logging.INFO) 141 | finder = AMIFinder(logger) 142 | 143 | if sys.version_info < (2, 7): 144 | logger.info('Using Python < 2.7') 145 | parser = argparse.ArgumentParser(description='Find an Amazon Windows Base AMI') 146 | parser.add_argument('-v', '--version', action='version', version='%(prog)s v' + 147 | str(__version__)) 148 | else: 149 | parser = argparse.ArgumentParser(description='Find an Amazon Windows Base AMI', version='%(prog)s v' + 150 | str(__version__)) 151 | parser.add_argument('-r', '--region', type=str, help='Region name (default to local region when run on EC2)') 152 | parser.add_argument('-a', '--amiversion', type=str, help='String to search in version name (for example "2008" or ' 153 | '"2012 SP1")', required=True) 154 | parser.add_argument('-l', '--locale', type=str, default='en', help='Two letters locale name') 155 | args = parser.parse_args() 156 | main(finder, **vars(args)) 157 | -------------------------------------------------------------------------------- /course/system_operations/AMIFinder/sh/aws-cfn-resource-bridge-0.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-training-demo/3aa80fd3daf4b761b6425154f3124e167b4c8541/course/system_operations/AMIFinder/sh/aws-cfn-resource-bridge-0.1.tar.gz -------------------------------------------------------------------------------- /course/system_operations/AMIFinder/sh/cfn-resource-bridge.conf: -------------------------------------------------------------------------------- 1 | [amifinder] 2 | resource_type=Custom::AMIFinder 3 | queue_url={{queue_url}} 4 | timeout=60 5 | create_action=/home/ec2-user/create.sh 6 | delete_action=/home/ec2-user/delete.sh 7 | update_action=/home/ec2-user/update.sh -------------------------------------------------------------------------------- /course/system_operations/AMIFinder/sh/create.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | # 5 | # http://aws.amazon.com/apache2.0/ 6 | # 7 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | 9 | 10 | #!/bin/bash -x 11 | 12 | exec 3>&1 # "save" stdout to fd 3 13 | exec &>> /home/ec2-user/create.log 14 | 15 | function error_exit() { 16 | echo "{\"Reason\": \"$1\"}" >&3 3>&- # echo reason to stdout (instead of log) and then close fd 3 17 | exit $2 18 | } 19 | 20 | if [ -z "${Event_ResourceProperties_Version}" ] 21 | then 22 | error_exit "Version is required." 64 23 | fi 24 | 25 | 26 | AMI=$(/home/ec2-user/findAMI.py -a ${Event_ResourceProperties_Version}) 27 | AMI_ret=$? 28 | if [ $AMI_ret -ne 0 ] 29 | then 30 | error_exit "findAMI.py failed." $AMI_ret 31 | else 32 | echo "{ \"PhysicalResourceId\" : \"$AMI\" }" >&3 3>&- # echo success to stdout (instead of log) and then close fd 3 33 | exit 0 34 | fi 35 | -------------------------------------------------------------------------------- /course/system_operations/AMIFinder/sh/delete.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | # 5 | # http://aws.amazon.com/apache2.0/ 6 | # 7 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | 9 | #!/bin/bash -x 10 | 11 | exec 3>&1 # "save" stdout to fd 3 12 | exec &>> /home/ec2-user/delete.log 13 | 14 | function error_exit() { 15 | echo "{\"Reason\": \"$1\"}" >&3 3>&- # echo reason to stdout (instead of log) and then close fd 3 16 | exit $2 17 | } 18 | 19 | echo "{}" >&3 3>&- # echo reason to stdout (instead of log) and then close fd 3 20 | exit 0 21 | -------------------------------------------------------------------------------- /course/system_operations/AMIFinder/sh/update.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | # 5 | # http://aws.amazon.com/apache2.0/ 6 | # 7 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | 9 | #!/bin/bash -x 10 | 11 | exec 3>&1 # "save" stdout to fd 3 12 | exec &>> /home/ec2-user/delete.log 13 | 14 | function error_exit() { 15 | echo "{\"Reason\": \"$1\"}" >&3 3>&- # echo reason to stdout (instead of log) and then close fd 3 16 | exit $2 17 | } 18 | 19 | echo "{}" >&3 3>&- # echo reason to stdout (instead of log) and then close fd 3 20 | exit 0 21 | -------------------------------------------------------------------------------- /course/system_operations/AMIFinder/test/test_amifinder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | # 5 | # http://aws.amazon.com/apache2.0/ 6 | # 7 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | 9 | 10 | __author__ = 'stormacq' 11 | 12 | import unittest 13 | import logging 14 | 15 | from findAMI import AMIFinder 16 | 17 | 18 | class AMIFinderTest(unittest.TestCase): 19 | 20 | @classmethod 21 | def setUpClass(cls): 22 | logging.basicConfig() 23 | cls.logger = logging.getLogger(__name__) 24 | cls.logger.setLevel(logging.INFO) 25 | cls.finder = AMIFinder(cls.logger) 26 | 27 | def test_FindAMI_BasicVersion_eu_west_1(self): 28 | image = self.finder.findWindowsAMIInRegion('eu-west-1', '2012', 'en') 29 | self.assertIsNotNone(image, 'Did not find a Windows 2012 AMI in eu-west-1 for English locale') 30 | 31 | def test_FindAMI_ComplexVersion_eu_west_1(self): 32 | image = self.finder.findWindowsAMIInRegion('eu-west-1', '2008 R2 SP1', 'en') 33 | self.assertIsNotNone(image, 'Did not find a Windows 2008 AMI in eu-west-1 for English locale') 34 | 35 | def test_FindAMI_BasicVersion_us_east_1(self): 36 | image = self.finder.findWindowsAMIInRegion('us-east-1', '2012', 'en') 37 | self.assertIsNotNone(image, 'Did not find a Windows 2012 AMI in us-east-1 for English locale') 38 | 39 | def test_FindAMI_ComplexVersion__us_east_1(self): 40 | image = self.finder.findWindowsAMIInRegion('us-east-1', '2008 R2 SP1', 'en') 41 | self.assertIsNotNone(image, 'Did not find a Windows 2008 AMI in us-east-1 for English locale') 42 | 43 | def test_FindAMI_BasicVersion__DifferentLocale(self): 44 | image = self.finder.findWindowsAMIInRegion('us-east-1', '2012', 'fr') 45 | self.assertIsNotNone(image, 'Did not find a Windows 2012 AMI in us-east-1 for French locale') 46 | 47 | def test_FindAMI_invalid_locale(self): 48 | image = self.finder.findWindowsAMIInRegion('us-east-1', '2012', 'xx') 49 | self.assertIsNone(image, 'xx locale does not exist') 50 | 51 | def test_FindAMI_invalid_region(self): 52 | image = self.finder.findWindowsAMIInRegion('xx-east-1', '2012', 'en') 53 | self.assertIsNone(image, 'xx-east-1 region does not exist') 54 | 55 | if __name__ == '__main__': 56 | unittest.main() 57 | -------------------------------------------------------------------------------- /course/system_operations/SGFinder/README.md: -------------------------------------------------------------------------------- 1 | SGFinder 2 | ======== 3 | 4 | SGFinder will list (find) and optionally cleanup unused security groups in your account accross all regions. 5 | 6 | Usage 7 | ----- 8 | Review the following parameters: 9 | * ```IncludeDefault``` should the default security groups be included or not 10 | * ```DeleteEmpty``` should empty groups be removed 11 | * ```AccessKey``` & ```SecretKey``` configure your keys to enable access. A security [best practice](http://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html#use-roles-with-ec2) is to use roles instead, to use roles simple removed the ```aws_access_key_id``` and ```aws_secret_access_key``` parameters on line 45. 12 | 13 | Run ```findSG.py``` -------------------------------------------------------------------------------- /course/system_operations/SGFinder/findSG.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 10 | 11 | 12 | # findSG.py: Prints out all security groups across regions and the amount of instances using them 13 | # (and optionally removes them), including/excluding the "default" ones too 14 | 15 | 16 | # Config part 17 | ## IncludeDefault: Change this to include/exclude the "default" SG and set your access+secrets 18 | ## DeleteEmpty: Remove the empty SG or not? 19 | ## Access/Secret: Only there because of testing on my box and not an instance with a role, remove if you put 'em on an EC2 20 | IncludeDefault = False 21 | DeleteEmpty = True 22 | AccessKey = 'PLEASE_CHANGE' 23 | SecretKey = 'BOTH_OF_US' 24 | 25 | 26 | # You know... 27 | import boto 28 | ec2 = boto.connect_ec2() 29 | rgs = boto.ec2.regions() 30 | tcn = 0 31 | rcn = 0 32 | dcn = 0 33 | 34 | 35 | # Go over all regions 36 | for rg in rgs: 37 | 38 | # Skipping China & GovCloud 39 | if rg.name.startswith( 'cn-' ) or rg.name.startswith( "us-gov" ): 40 | continue 41 | 42 | 43 | # Header + connect + get the collection of SG 44 | print 'Region: %s\n--------------------' % (rg.name) 45 | sgs = boto.ec2.connect_to_region( rg.name, aws_access_key_id = AccessKey, aws_secret_access_key = SecretKey ).get_all_security_groups() 46 | 47 | 48 | # SG iterator 49 | for sg in sgs: 50 | 51 | # Did you ask for the default? :| 52 | if IncludeDefault == False and sg.name == 'default': 53 | continue 54 | 55 | # "Table" + print 56 | if rcn == 0: 57 | print '%-68s\t%-12s' % ("Security Group Name:", "Used by [instance count]:") 58 | print '%-68s\t%-12s' % (sg.name, len(sg.instances())), 59 | 60 | # Cleanup sir? 61 | if( len(sg.instances()) == 0 and DeleteEmpty ): 62 | try: 63 | sg.delete( dry_run = False ) 64 | break 65 | except boto.exception.EC2ResponseError as e: 66 | print "[shhh, it's a dry run]", 67 | 68 | print "\tRemoved", 69 | dcn += 1 70 | 71 | print '' 72 | rcn += 1 73 | 74 | 75 | # Region sum 76 | if rcn == 0: 77 | print '\nNo unused security groups were found', 78 | else: 79 | print '\n%d unused security groups were found' % (rcn), 80 | 81 | print 'in %s\n' % (rg.name) 82 | tcn += rcn 83 | rcn = 0 84 | 85 | 86 | # General sum 87 | if tcn == 0: 88 | print '\n\nSummary: Great job! no unused security groups were found!' 89 | else: 90 | print '\n\nSummary: %d unused security groups were found' % (tcn) 91 | 92 | # Removal sum 93 | if DeleteEmpty: 94 | print '\t %d security groups were removed\n' % (dcn) 95 | -------------------------------------------------------------------------------- /course/system_operations/chef/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | -------------------------------------------------------------------------------- /course/system_operations/chef/README.md: -------------------------------------------------------------------------------- 1 | Summary 2 | ======= 3 | Bootstrap your instance automatically with chef-solo, The purpose is to demonstrate to students how *relatively* easy it is to use automation tools such as chef-solo to automate deployments on AWS. 4 | By all means this mini framework does not "compete" with a fully blown chef-server / OpsWorks , Its sole purpose is to provide a simple yet powerful first step into automating instance deployments @AWS Using chef-solo 5 | 6 | 7 | Components 8 | ========== 9 | 10 | * Admin Instance - The instance that will orchestrate the instance bootstrapping 11 | * Bootstrapped EC2 Instance - The instance that will be launched as part of the process 12 | * Chef-solo - The chef-client that will be executed on every Bootstrapped EC2 Instance, Executing cookbooks 13 | * Artifcats Repo - S3 Bucket that will be created by Cloudformation, Staging the artifcats: Chef Cookbooks , Bootstrap Scripts... 14 | * Instance Role/Instance Profile - Allow the Bootstrapped EC2 instance to make Authenticated API calls to the S3 bucket 15 | * Cookbooks - Contains the Recipes that will be executed on the bootstrapped instance 16 | 17 | 18 | Programs 19 | ======== 20 | 21 | newInstance.sh - Once the admin instance is up, this script is will be used to bootstrap EC2 Instances with chef-solo 22 | 23 | bootstrap.sh - newInstance.sh will inject userdata to The bootstrapped Instance, the userdata will download&execute bootstrap.sh 24 | Which in turn will install chef-solo and execute the cookbooks... 25 | 26 | solocron.sh - A cron job script running on The bootstrapped instance (every 20 mins) , the script will track the changes to the chef-solo 27 | configurations stored on s3 28 | 29 | *Step 1 - Create the Demo Environments* 30 | --------------------------------------- 31 | 32 | For the lazy: 33 | 34 | - Download the chef-solo-admin-instance.json cloudformation template 35 | - Choose the region you wish to deploy 36 | - Use cloudformation to deploy 37 | 38 | The above CFN template will create an Ubuntu Admin Instance, and will stage all artifacts under /home/ubuntu, Just login 39 | And execute newInstance.sh to bootstrap the ec2 instance with chef-solo (see next step for more details) 40 | 41 | 42 | *Step 2 - Bootstrap the Instance* 43 | --------------------------------- 44 | 45 | ****Important: I assume that you launch the instance into a pre-existing VPC , & Into a valid existing PUBLIC SUBNET that automatically assigns public IP 46 | to the newly launched instance**** 47 | 48 | - Login to the admin instance 49 | - Change dir to /home/ubuntu 50 | - As the ubuntu user: execute the newInstance.sh , provide all the needed arguments to the script. 51 | Several Argument Values shall be taken from the cloudformation's stack output (see below for more details) 52 | 53 | Usage: 54 | 55 | usage: ./newInstance.sh -a AMI-ID -s securityGroupID -k KeyPairName -i InstanceSize -n VPCsubnetID -m IAM-Instance-Profile(Arn=value) -c ChefRole 56 | 57 | Currently ChefRole can be "base" or "web" 58 | 59 | Example: 60 | 61 | Start an EC2 Instance and have chef deploy it as an apache webserver: 62 | 63 | ./newInstance.sh -a ami-423c0a5f -s sg-xxxxxxxx -k xxxxxxxx -i m3.medium -n subnet-xxxxxxxx -m Arn=arn:aws:iam::xxxxxxxxxxxxxxxx:instance-profile/s3-chef-solo -c web 64 | 65 | 1. *securityGroupID*: The bootstrapped instance Security Group Id, See the cloudformation output for the value : ChefSoloInstanceSecurityGroupID 66 | 2. *KeyPairName*: Your keypair name, the public part will be pushed to the instance 67 | 3. *InstanceSize*: Any instance size 68 | 4. *VPCsubnetID*: The VPC Subnet Id to start the instance into, Subnet MUST BE PUBLIC and auto assign public IPs unless you plan to use VPN 69 | 5. *IAM-Instance-Profile*: The IAM Role that will be attached to The bootstrapped instance, See the cloudformation output for the value of: ChefSoloInstanceProfileArn 70 | 6. *AMI-ID*: Only ubuntu based AMI's are supported ! 71 | 7. *ChefRole*: Supported Roles are: "base" or "web" 72 | 73 | - Login to the instance after a few minutes the chef-solo web role should have installed apache2 automatically!, browse to the ec2 instance public ip 74 | 75 | Troubleshooting 76 | =============== 77 | 78 | The first step to troubleshoot is logs , I made sure that all the bootstrap process will be logged for any possible error: 79 | 80 | SSH login to the bootstrapped instance and examine the following logs in that order: 81 | 82 | - /var/log/cloud-init-output ---> Very important log , the user-data execute stdout will be logged here so if for some reason 83 | bootstrap.sh could not be downloaded from s3 the error will appear here 84 | - /var/log/bootstrap.log ---> This script will install the chef-client ,create folders , install AWS CLI... 85 | - /var/log/solorun.log ---> This logs means that chef-solo has executed , used to troubleshoot cookbook installation issues 86 | 87 | Tip: When deploying the CFN Template DISABLE ROLLBACK this will enable you to login to the admin instance and investigate if needed 88 | 89 | Author 90 | ====== 91 | 92 | Kobi Biton: kobibito@amazon.lu 93 | 94 | Do not hesitate to contact me for feedback / questions 95 | 96 | 97 | -------------------------------------------------------------------------------- /course/system_operations/chef/admin-instance-resources.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-training-demo/3aa80fd3daf4b761b6425154f3124e167b4c8541/course/system_operations/chef/admin-instance-resources.tar.gz -------------------------------------------------------------------------------- /course/system_operations/cloudtrail-logstash/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | -------------------------------------------------------------------------------- /course/system_operations/cloudtrail-logstash/README.md: -------------------------------------------------------------------------------- 1 | Cloudtrail Logstash cloudformation Stack coming soon... 2 | kobibito@amazon.lu 3 | -------------------------------------------------------------------------------- /course/system_operations/cloudtrail-logstash/bootstrap.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | # 5 | # http://aws.amazon.com/apache2.0/ 6 | # 7 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | 9 | 10 | #!/bin/bash 11 | ###################### 12 | #Since Cloufromation will generate some env variabiles dynamically lets source the file 13 | source /root/.bootstrap.cfg 14 | ############################################################## 15 | #Functions 16 | error_n_exit() 17 | { 18 | echo "`date`:ERROR $1 Exiting..." >> $LOG 19 | /bin/cat /tmp/stderr >> $LOG 20 | } 21 | ok_n_cont() 22 | { 23 | echo "`date`:OK $1" >> $LOG 24 | } 25 | create_folders() 26 | { 27 | mkdir -p /var/chef-solo/cache /var/chef-solo/cache/cookbooks /etc/chef /var/chef-solo/roles /var/chef-solo/check-sum 28 | if [ "$?" == "0" ];then 29 | return 0 30 | else 31 | return 1 32 | fi 33 | } 34 | #Create Folders 35 | create_folders 36 | if [ "$?" != "0" ];then 37 | error_n_exit "Failed to create folders" 38 | else 39 | ok_n_cont "Created Folders" 40 | fi 41 | #Get Chef-Solo Artifacts from Artifacts Bucket 42 | ## 43 | ###Obtain the chef solo artifacts and copy them locally to the node 44 | aws s3 --region $REGION cp s3://$ARTIFACTS/chef-solo-all.tar.gz "/" 2>>/tmp/s3out 45 | cd / 46 | tar xfz chef-solo-all.tar.gz || error_n_exit "could not extract chef-solo-all.tar.gz" 47 | #Lets call OpsCode Chef-solo master install script , this will install chef-solo (omnibus) 48 | chmod +x /root/install.sh || error_n_exit "Could not set +x to /root/install.sh" 49 | source /root/install.sh 2>&1 >> $SOLOLOG 50 | if [ "$?" == "0" ];then 51 | ok_n_cont "Looks like Chef-Solo Was Installed Successfully Check $SOLOLOG for more info" 52 | else 53 | error_n_exit "Chef-solo failed to install, check $SOLOLOG for more info" 54 | fi 55 | ###Manipulating the chef attributes (we assume that cfn-init has already executed and ended successfuly) 56 | tb=`cat /home/ubuntu/.trailbucket` 57 | bk=`cat /home/ubuntu/.backupbucket` 58 | ak=`cat /home/ubuntu/.ak` 59 | sk=`cat /home/ubuntu/.sk` 60 | aggrprefix=`cat /home/ubuntu/.aggrprefix` 61 | echo "override_attributes(" >> /var/chef-solo/roles/logstash.rb 62 | echo " 'aws' => {" >> /var/chef-solo/roles/logstash.rb 63 | echo " 'bucket' => '"$tb"'," >> /var/chef-solo/roles/logstash.rb 64 | echo " 'backupbucket' => '"$bk"'," >> /var/chef-solo/roles/logstash.rb 65 | echo " 'accesskey' => '"$ak"'," >> /var/chef-solo/roles/logstash.rb 66 | echo " 'secretkey' => '"$sk"'," >> /var/chef-solo/roles/logstash.rb 67 | echo " 'region' => '"$REGION"'," >> /var/chef-solo/roles/logstash.rb 68 | echo " 'prefix' => '"$aggrprefix"'" >> /var/chef-solo/roles/logstash.rb 69 | echo "}" >> /var/chef-solo/roles/logstash.rb 70 | echo ")" >> /var/chef-solo/roles/logstash.rb 71 | ###Now lets create the log mv cron 72 | echo "*/1 * * * * /root/s3mvlog.sh" >> /tmp/cronme 73 | chmod +x /root/s3mvlog.sh || error_n_exit "Could not set +x to /root/s3mvlog.sh" 74 | ### lets end by setting permissions 75 | chown root.root "/etc/chef" -R 76 | chown root.root "/var/chef-solo" -R 77 | ## Execute first chef-solo run .... 78 | /usr/bin/chef-solo -L /var/log/solorun.log 79 | ##Setup the cronjob 80 | echo "*/20 * * * * /usr/bin/chef-solo -L /var/log/solorun.log" >> /tmp/cronme 81 | #Enable the cron 82 | crontab /tmp/cronme 83 | # 84 | exit 0 85 | -------------------------------------------------------------------------------- /course/system_operations/cloudtrail-logstash/cf1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-training-demo/3aa80fd3daf4b761b6425154f3124e167b4c8541/course/system_operations/cloudtrail-logstash/cf1.png -------------------------------------------------------------------------------- /course/system_operations/cloudtrail-logstash/cf2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-training-demo/3aa80fd3daf4b761b6425154f3124e167b4c8541/course/system_operations/cloudtrail-logstash/cf2.png -------------------------------------------------------------------------------- /course/system_operations/cloudtrail-logstash/changelog: -------------------------------------------------------------------------------- 1 | 1/Nov - Release Version 1.0 2 | 26/Nov - Version 1.1 3 | - Added support for multi-region 4 | - Fixed bug in kibana deployment via chef 5 | - Added ubuntu AMI IDs for eu-central and the APAC regions 6 | -------------------------------------------------------------------------------- /course/system_operations/cloudwatch/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | -------------------------------------------------------------------------------- /course/system_operations/cloudwatch/README.md: -------------------------------------------------------------------------------- 1 | Simple Custom Cloudwatch Statistics Set Example 2 | =============================================== 3 | Very simple but useful! example to show students how east it to publish a custom metric from statistics set type 4 | more info here: http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/publishingMetrics.html#publishingDataPoints1 #Publish Statistic Sets 5 | 6 | Custom Metrics costs can be high , to try and save costs customer can sample few business data points and aggregate them on the client side 7 | Then Publish them as "Already Aggregated Data Points" for example Say you need to sample the performance of your webapp which consists of 8 | 3,4 Counters such as : Average Time to Load the Page , Average Database Access ... you could simply get those counters and perform 1st level 9 | Aggregation which consists on sum/average/min/max then publish it to cloudwatch as Single Metric per instance (Using instanceID as dimension 10 | 11 | That is about it see you next time Trainers! 12 | 13 | To test it just: 14 | 15 | - Spin up your favorite nix flavour (I Use ubuntu) 16 | - You will need the aws cli tools installed (for ubuntu you need to install it) 17 | - You will need an IAM Role and a policy to allow the ec2 instance to post the metrics , the below will do the trick 18 | ``` 19 | { 20 | "Version": "2012-10-17", 21 | "Statement": [ 22 | { 23 | "Sid": "Stmt1396006753000", 24 | "Effect": "Allow", 25 | "Action": [ 26 | "cloudwatch:GetMetricStatistics", 27 | "cloudwatch:ListMetrics", 28 | "cloudwatch:PutMetricAlarm", 29 | "cloudwatch:PutMetricData" 30 | ], 31 | "Resource": [ 32 | "*" 33 | ] 34 | } 35 | ] 36 | } 37 | ``` 38 | - Install the sysstat binary, apt-get install sysstat , yum install sysstat 39 | - create a crontab from the ubuntu/ec2-user and execute the script every 1/5 whatever minutes you wish 40 | 41 | Enjoy! 42 | 43 | -------------------------------------------------------------------------------- /course/system_operations/cloudwatch/stat-set-cw.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | # 5 | # http://aws.amazon.com/apache2.0/ 6 | # 7 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | 9 | #!/bin/bash 10 | #Get instanceID 11 | IP=$(curl http://169.254.169.254/latest/meta-data/instance-id -s) 12 | #Check if sysstat was install 13 | /usr/bin/which sar 14 | if [ "$?" != "0" ];then 15 | echo "Sar Could be found?" 16 | exit 3 17 | fi 18 | #The below sysstat is just a POC of gathering and "Agregating metrics" which can be sent to cloudwatch already Aggregated 19 | #Of course in real prod scenarion you would proably pull multiple counters here and then perfrom your own math... 20 | cswch=`sar 1 5 -w | grep Average | awk '{ print $3 }'` 21 | /usr/bin/aws cloudwatch put-metric-data --metric-name ContextSwitchesPerSecAvg --namespace "System/Linux" --statistic-value Sum=${cswch},Minimum=${cswch},Maximum=${cswch},SampleCount=5 --dimensions "InstanceId=${IP}" --region eu-west-1 22 | 23 | 24 | if [ "$?" != "0" ];then 25 | 26 | echo "Not Cool, AWS Command failed, Path?IAM Role? do not be lazy just run it interactivly and capture the error" 27 | #DEbug here 28 | echo $IP 29 | echo $cswch 30 | exit 3 31 | 32 | fi 33 | exit 0 34 | -------------------------------------------------------------------------------- /course/system_operations/sns/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | -------------------------------------------------------------------------------- /course/system_operations/sns/README.md: -------------------------------------------------------------------------------- 1 | Summary 2 | ======= 3 | Just a simple SNS demo , you will need: 4 | 5 | * AWS CLI Tools (Python Version) 6 | * Access Keys / Secret Keys that needs a minimum of : Create Topics, Send Message to Topics 7 | 8 | Might be used an a short interactive demo, Just to show how powerful and simple SNS can be used in certain automation processes! 9 | 10 | Comments are welcome ! 11 | -------------------------------------------------------------------------------- /course/system_operations/sns/snsDemo.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | # 5 | # http://aws.amazon.com/apache2.0/ 6 | # 7 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | 9 | #!/bin/bash 10 | ### Script tp demostrate sns push message! 11 | ##ver 0.1 Kobi Biton Initial 12 | ### 13 | # trap ctrl-c and call ctrl_c() 14 | trap ctrl_c INT 15 | 16 | function ctrl_c() { 17 | echo "** Trapped CTRL-C ** Or end of the script , Lets go and clean stuff..." 18 | aws sns delete-topic --topic-arn $arnTopic || error_exit "Could DELETE SNS Topic $arnTopic" 19 | ok_cont "Program has ended and cleaned up after it self bye bye!" 20 | exit 0 21 | } 22 | 23 | 24 | function error_exit() 25 | { 26 | 27 | echo "Fatal Error: $1 Will exit now" 28 | ctrl_c 29 | exit 1 30 | } 31 | 32 | function ok_cont() 33 | { 34 | echo "$1" 35 | 36 | } 37 | 38 | usage() 39 | { 40 | cat << EOF 41 | 42 | usage: $0 -t TopicName -e ValidEmailAddress -m "Message To Publish" 43 | 44 | This script will create an SNS demo! , it will require an AWS IAM user with creds/temp creds that is able to create/delete topics and send messages to topics as well! 45 | You can use an IAM user , or an EC2 trusted role for this purpose 46 | 47 | OPTIONS: 48 | -t SNS Topic Name 49 | -e Subscriber email address 50 | -m Message to publish 51 | 52 | Message-->SNS Topic-->Email Subscriber 53 | 54 | EOF 55 | } 56 | 57 | while getopts “t:e:m:q:?” OPTION 58 | do 59 | case $OPTION in 60 | t) 61 | TOPIC=$OPTARG 62 | ;; 63 | e) 64 | EMAIL=$OPTARG 65 | ;; 66 | m) 67 | MESSAGE=$OPTARG 68 | ;; 69 | ?) 70 | usage 71 | exit 1 72 | ;; 73 | esac 74 | done 75 | 76 | if [[ -z $TOPIC ]] || [[ -z $EMAIL ]] || [[ -z $MESSAGE ]] 77 | then 78 | usage 79 | exit 1 80 | fi 81 | function post_message_to_topic() 82 | { 83 | #App Logic..... 84 | #App Logic .... 85 | #Then Publish Results! 86 | aws sns publish --topic-arn $arnTopic --message "$MESSAGE" || error_exit "Fail to publish to SNS Topic $arnTopic" 87 | ok_cont "Published message to Topic Lets wait for the mail!" 88 | } 89 | #Lets start... 90 | arnTopic=$(aws sns create-topic --name $TOPIC --output text || error_exit "Failed to create an SNS Topic, Check Output") 91 | ok_cont "Created Topic $arnTopic..." 92 | # Let Add Subscribers! 93 | # 94 | aws sns subscribe --topic-arn $arnTopic --protocol email --notification-endpoint $EMAIL || error_exit "Failed to subscribe email to topic" 95 | ok_cont "Registered Email Subscriber!" 96 | while true ; do 97 | echo "Waiting for the email confirmation of the subscription please press any key when EMAIL endpoint has confirmed..." 98 | read -t 1 -n 1 && break 99 | done 100 | ok_cont "key pressed, endpoint has approved the subscription..." 101 | #####Now lets generate a push message example 102 | post_message_to_topic 103 | ###Cleanup is very importat ! 104 | ctrl_c 105 | exit 0 106 | -------------------------------------------------------------------------------- /course/system_operations/tags/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | -------------------------------------------------------------------------------- /course/system_operations/tags/README.md: -------------------------------------------------------------------------------- 1 | Summary 2 | ======= 3 | 4 | Just few scripts that will make you life easier ... 5 | 6 | Goal 7 | ==== 8 | 9 | AWS API related scripts, Can be used as ref point for students in Architecting or better Sysops class 10 | 11 | PreReq 12 | ====== 13 | 14 | * AWS Python CLI tools 15 | * Access / Secert Key with IAM Policy (See specific policy requirments per script) 16 | 17 | Usage 18 | ===== 19 | 20 | Just execute the script with no arguments to list the usage and options 21 | -------------------------------------------------------------------------------- /course/system_operations/tags/tagBased.md: -------------------------------------------------------------------------------- 1 | Summary 2 | ======= 3 | 4 | Simple program which loops the describe ec2 API seeking for key:value pair and act upon: stop/terminate in addition will send SNS Messages to pre-defined topic 5 | 6 | Name 7 | ==== 8 | tagBased.sh 9 | 10 | Goal 11 | ==== 12 | Stop or Terminate EC2 resources based on key:value Tag Pair , currently only support instance as resource type , future will also suppot EBS volumes 13 | 14 | PreReq 15 | ====== 16 | 17 | * AWS Python CLI tools 18 | * Access / Secert Key with IAM Policy that at mimimum needs permissions to: Describe/Stop/Terminate Instanceas/Tags and optionally send SNS message to topic 19 | 20 | Usage 21 | ===== 22 | $ usage: ./tagBased.sh -t ResourceType (instance / volume) -a APIToExecute (Stop/Terminate) -k (TagKeyName) -v (TagValue) -s [SNSTopicARN-NAME] 23 | $ This script will Filter Tags based on resource type and then will loop all tag key:value pairs , based on a condition will perform API Operations (Currently Supports Only Terminate and Run Instances API) 24 | $ OPTIONS: 25 | $ -t Mandatory: Resource Type, Currently only "instance" is supported! 26 | $ -a Mandatory: API To execute aginst the instance Stop/Terminate 27 | $ -k Mandatory: The Tag Key Name 28 | $ -v Mandatory: The Tag Value 29 | $ -s OPTIONAL: SNSARNTopicName 30 | $ describe-tags with Filter --> Sort --> Loop based on specific Tag --> Perform Action Based On Tag Name --> Send message to SNS Topic 31 | -------------------------------------------------------------------------------- /course/system_operations/tags/tagBased.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | # 5 | # http://aws.amazon.com/apache2.0/ 6 | # 7 | # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | 9 | #!/bin/bash 10 | ### Script tp demostrate how to perform actions based on specific Tags! 11 | ##ver 0.1 Kobi Biton Initial 12 | ### 13 | byTag="/tmp/describeAPIByTag.out" 14 | report="/tmp/instances.report.log" 15 | function cleanup() 16 | { 17 | 18 | rm -f $byTag 19 | rm -f $report 20 | } 21 | function error_exit() 22 | { 23 | 24 | echo "Fatal Error: $1 Will exit now" 25 | exit 1 26 | } 27 | function error_cont() 28 | { 29 | echo "Error: $1" 30 | } 31 | 32 | function ok_cont() 33 | { 34 | echo "$1" 35 | 36 | } 37 | function send_to_sns_topic() 38 | { 39 | if [ ! -z $SNSARN ];then 40 | body=`cat $report` 41 | aws sns publish --topic-arn $SNSARN --message "$body" || error_cont "Fail to publish to SNS Topic $SNSARN, Permissions? Topic Exists?" 42 | ok_cont "Published message to Topic" 43 | fi 44 | } 45 | 46 | 47 | usage() 48 | { 49 | cat << EOF 50 | 51 | usage: $0 -t ResourceType (instance / volume) -a APIToExecute (Stop/Terminate) -k (TagKeyName) -v (TagValue) -s [SNSTopicARN-NAME] 52 | 53 | This script will Filter Tags based on resource type and then will loop all tag key:value pairs , based on a condition will perform API Operations (Currently Supports Only Terminate and Run Instances API) 54 | 55 | OPTIONS: 56 | 57 | -t Mandatory: Resource Type, Currently only "instance" is supported! 58 | -a Mandatory: API To execute aginst the instance Stop/Terminate 59 | -k Mandatory: The Tag Key Name 60 | -v Mandatory: The Tag Value 61 | -s OPTIONAL: SNSARNTopicName 62 | 63 | 64 | describe-tags with Filter --> Sort --> Loop based on specific Tag --> Perform Action Based On Tag Name --> Send message to SNS Topic 65 | 66 | EOF 67 | } 68 | 69 | while getopts “t:a:k:v:s:?” OPTION 70 | do 71 | case $OPTION in 72 | t) 73 | RESOURCETYPE=$OPTARG 74 | ;; 75 | a) 76 | API=$OPTARG 77 | ;; 78 | k) 79 | KEY=$OPTARG 80 | ;; 81 | v) 82 | VALUE=$OPTARG 83 | ;; 84 | s) 85 | SNSARN=$OPTARG 86 | ;; 87 | ?) 88 | usage 89 | exit 1 90 | ;; 91 | esac 92 | done 93 | 94 | if [[ -z $RESOURCETYPE ]] || [[ -z $API ]] || [[ -z $KEY ]] || [[ -z $VALUE ]] 95 | then 96 | usage 97 | exit 1 98 | fi 99 | 100 | # Need to check if aws cli tools are installed! 101 | if [ ! -f /usr/local/bin/aws ];then 102 | error_exit "Could not locate AWS CLI tools" 103 | fi 104 | ### 105 | ###Lets get the list of resources based on type 106 | case $RESOURCETYPE in 107 | 108 | instance) 109 | 110 | #aws ec2 describe-tags --filters 'Name=resource-type,Values=instance' --query 'Tags[*].[ResourceId, Key, Value]' --output text | sort | while read id myKey myValue ;do echo $id $myKey $myValue;done 111 | aws ec2 describe-tags --filters 'Name=resource-type,Values=instance' --query 'Tags[*].[ResourceId, Key, Value]' --output text | sort > $byTag 112 | for i in "${PIPESTATUS[@]}" 113 | do 114 | if [ $i != 0 ];then 115 | error_exit "Could not describe tags check error output" 116 | fi 117 | done 118 | 119 | while read id myKey myValue 120 | 121 | do 122 | echo $id $myKey $myValue 123 | if [[ $myKey == $KEY ]] && [[ $myValue == $VALUE ]];then 124 | ok_cont "Instance ID: $id is True for Tag Pair $myKey:$myValue, Going to $API Instance ID: $id" 125 | if [[ $API == "Stop" ]] || [[ $API == "stop" ]];then 126 | #but need to check we do not stop a stopped one :-) 127 | status=$(aws ec2 describe-instance-status --instance-ids $id --query 'InstanceStatuses[*].InstanceState.Name' --output text --include-all-instances) 128 | if [ "$status" != "stopped" ];then 129 | ok_cont "Stopping Instance : $id" 130 | aws ec2 stop-instances --instance-ids $id || error_cont "Could not stop instance ID: $id" 131 | echo "`date`: Instance ID:$id Key: $myKey Value: $myValue Was Stopped" >> $report 132 | send_to_sns_topic 133 | else 134 | ok_cont "Instance ID: $id is already stopped doing nada.." 135 | fi 136 | elif [[ $API == "Terminate" ]] || [[ $API == "terminate" ]];then 137 | #Terminate Instance Code here... 138 | #but First we need to check we do not terminate a terminated one :-) 139 | status=$(aws ec2 describe-instance-status --instance-ids $id --query 'InstanceStatuses[*].InstanceState.Name' --output text --include-all-instances) 140 | if [ "$status" != "terminated" ];then 141 | ok_cont "Terminating Instance : $id" 142 | aws ec2 terminate-instances --instance-ids $id || error_cont "Could not terminate instance ID: $id" 143 | echo "`date`: Instance ID:$id Key: $myKey Value: $myValue Was Terminated!" >> $report 144 | send_to_sns_topic 145 | else 146 | ok_cont "Instance ID: $id is already terminated doing nada.." 147 | fi 148 | else 149 | error_exit "$API was unknown exiting" 150 | fi 151 | 152 | fi 153 | done < $byTag 154 | 155 | ;; 156 | 157 | volume) 158 | exit 1 159 | ;; 160 | 161 | *) error_exit "I do not know this Resource Type or I simply do not support it yet , bye!" 162 | 163 | ;; 164 | 165 | esac 166 | cleanup 167 | exit 0 168 | -------------------------------------------------------------------------------- /misc/aws-daleks/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | -------------------------------------------------------------------------------- /misc/aws-daleks/README.md: -------------------------------------------------------------------------------- 1 | # Exterminate your AWS Account! 2 | 3 | This will remove (almost) all resources in your AWS Accout. 4 | 5 | 1- Install SBT: 6 | ``` 7 | brew install sbt 8 | ``` 9 | 10 | 2- Set your credentials: 11 | ``` 12 | $ export AWS_ACCESS_KEY_ID=XXX 13 | $ export AWS_SECRET_ACCESS_KEY=XXX 14 | ``` 15 | see: http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/credentials.html 16 | 17 | 3- Exterminate: 18 | ``` 19 | sbt run 20 | ``` 21 | 22 | As an alternative, you can setup a profile in 23 | ``` 24 | ~/.aws/config 25 | ``` 26 | See: http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-multiple-profiles 27 | 28 | and use that profile from the command line 29 | ``` 30 | sbt "run -profile " 31 | ``` 32 | 33 | This project is cloned from https://github.com/jfaerman/aws-daleks 34 | -------------------------------------------------------------------------------- /misc/aws-daleks/build.sbt: -------------------------------------------------------------------------------- 1 | libraryDependencies += "com.amazonaws" % "aws-java-sdk" % "1.9.13" 2 | 3 | mainClass := Some("aws.daleks.eager.EagerAWSDaleks") 4 | -------------------------------------------------------------------------------- /misc/aws-daleks/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "2.4.0") 2 | -------------------------------------------------------------------------------- /misc/aws-daleks/src/main/scala/aws/daleks/eager/EagerAWSDaleks.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package aws.daleks.eager 12 | 13 | import com.amazonaws.regions.Regions 14 | import com.amazonaws.regions.Regions._ 15 | import com.amazonaws.auth.ClasspathPropertiesFileCredentialsProvider 16 | import com.amazonaws.regions.Region 17 | import com.amazonaws.auth.DefaultAWSCredentialsProviderChain 18 | import com.amazonaws.auth.profile.ProfileCredentialsProvider 19 | 20 | object EagerAWSDaleks extends App { 21 | println("EXTERMINATE!") 22 | 23 | def findArg(arg:String):Option[String] = { 24 | val i = args.indexOf(s"-$arg") 25 | if ( i >= 0) 26 | Option(args(i+1)) 27 | else None 28 | } 29 | 30 | val profile = findArg("profile") 31 | 32 | implicit val credentials = profile match { 33 | case Some(prf) => new ProfileCredentialsProvider(prf) 34 | case None => new DefaultAWSCredentialsProviderChain 35 | } 36 | 37 | val excludedRegions = List(GovCloud,CN_NORTH_1) 38 | val regions = Regions.values diff excludedRegions 39 | 40 | println(s"Exterminating regions [${regions.mkString(",")}]") 41 | 42 | val globals = List( 43 | new EagerRoute53Dalek(), 44 | new EagerIAMDalek()) 45 | 46 | val regionals = regions 47 | .map { Region.getRegion(_) } 48 | .par 49 | .foreach { implicit region => 50 | println("Preparing extermination of region ["+region+"]") 51 | List(new EagerS3Dalek, 52 | new EagerBeanstalkDalek, 53 | new EagerCloudFormationDalek, 54 | new EagerDynamoDBDalek, 55 | new EagerElastiCacheDalek, 56 | new EagerEMRDalek, 57 | new EagerRDSDalek, 58 | new EagerS3Dalek, 59 | new EagerSNSDalek, 60 | new EagerSQSDalek, 61 | new EagerEC2Dalek) foreach {_.exterminate} 62 | } 63 | 64 | globals foreach { _.exterminate } 65 | 66 | println("EXTERMINATE!") 67 | 68 | } 69 | -------------------------------------------------------------------------------- /misc/aws-daleks/src/main/scala/aws/daleks/eager/EagerBeanstalkDalek.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package aws.daleks.eager 12 | 13 | import com.amazonaws.auth.AWSCredentialsProvider 14 | import com.amazonaws.services.elasticbeanstalk.AWSElasticBeanstalkClient 15 | import com.amazonaws.regions.Region 16 | import scala.collection.JavaConverters._ 17 | import com.amazonaws.services.elasticbeanstalk.model.EnvironmentStatus 18 | import com.amazonaws.services.elasticbeanstalk.model.ApplicationDescription 19 | import com.amazonaws.services.elasticbeanstalk.model.DeleteApplicationRequest 20 | import com.amazonaws.services.elasticbeanstalk.model.EnvironmentDescription 21 | import com.amazonaws.services.elasticbeanstalk.model.TerminateEnvironmentRequest 22 | import aws.daleks.util.Humid 23 | 24 | class EagerBeanstalkDalek(implicit region: Region, credentials: AWSCredentialsProvider) extends Dalek { 25 | val beanstalk = withRegion(new AWSElasticBeanstalkClient(credentials), region) 26 | 27 | def exterminate = { 28 | val TERMINATED = EnvironmentStatus.Terminated.toString() 29 | val envs = beanstalk.describeEnvironments().getEnvironments().asScala filter { e => 30 | !TERMINATED.equalsIgnoreCase(e.getStatus()) 31 | } 32 | 33 | val apps = try { 34 | beanstalk.describeApplications.getApplications asScala 35 | } catch { 36 | case e: Exception => { 37 | println("Could not fectch beanstalk applications: " + e.getMessage()); 38 | List.empty 39 | } 40 | } 41 | 42 | envs foreach exterminateEnv 43 | 44 | apps foreach exterminateApp 45 | } 46 | 47 | def exterminateEnv(env: EnvironmentDescription) = 48 | try { 49 | val envName = env.getEnvironmentName() 50 | println(s"** Exterminating Beanstalk Environment ${envName} [${env.getStatus()} ] ") 51 | Humid { 52 | beanstalk.terminateEnvironment(new TerminateEnvironmentRequest() 53 | .withEnvironmentName(envName) 54 | .withTerminateResources(true)) 55 | } 56 | } catch { 57 | case e: Exception => println(s"! Failed to exterminate Beanstalk Environment ${env.getEnvironmentName()} [id: ${env.getEnvironmentId} ]: ${e.getMessage()}"); 58 | } 59 | 60 | def exterminateApp(app: ApplicationDescription) = 61 | try { 62 | println("** Exterminating Beanstalk Application " + app.getApplicationName()) 63 | Humid { 64 | beanstalk.deleteApplication(new DeleteApplicationRequest().withApplicationName(app.getApplicationName())) 65 | } 66 | } catch { 67 | case e: Exception => println(s"! Failed to exterminate Beanstalk Application ${app.getApplicationName()}: ${e.getMessage()}") 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /misc/aws-daleks/src/main/scala/aws/daleks/eager/EagerCloudFormationDalek.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package aws.daleks.eager 12 | 13 | import com.amazonaws.auth.AWSCredentialsProvider 14 | import com.amazonaws.regions.Region 15 | import scala.collection.JavaConverters._ 16 | import com.amazonaws.services.sqs.AmazonSQSClient 17 | import com.amazonaws.services.sqs.model.DeleteQueueRequest 18 | import com.amazonaws.services.cloudformation.AmazonCloudFormationClient 19 | import com.amazonaws.services.cloudformation.model.DeleteStackRequest 20 | import aws.daleks.util.Humid 21 | 22 | class EagerCloudFormationDalek(implicit region: Region, credentials: AWSCredentialsProvider) extends Dalek { 23 | val cloudformation = withRegion(new AmazonCloudFormationClient(credentials), region) 24 | 25 | def exterminate = { 26 | val stacks = cloudformation.describeStacks.getStacks asScala 27 | 28 | stacks foreach { stack => 29 | try { 30 | info(this,s"** Exterminating CloudFormation Stack " + stack.getStackName()) 31 | Humid { 32 | cloudformation.deleteStack(new DeleteStackRequest().withStackName(stack.getStackName())) 33 | } 34 | } catch { 35 | case e: Exception => println(s"! Failed to exterminate Beanstalk Application ${stack.getStackName}: ${e.getMessage()}") 36 | } 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /misc/aws-daleks/src/main/scala/aws/daleks/eager/EagerDalek.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package aws.daleks.eager 12 | 13 | import java.util.logging.Logger 14 | import aws.daleks.util.Humid 15 | 16 | trait Dalek { 17 | val logger = Logger.getLogger(classOf[Dalek].getName) 18 | def humidity = if (Humid.isDry) "[DRY]" else "[WET]" 19 | def exterminate 20 | def info(dalek:Dalek,msg:String) = logger.info(s"[$humidity] [$msg]") 21 | } 22 | -------------------------------------------------------------------------------- /misc/aws-daleks/src/main/scala/aws/daleks/eager/EagerDynamoDBDalek.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package aws.daleks.eager 12 | 13 | import com.amazonaws.auth.AWSCredentialsProvider 14 | import com.amazonaws.regions.Region 15 | import com.amazonaws.services.s3.AmazonS3Client 16 | import com.amazonaws.regions.ServiceAbbreviations 17 | import scala.collection.JavaConverters._ 18 | import com.amazonaws.services.s3.model.{ Region => S3Region } 19 | import com.amazonaws.services.s3.model.S3ObjectSummary 20 | import com.amazonaws.services.s3.model.Bucket 21 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient 22 | import aws.daleks.util.Humid 23 | 24 | class EagerDynamoDBDalek(implicit region: Region, credentials: AWSCredentialsProvider) extends Dalek { 25 | val dynamo = withRegion(new AmazonDynamoDBClient(credentials), region) 26 | 27 | def exterminate = { 28 | val tables: Seq[String] = dynamo.listTables.getTableNames asScala 29 | 30 | tables foreach { t => 31 | info(this,s"Exterminating DyanmoDB Table ${t}") 32 | Humid { dynamo.deleteTable(t) } 33 | } 34 | 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /misc/aws-daleks/src/main/scala/aws/daleks/eager/EagerEC2Dalek.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package aws.daleks.eager 12 | 13 | import com.amazonaws.auth.AWSCredentialsProvider 14 | import com.amazonaws.services.ec2.AmazonEC2Client 15 | import com.amazonaws.regions.Region 16 | import scala.collection.JavaConverters._ 17 | import com.amazonaws.services.ec2.model.StopInstancesRequest 18 | import com.amazonaws.services.ec2.model.TerminateInstancesRequest 19 | import com.amazonaws.services.ec2.model.Instance 20 | import com.amazonaws.services.elasticloadbalancing.AmazonElasticLoadBalancingClient 21 | import com.amazonaws.services.elasticloadbalancing.model.DeleteLoadBalancerRequest 22 | import com.amazonaws.services.ec2.model.DeleteVolumeRequest 23 | import com.amazonaws.services.ec2.model.DeleteSecurityGroupRequest 24 | import com.amazonaws.services.ec2.model.RevokeSecurityGroupIngressRequest 25 | import com.amazonaws.services.ec2.model.RevokeSecurityGroupEgressRequest 26 | import com.amazonaws.services.ec2.model.DeleteKeyPairRequest 27 | import com.amazonaws.services.ec2.model.InstanceState 28 | import com.amazonaws.services.ec2.model.DeregisterImageRequest 29 | import com.amazonaws.services.ec2.model.DescribeImagesRequest 30 | import aws.daleks.util.Humid 31 | 32 | class EagerEC2Dalek(implicit region: Region, credentials: AWSCredentialsProvider) extends Dalek { 33 | val ec2 = withRegion(new AmazonEC2Client(credentials), region) 34 | val elb = withRegion(new AmazonElasticLoadBalancingClient(credentials), region) 35 | 36 | def TerminateOrStop(i: Instance) = try { 37 | info(this, s"** Exterminating EC2 Instance ${i.getInstanceId} [${i.getState.getName}] on region ${region}") 38 | Humid { 39 | ec2.terminateInstances(new TerminateInstancesRequest().withInstanceIds(i.getInstanceId)) 40 | } 41 | } catch { 42 | case e: Exception => { 43 | info(this, "! Failed to terminate EC2 Instance" + i.getInstanceId()) 44 | if ("Running".equalsIgnoreCase(i.getState.getName())) { 45 | info(this, s"** Stopping EC2 Instance ${i.getInstanceId} [${i.getState.getName}] on region ${region}") 46 | ec2.stopInstances(new StopInstancesRequest().withInstanceIds(i.getInstanceId())) 47 | } 48 | } 49 | } 50 | 51 | def exterminateKeypairs = { 52 | val keypairs = ec2.describeKeyPairs().getKeyPairs().asScala 53 | keypairs foreach { k => 54 | try { 55 | info(this, "** Exterminating KeyPair " + k.getKeyName()) 56 | Humid { 57 | ec2.deleteKeyPair(new DeleteKeyPairRequest(k.getKeyName())) 58 | } 59 | } catch { 60 | case e: Exception => info(this, s"! Failed to exterminate KeyPair ${k.getKeyName()}: ${e.getMessage()}") 61 | } 62 | } 63 | } 64 | 65 | def exterminateInstances = { 66 | val reservations = ec2.describeInstances.getReservations asScala 67 | val instances = reservations 68 | .flatMap { r => r.getInstances asScala } 69 | .filter { i => !i.getState.getName().equalsIgnoreCase("terminated") } 70 | 71 | instances foreach TerminateOrStop 72 | } 73 | 74 | //TODO: Recurse dependencies 75 | def exterminateSecurityGroups = { 76 | val secGroups = ec2.describeSecurityGroups().getSecurityGroups().asScala.filter(_.getGroupName != "default") 77 | secGroups foreach { sg => 78 | try { 79 | 80 | val ingress = sg.getIpPermissions 81 | info(this, s"** Revoking [${ingress.size}] ingress rules for [${sg.getGroupId}]") 82 | Humid { 83 | ec2.revokeSecurityGroupIngress( 84 | new RevokeSecurityGroupIngressRequest() 85 | .withGroupId(sg.getGroupId()) 86 | .withIpPermissions(ingress)) 87 | } 88 | 89 | val egress = sg.getIpPermissionsEgress 90 | info(this, s"** Revoking [${egress.size}] egress rules for [${sg.getGroupId}]") 91 | Humid { 92 | ec2.revokeSecurityGroupEgress( 93 | new RevokeSecurityGroupEgressRequest() 94 | //.withGroupId(sg.getGroupId()) 95 | .withIpPermissions(egress)) 96 | } 97 | 98 | } catch { 99 | case e: Exception => info(this, s"! Failed to clean Security Group ${sg.getGroupId()}: ${e.getMessage()}") 100 | } 101 | } 102 | 103 | secGroups.foreach { sg => 104 | try { 105 | info(this, "** Exterminating Security Group " + sg.getGroupId()) 106 | Humid { ec2.deleteSecurityGroup(new DeleteSecurityGroupRequest().withGroupId(sg.getGroupId())) } 107 | } catch { 108 | case e: Exception => info(this, s"! Failed to exterminate Security Group ${sg.getGroupId()}: ${e.getMessage()}") 109 | } 110 | 111 | } 112 | } 113 | 114 | def exterminateVolumes = { 115 | val volumes = ec2.describeVolumes.getVolumes.asScala.filter { 116 | v => !"in-use".equals(v.getState) 117 | } 118 | volumes filter { "in-use" != _.getState } foreach { v => 119 | info(this, s"** Exterminating Volume ${v.getVolumeId}[${v.getState}]") 120 | Humid { ec2.deleteVolume(new DeleteVolumeRequest().withVolumeId(v.getVolumeId)) } 121 | } 122 | } 123 | 124 | def exterminateELBs = { 125 | val elbs = elb.describeLoadBalancers().getLoadBalancerDescriptions().asScala 126 | elbs foreach { lb => 127 | try { 128 | info(this, "** Exterminating Elastic Load Balancer " + lb.getLoadBalancerName()) 129 | Humid { elb.deleteLoadBalancer(new DeleteLoadBalancerRequest().withLoadBalancerName(lb.getLoadBalancerName())) } 130 | } catch { 131 | case e: Exception => info(this, s"! Failed to exterminate Load Balancer ${lb.getLoadBalancerName()}: ${e.getMessage()}") 132 | } 133 | } 134 | } 135 | 136 | def exterminateAMIs = { 137 | val amis = ec2.describeImages(new DescribeImagesRequest().withOwners("self")).getImages().asScala 138 | amis foreach { ami => 139 | try { 140 | info(this, s"** Exterminating Image [${ami.getImageId}] [${ami.getName}]") 141 | Humid { 142 | ec2.deregisterImage(new DeregisterImageRequest().withImageId(ami.getImageId())) 143 | } 144 | } catch { 145 | case e: Exception => info(this, s"! Failed to exterminate Image [${ami.getImageId()}]: ${e.getMessage()}") 146 | } 147 | 148 | } 149 | } 150 | 151 | def exterminate = { 152 | exterminateInstances 153 | exterminateKeypairs 154 | exterminateVolumes 155 | exterminateELBs 156 | exterminateSecurityGroups 157 | exterminateAMIs 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /misc/aws-daleks/src/main/scala/aws/daleks/eager/EagerEMRDalek.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package aws.daleks.eager 12 | 13 | import com.amazonaws.auth.AWSCredentialsProvider 14 | import com.amazonaws.regions.Region 15 | import scala.collection.JavaConverters._ 16 | import com.amazonaws.services.sqs.AmazonSQSClient 17 | import com.amazonaws.services.sqs.model.DeleteQueueRequest 18 | import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClient 19 | import com.amazonaws.services.elasticmapreduce.model.TerminateJobFlowsRequest 20 | import aws.daleks.util.Humid 21 | 22 | class EagerEMRDalek(implicit region: Region, credentials: AWSCredentialsProvider) extends Dalek { 23 | val emr = withRegion(new AmazonElasticMapReduceClient(credentials), region) 24 | 25 | def exterminate = { 26 | val clusters = emr.listClusters.getClusters.asScala 27 | 28 | clusters map { _.getId() } foreach { id => 29 | try { 30 | info(this,s"Exterminating Clusters $id") 31 | val req = new TerminateJobFlowsRequest 32 | req.setJobFlowIds(List(id).asJava) 33 | Humid { 34 | emr.terminateJobFlows(req) 35 | } 36 | } catch { 37 | case e: Exception => println(s"! Failed to exterminate Clusters ${id}: ${e.getMessage()}") 38 | } 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /misc/aws-daleks/src/main/scala/aws/daleks/eager/EagerElastiCacheDalek.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package aws.daleks.eager 12 | 13 | import com.amazonaws.auth.AWSCredentialsProvider 14 | import com.amazonaws.regions.Region 15 | import scala.collection.JavaConverters._ 16 | import com.amazonaws.services.sqs.AmazonSQSClient 17 | import com.amazonaws.services.sqs.model.DeleteQueueRequest 18 | import com.amazonaws.services.elasticache.AmazonElastiCacheClient 19 | import com.amazonaws.services.elasticache.model.DeleteCacheClusterRequest 20 | import aws.daleks.util.Humid 21 | 22 | class EagerElastiCacheDalek(implicit region: Region, credentials: AWSCredentialsProvider) extends Dalek { 23 | val ecache = withRegion(new AmazonElastiCacheClient(credentials), region) 24 | 25 | def exterminate = { 26 | val caches = ecache.describeCacheClusters.getCacheClusters asScala 27 | 28 | caches foreach { c => 29 | try { 30 | info(this,"Exterminating Cache Cluster " + c.getCacheClusterId) 31 | Humid{ 32 | ecache.deleteCacheCluster(new DeleteCacheClusterRequest().withCacheClusterId(c.getCacheClusterId())) 33 | } 34 | } catch { 35 | case e: Exception => println(s"! Failed to exterminate Cache Cluster ${c.getCacheClusterId()}: ${e.getMessage()}") 36 | } 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /misc/aws-daleks/src/main/scala/aws/daleks/eager/EagerRDSDalek.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package aws.daleks.eager 12 | 13 | import com.amazonaws.services.rds.AmazonRDSClient 14 | import com.amazonaws.auth.AWSCredentialsProvider 15 | import com.amazonaws.regions.Region 16 | import scala.collection.JavaConverters._ 17 | import com.amazonaws.services.rds.model.DeleteDBInstanceRequest 18 | import aws.daleks.util.Humid 19 | 20 | class EagerRDSDalek(implicit region: Region, credentials: AWSCredentialsProvider) extends Dalek { 21 | val rds = withRegion(new AmazonRDSClient(credentials), region) 22 | 23 | def exterminate = { 24 | val databases = rds.describeDBInstances.getDBInstances asScala 25 | 26 | databases foreach { db => 27 | println("** Exterminating RDS Database " + db.getDBInstanceIdentifier) 28 | val delReq = new DeleteDBInstanceRequest 29 | delReq.setDBInstanceIdentifier(db.getDBInstanceIdentifier()) 30 | delReq.setSkipFinalSnapshot(true); 31 | Humid { 32 | rds.deleteDBInstance(delReq) 33 | } 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /misc/aws-daleks/src/main/scala/aws/daleks/eager/EagerRoute53Dalek.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package aws.daleks.eager 12 | 13 | import com.amazonaws.auth.AWSCredentialsProvider 14 | import com.amazonaws.services.route53.AmazonRoute53Client 15 | import scala.collection.JavaConverters._ 16 | 17 | class EagerRoute53Dalek(implicit credentials: AWSCredentialsProvider) extends Dalek { 18 | val r53 = new AmazonRoute53Client(credentials) 19 | 20 | def zones = r53.listHostedZones.getHostedZones.asScala 21 | 22 | def exterminate = { 23 | println("Exterminating Hosted Zones") 24 | zones.foreach { z => 25 | try { 26 | println("** Exterminating HostedZone " + z.getName) 27 | // val records = r53.listResourceRecordSets(new ListResourceRecordSetsRequest().withHostedZoneId(z.getId())).getResourceRecordSets() asScala 28 | // records.foreach 29 | // TODO 30 | } catch { 31 | case e: Exception => println(s"! Failed to exterminate Zone ${z.getName()}: ${e.getMessage()}") 32 | } 33 | } 34 | 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /misc/aws-daleks/src/main/scala/aws/daleks/eager/EagerS3Dalek.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package aws.daleks.eager 12 | 13 | import com.amazonaws.auth.AWSCredentialsProvider 14 | import com.amazonaws.regions.Region 15 | import com.amazonaws.services.s3.AmazonS3Client 16 | import com.amazonaws.regions.ServiceAbbreviations 17 | import scala.collection.JavaConverters._ 18 | import com.amazonaws.services.s3.model.{ Region => S3Region } 19 | import com.amazonaws.services.s3.model.S3ObjectSummary 20 | import com.amazonaws.services.s3.model.Bucket 21 | import aws.daleks.util.Humid 22 | 23 | class EagerS3Dalek(implicit region: Region, credentials: AWSCredentialsProvider) 24 | extends Dalek { 25 | 26 | val s3 = { 27 | val s3 = new AmazonS3Client(credentials); 28 | val endpoint = region.getServiceEndpoint(ServiceAbbreviations.S3); 29 | s3.setEndpoint(endpoint); 30 | withRegion(s3, region) 31 | } 32 | 33 | def buckets = (s3.listBuckets.asScala).filter { bucket => 34 | val name = bucket.getName 35 | val keep = name.startsWith("logs") || name.startsWith("billing") || name.startsWith("share") 36 | !keep 37 | }.filter { bucket => 38 | val locStr = s3.getBucketLocation(bucket.getName) 39 | val bucketRegion = S3Region.fromValue(locStr).toAWSRegion() 40 | bucketRegion.equals(region) 41 | } 42 | 43 | def exterminate = buckets foreach { bucket => 44 | val bucketName = bucket.getName 45 | //TODO: Support > 1000 Objects 46 | val objects = s3.listObjects(bucketName).getObjectSummaries.asScala.par 47 | objects.foreach { o => 48 | println("** Exterminating S3 Object " + bucket.getName + "/" + o.getKey); 49 | Humid { 50 | s3.deleteObject(o.getBucketName, o.getKey) 51 | } 52 | } 53 | val versions = s3.listVersions(bucketName, "").getVersionSummaries().asScala.par 54 | versions.foreach { v => 55 | println("** Exterminating S3 Version " + bucket.getName + "/" + v.getKey() + " v" + v.getVersionId); 56 | Humid { 57 | s3.deleteVersion(bucketName, v.getKey, v.getVersionId) 58 | } 59 | } 60 | 61 | try { 62 | println("** Exterminating S3 Bucket Policy " + bucket.getName) 63 | Humid { s3.deleteBucketPolicy(bucket.getName()) } 64 | println("** Exterminating S3 Bucket " + bucket.getName) 65 | Humid { s3.deleteBucket(bucket.getName) } 66 | } catch { 67 | case e: Exception => println(s"! Failed to exterminate S3 Bucket ${bucket.getName}: ${e.getMessage()}") 68 | } 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /misc/aws-daleks/src/main/scala/aws/daleks/eager/EagerSNSDalek.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package aws.daleks.eager 12 | 13 | import com.amazonaws.auth.AWSCredentialsProvider 14 | import com.amazonaws.regions.Region 15 | import scala.collection.JavaConverters._ 16 | import com.amazonaws.services.sqs.AmazonSQSClient 17 | import com.amazonaws.services.sqs.model.DeleteQueueRequest 18 | import com.amazonaws.services.sns.AmazonSNSClient 19 | import scala.collection.JavaConverters._ 20 | import aws.daleks.util.Humid 21 | 22 | class EagerSNSDalek(implicit region: Region, credentials: AWSCredentialsProvider) extends Dalek { 23 | val sns = withRegion(new AmazonSNSClient(credentials),region) 24 | 25 | def exterminate = { 26 | val topics = sns.listTopics.getTopics asScala 27 | 28 | topics.foreach { t => 29 | println("** Exterminating SNS Topic " + t.getTopicArn()) 30 | Humid {sns.deleteTopic(t.getTopicArn())} 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /misc/aws-daleks/src/main/scala/aws/daleks/eager/EagerSQSDalek.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package aws.daleks.eager 12 | 13 | import com.amazonaws.auth.AWSCredentialsProvider 14 | import com.amazonaws.regions.Region 15 | import scala.collection.JavaConverters._ 16 | import com.amazonaws.services.sqs.AmazonSQSClient 17 | import com.amazonaws.services.sqs.model.DeleteQueueRequest 18 | import aws.daleks.util.Humid 19 | 20 | class EagerSQSDalek(implicit region: Region, credentials: AWSCredentialsProvider) extends Dalek { 21 | val sqs = withRegion(new AmazonSQSClient(credentials), region) 22 | 23 | def exterminate = { 24 | val queues = sqs.listQueues.getQueueUrls asScala 25 | 26 | queues foreach { q => 27 | println("Esterminating SQS Queue " + q) 28 | Humid { 29 | sqs.deleteQueue(new DeleteQueueRequest().withQueueUrl(q)) 30 | } 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /misc/aws-daleks/src/main/scala/aws/daleks/eager/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package aws.daleks 12 | 13 | import com.amazonaws.AmazonWebServiceClient 14 | import com.amazonaws.regions.Region 15 | package object eager { 16 | def withRegion[T <: AmazonWebServiceClient](client: T, region:Region): T = { 17 | client.setRegion(region) 18 | client 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /misc/aws-daleks/src/main/scala/aws/daleks/util/Humid.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package aws.daleks.util 12 | 13 | import java.util.concurrent.atomic.AtomicBoolean 14 | 15 | sealed trait Humid[+A] extends Traversable[A] { 16 | def flatMap[B](f: A => Humid[B]): Humid[B] 17 | def get:A 18 | def isEmpty:Boolean 19 | 20 | override def foreach[U](f: A => U): Unit = 21 | if (! isEmpty) f(get) 22 | } 23 | 24 | case class Wet[+A](a: A) extends Humid[A] { 25 | override def flatMap[B](f: A => Humid[B]): Humid[B] = f(a) 26 | override def get:A = a 27 | override def isEmpty:Boolean = false 28 | } 29 | 30 | case object Dry extends Humid[Nothing] { 31 | override def flatMap[B](f: Nothing => Humid[B]): Humid[B] = Dry 32 | override def isEmpty = true 33 | override def get = throw new NoSuchElementException("Dry.get") 34 | } 35 | 36 | object Humid { 37 | val globalDry = new AtomicBoolean(true) 38 | def isDry = globalDry.get 39 | def apply[A](x:A):Humid[A] = if (isDry) Dry else Wet(x) 40 | } 41 | 42 | object HumidTest extends App { 43 | Humid("Uala !!! 1").foreach(println(_)); 44 | Humid.globalDry.set(false) 45 | Humid("Uala !!! 2").foreach(println(_)); 46 | val t = Humid("Something").map {i => i.toUpperCase() + " uala" } 47 | t.foreach(println(_)) 48 | 49 | 50 | } 51 | -------------------------------------------------------------------------------- /misc/video/README.md: -------------------------------------------------------------------------------- 1 | # Media content to share during classes 2 | 3 | Showing brief videos might help students to maintain their focus, while bringing a touch of fun and humor. 4 | 5 | We typically show short videos either to illustrate our points when talking about a specific service or when students return from break, lab or lunch. 6 | 7 | ## Case Studies 8 | 9 | - Zones: [https://aws.amazon.com/solutions/case-studies/pinterest](https://aws.amazon.com/solutions/case-studies/pinterest) 10 | - Search: [https://aws.amazon.com/solutions/case-studies/smugmug-search/](https://aws.amazon.com/solutions/case-studies/smugmug-search/) 11 | - Search (Viddy): [http://www.youtube.com/watch?v=gud9Pvt2jJM](http://www.youtube.com/watch?v=gud9Pvt2jJM) 12 | - Adobe: [https://aws.amazon.com/solutions/case-studies/adobe/](https://aws.amazon.com/solutions/case-studies/adobe/) 13 | - Reddit: [https://aws.amazon.com/solutions/case-studies/reddit/](https://aws.amazon.com/solutions/case-studies/reddit/) 14 | - SAP: [https://aws.amazon.com/solutions/case-studies/sap/](https://aws.amazon.com/solutions/case-studies/sap/) 15 | - Hadoop (Mortar): [http://www.youtube.com/watch?v=nmbh8X9goTE&feature=youtu.be](http://www.youtube.com/watch?v=nmbh8X9goTE&feature=youtu.be) 16 | - NASA / JPL Mars Roover Landing: [http://www.youtube.com/watch?v=8FJ5DBLSFe4](http://www.youtube.com/watch?v=8FJ5DBLSFe4) (starts at 11:46) 17 | - SONOS Music System (about the cost to experiment and the cost of failure) : [https://www.youtube.com/watch?v=C9UVrbOMIZw](https://www.youtube.com/watch?v=C9UVrbOMIZw) 18 | 19 | 20 | When talking About Animoto during Essentials and Architect class 21 | 22 | - Animoto (Marketing): [https://www.youtube.com/embed/VwDS6MexKEo?rel=0;hd=1;autoplay=1](https://www.youtube.com/embed/VwDS6MexKEo?rel=0;hd=1;autoplay=1) 23 | - Animoto (Technical):[https://www.youtube.com/embed/VwDS6MexKEo?rel=0;hd=1;autoplay=1](https://www.youtube.com/embed/VwDS6MexKEo?rel=0;hd=1;autoplay=1) 24 | 25 | ## Re:Invent 26 | 27 | - 2014 : [Incident Response in the Cloud](https://www.youtube.com/watch?v=nzSrRvADh6g) 28 | - 2013 : [Building a DDos resilient architecture on AWS](https://www.youtube.com/watch?v=OT2y3DzMEmQ) 29 | 30 | 31 | ## Certification 32 | 33 | - How to study for AWS Architect certification : [http://www.youtube.com/watch?v=nt1-ZIX_s5U](http://www.youtube.com/watch?v=nt1-ZIX_s5U) 34 | 35 | ## Humor 36 | 37 | - Fun: [From the CIA to the Freezer : How big is Amazon ?](http://www.businessweek.com/videos/2013-06-27/from-the-cia-to-your-freezer-how-big-is-amazon) 38 | Beware - not exactly the "Amazon" tone 39 | 40 | ### Dilbert 41 | 42 | - Talking about [Glacier](http://dilbert.com/dyn/str_strip/000000000/00000000/0000000/100000/90000/0000/400/190493/190493.strip.gif) 43 | - Talking about [Ephemeral Storage](http://dilbert.com/strips/comic/2013-06-05/) 44 | - Talking about [Data Center Migration](http://dilbert.com/dyn/str_strip/000000000/00000000/0000000/100000/80000/9000/400/189416/189416.strip.gif) 45 | 46 | ### Misc. 47 | 48 | - The Arduino based [Management Console](https://github.com/jonathanhickford/aws-dash-gauge) 49 | 50 | ## Credits 51 | 52 | Credits to John Rotenstein (AWS Technical Trainer @ Melbourne) to have created this list of videos 53 | -------------------------------------------------------------------------------- /self-paced-lab/spl49/NOTICE.txt: -------------------------------------------------------------------------------- 1 | 2 | Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | 10 | ********************** 11 | THIRD PARTY COMPONENTS 12 | ********************** 13 | This software includes third party software subject to the following copyrights: 14 | 15 | - javax.ejb : COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 16 | - javax.faces : COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 17 | - jstl : COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 18 | 19 | The licenses for these third party components are included in their respective LICENSE.txt 20 | -------------------------------------------------------------------------------- /self-paced-lab/spl49/README.md: -------------------------------------------------------------------------------- 1 | JavaEEDemo 2 | ========== 3 | 4 | Simple Java EE Web application to deploy on any Java EE 7 compliant container. 5 | 6 | This Web App consists of 7 | 8 | - A stateless EJB (GreeterEJBBean.java) 9 | - A managed bean (Greeter.java) 10 | - A Java Server Page (index.xhtml) 11 | 12 | Java EE technologies used : 13 | 14 | - EJB (SLSB) 15 | - CDI 16 | - JSF 17 | - JSTL 18 | 19 | Non Java EE Technologies used : 20 | 21 | - Bootstrap CSS 22 | 23 | Tested on JBoss WildFly 8.1.0 24 | 25 | How to build ? 26 | -------------- 27 | 28 | Using Maven, type : 29 | 30 | ```mvn3 package``` 31 | 32 | to compile and generate a WAR file ready to deploy 33 | 34 | TODO 35 | ---- 36 | 37 | - Test on GlassFish 4.x 38 | - Test on Tomee 39 | -------------------------------------------------------------------------------- /self-paced-lab/spl49/lib/javax.ejb-api.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-training-demo/3aa80fd3daf4b761b6425154f3124e167b4c8541/self-paced-lab/spl49/lib/javax.ejb-api.jar -------------------------------------------------------------------------------- /self-paced-lab/spl49/lib/javax.faces-2.2.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-training-demo/3aa80fd3daf4b761b6425154f3124e167b4c8541/self-paced-lab/spl49/lib/javax.faces-2.2.1.jar -------------------------------------------------------------------------------- /self-paced-lab/spl49/lib/jstl-1.1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-training-demo/3aa80fd3daf4b761b6425154f3124e167b4c8541/self-paced-lab/spl49/lib/jstl-1.1.0.jar -------------------------------------------------------------------------------- /self-paced-lab/spl49/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.amazon.aws.training 8 | JavaEEDemo 9 | 1.0-SNAPSHOT 10 | 11 | AWS Hands on Lab : Java EE Demo 12 | Demo Application for AWS HoL -INSERT URL- 13 | 14 | war 15 | 16 | 17 | 18 | 20 | UTF-8 21 | 22 | 23 | 3.1 24 | 2.4 25 | 26 | 27 | 1.7 28 | 1.7 29 | 30 | 31 | 32 | 33 | javax 34 | javaee-api 35 | 7.0 36 | 37 | 38 | 39 | 40 | ${project.artifactId} 41 | 42 | 43 | maven-compiler-plugin 44 | ${version.compiler.plugin} 45 | 46 | ${maven.compiler.source} 47 | ${maven.compiler.target} 48 | 49 | 50 | 51 | maven-war-plugin 52 | ${version.war.plugin} 53 | 54 | 55 | false 56 | WEB-INF/lib/*.jar 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /self-paced-lab/spl49/src/main/java/com/amazon/aws/training/backend/GreeterEJBBean.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License.A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package com.amazon.aws.training.backend; 12 | 13 | import javax.ejb.Stateless; 14 | 15 | /** 16 | * Created by stormacq on 04/08/14. 17 | */ 18 | @Stateless(name = "GreeterEJBEJB") 19 | public class GreeterEJBBean { 20 | public GreeterEJBBean() { 21 | } 22 | public String sayHello(String name) { 23 | return "Well done " + name + " !"; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /self-paced-lab/spl49/src/main/java/com/amazon/aws/training/controler/Greeter.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License.A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | */ 10 | 11 | package com.amazon.aws.training.controler; 12 | 13 | import com.amazon.aws.training.backend.GreeterEJBBean; 14 | 15 | import javax.ejb.EJB; 16 | import javax.enterprise.context.SessionScoped; 17 | import javax.inject.Named; 18 | 19 | import java.io.Serializable; 20 | 21 | /** 22 | * A simple managed bean that is used to invoke the GreeterEJB and store the 23 | * response. The response is obtained by invoking getMessage(). 24 | * 25 | */ 26 | @Named("greeter") 27 | @SessionScoped 28 | public class Greeter implements Serializable { 29 | 30 | @EJB 31 | private GreeterEJBBean greeterEJB; 32 | 33 | private String message; 34 | 35 | public void setName(String name) { 36 | if (name.equalsIgnoreCase("")) 37 | message = ""; 38 | else 39 | message = greeterEJB.sayHello(name); 40 | } 41 | 42 | public String getMessage() { 43 | return message; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /self-paced-lab/spl49/src/main/webapp/WEB-INF/beans.xml: -------------------------------------------------------------------------------- 1 | 2 | 8 | -------------------------------------------------------------------------------- /self-paced-lab/spl49/src/main/webapp/WEB-INF/faces-config.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | -------------------------------------------------------------------------------- /self-paced-lab/spl49/src/main/webapp/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | Faces Servlet 8 | javax.faces.webapp.FacesServlet 9 | 1 10 | 11 | 12 | Faces Servlet 13 | *.xhtml 14 | 15 | 16 | 17 | index.xhtml 18 | 19 | -------------------------------------------------------------------------------- /self-paced-lab/spl49/src/main/webapp/css/justified-nav.css: -------------------------------------------------------------------------------- 1 | body { 2 | padding-top: 20px; 3 | } 4 | 5 | .footer { 6 | padding-top: 40px; 7 | padding-bottom: 40px; 8 | margin-top: 40px; 9 | border-top: 1px solid #eee; 10 | } 11 | 12 | /* Main marketing message and sign up button */ 13 | .jumbotron { 14 | text-align: center; 15 | background-color: transparent; 16 | } 17 | .jumbotron .btn { 18 | padding: 14px 24px; 19 | font-size: 21px; 20 | } 21 | 22 | /* Customize the nav-justified links to be fill the entire space of the .navbar */ 23 | 24 | .nav-justified { 25 | background-color: #eee; 26 | border: 1px solid #ccc; 27 | border-radius: 5px; 28 | } 29 | .nav-justified > li > a { 30 | padding-top: 15px; 31 | padding-bottom: 15px; 32 | margin-bottom: 0; 33 | font-weight: bold; 34 | color: #777; 35 | text-align: center; 36 | background-color: #e5e5e5; /* Old browsers */ 37 | background-image: -webkit-gradient(linear, left top, left bottom, from(#f5f5f5), to(#e5e5e5)); 38 | background-image: -webkit-linear-gradient(top, #f5f5f5 0%, #e5e5e5 100%); 39 | background-image: -o-linear-gradient(top, #f5f5f5 0%, #e5e5e5 100%); 40 | background-image: -webkit-gradient(linear, left top, left bottom, from(top), color-stop(0%, #f5f5f5), to(#e5e5e5)); 41 | background-image: linear-gradient(top, #f5f5f5 0%, #e5e5e5 100%); 42 | filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#f5f5f5', endColorstr='#e5e5e5',GradientType=0 ); /* IE6-9 */ 43 | background-repeat: repeat-x; /* Repeat the gradient */ 44 | border-bottom: 1px solid #d5d5d5; 45 | } 46 | .nav-justified > .active > a, 47 | .nav-justified > .active > a:hover, 48 | .nav-justified > .active > a:focus { 49 | background-color: #ddd; 50 | background-image: none; 51 | -webkit-box-shadow: inset 0 3px 7px rgba(0,0,0,.15); 52 | box-shadow: inset 0 3px 7px rgba(0,0,0,.15); 53 | } 54 | .nav-justified > li:first-child > a { 55 | border-radius: 5px 5px 0 0; 56 | } 57 | .nav-justified > li:last-child > a { 58 | border-bottom: 0; 59 | border-radius: 0 0 5px 5px; 60 | } 61 | 62 | @media (min-width: 768px) { 63 | .nav-justified { 64 | max-height: 52px; 65 | } 66 | .nav-justified > li > a { 67 | border-right: 1px solid #d5d5d5; 68 | border-left: 1px solid #fff; 69 | } 70 | .nav-justified > li:first-child > a { 71 | border-left: 0; 72 | border-radius: 5px 0 0 5px; 73 | } 74 | .nav-justified > li:last-child > a { 75 | border-right: 0; 76 | border-radius: 0 5px 5px 0; 77 | } 78 | } 79 | 80 | /* Responsive: Portrait tablets and up */ 81 | @media screen and (min-width: 768px) { 82 | /* Remove the padding we set earlier */ 83 | .masthead, 84 | .marketing, 85 | .footer { 86 | padding-right: 0; 87 | padding-left: 0; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /self-paced-lab/spl49/src/main/webapp/img/amazon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-training-demo/3aa80fd3daf4b761b6425154f3124e167b4c8541/self-paced-lab/spl49/src/main/webapp/img/amazon.png -------------------------------------------------------------------------------- /self-paced-lab/spl49/src/main/webapp/index.xhtml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | Hand on Lab : Deploy Java EE applications on AWS Elastic Beanstalk using Docker Containers 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 |