├── .DS_Store ├── .gitignore ├── .ipynb_checkpoints └── Section-13-Feature-Engineering-checkpoint.ipynb ├── README.md ├── Section-10-CodePipeline ├── .DS_Store ├── CODEBUILD.md ├── CODEDEPLOY.md ├── HELP.md ├── appspec.yml ├── buildspec.yml ├── index.html └── scripts │ ├── after_install.sh │ ├── install_dependencies.sh │ ├── start_server.sh │ ├── stop_server.sh │ └── validate_service.sh ├── Section-11-Docker ├── .DS_Store ├── CodeBuild-ECR-Hands-On │ ├── Dockerfile │ └── buildspec.yml ├── README.md ├── docker_cheatsheet.pdf ├── flask-app │ ├── .DS_Store │ ├── Dockerfile │ ├── app.py │ ├── requirements.txt │ └── templates │ │ └── index.html └── images │ ├── image1.gif │ ├── image2.gif │ ├── image3.gif │ ├── image4.gif │ └── image5.gif ├── Section-12-Intro-to-Sagemaker └── section-12-presentation.pdf ├── Section-13-Feature-Engineering ├── .DS_Store ├── .ipynb_checkpoints │ ├── 1.sagemaker_feature_engineering-checkpoint.ipynb │ ├── sagemaker-feature-engineering-feature-store-checkpoint.ipynb │ ├── sagemaker-feature-model-building-checkpoint.ipynb │ └── sagemaker_feature_engineering-checkpoint.ipynb ├── Dataset │ ├── bank-additional-full.csv │ ├── bank-additional-names.txt │ └── bank-additional.csv ├── Readme.md ├── custom-transformer.py ├── data-wrangler-insights-report.png ├── feature-engg-script.py ├── feature-engineering.flow ├── feature-store.json ├── image.png └── sagemaker-feature-model-building.ipynb ├── Section-13.1-Feature-Store ├── README.md ├── feature-engineering-fs.ipynb ├── feature-store-demo.ipynb ├── raw │ ├── customers.csv │ ├── customers_updated.csv │ └── orders.csv └── transformed │ ├── customers.csv │ ├── customers_updated.csv │ └── orders.csv ├── Section-14-Train-tune-deploy └── sagemaker-feature-model-building.ipynb ├── Section-15-Custom-models ├── .DS_Store ├── README.md ├── bring-custom-container.ipynb ├── bring-custom-script-pytorch.ipynb ├── custom-script-sagemaker-tensorflow.ipynb ├── iam-policy.json ├── mnist-pytorch.py ├── mnist-tf2.py ├── scikit-byoc.zip ├── scikit-byoc │ ├── .DS_Store │ ├── container │ │ ├── .DS_Store │ │ ├── Dockerfile │ │ ├── ReadMe.md │ │ ├── build_and_push.sh │ │ ├── decision_trees │ │ │ ├── nginx.conf │ │ │ ├── predictor.py │ │ │ ├── serve │ │ │ ├── train │ │ │ └── wsgi.py │ │ └── local_test │ │ │ ├── payload.csv │ │ │ ├── predict.sh │ │ │ ├── serve_local.sh │ │ │ ├── test_dir │ │ │ ├── input │ │ │ │ ├── config │ │ │ │ │ ├── hyperparameters.json │ │ │ │ │ └── resourceConfig.json │ │ │ │ └── data │ │ │ │ │ └── training │ │ │ │ │ └── iris.csv │ │ │ ├── model │ │ │ │ └── decision-tree-model.pkl │ │ │ └── output │ │ │ │ └── success │ │ │ └── train_local.sh │ ├── data │ │ └── iris.csv │ └── stack.png └── trust-policy.json ├── Section-15.1-CloudFormation ├── .DS_Store ├── 1.1.v2-ec2.yaml ├── 1.getting-started-ec2.yaml ├── 15.resource-dependencies-without.yaml ├── 16.resource-dependencies-with.yaml ├── 17.resource-dependencies-with-intrinsic-functions.yaml ├── 18.1.helper-scripts-failure.yaml ├── 18.helper-scripts.yaml ├── 2.s3-bucket-yaml.yaml ├── 20.custom-s3-lambda-backed.yaml ├── 21.dynamic-references.MD ├── 21.ec2-instance-parameter-store.yaml ├── 3.s3-bucket-json.json ├── 4.ec2-intrinsic.yaml ├── 5.ec2-intrinsic-v2.yaml ├── 6.s3-intrinsic-parameters.yaml ├── 7.resource-getatt-sub.yaml ├── 8.conditions-cf.yaml ├── 9.conditons-property.yaml ├── attributes-cf.pdf └── intrinsic-functions.yaml ├── Section-15.2-StepFunctions ├── .DS_Store ├── 1.AWS-Step │ ├── README.md │ ├── lambda.py │ └── step.json └── 2.AWS-Step-Function-Serverless │ ├── demo-test1.json │ ├── demo-test2.json │ ├── demo-test3.json │ └── step_function_definition.json ├── Section-15.3-Sagemaker-mlops └── cfnStudioDomain.yaml ├── Section-16-mlops-pipeline ├── .DS_Store ├── README.md ├── customer-retention-model.tar.gz ├── dataset │ ├── abalone.csv │ └── storedata_total.xlsx ├── evaluate-churn.py ├── image-1.png ├── image-2.png ├── image-3.png ├── image-4.png ├── image.png ├── mlops-tutorial.zip ├── mlops-tutorial │ ├── .DS_Store │ ├── etl │ │ ├── etljob.json │ │ └── preprocess.py │ ├── model │ │ ├── Dockerfile │ │ ├── app.py │ │ ├── assets │ │ │ ├── Dev │ │ │ │ ├── Dev-config.json │ │ │ │ └── deploy-model-Dev.yml │ │ │ └── Prd │ │ │ │ ├── Prd-config.json │ │ │ │ └── deploy-model-Prd.yml │ │ ├── build.py │ │ ├── buildspec.yml │ │ ├── model.py │ │ ├── nginx.conf │ │ ├── trainingjob.json │ │ └── wsgi.py │ ├── pipeline │ │ ├── EtlJobMonitor │ │ │ └── lambda.py │ │ ├── EtlLaunchJob │ │ │ └── lambda.py │ │ ├── ModelGroup │ │ │ └── lambda.py │ │ ├── TrainingJobMonitor │ │ │ └── lambda.py │ │ ├── TrainingLaunchJob │ │ │ └── lambda.py │ │ └── mlops-pipeline.yml │ ├── tests │ │ ├── system_test │ │ │ ├── assets │ │ │ │ ├── evaluateEndpoint │ │ │ │ │ └── lambda.py │ │ │ │ ├── registerModel │ │ │ │ │ └── lambda.py │ │ │ │ ├── requirements.txt │ │ │ │ └── workflow-resources.yml │ │ │ ├── build.py │ │ │ └── buildspec.yml │ │ └── unit_test │ │ │ ├── app_test.py │ │ │ └── input │ │ │ ├── config │ │ │ └── hyperparameters.json │ │ │ └── data │ │ │ └── training │ │ │ ├── train.csv │ │ │ └── validate.csv │ └── utils │ │ ├── c9_resize.sh │ │ ├── load_sim.py │ │ └── repository_validation.py ├── preprocess-churn.py ├── sagemaker-pipelines-inference-pipeline.ipynb ├── sagemaker-pipelines-train-pipeline.ipynb └── statistics.json ├── Section-4-Getting-Started-with-AWS ├── NLP.png └── README.md ├── Section-5-Linux-Basics └── README.md ├── Section-6-CodeCommit ├── .DS_Store ├── CI CD │ ├── .DS_Store │ ├── CODEDEPLOY.md │ ├── HELP.md │ ├── appspec.yml │ ├── buildspec.yml │ ├── code.py │ ├── index.html │ └── scripts │ │ ├── after_install.sh │ │ ├── install_dependencies.sh │ │ ├── start_server.sh │ │ ├── stop_server.sh │ │ └── validate_service.sh ├── README.md └── deny-master-policy.json ├── Section-6.1-Github ├── README.md ├── file1.py ├── git4mlops.png ├── image-1.png └── image.png ├── Section-7-YAML-tutorial └── yaml-basics.yml ├── Section-8-CodeBuild ├── .DS_Store ├── build-v1-no-artifacts │ ├── .DS_Store │ ├── CODEBUILD.md │ ├── CODEDEPLOY.md │ ├── HELP.md │ ├── appspec.yml │ ├── buildspec.yml │ ├── index.html │ └── scripts │ │ ├── after_install.sh │ │ ├── install_dependencies.sh │ │ ├── start_server.sh │ │ ├── stop_server.sh │ │ └── validate_service.sh ├── build-v2-env-variables │ ├── .DS_Store │ ├── CODEBUILD.md │ ├── CODEDEPLOY.md │ ├── HELP.md │ ├── appspec.yml │ ├── buildspec.yml │ ├── index.html │ └── scripts │ │ ├── after_install.sh │ │ ├── install_dependencies.sh │ │ ├── start_server.sh │ │ ├── stop_server.sh │ │ └── validate_service.sh └── build-v3-push-artifacts │ ├── .DS_Store │ ├── CODEBUILD.md │ ├── CODEDEPLOY.md │ ├── HELP.md │ ├── appspec.yml │ ├── buildspec.yml │ ├── index.html │ └── scripts │ ├── after_install.sh │ ├── install_dependencies.sh │ ├── start_server.sh │ ├── stop_server.sh │ └── validate_service.sh └── Section-9-CodeDeploy ├── .DS_Store ├── CODEDEPLOY.md ├── HELP.md ├── SLDC-Automation-AWS.pdf ├── appspec.yml ├── buildspec.yml ├── index.html └── scripts ├── after_install.sh ├── install_dependencies.sh ├── start_server.sh ├── stop_server.sh └── validate_service.sh /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/.DS_Store -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | venv_package/ 3 | .pytest_cache 4 | .ipynb_checkpoints 5 | .ipynb_checkpoints/* 6 | */.ipynb_checkpoints/* 7 | .DS_Store 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # This is a Repository for MLOps Course with AWS 2 | A Course by Manifold AI Learning to help the practitioners to Master MLOps tasks with AWS - this is hands-on course! 3 | 4 | Course link - 5 | 6 | https://www.manifoldailearning.in/courses/Master-Practical-MLOps-for-Data-Scientists--DevOps-on-AWS-65351f01e4b08600bc438698 7 | 8 | Be a part of Workshops conducted Live for free - 9 | https://www.manifoldailearning.in/courses/Webinar-Learning-Materials--Slides-64df2bc5e4b0267d331f9998 10 | 11 | 12 | Our other best courses: 13 | 14 | MLOps Bootcamp - https://www.manifoldailearning.in/courses/Complete-MLOps-BootCamp-654ddf11e4b004edc19e2649 15 | 16 | Python for MLOps DevOps AIOps - 17 | 18 | https://www.manifoldailearning.in/courses/Python-Programming-for-DevOps---MLOps-Engineers-6605b6973e488b5c28072d2e 19 | 20 | Created by the Top Instructor of Manifold AI Learning - Nachiketh Murthy (https://www.linkedin.com/in/nachiketh-murthy/) 21 | 22 | For support - support@manifoldailearning.in 23 | 24 | Thank you for all your support. Happy Learning!!! 25 | -------------------------------------------------------------------------------- /Section-10-CodePipeline/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-10-CodePipeline/.DS_Store -------------------------------------------------------------------------------- /Section-10-CodePipeline/CODEBUILD.md: -------------------------------------------------------------------------------- 1 | # Some System Settings to consider 2 | ## For CodeBuild Environment, Use the settings as below 3 | OS - Amazon Linux 2 4 | Runtime - Standard 5 | Image - aws/codebuild/amazonlinux2-aarch64-standard:2.0 6 | -------------------------------------------------------------------------------- /Section-10-CodePipeline/CODEDEPLOY.md: -------------------------------------------------------------------------------- 1 | 2 | # Installing the CodeDeploy agent on EC2 3 | ** Updated way to install with SSM ** 4 | ```YAML 5 | 6 | - Create EC2 Instance with IAM Role attached 7 | - Assign the Policy `AmazonS3ReadOnlyAccess` to allow CodeDeploy agent to read the version from S3 Bucket 8 | - Add User Data as mentioned below 9 | - Assign tags to EC2 Instances 10 | - Launch the instance and run the command - `sudo service codedeploy-agent status` to validate - CodeDeploy Agent is not running in EC2 instance 11 | - Create an Application in CodeDeploy 12 | - Push app revision to S3 Bucket (create a S3 bucket with versioning if its not created) - see section - **deploy the files into S3** below 13 | - Create a Service Role for CodeDeploy and assign Codedeploy policy 14 | - Create CodeDeployment Group and assign IAM role created above 15 | - Do necessary settings and create Code Deployment 16 | - Now validate in EC2 after few seconds to see whether codeDeploy agent has been installed `sudo service codedeploy-agent status` 17 | - Run the Deployment 18 | - Verify whether the website is working (Make sure to check the security group of ec2 instance) 19 | ``` 20 | # User data 21 | ``` 22 | #!bin/bash 23 | sudo yum update -y 24 | sudo yum install -y ruby wget 25 | wget https://aws-codedeploy-eu-west-1.s3.eu-west-1.amazonaws.com/latest/install 26 | chmod +x ./install 27 | sudo ./install auto 28 | sudo service codedeploy-agent status 29 | ``` 30 | -------------------------------------------------------------------------------- /Section-10-CodePipeline/HELP.md: -------------------------------------------------------------------------------- 1 | # Help for BuildSpec 2 | 3 | https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html#build-spec-ref-example 4 | 5 | # Appspec file 6 | 7 | https://docs.aws.amazon.com/codedeploy/latest/userguide/reference-appspec-file-example.html 8 | 9 | # Help for Metadata 10 | https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-categories.html 11 | 12 | # AWS CLI Profile 13 | 14 | **Create Profile** 15 | aws configure --profile profile_name 16 | *you need access key for cli* 17 | ~/.aws/credentials - contains stored creds 18 | ~/.aws/config - contains stored profile 19 | 20 | *If Getting Error as Unable to commit* 21 | - Goto Credential Manager in Windows & Remove the Existing credentials for codecommit 22 | - Extra information - https://stackoverflow.com/questions/34517534/running-git-clone-against-aws-codecommits-gets-me-a-403-error 23 | 24 | 25 | 26 | 27 | 28 | *Display Profile Names:* 29 | aws configure list-profiles 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /Section-10-CodePipeline/appspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.0 2 | os: linux 3 | files: 4 | - source: /index.html 5 | destination: /var/www/html/ 6 | hooks: 7 | ApplicationStop: 8 | - location: scripts/stop_server.sh 9 | timeout: 300 10 | runas: root 11 | 12 | BeforeInstall: 13 | - location: scripts/install_dependencies.sh 14 | timeout: 300 15 | runas: root 16 | 17 | AfterInstall: 18 | - location: scripts/after_install.sh 19 | timeout: 300 20 | runas: root 21 | 22 | ApplicationStart: 23 | - location: scripts/start_server.sh 24 | timeout: 300 25 | runas: root 26 | 27 | ValidateService: 28 | - location: scripts/validate_service.sh 29 | timeout: 300 30 | 31 | -------------------------------------------------------------------------------- /Section-10-CodePipeline/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | env: 4 | variables: 5 | MY_AWS_REGION: "ap-south-1" 6 | 7 | 8 | phases: 9 | install: 10 | runtime-versions: 11 | nodejs: 12 12 | commands: 13 | - echo "Running on $MY_AWS_REGION" 14 | - echo "Print Environment Variables" 15 | - printenv 16 | - echo "Start the installation process" 17 | - echo "Installation Process Completed" 18 | - echo "hey welcome to AWS $name" 19 | - echo "My code build id is $CODEBUILD_BUILD_ID" 20 | - echo "My Golang version is $GOLANG_12_VERSIONGOLANG_12_VERSION" 21 | pre_build: 22 | commands: 23 | - echo "This is the start of Pre-Build phase" 24 | - echo "Pre-Build phase is now completed" 25 | build: 26 | commands: 27 | - echo "we are currently on build phase" 28 | - echo "Build Activity is completed" 29 | - echo "Run Basic check whether Website is ready for use" 30 | - grep -Fq "AWS Certified DevOps Engineer" index.html 31 | - echo "Test was successful" 32 | - echo "proceed with next steps" 33 | post_build: 34 | commands: 35 | - echo "We are currently on post_build phase" 36 | - echo "post_build phase activity is completed. Moving to Artifacts Phase" 37 | 38 | artifacts: 39 | files: 40 | - '**/*' -------------------------------------------------------------------------------- /Section-10-CodePipeline/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Page Title 5 | 6 | 7 | 112 | 113 | 114 | 115 |
116 |

Manifold AI Learning

117 |

AWS Certified DevOps Engineer – Professional exam (DOP-C01)

118 |

Congraluations you are on Track

119 |
120 | 121 |
122 | Homepage 123 | Youtube 124 | Udemy 125 | Facebook 126 |
127 | 128 |
129 |
130 |

About Us

131 |

Manifold AI Learning ® is an online Academy with the goal to empower the students with the knowledge and skills that can be directly applied to solving the Real world problems in Data Science, Machine Learning and Artificial intelligence.

132 |

Our Courses

133 |

Join us on Youtube and start learning for free

134 |
Youtube Hands-On ML Series

135 |
Youtube Deep Learning Series

136 |
Deep Learning Tips and Tricks

137 |
Python for Data Science

138 |
Deep Learning with Pytorch

139 |
140 |
141 |

TITLE HEADING

142 |
Title description, Dec 7, 2017
143 |
Image
144 |

Some text..

145 |

Sunt in culpa qui officia deserunt mollit anim id est laborum consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco.

146 |
147 |

What will you get!!

148 |
Hands-On courses
149 |
Image
150 |

Some text..

151 |

Sunt in culpa qui officia deserunt mollit anim id est laborum consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco.

152 |
153 |
154 | 155 |
156 |

For more information, reach us at: support@manifoldailearning.in

157 |
158 | 159 | 160 | 161 | -------------------------------------------------------------------------------- /Section-10-CodePipeline/scripts/after_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | EC2_INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) 3 | EC2_INSTANCE_TYPE=$(curl -s http://169.254.169.254/latest/meta-data/instance-type) 4 | EC2_AZ=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) 5 | sed -i "s/you are on Track/This is currently runnining on $EC2_INSANCE_TYPE with id - $EC2_INSTANCE_ID on AZ - $EC2_AZ/g" /var/www/html/index.html 6 | chmod 664 /var/www/html/index.html -------------------------------------------------------------------------------- /Section-10-CodePipeline/scripts/install_dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | yum install -y httpd 3 | 4 | -------------------------------------------------------------------------------- /Section-10-CodePipeline/scripts/start_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | service httpd start 3 | 4 | -------------------------------------------------------------------------------- /Section-10-CodePipeline/scripts/stop_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | isExistApp = `pgrep httpd` 3 | if [[ -n $isExistApp ]]; then 4 | service httpd stop 5 | fi 6 | 7 | -------------------------------------------------------------------------------- /Section-10-CodePipeline/scripts/validate_service.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # verify we can access our webpage successfully 4 | curl -v --silent localhost:80 2>&1 | grep AWS -------------------------------------------------------------------------------- /Section-11-Docker/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-11-Docker/.DS_Store -------------------------------------------------------------------------------- /Section-11-Docker/CodeBuild-ECR-Hands-On/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.12-alpine AS build 2 | #Install git 3 | RUN apk add --no-cache git 4 | #Get the hello world package from a GitHub repository 5 | RUN go get github.com/golang/example/hello 6 | WORKDIR /go/src/github.com/golang/example/hello 7 | # Build the project and send the output to /bin/HelloWorld 8 | RUN go build -o /bin/HelloWorld 9 | 10 | FROM golang:1.12-alpine 11 | #Copy the build's output binary from the previous build container 12 | COPY --from=build /bin/HelloWorld /bin/HelloWorld 13 | ENTRYPOINT ["/bin/HelloWorld"] -------------------------------------------------------------------------------- /Section-11-Docker/CodeBuild-ECR-Hands-On/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | #Test 3 | 4 | 5 | phases: 6 | pre_build: 7 | commands: 8 | - echo Logging in to Amazon ECR... 9 | - aws ecr get-login-password --region ap-south-1 | docker login --username AWS --password-stdin 056247753720.dkr.ecr.ap-south-1.amazonaws.com 10 | build: 11 | commands: 12 | - echo Build started on `date` 13 | - echo Building the Docker image... 14 | - docker build -t ecr-demo . 15 | - docker tag ecr-demo:latest 056247753720.dkr.ecr.ap-south-1.amazonaws.com/ecr-demo:latest 16 | post_build: 17 | commands: 18 | - echo Build completed on `date` 19 | - echo Pushing the Docker image... 20 | - docker push 056247753720.dkr.ecr.ap-south-1.amazonaws.com/ecr-demo:latest -------------------------------------------------------------------------------- /Section-11-Docker/README.md: -------------------------------------------------------------------------------- 1 | # Docker Installation 2 | 3 | https://docs.docker.com/desktop/ 4 | 5 | # Commands on Hands On 6 | 7 | ``` 8 | docker --version 9 | docker run hello-world 10 | docker pull busybox 11 | docker images 12 | docker run busybox 13 | docker run busybox echo "hello from busybox mlops" 14 | docker ps 15 | docker ps -a 16 | docker run -it busybox sh 17 | docker rm 18 | ``` 19 | 20 | # 2. Hands On 21 | 22 | 23 | ``` 24 | docker run -d -P --name catgif manifoldailearning/catgif 25 | docker ps 26 | docker port catgif 27 | docker stop catgif 28 | docker run -p 8888:5000 manifoldailearning/catgif 29 | docker build -t catgifv2 . 30 | docker run -p 8888:5000 catgifv2 31 | docker login 32 | docker build -t yourusername/catgif . 33 | docker push yourusername/catgif 34 | ``` 35 | 36 | -------------------------------------------------------------------------------- /Section-11-Docker/docker_cheatsheet.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-11-Docker/docker_cheatsheet.pdf -------------------------------------------------------------------------------- /Section-11-Docker/flask-app/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-11-Docker/flask-app/.DS_Store -------------------------------------------------------------------------------- /Section-11-Docker/flask-app/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8 2 | 3 | # set a directory for the app 4 | WORKDIR /usr/src/app 5 | 6 | # copy all the files to the container 7 | COPY . . 8 | 9 | # install dependencies 10 | RUN pip install --no-cache-dir -r requirements.txt 11 | 12 | # tell the port number the container should expose 13 | EXPOSE 5000 14 | 15 | # run the command 16 | CMD ["python", "./app.py"] -------------------------------------------------------------------------------- /Section-11-Docker/flask-app/app.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, render_template 2 | import os 3 | import random 4 | 5 | app = Flask(__name__) 6 | 7 | # list of cat images 8 | images = [ 9 | "https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/main/Section-11-Docker/images/image1.gif", 10 | "https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/main/Section-11-Docker/images/image2.gif", 11 | "https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/main/Section-11-Docker/images/image3.gif", 12 | "https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/main/Section-11-Docker/images/image4.gif", 13 | "https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/main/Section-11-Docker/images/image5.gif" 14 | ] 15 | 16 | 17 | @app.route("/") 18 | def index(): 19 | url = random.choice(images) 20 | return render_template("index.html", url=url) 21 | 22 | 23 | if __name__ == "__main__": 24 | app.run(host="0.0.0.0", port=int(os.environ.get("PORT", 5000))) -------------------------------------------------------------------------------- /Section-11-Docker/flask-app/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==2.0.2 2 | Werkzeug==2.2.2 3 | -------------------------------------------------------------------------------- /Section-11-Docker/flask-app/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 19 | 20 | 21 |
22 |

Cat Gif of the day

23 | 24 |

Courtesy: Buzzfeed

25 |
26 | 27 | 28 | -------------------------------------------------------------------------------- /Section-11-Docker/images/image1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-11-Docker/images/image1.gif -------------------------------------------------------------------------------- /Section-11-Docker/images/image2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-11-Docker/images/image2.gif -------------------------------------------------------------------------------- /Section-11-Docker/images/image3.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-11-Docker/images/image3.gif -------------------------------------------------------------------------------- /Section-11-Docker/images/image4.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-11-Docker/images/image4.gif -------------------------------------------------------------------------------- /Section-11-Docker/images/image5.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-11-Docker/images/image5.gif -------------------------------------------------------------------------------- /Section-12-Intro-to-Sagemaker/section-12-presentation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-12-Intro-to-Sagemaker/section-12-presentation.pdf -------------------------------------------------------------------------------- /Section-13-Feature-Engineering/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-13-Feature-Engineering/.DS_Store -------------------------------------------------------------------------------- /Section-13-Feature-Engineering/Readme.md: -------------------------------------------------------------------------------- 1 | # Hands On 2 | 3 | ## Pre-requisite 4 | - Upload all the files in the dataset folder into the S3 bucket 5 | - Open the Data Wrangler and Follow the steps mentioned in the video 6 | 7 | ![Alt text](image.png) -------------------------------------------------------------------------------- /Section-13-Feature-Engineering/custom-transformer.py: -------------------------------------------------------------------------------- 1 | import time 2 | import pandas as pd 3 | from statsmodels.stats.outliers_influence import variance_inflation_factor 4 | from statsmodels.tools.tools import add_constant 5 | import numpy as np 6 | 7 | # Add two new indicators 8 | df["no_previous_contact"] = (df["pdays"] == 999).astype("int8") 9 | df["not_working"] = df["job"].isin(["student", "retired", "unemployed"]).astype("int8") 10 | df['pdays']=df['pdays'].astype(np.float64) #cast pdays column type to double precision 11 | 12 | # Add unique ID and event time for features store 13 | df['FS_ID'] = df.index + 1000 14 | current_time_sec = int(round(time.time())) 15 | df['FS_time'] = pd.Series([current_time_sec]*len(df), dtype="float64") 16 | 17 | # compute the vif and drop columns greater than 1.2 threshold 18 | def compute_vif(df, threshold): 19 | names=['age','euribor3m','campaign','not_working', 'no_previous_contact'] 20 | considered_features= [name for name in names if name in df.columns] 21 | subset=df[considered_features] 22 | subset=subset.dropna() 23 | subset['intercept'] = 1 24 | vif = pd.DataFrame() 25 | vif["Variable"] = subset.columns 26 | vif["VIF"] = [variance_inflation_factor(subset.values, i) for i in range(subset.shape[1])] 27 | vif = vif[vif['Variable']!='intercept'] 28 | vif=vif.sort_values('VIF', ascending=False) 29 | print(vif) 30 | drop_clm=vif.index[vif.VIF.gt(threshold)].tolist() 31 | drop_clm_names=vif.loc[drop_clm]['Variable'].tolist() 32 | df.drop(drop_clm_names, axis=1, inplace=True) 33 | return df 34 | df=compute_vif(df, 1.2) 35 | -------------------------------------------------------------------------------- /Section-13-Feature-Engineering/data-wrangler-insights-report.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-13-Feature-Engineering/data-wrangler-insights-report.png -------------------------------------------------------------------------------- /Section-13-Feature-Engineering/feature-engg-script.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import argparse 4 | import os 5 | from sklearn.preprocessing import OrdinalEncoder 6 | 7 | def _parse_args(): 8 | 9 | parser = argparse.ArgumentParser() 10 | 11 | # Data, model, and output directories 12 | # model_dir is always passed in from SageMaker. By default this is a S3 path under the default bucket. 13 | parser.add_argument('--filepath', type=str, default='/opt/ml/processing/input/') 14 | parser.add_argument('--filename', type=str, default='bank-additional-full.csv') 15 | parser.add_argument('--outputpath', type=str, default='/opt/ml/processing/output/') 16 | parser.add_argument('--categorical_features', type=str, default='y, job, marital, education, default, housing, loan, contact, month, day_of_week, poutcome') 17 | 18 | return parser.parse_known_args() 19 | 20 | if __name__=="__main__": 21 | args, _ = _parse_args() 22 | df = pd.read_csv(os.path.join(args.filepath, args.filename)) 23 | df = df.replace(regex=r'\.', value='_') 24 | df = df.replace(regex=r'\_$', value='') 25 | # Add two new indicators 26 | df["no_previous_contact"] = (df["pdays"] == 999).astype(int) 27 | df["not_working"] = df["job"].isin(["student", "retired", "unemployed"]).astype(int) 28 | df = df.drop(['duration', 'emp.var.rate', 'cons.price.idx', 'cons.conf.idx', 'euribor3m', 'nr.employed'], axis=1) 29 | # Encode the categorical features 30 | df = pd.get_dummies(df) 31 | # Train, test, validation split 32 | train_data, validation_data, test_data = np.split(df.sample(frac=1, random_state=42), [int(0.7 * len(df)), int(0.9 * len(df))]) # Randomly sort the data then split out first 70%, second 20%, and last 10% 33 | # Local store 34 | pd.concat([train_data['y_yes'], train_data.drop(['y_yes','y_no'], axis=1)], axis=1).to_csv(os.path.join(args.outputpath, 'train/train_script.csv'), index=False, header=False) 35 | pd.concat([validation_data['y_yes'], validation_data.drop(['y_yes','y_no'], axis=1)], axis=1).to_csv(os.path.join(args.outputpath, 'validation/validation_script.csv'), index=False, header=False) 36 | test_data['y_yes'].to_csv(os.path.join(args.outputpath, 'test/test_script_y.csv'), index=False, header=False) 37 | test_data.drop(['y_yes','y_no'], axis=1).to_csv(os.path.join(args.outputpath, 'test/test_script_x.csv'), index=False, header=False) 38 | print("## Processing completed. Exiting.") -------------------------------------------------------------------------------- /Section-13-Feature-Engineering/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-13-Feature-Engineering/image.png -------------------------------------------------------------------------------- /Section-13.1-Feature-Store/README.md: -------------------------------------------------------------------------------- 1 | ## Amazon SageMaker Feature Store Introduction 2 | 3 | 4 | In the Hands on journey of Amazon SageMaker Feature Store by demonstrating how to create and use feature groups. Feature groups are logical groupings of features used to store data for machine learning models. 5 | 6 | In the notebook, we create two feature groups: 7 | 8 | * A feature group for customer data 9 | * A feature group for order data 10 | 11 | We then ingest data from CSV files into the respective feature groups. 12 | 13 | -------------------------------------------------------------------------------- /Section-15-Custom-models/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-15-Custom-models/.DS_Store -------------------------------------------------------------------------------- /Section-15-Custom-models/README.md: -------------------------------------------------------------------------------- 1 | # Activity on Bring your Own Container 2 | 3 | - Edit and add the new trust policy for sagemaker execution role using contents of `trust-policy.json` 4 | - add the inline policy using the contents from `iam-policy.json` 5 | - Follow the instructions on the video -------------------------------------------------------------------------------- /Section-15-Custom-models/iam-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Action": [ 7 | "codebuild:DeleteProject", 8 | "codebuild:CreateProject", 9 | "codebuild:BatchGetBuilds", 10 | "codebuild:StartBuild" 11 | ], 12 | "Resource": "arn:aws:codebuild:*:*:project/sagemaker-studio*" 13 | }, 14 | { 15 | "Effect": "Allow", 16 | "Action": "logs:CreateLogStream", 17 | "Resource": "arn:aws:logs:*:*:log-group:/aws/codebuild/sagemaker-studio*" 18 | }, 19 | { 20 | "Effect": "Allow", 21 | "Action": [ 22 | "logs:GetLogEvents", 23 | "logs:PutLogEvents" 24 | ], 25 | "Resource": "arn:aws:logs:*:*:log-group:/aws/codebuild/sagemaker-studio*:log-stream:*" 26 | }, 27 | { 28 | "Effect": "Allow", 29 | "Action": "logs:CreateLogGroup", 30 | "Resource": "*" 31 | }, 32 | { 33 | "Effect": "Allow", 34 | "Action": [ 35 | "ecr:CreateRepository", 36 | "ecr:BatchGetImage", 37 | "ecr:CompleteLayerUpload", 38 | "ecr:DescribeImages", 39 | "ecr:DescribeRepositories", 40 | "ecr:UploadLayerPart", 41 | "ecr:ListImages", 42 | "ecr:InitiateLayerUpload", 43 | "ecr:BatchCheckLayerAvailability", 44 | "ecr:PutImage" 45 | ], 46 | "Resource": "arn:aws:ecr:*:*:repository/sagemaker-studio*" 47 | }, 48 | { 49 | "Effect": "Allow", 50 | "Action": "ecr:GetAuthorizationToken", 51 | "Resource": "*" 52 | }, 53 | { 54 | "Effect": "Allow", 55 | "Action": [ 56 | "s3:GetObject", 57 | "s3:DeleteObject", 58 | "s3:PutObject" 59 | ], 60 | "Resource": "arn:aws:s3:::sagemaker-*/*" 61 | }, 62 | { 63 | "Effect": "Allow", 64 | "Action": [ 65 | "s3:CreateBucket" 66 | ], 67 | "Resource": "arn:aws:s3:::sagemaker*" 68 | }, 69 | { 70 | "Effect": "Allow", 71 | "Action": [ 72 | "iam:GetRole", 73 | "iam:ListRoles" 74 | ], 75 | "Resource": "*" 76 | }, 77 | { 78 | "Effect": "Allow", 79 | "Action": "iam:PassRole", 80 | "Resource": "arn:aws:iam::*:role/*", 81 | "Condition": { 82 | "StringLikeIfExists": { 83 | "iam:PassedToService": "codebuild.amazonaws.com" 84 | } 85 | } 86 | } 87 | ] 88 | } -------------------------------------------------------------------------------- /Section-15-Custom-models/mnist-tf2.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import argparse 3 | import os 4 | import numpy as np 5 | import json 6 | 7 | 8 | def model(x_train, y_train, x_test, y_test): 9 | """Generate a simple model""" 10 | model = tf.keras.models.Sequential([ 11 | tf.keras.layers.Flatten(), 12 | tf.keras.layers.Dense(1024, activation=tf.nn.relu), 13 | tf.keras.layers.Dropout(0.4), 14 | tf.keras.layers.Dense(10, activation=tf.nn.softmax) 15 | ]) 16 | 17 | model.compile(optimizer='adam', 18 | loss='sparse_categorical_crossentropy', 19 | metrics=['accuracy']) 20 | model.fit(x_train, y_train) 21 | model.evaluate(x_test, y_test) 22 | 23 | return model 24 | 25 | 26 | def _load_training_data(base_dir): 27 | """Load MNIST training data""" 28 | x_train = np.load(os.path.join(base_dir, 'train_data.npy')) 29 | y_train = np.load(os.path.join(base_dir, 'train_labels.npy')) 30 | return x_train, y_train 31 | 32 | 33 | def _load_testing_data(base_dir): 34 | """Load MNIST testing data""" 35 | x_test = np.load(os.path.join(base_dir, 'eval_data.npy')) 36 | y_test = np.load(os.path.join(base_dir, 'eval_labels.npy')) 37 | return x_test, y_test 38 | 39 | 40 | def _parse_args(): 41 | parser = argparse.ArgumentParser() 42 | 43 | # Data, model, and output directories 44 | # model_dir is always passed in from SageMaker. By default this is a S3 path under the default bucket. 45 | parser.add_argument('--model_dir', type=str) 46 | parser.add_argument('--sm-model-dir', type=str, default=os.environ.get('SM_MODEL_DIR')) 47 | parser.add_argument('--train', type=str, default=os.environ.get('SM_CHANNEL_TRAINING')) 48 | parser.add_argument('--hosts', type=list, default=json.loads(os.environ.get('SM_HOSTS'))) 49 | parser.add_argument('--current-host', type=str, default=os.environ.get('SM_CURRENT_HOST')) 50 | 51 | return parser.parse_known_args() 52 | 53 | 54 | if __name__ == "__main__": 55 | args, unknown = _parse_args() 56 | 57 | train_data, train_labels = _load_training_data(args.train) 58 | eval_data, eval_labels = _load_testing_data(args.train) 59 | 60 | mnist_classifier = model(train_data, train_labels, eval_data, eval_labels) 61 | 62 | if args.current_host == args.hosts[0]: 63 | # save model to an S3 directory with version number '00000001' in Tensorflow SavedModel Format 64 | # To export the model as h5 format use model.save('my_model.h5') 65 | mnist_classifier.save(os.path.join(args.sm_model_dir, '000000001')) 66 | -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-15-Custom-models/scikit-byoc.zip -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-15-Custom-models/scikit-byoc/.DS_Store -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-15-Custom-models/scikit-byoc/container/.DS_Store -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/Dockerfile: -------------------------------------------------------------------------------- 1 | # Build an image that can do training and inference in SageMaker 2 | # This is a Python 3 image that uses the nginx, gunicorn, flask stack 3 | # for serving inferences in a stable way. 4 | 5 | FROM ubuntu:18.04 6 | 7 | MAINTAINER Amazon AI 8 | 9 | 10 | RUN apt-get -y update && apt-get install -y --no-install-recommends \ 11 | wget \ 12 | python3-pip \ 13 | python3-setuptools \ 14 | nginx \ 15 | ca-certificates \ 16 | && rm -rf /var/lib/apt/lists/* 17 | 18 | RUN ln -s /usr/bin/python3 /usr/bin/python 19 | RUN ln -s /usr/bin/pip3 /usr/bin/pip 20 | 21 | # Here we get all python packages. 22 | # There's substantial overlap between scipy and numpy that we eliminate by 23 | # linking them together. Likewise, pip leaves the install caches populated which uses 24 | # a significant amount of space. These optimizations save a fair amount of space in the 25 | # image, which reduces start up time. 26 | RUN pip --no-cache-dir install numpy==1.16.2 scipy==1.2.1 scikit-learn==0.20.2 pandas flask gunicorn 27 | 28 | # Set some environment variables. PYTHONUNBUFFERED keeps Python from buffering our standard 29 | # output stream, which means that logs can be delivered to the user quickly. PYTHONDONTWRITEBYTECODE 30 | # keeps Python from writing the .pyc files which are unnecessary in this case. We also update 31 | # PATH so that the train and serve programs are found when the container is invoked. 32 | 33 | ENV PYTHONUNBUFFERED=TRUE 34 | ENV PYTHONDONTWRITEBYTECODE=TRUE 35 | ENV PATH="/opt/program:${PATH}" 36 | 37 | # Set up the program in the image 38 | COPY decision_trees /opt/program 39 | WORKDIR /opt/program 40 | 41 | -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/build_and_push.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script shows how to build the Docker image and push it to ECR to be ready for use 4 | # by SageMaker. 5 | 6 | # The argument to this script is the image name. This will be used as the image on the local 7 | # machine and combined with the account and region to form the repository name for ECR. 8 | image=$1 9 | 10 | if [ "$image" == "" ] 11 | then 12 | echo "Usage: $0 " 13 | exit 1 14 | fi 15 | 16 | chmod +x decision_trees/train 17 | chmod +x decision_trees/serve 18 | 19 | # Get the account number associated with the current IAM credentials 20 | account=$(aws sts get-caller-identity --query Account --output text) 21 | 22 | if [ $? -ne 0 ] 23 | then 24 | exit 255 25 | fi 26 | 27 | 28 | # Get the region defined in the current configuration (default to us-west-2 if none defined) 29 | region=$(aws configure get region) 30 | region=${region:-us-east-1} 31 | 32 | 33 | fullname="${account}.dkr.ecr.${region}.amazonaws.com/${image}:latest" 34 | 35 | # If the repository doesn't exist in ECR, create it. 36 | 37 | aws ecr describe-repositories --repository-names "${image}" > /dev/null 2>&1 38 | 39 | if [ $? -ne 0 ] 40 | then 41 | aws ecr create-repository --repository-name "${image}" > /dev/null 42 | fi 43 | 44 | # Get the login command from ECR and execute it directly 45 | aws ecr get-login-password --region "${region}" | docker login --username AWS --password-stdin "${account}".dkr.ecr."${region}".amazonaws.com 46 | 47 | # Build the docker image locally with the image name and then push it to ECR 48 | # with the full name. 49 | 50 | docker build -t ${image} . 51 | docker tag ${image} ${fullname} 52 | 53 | docker push ${fullname} 54 | -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/decision_trees/nginx.conf: -------------------------------------------------------------------------------- 1 | worker_processes 1; 2 | daemon off; # Prevent forking 3 | 4 | 5 | pid /tmp/nginx.pid; 6 | error_log /var/log/nginx/error.log; 7 | 8 | events { 9 | # defaults 10 | } 11 | 12 | http { 13 | include /etc/nginx/mime.types; 14 | default_type application/octet-stream; 15 | access_log /var/log/nginx/access.log combined; 16 | 17 | upstream gunicorn { 18 | server unix:/tmp/gunicorn.sock; 19 | } 20 | 21 | server { 22 | listen 8080 deferred; 23 | client_max_body_size 5m; 24 | 25 | keepalive_timeout 5; 26 | proxy_read_timeout 1200s; 27 | 28 | location ~ ^/(ping|invocations) { 29 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 30 | proxy_set_header Host $http_host; 31 | proxy_redirect off; 32 | proxy_pass http://gunicorn; 33 | } 34 | 35 | location / { 36 | return 404 "{}"; 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/decision_trees/predictor.py: -------------------------------------------------------------------------------- 1 | # This is the file that implements a flask server to do inferences. It's the file that you will modify to 2 | # implement the scoring for your own algorithm. 3 | 4 | from __future__ import print_function 5 | 6 | import io 7 | import json 8 | import os 9 | import pickle 10 | import signal 11 | import sys 12 | import traceback 13 | 14 | import flask 15 | import pandas as pd 16 | 17 | prefix = "/opt/ml/" 18 | model_path = os.path.join(prefix, "model") 19 | 20 | # A singleton for holding the model. This simply loads the model and holds it. 21 | # It has a predict function that does a prediction based on the model and the input data. 22 | 23 | 24 | class ScoringService(object): 25 | model = None # Where we keep the model when it's loaded 26 | 27 | @classmethod 28 | def get_model(cls): 29 | """Get the model object for this instance, loading it if it's not already loaded.""" 30 | if cls.model == None: 31 | with open(os.path.join(model_path, "decision-tree-model.pkl"), "rb") as inp: 32 | cls.model = pickle.load(inp) 33 | return cls.model 34 | 35 | @classmethod 36 | def predict(cls, input): 37 | """For the input, do the predictions and return them. 38 | 39 | Args: 40 | input (a pandas dataframe): The data on which to do the predictions. There will be 41 | one prediction per row in the dataframe""" 42 | clf = cls.get_model() 43 | return clf.predict(input) 44 | 45 | 46 | # The flask app for serving predictions 47 | app = flask.Flask(__name__) 48 | 49 | 50 | @app.route("/ping", methods=["GET"]) 51 | def ping(): 52 | """Determine if the container is working and healthy. In this sample container, we declare 53 | it healthy if we can load the model successfully.""" 54 | health = ScoringService.get_model() is not None # You can insert a health check here 55 | 56 | status = 200 if health else 404 57 | return flask.Response(response="\n", status=status, mimetype="application/json") 58 | 59 | 60 | @app.route("/invocations", methods=["POST"]) 61 | def transformation(): 62 | """Do an inference on a single batch of data. In this sample server, we take data as CSV, convert 63 | it to a pandas data frame for internal use and then convert the predictions back to CSV (which really 64 | just means one prediction per line, since there's a single column. 65 | """ 66 | data = None 67 | 68 | # Convert from CSV to pandas 69 | if flask.request.content_type == "text/csv": 70 | data = flask.request.data.decode("utf-8") 71 | s = io.StringIO(data) 72 | data = pd.read_csv(s, header=None) 73 | else: 74 | return flask.Response( 75 | response="This predictor only supports CSV data", status=415, mimetype="text/plain" 76 | ) 77 | 78 | print("Invoked with {} records".format(data.shape[0])) 79 | 80 | # Do the prediction 81 | predictions = ScoringService.predict(data) 82 | 83 | # Convert from numpy back to CSV 84 | out = io.StringIO() 85 | pd.DataFrame({"results": predictions}).to_csv(out, header=False, index=False) 86 | result = out.getvalue() 87 | 88 | return flask.Response(response=result, status=200, mimetype="text/csv") 89 | -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/decision_trees/serve: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This file implements the scoring service shell. You don't necessarily need to modify it for various 4 | # algorithms. It starts nginx and gunicorn with the correct configurations and then simply waits until 5 | # gunicorn exits. 6 | # 7 | # The flask server is specified to be the app object in wsgi.py 8 | # 9 | # We set the following parameters: 10 | # 11 | # Parameter Environment Variable Default Value 12 | # --------- -------------------- ------------- 13 | # number of workers MODEL_SERVER_WORKERS the number of CPU cores 14 | # timeout MODEL_SERVER_TIMEOUT 60 seconds 15 | 16 | import multiprocessing 17 | import os 18 | import signal 19 | import subprocess 20 | import sys 21 | 22 | cpu_count = multiprocessing.cpu_count() 23 | 24 | model_server_timeout = os.environ.get('MODEL_SERVER_TIMEOUT', 60) 25 | model_server_workers = int(os.environ.get('MODEL_SERVER_WORKERS', cpu_count)) 26 | 27 | def sigterm_handler(nginx_pid, gunicorn_pid): 28 | try: 29 | os.kill(nginx_pid, signal.SIGQUIT) 30 | except OSError: 31 | pass 32 | try: 33 | os.kill(gunicorn_pid, signal.SIGTERM) 34 | except OSError: 35 | pass 36 | 37 | sys.exit(0) 38 | 39 | def start_server(): 40 | print('Starting the inference server with {} workers.'.format(model_server_workers)) 41 | 42 | 43 | # link the log streams to stdout/err so they will be logged to the container logs 44 | subprocess.check_call(['ln', '-sf', '/dev/stdout', '/var/log/nginx/access.log']) 45 | subprocess.check_call(['ln', '-sf', '/dev/stderr', '/var/log/nginx/error.log']) 46 | 47 | nginx = subprocess.Popen(['nginx', '-c', '/opt/program/nginx.conf']) 48 | gunicorn = subprocess.Popen(['gunicorn', 49 | '--timeout', str(model_server_timeout), 50 | '-k', 'sync', 51 | '-b', 'unix:/tmp/gunicorn.sock', 52 | '-w', str(model_server_workers), 53 | 'wsgi:app']) 54 | 55 | signal.signal(signal.SIGTERM, lambda a, b: sigterm_handler(nginx.pid, gunicorn.pid)) 56 | 57 | # If either subprocess exits, so do we. 58 | pids = set([nginx.pid, gunicorn.pid]) 59 | while True: 60 | pid, _ = os.wait() 61 | if pid in pids: 62 | break 63 | 64 | sigterm_handler(nginx.pid, gunicorn.pid) 65 | print('Inference server exiting') 66 | 67 | # The main routine just invokes the start function. 68 | 69 | if __name__ == '__main__': 70 | start_server() 71 | -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/decision_trees/train: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # A sample training component that trains a simple scikit-learn decision tree model. 4 | # This implementation works in File mode and makes no assumptions about the input file names. 5 | # Input is specified as CSV with a data point in each row and the labels in the first column. 6 | 7 | from __future__ import print_function 8 | 9 | import json 10 | import os 11 | import pickle 12 | import sys 13 | import traceback 14 | 15 | import pandas as pd 16 | from sklearn import tree 17 | 18 | # These are the paths to where SageMaker mounts interesting things in your container. 19 | 20 | prefix = '/opt/ml/' 21 | 22 | input_path = prefix + 'input/data' 23 | output_path = os.path.join(prefix, 'output') 24 | model_path = os.path.join(prefix, 'model') 25 | param_path = os.path.join(prefix, 'input/config/hyperparameters.json') 26 | 27 | # This algorithm has a single channel of input data called 'training'. Since we run in 28 | # File mode, the input files are copied to the directory specified here. 29 | channel_name='training' 30 | training_path = os.path.join(input_path, channel_name) 31 | 32 | # The function to execute the training. 33 | def train(): 34 | print('Starting the training.') 35 | try: 36 | # Read in any hyperparameters that the user passed with the training job 37 | with open(param_path, 'r') as tc: 38 | trainingParams = json.load(tc) 39 | 40 | # Take the set of files and read them all into a single pandas dataframe 41 | input_files = [ os.path.join(training_path, file) for file in os.listdir(training_path) ] 42 | if len(input_files) == 0: 43 | raise ValueError(('There are no files in {}.\n' + 44 | 'This usually indicates that the channel ({}) was incorrectly specified,\n' + 45 | 'the data specification in S3 was incorrectly specified or the role specified\n' + 46 | 'does not have permission to access the data.').format(training_path, channel_name)) 47 | raw_data = [ pd.read_csv(file, header=None) for file in input_files ] 48 | train_data = pd.concat(raw_data) 49 | 50 | # labels are in the first column 51 | train_y = train_data.iloc[:,0] 52 | train_X = train_data.iloc[:,1:] 53 | 54 | # Here we only support a single hyperparameter. Note that hyperparameters are always passed in as 55 | # strings, so we need to do any necessary conversions. 56 | max_leaf_nodes = trainingParams.get('max_leaf_nodes', None) 57 | if max_leaf_nodes is not None: 58 | max_leaf_nodes = int(max_leaf_nodes) 59 | 60 | # Now use scikit-learn's decision tree classifier to train the model. 61 | clf = tree.DecisionTreeClassifier(max_leaf_nodes=max_leaf_nodes) 62 | clf = clf.fit(train_X, train_y) 63 | 64 | # save the model 65 | with open(os.path.join(model_path, 'decision-tree-model.pkl'), 'wb') as out: 66 | pickle.dump(clf, out) 67 | print('Training complete.') 68 | except Exception as e: 69 | # Write out an error file. This will be returned as the failureReason in the 70 | # DescribeTrainingJob result. 71 | trc = traceback.format_exc() 72 | with open(os.path.join(output_path, 'failure'), 'w') as s: 73 | s.write('Exception during training: ' + str(e) + '\n' + trc) 74 | # Printing this causes the exception to be in the training job logs, as well. 75 | print('Exception during training: ' + str(e) + '\n' + trc, file=sys.stderr) 76 | # A non-zero exit code causes the training job to be marked as Failed. 77 | sys.exit(255) 78 | 79 | if __name__ == '__main__': 80 | train() 81 | 82 | # A zero exit code causes the job to be marked a Succeeded. 83 | sys.exit(0) 84 | -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/decision_trees/wsgi.py: -------------------------------------------------------------------------------- 1 | import predictor as myapp 2 | 3 | # This is just a simple wrapper for gunicorn to find your app. 4 | # If you want to change the algorithm file, simply change "predictor" above to the 5 | # new file. 6 | 7 | app = myapp.app 8 | -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/local_test/payload.csv: -------------------------------------------------------------------------------- 1 | 5.000000000000000000e+00,3.500000000000000000e+00,1.300000000000000044e+00,2.999999999999999889e-01 2 | 4.500000000000000000e+00,2.299999999999999822e+00,1.300000000000000044e+00,2.999999999999999889e-01 3 | 4.400000000000000355e+00,3.200000000000000178e+00,1.300000000000000044e+00,2.000000000000000111e-01 4 | 5.000000000000000000e+00,3.500000000000000000e+00,1.600000000000000089e+00,5.999999999999999778e-01 5 | 5.099999999999999645e+00,3.799999999999999822e+00,1.899999999999999911e+00,4.000000000000000222e-01 6 | 4.799999999999999822e+00,3.000000000000000000e+00,1.399999999999999911e+00,2.999999999999999889e-01 7 | 5.099999999999999645e+00,3.799999999999999822e+00,1.600000000000000089e+00,2.000000000000000111e-01 8 | 4.599999999999999645e+00,3.200000000000000178e+00,1.399999999999999911e+00,2.000000000000000111e-01 9 | 5.299999999999999822e+00,3.700000000000000178e+00,1.500000000000000000e+00,2.000000000000000111e-01 10 | 5.000000000000000000e+00,3.299999999999999822e+00,1.399999999999999911e+00,2.000000000000000111e-01 11 | 5.500000000000000000e+00,2.600000000000000089e+00,4.400000000000000355e+00,1.199999999999999956e+00 12 | 6.099999999999999645e+00,3.000000000000000000e+00,4.599999999999999645e+00,1.399999999999999911e+00 13 | 5.799999999999999822e+00,2.600000000000000089e+00,4.000000000000000000e+00,1.199999999999999956e+00 14 | 5.000000000000000000e+00,2.299999999999999822e+00,3.299999999999999822e+00,1.000000000000000000e+00 15 | 5.599999999999999645e+00,2.700000000000000178e+00,4.200000000000000178e+00,1.300000000000000044e+00 16 | 5.700000000000000178e+00,3.000000000000000000e+00,4.200000000000000178e+00,1.199999999999999956e+00 17 | 5.700000000000000178e+00,2.899999999999999911e+00,4.200000000000000178e+00,1.300000000000000044e+00 18 | 6.200000000000000178e+00,2.899999999999999911e+00,4.299999999999999822e+00,1.300000000000000044e+00 19 | 5.099999999999999645e+00,2.500000000000000000e+00,3.000000000000000000e+00,1.100000000000000089e+00 20 | 5.700000000000000178e+00,2.799999999999999822e+00,4.099999999999999645e+00,1.300000000000000044e+00 21 | 6.700000000000000178e+00,3.100000000000000089e+00,5.599999999999999645e+00,2.399999999999999911e+00 22 | 6.900000000000000355e+00,3.100000000000000089e+00,5.099999999999999645e+00,2.299999999999999822e+00 23 | 5.799999999999999822e+00,2.700000000000000178e+00,5.099999999999999645e+00,1.899999999999999911e+00 24 | 6.799999999999999822e+00,3.200000000000000178e+00,5.900000000000000355e+00,2.299999999999999822e+00 25 | 6.700000000000000178e+00,3.299999999999999822e+00,5.700000000000000178e+00,2.500000000000000000e+00 26 | 6.700000000000000178e+00,3.000000000000000000e+00,5.200000000000000178e+00,2.299999999999999822e+00 27 | 6.299999999999999822e+00,2.500000000000000000e+00,5.000000000000000000e+00,1.899999999999999911e+00 28 | 6.500000000000000000e+00,3.000000000000000000e+00,5.200000000000000178e+00,2.000000000000000000e+00 29 | 6.200000000000000178e+00,3.399999999999999911e+00,5.400000000000000355e+00,2.299999999999999822e+00 30 | -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/local_test/predict.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | payload=$1 4 | content=${2:-text/csv} 5 | 6 | curl --data-binary @${payload} -H "Content-Type: ${content}" -v http://localhost:8080/invocations 7 | -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/local_test/serve_local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | image=$1 4 | 5 | docker run -v $(pwd)/test_dir:/opt/ml -p 8080:8080 --rm ${image} serve 6 | -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/local_test/test_dir/input/config/hyperparameters.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/local_test/test_dir/input/config/resourceConfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-15-Custom-models/scikit-byoc/container/local_test/test_dir/input/config/resourceConfig.json -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/local_test/test_dir/input/data/training/iris.csv: -------------------------------------------------------------------------------- 1 | setosa,5.1,3.5,1.4,0.2 2 | setosa,4.9,3,1.4,0.2 3 | setosa,4.7,3.2,1.3,0.2 4 | setosa,4.6,3.1,1.5,0.2 5 | setosa,5,3.6,1.4,0.2 6 | setosa,5.4,3.9,1.7,0.4 7 | setosa,4.6,3.4,1.4,0.3 8 | setosa,5,3.4,1.5,0.2 9 | setosa,4.4,2.9,1.4,0.2 10 | setosa,4.9,3.1,1.5,0.1 11 | setosa,5.4,3.7,1.5,0.2 12 | setosa,4.8,3.4,1.6,0.2 13 | setosa,4.8,3,1.4,0.1 14 | setosa,4.3,3,1.1,0.1 15 | setosa,5.8,4,1.2,0.2 16 | setosa,5.7,4.4,1.5,0.4 17 | setosa,5.4,3.9,1.3,0.4 18 | setosa,5.1,3.5,1.4,0.3 19 | setosa,5.7,3.8,1.7,0.3 20 | setosa,5.1,3.8,1.5,0.3 21 | setosa,5.4,3.4,1.7,0.2 22 | setosa,5.1,3.7,1.5,0.4 23 | setosa,4.6,3.6,1,0.2 24 | setosa,5.1,3.3,1.7,0.5 25 | setosa,4.8,3.4,1.9,0.2 26 | setosa,5,3,1.6,0.2 27 | setosa,5,3.4,1.6,0.4 28 | setosa,5.2,3.5,1.5,0.2 29 | setosa,5.2,3.4,1.4,0.2 30 | setosa,4.7,3.2,1.6,0.2 31 | setosa,4.8,3.1,1.6,0.2 32 | setosa,5.4,3.4,1.5,0.4 33 | setosa,5.2,4.1,1.5,0.1 34 | setosa,5.5,4.2,1.4,0.2 35 | setosa,4.9,3.1,1.5,0.2 36 | setosa,5,3.2,1.2,0.2 37 | setosa,5.5,3.5,1.3,0.2 38 | setosa,4.9,3.6,1.4,0.1 39 | setosa,4.4,3,1.3,0.2 40 | setosa,5.1,3.4,1.5,0.2 41 | setosa,5,3.5,1.3,0.3 42 | setosa,4.5,2.3,1.3,0.3 43 | setosa,4.4,3.2,1.3,0.2 44 | setosa,5,3.5,1.6,0.6 45 | setosa,5.1,3.8,1.9,0.4 46 | setosa,4.8,3,1.4,0.3 47 | setosa,5.1,3.8,1.6,0.2 48 | setosa,4.6,3.2,1.4,0.2 49 | setosa,5.3,3.7,1.5,0.2 50 | setosa,5,3.3,1.4,0.2 51 | versicolor,7,3.2,4.7,1.4 52 | versicolor,6.4,3.2,4.5,1.5 53 | versicolor,6.9,3.1,4.9,1.5 54 | versicolor,5.5,2.3,4,1.3 55 | versicolor,6.5,2.8,4.6,1.5 56 | versicolor,5.7,2.8,4.5,1.3 57 | versicolor,6.3,3.3,4.7,1.6 58 | versicolor,4.9,2.4,3.3,1 59 | versicolor,6.6,2.9,4.6,1.3 60 | versicolor,5.2,2.7,3.9,1.4 61 | versicolor,5,2,3.5,1 62 | versicolor,5.9,3,4.2,1.5 63 | versicolor,6,2.2,4,1 64 | versicolor,6.1,2.9,4.7,1.4 65 | versicolor,5.6,2.9,3.6,1.3 66 | versicolor,6.7,3.1,4.4,1.4 67 | versicolor,5.6,3,4.5,1.5 68 | versicolor,5.8,2.7,4.1,1 69 | versicolor,6.2,2.2,4.5,1.5 70 | versicolor,5.6,2.5,3.9,1.1 71 | versicolor,5.9,3.2,4.8,1.8 72 | versicolor,6.1,2.8,4,1.3 73 | versicolor,6.3,2.5,4.9,1.5 74 | versicolor,6.1,2.8,4.7,1.2 75 | versicolor,6.4,2.9,4.3,1.3 76 | versicolor,6.6,3,4.4,1.4 77 | versicolor,6.8,2.8,4.8,1.4 78 | versicolor,6.7,3,5,1.7 79 | versicolor,6,2.9,4.5,1.5 80 | versicolor,5.7,2.6,3.5,1 81 | versicolor,5.5,2.4,3.8,1.1 82 | versicolor,5.5,2.4,3.7,1 83 | versicolor,5.8,2.7,3.9,1.2 84 | versicolor,6,2.7,5.1,1.6 85 | versicolor,5.4,3,4.5,1.5 86 | versicolor,6,3.4,4.5,1.6 87 | versicolor,6.7,3.1,4.7,1.5 88 | versicolor,6.3,2.3,4.4,1.3 89 | versicolor,5.6,3,4.1,1.3 90 | versicolor,5.5,2.5,4,1.3 91 | versicolor,5.5,2.6,4.4,1.2 92 | versicolor,6.1,3,4.6,1.4 93 | versicolor,5.8,2.6,4,1.2 94 | versicolor,5,2.3,3.3,1 95 | versicolor,5.6,2.7,4.2,1.3 96 | versicolor,5.7,3,4.2,1.2 97 | versicolor,5.7,2.9,4.2,1.3 98 | versicolor,6.2,2.9,4.3,1.3 99 | versicolor,5.1,2.5,3,1.1 100 | versicolor,5.7,2.8,4.1,1.3 101 | virginica,6.3,3.3,6,2.5 102 | virginica,5.8,2.7,5.1,1.9 103 | virginica,7.1,3,5.9,2.1 104 | virginica,6.3,2.9,5.6,1.8 105 | virginica,6.5,3,5.8,2.2 106 | virginica,7.6,3,6.6,2.1 107 | virginica,4.9,2.5,4.5,1.7 108 | virginica,7.3,2.9,6.3,1.8 109 | virginica,6.7,2.5,5.8,1.8 110 | virginica,7.2,3.6,6.1,2.5 111 | virginica,6.5,3.2,5.1,2 112 | virginica,6.4,2.7,5.3,1.9 113 | virginica,6.8,3,5.5,2.1 114 | virginica,5.7,2.5,5,2 115 | virginica,5.8,2.8,5.1,2.4 116 | virginica,6.4,3.2,5.3,2.3 117 | virginica,6.5,3,5.5,1.8 118 | virginica,7.7,3.8,6.7,2.2 119 | virginica,7.7,2.6,6.9,2.3 120 | virginica,6,2.2,5,1.5 121 | virginica,6.9,3.2,5.7,2.3 122 | virginica,5.6,2.8,4.9,2 123 | virginica,7.7,2.8,6.7,2 124 | virginica,6.3,2.7,4.9,1.8 125 | virginica,6.7,3.3,5.7,2.1 126 | virginica,7.2,3.2,6,1.8 127 | virginica,6.2,2.8,4.8,1.8 128 | virginica,6.1,3,4.9,1.8 129 | virginica,6.4,2.8,5.6,2.1 130 | virginica,7.2,3,5.8,1.6 131 | virginica,7.4,2.8,6.1,1.9 132 | virginica,7.9,3.8,6.4,2 133 | virginica,6.4,2.8,5.6,2.2 134 | virginica,6.3,2.8,5.1,1.5 135 | virginica,6.1,2.6,5.6,1.4 136 | virginica,7.7,3,6.1,2.3 137 | virginica,6.3,3.4,5.6,2.4 138 | virginica,6.4,3.1,5.5,1.8 139 | virginica,6,3,4.8,1.8 140 | virginica,6.9,3.1,5.4,2.1 141 | virginica,6.7,3.1,5.6,2.4 142 | virginica,6.9,3.1,5.1,2.3 143 | virginica,5.8,2.7,5.1,1.9 144 | virginica,6.8,3.2,5.9,2.3 145 | virginica,6.7,3.3,5.7,2.5 146 | virginica,6.7,3,5.2,2.3 147 | virginica,6.3,2.5,5,1.9 148 | virginica,6.5,3,5.2,2 149 | virginica,6.2,3.4,5.4,2.3 150 | virginica,5.9,3,5.1,1.8 151 | -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/local_test/test_dir/output/success: -------------------------------------------------------------------------------- 1 | Done -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/container/local_test/train_local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | image=$1 4 | 5 | mkdir -p test_dir/model 6 | mkdir -p test_dir/output 7 | 8 | rm test_dir/model/* 9 | rm test_dir/output/* 10 | 11 | docker run -v $(pwd)/test_dir:/opt/ml --rm ${image} train 12 | -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/data/iris.csv: -------------------------------------------------------------------------------- 1 | setosa,5.1,3.5,1.4,0.2 2 | setosa,4.9,3,1.4,0.2 3 | setosa,4.7,3.2,1.3,0.2 4 | setosa,4.6,3.1,1.5,0.2 5 | setosa,5,3.6,1.4,0.2 6 | setosa,5.4,3.9,1.7,0.4 7 | setosa,4.6,3.4,1.4,0.3 8 | setosa,5,3.4,1.5,0.2 9 | setosa,4.4,2.9,1.4,0.2 10 | setosa,4.9,3.1,1.5,0.1 11 | setosa,5.4,3.7,1.5,0.2 12 | setosa,4.8,3.4,1.6,0.2 13 | setosa,4.8,3,1.4,0.1 14 | setosa,4.3,3,1.1,0.1 15 | setosa,5.8,4,1.2,0.2 16 | setosa,5.7,4.4,1.5,0.4 17 | setosa,5.4,3.9,1.3,0.4 18 | setosa,5.1,3.5,1.4,0.3 19 | setosa,5.7,3.8,1.7,0.3 20 | setosa,5.1,3.8,1.5,0.3 21 | setosa,5.4,3.4,1.7,0.2 22 | setosa,5.1,3.7,1.5,0.4 23 | setosa,4.6,3.6,1,0.2 24 | setosa,5.1,3.3,1.7,0.5 25 | setosa,4.8,3.4,1.9,0.2 26 | setosa,5,3,1.6,0.2 27 | setosa,5,3.4,1.6,0.4 28 | setosa,5.2,3.5,1.5,0.2 29 | setosa,5.2,3.4,1.4,0.2 30 | setosa,4.7,3.2,1.6,0.2 31 | setosa,4.8,3.1,1.6,0.2 32 | setosa,5.4,3.4,1.5,0.4 33 | setosa,5.2,4.1,1.5,0.1 34 | setosa,5.5,4.2,1.4,0.2 35 | setosa,4.9,3.1,1.5,0.2 36 | setosa,5,3.2,1.2,0.2 37 | setosa,5.5,3.5,1.3,0.2 38 | setosa,4.9,3.6,1.4,0.1 39 | setosa,4.4,3,1.3,0.2 40 | setosa,5.1,3.4,1.5,0.2 41 | setosa,5,3.5,1.3,0.3 42 | setosa,4.5,2.3,1.3,0.3 43 | setosa,4.4,3.2,1.3,0.2 44 | setosa,5,3.5,1.6,0.6 45 | setosa,5.1,3.8,1.9,0.4 46 | setosa,4.8,3,1.4,0.3 47 | setosa,5.1,3.8,1.6,0.2 48 | setosa,4.6,3.2,1.4,0.2 49 | setosa,5.3,3.7,1.5,0.2 50 | setosa,5,3.3,1.4,0.2 51 | versicolor,7,3.2,4.7,1.4 52 | versicolor,6.4,3.2,4.5,1.5 53 | versicolor,6.9,3.1,4.9,1.5 54 | versicolor,5.5,2.3,4,1.3 55 | versicolor,6.5,2.8,4.6,1.5 56 | versicolor,5.7,2.8,4.5,1.3 57 | versicolor,6.3,3.3,4.7,1.6 58 | versicolor,4.9,2.4,3.3,1 59 | versicolor,6.6,2.9,4.6,1.3 60 | versicolor,5.2,2.7,3.9,1.4 61 | versicolor,5,2,3.5,1 62 | versicolor,5.9,3,4.2,1.5 63 | versicolor,6,2.2,4,1 64 | versicolor,6.1,2.9,4.7,1.4 65 | versicolor,5.6,2.9,3.6,1.3 66 | versicolor,6.7,3.1,4.4,1.4 67 | versicolor,5.6,3,4.5,1.5 68 | versicolor,5.8,2.7,4.1,1 69 | versicolor,6.2,2.2,4.5,1.5 70 | versicolor,5.6,2.5,3.9,1.1 71 | versicolor,5.9,3.2,4.8,1.8 72 | versicolor,6.1,2.8,4,1.3 73 | versicolor,6.3,2.5,4.9,1.5 74 | versicolor,6.1,2.8,4.7,1.2 75 | versicolor,6.4,2.9,4.3,1.3 76 | versicolor,6.6,3,4.4,1.4 77 | versicolor,6.8,2.8,4.8,1.4 78 | versicolor,6.7,3,5,1.7 79 | versicolor,6,2.9,4.5,1.5 80 | versicolor,5.7,2.6,3.5,1 81 | versicolor,5.5,2.4,3.8,1.1 82 | versicolor,5.5,2.4,3.7,1 83 | versicolor,5.8,2.7,3.9,1.2 84 | versicolor,6,2.7,5.1,1.6 85 | versicolor,5.4,3,4.5,1.5 86 | versicolor,6,3.4,4.5,1.6 87 | versicolor,6.7,3.1,4.7,1.5 88 | versicolor,6.3,2.3,4.4,1.3 89 | versicolor,5.6,3,4.1,1.3 90 | versicolor,5.5,2.5,4,1.3 91 | versicolor,5.5,2.6,4.4,1.2 92 | versicolor,6.1,3,4.6,1.4 93 | versicolor,5.8,2.6,4,1.2 94 | versicolor,5,2.3,3.3,1 95 | versicolor,5.6,2.7,4.2,1.3 96 | versicolor,5.7,3,4.2,1.2 97 | versicolor,5.7,2.9,4.2,1.3 98 | versicolor,6.2,2.9,4.3,1.3 99 | versicolor,5.1,2.5,3,1.1 100 | versicolor,5.7,2.8,4.1,1.3 101 | virginica,6.3,3.3,6,2.5 102 | virginica,5.8,2.7,5.1,1.9 103 | virginica,7.1,3,5.9,2.1 104 | virginica,6.3,2.9,5.6,1.8 105 | virginica,6.5,3,5.8,2.2 106 | virginica,7.6,3,6.6,2.1 107 | virginica,4.9,2.5,4.5,1.7 108 | virginica,7.3,2.9,6.3,1.8 109 | virginica,6.7,2.5,5.8,1.8 110 | virginica,7.2,3.6,6.1,2.5 111 | virginica,6.5,3.2,5.1,2 112 | virginica,6.4,2.7,5.3,1.9 113 | virginica,6.8,3,5.5,2.1 114 | virginica,5.7,2.5,5,2 115 | virginica,5.8,2.8,5.1,2.4 116 | virginica,6.4,3.2,5.3,2.3 117 | virginica,6.5,3,5.5,1.8 118 | virginica,7.7,3.8,6.7,2.2 119 | virginica,7.7,2.6,6.9,2.3 120 | virginica,6,2.2,5,1.5 121 | virginica,6.9,3.2,5.7,2.3 122 | virginica,5.6,2.8,4.9,2 123 | virginica,7.7,2.8,6.7,2 124 | virginica,6.3,2.7,4.9,1.8 125 | virginica,6.7,3.3,5.7,2.1 126 | virginica,7.2,3.2,6,1.8 127 | virginica,6.2,2.8,4.8,1.8 128 | virginica,6.1,3,4.9,1.8 129 | virginica,6.4,2.8,5.6,2.1 130 | virginica,7.2,3,5.8,1.6 131 | virginica,7.4,2.8,6.1,1.9 132 | virginica,7.9,3.8,6.4,2 133 | virginica,6.4,2.8,5.6,2.2 134 | virginica,6.3,2.8,5.1,1.5 135 | virginica,6.1,2.6,5.6,1.4 136 | virginica,7.7,3,6.1,2.3 137 | virginica,6.3,3.4,5.6,2.4 138 | virginica,6.4,3.1,5.5,1.8 139 | virginica,6,3,4.8,1.8 140 | virginica,6.9,3.1,5.4,2.1 141 | virginica,6.7,3.1,5.6,2.4 142 | virginica,6.9,3.1,5.1,2.3 143 | virginica,5.8,2.7,5.1,1.9 144 | virginica,6.8,3.2,5.9,2.3 145 | virginica,6.7,3.3,5.7,2.5 146 | virginica,6.7,3,5.2,2.3 147 | virginica,6.3,2.5,5,1.9 148 | virginica,6.5,3,5.2,2 149 | virginica,6.2,3.4,5.4,2.3 150 | virginica,5.9,3,5.1,1.8 151 | -------------------------------------------------------------------------------- /Section-15-Custom-models/scikit-byoc/stack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-15-Custom-models/scikit-byoc/stack.png -------------------------------------------------------------------------------- /Section-15-Custom-models/trust-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Principal": { 7 | "Service": "sagemaker.amazonaws.com" 8 | }, 9 | "Action": "sts:AssumeRole" 10 | }, 11 | { 12 | "Effect": "Allow", 13 | "Principal": { 14 | "Service": "codebuild.amazonaws.com" 15 | }, 16 | "Action": "sts:AssumeRole" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-15.1-CloudFormation/.DS_Store -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/1.1.v2-ec2.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Description: 'AWS DevOps Manifold AI Learning' 3 | Resources: 4 | MyInstance: 5 | Type: AWS::EC2::Instance 6 | Properties: 7 | AvailabilityZone: us-east-1a 8 | ImageId: ami-09538990a0c4fe9be 9 | InstanceType: t2.micro 10 | KeyName: aws-devops 11 | Tags: 12 | - Key: "Name" 13 | Value: "Manifold-AI-Learning" 14 | - Key: "Course" 15 | Value: "DevOps" 16 | 17 | NewInstance: 18 | Type: AWS::EC2::Instance 19 | Properties: 20 | AvailabilityZone: us-east-1b 21 | ImageId: ami-09538990a0c4fe9be 22 | InstanceType: t2.micro 23 | KeyName: aws-devops 24 | Tags: 25 | - Key: "Name" 26 | Value: "V2-Manifold-AI-Learning" 27 | - Key: "Course" 28 | Value: "V2-DevOps" -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/1.getting-started-ec2.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Description: 'AWS DevOps Course Manifold AI Learning' 3 | Resources: 4 | MyInstance: 5 | Type: AWS::EC2::Instance 6 | Properties: 7 | AvailabilityZone: us-east-1a 8 | ImageId: ami-09538990a0c4fe9be 9 | InstanceType: t2.micro 10 | KeyName: aws-devops 11 | Tags: 12 | - Key: "Name" 13 | Value: "Manifold-AI-Learning" 14 | - Key: "Course" 15 | Value: "DevOps" -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/15.resource-dependencies-without.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | S3Bucket: 3 | Type: AWS::S3::Bucket 4 | Properties: 5 | Tags: 6 | - Key: Name 7 | Value: AWSDevOps-Resource-dependencies-ManifoldAILearning 8 | 9 | SNSTopic: 10 | Type: AWS::SNS::Topic 11 | Properties: 12 | Tags: 13 | - Key: Name 14 | Value: AWSDevOps-Resource-dependencies-ManifoldAILearning 15 | -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/16.resource-dependencies-with.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | S3Bucket: 3 | Type: AWS::S3::Bucket 4 | Properties: 5 | Tags: 6 | - Key: Name 7 | Value: AWSDevOps-Resource-dependencies-ManifoldAILearning 8 | 9 | SNSTopic: 10 | Type: AWS::SNS::Topic 11 | DependsOn: S3Bucket 12 | Properties: 13 | Tags: 14 | - Key: Name 15 | Value: AWSDevOps-Resource-dependencies-ManifoldAILearning 16 | -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/17.resource-dependencies-with-intrinsic-functions.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | 3 | Description: AWS DevOps Course Manifold AI Learning 4 | 5 | Parameters: 6 | EmailAddress: 7 | Description: Enter an email address to subscribe to your Amazon SNS topic. 8 | Type: String 9 | 10 | Resources: 11 | SNSTopic: 12 | Type: AWS::SNS::Topic 13 | Properties: 14 | Tags: 15 | - Key: Name 16 | Value: Resource-dependencies-ManifoldAILearning 17 | 18 | SNSTopicSubscription: 19 | Type: AWS::SNS::Subscription 20 | Properties: 21 | Endpoint: !Ref EmailAddress 22 | Protocol: email 23 | TopicArn: !Ref SNSTopic 24 | 25 | SecurityGroup: 26 | Type: AWS::EC2::SecurityGroup 27 | Metadata: 28 | cfn_nag: 29 | rules_to_suppress: 30 | - id: F1000 31 | reason: This is using default VPC where we dont know VpcId to support egress. Missing egress rule means all traffic is allowed outbound. 32 | Properties: 33 | GroupDescription: AWS DevOps Course Manifold AI Learning SG 34 | Tags: 35 | - Key: Name 36 | Value: Resource-dependencies-ManifoldAILearning 37 | 38 | SecurityGroupIngress: 39 | Type: AWS::EC2::SecurityGroupIngress 40 | Properties: 41 | GroupId: !GetAtt SecurityGroup.GroupId 42 | IpProtocol: tcp 43 | FromPort: 80 44 | ToPort: 80 45 | CidrIp: 0.0.0.0/0 46 | -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/2.s3-bucket-yaml.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | HelloBucket: 3 | Type: 'AWS::S3::Bucket' 4 | -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/20.custom-s3-lambda-backed.yaml: -------------------------------------------------------------------------------- 1 | Parameters: 2 | S3BucketName: 3 | Type: String 4 | Description: "Specify the S3 bucket to create" 5 | AllowedPattern: "[a-zA-Z][a-zA-Z0-9_-]*" 6 | 7 | Resources: 8 | SampleS3Bucket: 9 | Type: AWS::S3::Bucket 10 | Properties: 11 | BucketName: !Ref S3BucketName 12 | DeletionPolicy: Delete 13 | 14 | S3CustomResource: 15 | Type: Custom::S3CustomResource # AWS::CloudFormation::CustomResource 16 | Properties: 17 | ServiceToken: !GetAtt AWSLambdaFunction.Arn 18 | bucket_name: !Ref SampleS3Bucket 19 | 20 | AWSLambdaFunction: 21 | Type: AWS::Lambda::Function 22 | Properties: 23 | Description: "Lambda Function to Empty an S3 bucket!" 24 | FunctionName: !Sub '${AWS::StackName}-${AWS::Region}-lambda' 25 | Handler: index.handler 26 | Role: !GetAtt AWSLambdaExecutionRole.Arn 27 | Timeout: 360 28 | Runtime: python3.8 29 | Code: 30 | ZipFile: | 31 | import boto3 32 | import cfnresponse 33 | ### cfnresponse module help in sending responses to CloudFormation 34 | ### instead of writing your own code 35 | 36 | def handler(event, context): 37 | # Get request type 38 | the_event = event['RequestType'] 39 | print("The event is: ", str(the_event)) 40 | 41 | response_data = {} 42 | s3 = boto3.client('s3') 43 | 44 | # Retrieve parameters (bucket name) 45 | bucket_name = event['ResourceProperties']['bucket_name'] 46 | 47 | try: 48 | if the_event == 'Delete': 49 | print("Deleting contents of S3 Bucket") 50 | b_operator = boto3.resource('s3') 51 | b_operator.Bucket(str(bucket_name)).objects.all().delete() 52 | print("Execution succesfull!") 53 | cfnresponse.send(event, context, cfnresponse.SUCCESS, response_data) 54 | except Exception as e: 55 | print("Execution failed... some error") 56 | print(str(e)) 57 | response_data['Data'] = str(e) 58 | cfnresponse.send(event, context, cfnresponse.FAILED, response_data) 59 | 60 | AWSLambdaExecutionRole: 61 | Type: AWS::IAM::Role 62 | Properties: 63 | AssumeRolePolicyDocument: 64 | Statement: 65 | - Action: 66 | - sts:AssumeRole 67 | Effect: Allow 68 | Principal: 69 | Service: 70 | - lambda.amazonaws.com 71 | Version: '2012-10-17' 72 | Path: "/" 73 | Policies: 74 | - PolicyDocument: 75 | Statement: 76 | - Action: 77 | - logs:CreateLogGroup 78 | - logs:CreateLogStream 79 | - logs:PutLogEvents 80 | Effect: Allow 81 | Resource: arn:aws:logs:*:*:* 82 | Version: '2012-10-17' 83 | PolicyName: !Sub ${AWS::StackName}-${AWS::Region}-AWSLambda-CW 84 | - PolicyDocument: 85 | Statement: 86 | - Action: 87 | - s3:PutObject 88 | - s3:DeleteObject 89 | - s3:List* 90 | Effect: Allow 91 | Resource: 92 | - !Sub arn:aws:s3:::${SampleS3Bucket} 93 | - !Sub arn:aws:s3:::${SampleS3Bucket}/* 94 | Version: '2012-10-17' 95 | PolicyName: !Sub ${AWS::StackName}-${AWS::Region}-AWSLambda-S3 96 | RoleName: !Sub ${AWS::StackName}-${AWS::Region}-AWSLambdaExecutionRole -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/21.dynamic-references.MD: -------------------------------------------------------------------------------- 1 | # Dynamic References for Parameter Store 2 | 3 | 1. Store the AMI ID in the parameter Store using console 4 | - name : /golden-images/amazon-linux-2 5 | - value : #Example: ami-08a52ddb321b32a8c 6 | 7 | ``` 8 | aws ec2 describe-instances \ 9 | --instance-ids YOUR_INSTANCE_ID \ 10 | --region us-east-1 \ 11 | --query 'Reservations[0].Instances[0].ImageId' 12 | ``` -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/21.ec2-instance-parameter-store.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | 3 | Description: AWS DevOps Course Manifold AI Learning 4 | 5 | Resources: 6 | Instance: 7 | Type: AWS::EC2::Instance 8 | Properties: 9 | AvailabilityZone: !Select ["0", !GetAZs ""] 10 | InstanceType: t2.micro 11 | ImageId: '{{resolve:ssm:/golden-images/amazon-linux-2}}' 12 | -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/3.s3-bucket-json.json: -------------------------------------------------------------------------------- 1 | { 2 | "Resources": { 3 | "HelloBucket": { 4 | "Type": "AWS::S3::Bucket" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/4.ec2-intrinsic.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Description: 'AWS DevOps Course Manifold AI Learning' 3 | 4 | Mappings: 5 | RegionMap: 6 | us-east-1: 7 | HVM64: "ami-0ff8a91507f77f867" 8 | HVMG2: "ami-0a584ac55a7631c0c" 9 | us-west-1: 10 | HVM64: "ami-0bdb828fd58c52235" 11 | HVMG2: "ami-066ee5fd4a9ef77f1" 12 | eu-west-1: 13 | HVM64: "ami-047bb4163c506cd98" 14 | HVMG2: "ami-31c2f645" 15 | ap-southeast-1: 16 | HVM64: "ami-08569b978cc4dfa10" 17 | HVMG2: "ami-0be9df32ae9f92309" 18 | ap-northeast-1: 19 | HVM64: "ami-06cd52961ce9f0d85" 20 | HVMG2: "ami-053cdd503598e4a9d" 21 | 22 | Resources: 23 | MyInstance: 24 | Type: AWS::EC2::Instance 25 | Properties: 26 | ImageId: !FindInMap 27 | - RegionMap 28 | - !Ref 'AWS::Region' 29 | - HVM64 30 | InstanceType: t2.micro 31 | KeyName: aws-devops 32 | Tags: 33 | - Key: "Name" 34 | Value: "intrinsic-functions-Manifold-AI-Learning" 35 | - Key: "Course" 36 | Value: "DevOps" 37 | SecurityGroups: 38 | - Ref: WebServerSecurityGroup 39 | UserData: 40 | Fn::Base64: | 41 | #!bin/bash 42 | sudo yum update -y 43 | sudo yum install -y ruby wget 44 | wget https://aws-codedeploy-eu-west-1.s3.eu-west-1.amazonaws.com/latest/install 45 | chmod +x ./install 46 | sudo ./install auto 47 | sudo service codedeploy-agent status 48 | WebServerSecurityGroup: 49 | Type: AWS::EC2::SecurityGroup 50 | Properties: 51 | GroupDescription: "Enable HTTP access via port 80 + SSH access" 52 | SecurityGroupIngress: 53 | - CidrIp: 0.0.0.0/0 54 | FromPort: '80' 55 | IpProtocol: tcp 56 | ToPort: '80' 57 | - CidrIp: 0.0.0.0/0 58 | FromPort: '22' 59 | IpProtocol: tcp 60 | ToPort: '22' -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/5.ec2-intrinsic-v2.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Description: 'AWS DevOps Course Manifold AI Learning' 3 | 4 | Mappings: 5 | RegionMap: 6 | us-east-1: 7 | HVM64: "ami-0ff8a91507f77f867" 8 | HVMG2: "ami-0a584ac55a7631c0c" 9 | us-west-1: 10 | HVM64: "ami-0bdb828fd58c52235" 11 | HVMG2: "ami-066ee5fd4a9ef77f1" 12 | eu-west-1: 13 | HVM64: "ami-047bb4163c506cd98" 14 | HVMG2: "ami-31c2f645" 15 | ap-southeast-1: 16 | HVM64: "ami-08569b978cc4dfa10" 17 | HVMG2: "ami-0be9df32ae9f92309" 18 | ap-northeast-1: 19 | HVM64: "ami-06cd52961ce9f0d85" 20 | HVMG2: "ami-053cdd503598e4a9d" 21 | 22 | Resources: 23 | MyInstance: 24 | Type: AWS::EC2::Instance 25 | Properties: 26 | ImageId: !FindInMap 27 | - RegionMap 28 | - !Ref 'AWS::Region' 29 | - HVM64 30 | InstanceType: t2.micro 31 | KeyName: aws-devops 32 | Tags: 33 | - Key: "Name" 34 | Value: "intrinsic-functions-Manifold-AI-Learning-v2" 35 | - Key: "Course" 36 | Value: "DevOps" 37 | SecurityGroups: 38 | - Ref: WebServerSecurityGroup 39 | UserData: 40 | Fn::Base64: | 41 | #!bin/bash 42 | sudo yum update -y 43 | sudo yum install -y ruby wget 44 | wget https://aws-codedeploy-eu-west-1.s3.eu-west-1.amazonaws.com/latest/install 45 | chmod +x ./install 46 | sudo ./install auto 47 | sudo service codedeploy-agent status 48 | AvailabilityZone: !Select 49 | - 0 50 | - Fn::GetAZs: !Ref 'AWS::Region' 51 | WebServerSecurityGroup: 52 | Type: AWS::EC2::SecurityGroup 53 | Properties: 54 | GroupDescription: "Enable HTTP access via port 80 + SSH access" 55 | SecurityGroupIngress: 56 | - CidrIp: 0.0.0.0/0 57 | FromPort: '80' 58 | IpProtocol: tcp 59 | ToPort: '80' 60 | - CidrIp: 0.0.0.0/0 61 | FromPort: '22' 62 | IpProtocol: tcp 63 | ToPort: '22' 64 | 65 | Outputs: 66 | EC2InstanceType: 67 | Description: Availaibility Zone of EC2 68 | Value: !GetAtt MyInstance.AvailabilityZone -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/6.s3-intrinsic-parameters.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: 2010-09-09 2 | Description: Using intrinsic Functions and parameters - AWS DevOps Course Manifold AI Learning 3 | 4 | Parameters: 5 | s3NamePrefix: 6 | Description: Enter the unique prefix to the name of the S3 bucket you need to create 7 | Type: String 8 | Default: mybucket 9 | Resources: 10 | s3Bucket: 11 | Type: 'AWS::S3::Bucket' 12 | Properties: 13 | BucketName: !Join 14 | - '-' 15 | - - !Ref s3NamePrefix 16 | - !Ref 'AWS::StackName' 17 | - !Ref 'AWS::Region' 18 | 19 | -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/7.resource-getatt-sub.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | 3 | Description: AWS DevOps Course Manifold AI Learning 4 | 5 | Resources: 6 | S3Bucket: 7 | Type: AWS::S3::Bucket 8 | Properties: 9 | Tags: 10 | - Key: Purpose 11 | Value: Manfiold AI Learning Student 12 | 13 | S3BucketPolicy: 14 | Type: AWS::S3::BucketPolicy 15 | Properties: 16 | Bucket: !Ref S3Bucket 17 | PolicyDocument: 18 | Version: "2012-10-17" 19 | Statement: 20 | - Action: 21 | - s3:* 22 | Effect: Deny 23 | Resource: 24 | - !GetAtt S3Bucket.Arn 25 | - !Sub '${S3Bucket.Arn}/*' 26 | Principal: '*' 27 | Condition: 28 | Bool: 29 | aws:SecureTransport: false 30 | 31 | Outputs: 32 | S3BucketDomainName: 33 | Description: IPv4 DNS name of the bucket. 34 | Value: !GetAtt S3Bucket.DomainName 35 | -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/8.conditions-cf.yaml: -------------------------------------------------------------------------------- 1 | Parameters: 2 | LatestAmiId: 3 | Type: AWS::SSM::Parameter::Value 4 | Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2 5 | 6 | EnvType: 7 | Description: Specify the Environment type of the stack. 8 | Type: String 9 | AllowedValues: 10 | - test 11 | - prod 12 | Default: test 13 | ConstraintDescription: Specify either test or prod. 14 | 15 | Conditions: 16 | IsProduction: !Equals 17 | - !Ref EnvType 18 | - prod 19 | Resources: 20 | EC2Instance: 21 | Type: AWS::EC2::Instance 22 | Properties: 23 | ImageId: !Ref LatestAmiId 24 | InstanceType: t2.micro 25 | 26 | MountPoint: 27 | Type: AWS::EC2::VolumeAttachment 28 | Properties: 29 | InstanceId: !Ref EC2Instance 30 | VolumeId: !Ref Volume 31 | Device: /dev/sdh 32 | Condition: IsProduction 33 | 34 | Volume: 35 | Type: AWS::EC2::Volume 36 | Properties: 37 | Size: 2 38 | AvailabilityZone: !GetAtt EC2Instance.AvailabilityZone 39 | Encrypted: true 40 | Condition: IsProduction 41 | -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/9.conditons-property.yaml: -------------------------------------------------------------------------------- 1 | Parameters: 2 | LatestAmiId: 3 | Type: AWS::SSM::Parameter::Value 4 | Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2 5 | 6 | EnvType: 7 | Description: Specify the Environment type of the stack. 8 | Type: String 9 | AllowedValues: 10 | - test 11 | - prod 12 | Default: test 13 | ConstraintDescription: Specify either test or prod. 14 | 15 | Conditions: 16 | IsProduction: !Equals 17 | - !Ref EnvType 18 | - prod 19 | 20 | Resources: 21 | EC2Instance: 22 | Type: AWS::EC2::Instance 23 | Properties: 24 | ImageId: !Ref LatestAmiId 25 | InstanceType: !If [IsProduction, t2.nano, t2.micro] 26 | 27 | MountPoint: 28 | Type: AWS::EC2::VolumeAttachment 29 | Properties: 30 | InstanceId: !Ref EC2Instance 31 | VolumeId: !Ref Volume 32 | Device: /dev/sdh 33 | Condition: IsProduction 34 | 35 | Volume: 36 | Type: AWS::EC2::Volume 37 | Properties: 38 | Size: 2 39 | AvailabilityZone: !GetAtt EC2Instance.AvailabilityZone 40 | Encrypted: true 41 | Condition: IsProduction 42 | 43 | Outputs: 44 | VolumeId: 45 | Value: !Ref Volume 46 | Condition: IsProduction 47 | 48 | -------------------------------------------------------------------------------- /Section-15.1-CloudFormation/attributes-cf.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-15.1-CloudFormation/attributes-cf.pdf -------------------------------------------------------------------------------- /Section-15.2-StepFunctions/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-15.2-StepFunctions/.DS_Store -------------------------------------------------------------------------------- /Section-15.2-StepFunctions/1.AWS-Step/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-15.2-StepFunctions/1.AWS-Step/README.md -------------------------------------------------------------------------------- /Section-15.2-StepFunctions/1.AWS-Step/lambda.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | def lambda_handler(event, context): 4 | 5 | # TODO implement 6 | 7 | return { 8 | 9 | 'statusCode': 200, 10 | 11 | 'body': json.dumps('AWS ML course, the lambda function has been invoked from Step Function.') 12 | 13 | } -------------------------------------------------------------------------------- /Section-15.2-StepFunctions/1.AWS-Step/step.json: -------------------------------------------------------------------------------- 1 | { 2 | "Comment": "Sample State machine code to invoke a lambda function", 3 | "StartAt": "invokeLambda", 4 | "States": { 5 | "invokeLambda": { 6 | "Type": "Task", 7 | "Resource": "arn:aws:lambda:us-east-1:866824485776:function:myStepFunction_lambda", 8 | "End": true 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /Section-15.2-StepFunctions/2.AWS-Step-Function-Serverless/demo-test1.json: -------------------------------------------------------------------------------- 1 | { 2 | "TableOperation": "Write", 3 | "ID": "100", 4 | "Name": "Dexter" 5 | } -------------------------------------------------------------------------------- /Section-15.2-StepFunctions/2.AWS-Step-Function-Serverless/demo-test2.json: -------------------------------------------------------------------------------- 1 | { 2 | "TableOperation": "Read", 3 | "ID": "100" 4 | } -------------------------------------------------------------------------------- /Section-15.2-StepFunctions/2.AWS-Step-Function-Serverless/demo-test3.json: -------------------------------------------------------------------------------- 1 | { 2 | "TableOperation": "Update", 3 | "ID": "200", 4 | "Name": "Sam" 5 | } -------------------------------------------------------------------------------- /Section-15.2-StepFunctions/2.AWS-Step-Function-Serverless/step_function_definition.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "Comment": "Read or write data from DynamoDB and send Success or Failed message", 4 | 5 | "StartAt": "TableOperationType", 6 | 7 | "States": { 8 | 9 | "TableOperationType": { 10 | 11 | "Type": "Choice", 12 | 13 | "Choices": [ 14 | 15 | { 16 | 17 | "Variable": "$.TableOperation", 18 | 19 | "StringEquals": "Read", 20 | 21 | "Next": "ReadDataFromTable" 22 | 23 | }, 24 | 25 | { 26 | 27 | "Variable": "$.TableOperation", 28 | 29 | "StringEquals": "Write", 30 | 31 | "Next": "WriteDataToTable" 32 | 33 | } 34 | 35 | ], 36 | 37 | "Default": "NoMatch" 38 | 39 | }, 40 | 41 | "ReadDataFromTable": { 42 | 43 | "Type": "Task", 44 | 45 | "Resource": "arn:aws:states:::dynamodb:getItem", 46 | 47 | "Parameters": { 48 | 49 | "TableName": "awsmltable", 50 | 51 | "Key": { 52 | 53 | "ID": { 54 | 55 | "S.$": "$.ID" 56 | 57 | } 58 | 59 | } 60 | 61 | }, 62 | 63 | "Catch": [ 64 | 65 | { 66 | 67 | "ErrorEquals": [ 68 | 69 | "States.TaskFailed" 70 | 71 | ], 72 | 73 | "Next": "FailureNotification" 74 | 75 | } 76 | 77 | ], 78 | 79 | "ResultPath": "$.DynamoDB", 80 | 81 | "Next": "CheckStatus" 82 | 83 | }, 84 | 85 | "WriteDataToTable": { 86 | 87 | "Type": "Task", 88 | 89 | "Resource": "arn:aws:states:::dynamodb:putItem", 90 | 91 | "Parameters": { 92 | 93 | "TableName": "awsmltable", 94 | 95 | "Item": { 96 | 97 | "ID": { 98 | 99 | "S.$": "$.ID" 100 | 101 | }, 102 | 103 | "Name": { 104 | 105 | "S.$": "$.Name" 106 | 107 | } 108 | 109 | } 110 | 111 | }, 112 | 113 | "Catch": [ 114 | 115 | { 116 | 117 | "ErrorEquals": [ 118 | 119 | "States.TaskFailed" 120 | 121 | ], 122 | 123 | "Next": "FailureNotification" 124 | 125 | } 126 | 127 | ], 128 | 129 | "ResultPath": "$.DynamoDB", 130 | 131 | "Next": "CheckStatus" 132 | 133 | }, 134 | 135 | "NoMatch": { 136 | 137 | "Type": "Fail", 138 | 139 | "Error": "Unknown Table Operation", 140 | 141 | "Cause": "No Matches!" 142 | 143 | }, 144 | 145 | "CheckStatus": { 146 | 147 | "Type": "Choice", 148 | 149 | "Choices": [ 150 | 151 | { 152 | 153 | "Variable": "$.DynamoDB.SdkHttpMetadata.HttpStatusCode", 154 | 155 | "NumericEquals": 200, 156 | 157 | "Next": "SuccessNotification" 158 | 159 | }, 160 | 161 | { 162 | 163 | "Not": { 164 | 165 | "Variable": "$.DynamoDB.HttpStatusCode", 166 | 167 | "NumericEquals": 200 168 | 169 | }, 170 | 171 | "Next": "FailureNotification" 172 | 173 | } 174 | 175 | ] 176 | 177 | }, 178 | 179 | "SuccessNotification": { 180 | 181 | "Type": "Task", 182 | 183 | "Resource": "arn:aws:states:::sns:publish", 184 | 185 | "Parameters": { 186 | 187 | "Message": "DynamoDB Operation Success !", 188 | 189 | "TopicArn": "arn:aws:sns:us-east-1:866824485776:Step_function_AWSML" 190 | 191 | }, 192 | 193 | "End": true 194 | 195 | }, 196 | 197 | "FailureNotification": { 198 | 199 | "Type": "Task", 200 | 201 | "Resource": "arn:aws:states:::sns:publish", 202 | 203 | "Parameters": { 204 | 205 | "Message": "DynamoDB Operation Failed !", 206 | 207 | "TopicArn": "arn:aws:sns:us-east-1:866824485776:Step_function_AWSML" 208 | 209 | }, 210 | 211 | "End": true 212 | 213 | } 214 | 215 | } 216 | 217 | } 218 | -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-16-mlops-pipeline/.DS_Store -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/customer-retention-model.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-16-mlops-pipeline/customer-retention-model.tar.gz -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/dataset/storedata_total.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-16-mlops-pipeline/dataset/storedata_total.xlsx -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/evaluate-churn.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pathlib 3 | import pickle 4 | import tarfile 5 | import joblib 6 | import numpy as np 7 | import pandas as pd 8 | import xgboost 9 | import datetime as dt 10 | from sklearn.metrics import roc_curve,auc 11 | if __name__ == "__main__": 12 | #Read Model Tar File 13 | model_path = f"/opt/ml/processing/model/model.tar.gz" 14 | with tarfile.open(model_path) as tar: 15 | tar.extractall(path=".") 16 | model = pickle.load(open("xgboost-model", "rb")) 17 | #Read Test Data using which we evaluate the model 18 | test_path = "/opt/ml/processing/test/test.csv" 19 | df = pd.read_csv(test_path, header=None) 20 | y_test = df.iloc[:, 0].to_numpy() 21 | df.drop(df.columns[0], axis=1, inplace=True) 22 | X_test = xgboost.DMatrix(df.values) 23 | #Run Predictions 24 | predictions = model.predict(X_test) 25 | #Evaluate Predictions 26 | fpr, tpr, thresholds = roc_curve(y_test, predictions) 27 | auc_score = auc(fpr, tpr) 28 | report_dict = { 29 | "classification_metrics": { 30 | "auc_score": { 31 | "value": auc_score, 32 | }, 33 | }, 34 | } 35 | #Save Evaluation Report 36 | output_dir = "/opt/ml/processing/evaluation" 37 | pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True) 38 | evaluation_path = f"{output_dir}/evaluation.json" 39 | with open(evaluation_path, "w") as f: 40 | f.write(json.dumps(report_dict)) -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/image-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-16-mlops-pipeline/image-1.png -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/image-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-16-mlops-pipeline/image-2.png -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/image-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-16-mlops-pipeline/image-3.png -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/image-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-16-mlops-pipeline/image-4.png -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-16-mlops-pipeline/image.png -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-16-mlops-pipeline/mlops-tutorial.zip -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-16-mlops-pipeline/mlops-tutorial/.DS_Store -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/etl/etljob.json: -------------------------------------------------------------------------------- 1 | { 2 | "Name": "", 3 | "Description": "Python ETL Shell Job to featurize the Abalone dataset", 4 | "Role": "", 5 | "ExecutionProperty": { 6 | "MaxConcurrentRuns": 1 7 | }, 8 | "Command": { 9 | "Name": "pythonshell", 10 | "ScriptLocation": "", 11 | "PythonVersion": "3" 12 | }, 13 | "DefaultArguments": { 14 | "--job-language": "python" 15 | }, 16 | "Timeout": 15, 17 | "MaxCapacity": 0.0625 18 | } -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/etl/preprocess.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import boto3 4 | import numpy as np 5 | import pandas as pd 6 | import sklearn 7 | from awsglue.utils import getResolvedOptions 8 | from io import StringIO 9 | 10 | # Helper function to split dataset (80/19/1) 11 | def split_data(df, train_percent=0.8, validate_percent=0.19, seed=None): 12 | np.random.seed() 13 | perm = np.random.permutation(df.index) 14 | m = len(df.index) 15 | train_end = int(train_percent * m) 16 | validate_end = int(validate_percent * m) + train_end 17 | train = df.iloc[perm[:train_end]] 18 | validate = df.iloc[perm[train_end:validate_end]] 19 | test = df.iloc[perm[validate_end:]] 20 | return [('train', train), ('test', test), ('validate', validate), ('baseline', train)] 21 | 22 | # Get job args 23 | args = getResolvedOptions(sys.argv, ['S3_INPUT_BUCKET', 'S3_INPUT_KEY_PREFIX', 'S3_OUTPUT_BUCKET', 'S3_OUTPUT_KEY_PREFIX']) 24 | 25 | # Downloading the data from S3 into a Dataframe 26 | column_names = ["sex", "length", "diameter", "height", "whole weight", 27 | "shucked weight", "viscera weight", "shell weight", "rings"] 28 | client = boto3.client('s3') 29 | bucket_name = args['S3_INPUT_BUCKET'] 30 | object_key = os.path.join(args['S3_INPUT_KEY_PREFIX'], 'abalone.csv') 31 | print("Downloading input data from S3 ...\n") 32 | csv_obj = client.get_object(Bucket=bucket_name, Key=object_key) 33 | body = csv_obj['Body'] 34 | csv_string = body.read().decode('utf-8') 35 | data = pd.read_csv(StringIO(csv_string), sep=',', names=column_names) 36 | 37 | # Re-order data to better separate features 38 | data = data[["rings", "sex", "length", "diameter", "height", "whole weight", 39 | "shucked weight", "viscera weight", "shell weight"]] 40 | 41 | # Create dummy variables for categorical `sex` feature using pandas 42 | print("Encoding Features ...\n") 43 | data = pd.get_dummies(data) 44 | 45 | # Create train, test and validate datasets 46 | print("Creating dataset splits ...\n") 47 | datasets = split_data(data) 48 | 49 | # Upload data to S3 as .csv file while separating validation set 50 | for file_name, partition_name in datasets: 51 | if file_name == 'test': 52 | print("Writing {} data ...\n".format(file_name)) 53 | np.savetxt(file_name+'.csv', partition_name, delimiter=',') 54 | boto3.Session().resource('s3').Bucket(args['S3_OUTPUT_BUCKET']).Object(os.path.join(args['S3_OUTPUT_KEY_PREFIX'], 'testing', file_name+'.csv')).upload_file(file_name+'.csv') 55 | elif file_name == 'baseline': 56 | print("Writing {} data ...\n".format(file_name)) 57 | np.savetxt( 58 | file_name+'.csv', 59 | partition_name, 60 | delimiter=',', 61 | header="rings,length,diameter,height,whole weight,shucked weight,viscera weight,shell weight,sex_F,sex_I,sex_M" 62 | ) 63 | boto3.Session().resource('s3').Bucket(args['S3_OUTPUT_BUCKET']).Object(os.path.join(args['S3_OUTPUT_KEY_PREFIX'], 'baseline', file_name+'.csv')).upload_file(file_name+'.csv') 64 | else: 65 | print("Writing {} data ...\n".format(file_name)) 66 | np.savetxt(file_name+'.csv', partition_name, delimiter=',') 67 | boto3.Session().resource('s3').Bucket(args['S3_OUTPUT_BUCKET']).Object(os.path.join(args['S3_OUTPUT_KEY_PREFIX'], 'training', file_name+'.csv')).upload_file(file_name+'.csv') 68 | 69 | print("Done writing to S3 ...\n") -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/model/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG REGION 2 | 3 | FROM 763104351884.dkr.ecr.${REGION}.amazonaws.com/tensorflow-training:2.6.0-cpu-py38-ubuntu20.04-v1.0 4 | 5 | RUN apt-get update && apt-get install -y --no-install-recommends \ 6 | nginx \ 7 | && rm -rf /var/lib/apt/lists/* 8 | 9 | RUN pip install --upgrade pip 10 | 11 | RUN pip install --no-cache-dir -U \ 12 | flask \ 13 | gevent \ 14 | gunicorn 15 | 16 | RUN mkdir -p /opt/program 17 | RUN mkdir -p /opt/ml 18 | 19 | COPY app.py /opt/program 20 | COPY model.py /opt/program 21 | COPY nginx.conf /opt/program 22 | COPY wsgi.py /opt/program 23 | WORKDIR /opt/program 24 | 25 | EXPOSE 8080 26 | 27 | ENTRYPOINT ["python", "app.py"] -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/model/app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | import io 5 | import sys 6 | import os 7 | import signal 8 | import traceback 9 | import flask 10 | import multiprocessing 11 | import subprocess 12 | import tarfile 13 | import model 14 | import pandas as pd 15 | import numpy as np 16 | import tensorflow as tf 17 | from tensorflow import keras 18 | from tensorflow.keras.models import Sequential 19 | from tensorflow.keras.layers import Dense 20 | from tensorflow.keras.optimizers import Adam 21 | from sklearn import preprocessing 22 | 23 | # Adds the model.py path to the list 24 | prefix = '/opt/ml' 25 | model_path = os.path.join(prefix, 'model') 26 | sys.path.insert(0,model_path) 27 | model_cache = {} 28 | 29 | class PredictionService(object): 30 | tf_model = None 31 | @classmethod 32 | def get_model(cls): 33 | if cls.tf_model is None: 34 | cls.tf_model = load_model() 35 | return cls.tf_model 36 | 37 | @classmethod 38 | def predict(cls, input): 39 | tf_model = cls.get_model() 40 | return tf_model.predict(input) 41 | 42 | def load_model(): 43 | # Load 'h5' keras model 44 | model = tf.keras.models.load_model(os.path.join(model_path, 'model.h5')) 45 | model.compile(optimizer='adam', loss='mse') 46 | return model 47 | 48 | def sigterm_handler(nginx_pid, gunicorn_pid): 49 | try: 50 | os.kill(nginx_pid, signal.SIGQUIT) 51 | except OSError: 52 | pass 53 | try: 54 | os.kill(gunicorn_pid, signal.SIGTERM) 55 | except OSError: 56 | pass 57 | 58 | sys.exit(0) 59 | 60 | def start_server(timeout, workers): 61 | print('Starting the inference server with {} workers.'.format(model_server_workers)) 62 | # link the log streams to stdout/err so they will be logged to the container logs 63 | subprocess.check_call(['ln', '-sf', '/dev/stdout', '/var/log/nginx/access.log']) 64 | subprocess.check_call(['ln', '-sf', '/dev/stderr', '/var/log/nginx/error.log']) 65 | 66 | nginx = subprocess.Popen(['nginx', '-c', '/opt/program/nginx.conf']) 67 | gunicorn = subprocess.Popen(['gunicorn', 68 | '--timeout', str(timeout), 69 | '-k', 'gevent', 70 | '-b', 'unix:/tmp/gunicorn.sock', 71 | '-w', str(workers), 72 | 'wsgi:app']) 73 | 74 | signal.signal(signal.SIGTERM, lambda a, b: sigterm_handler(nginx.pid, gunicorn.pid)) 75 | 76 | # If either subprocess exits, so do we. 77 | pids = set([nginx.pid, gunicorn.pid]) 78 | while True: 79 | pid, _ = os.wait() 80 | if pid in pids: 81 | break 82 | 83 | sigterm_handler(nginx.pid, gunicorn.pid) 84 | print('Inference server exiting') 85 | 86 | # The flask app for serving predictions 87 | app = flask.Flask(__name__) 88 | 89 | @app.route('/ping', methods=['GET']) 90 | def ping(): 91 | health = PredictionService.get_model() is not None 92 | status = 200 if health else 404 93 | return flask.Response(response='\n', status=status, mimetype='application/json') 94 | 95 | @app.route('/invocations', methods=['POST']) 96 | def invoke(): 97 | data = None 98 | if flask.request.content_type == 'text/csv': 99 | """ 100 | NOTE: print(flask.request.data) --> Bytes string 101 | """ 102 | payload = np.fromstring(flask.request.data.decode('utf-8'), sep=',') # Convert `str` to `Numpy` 103 | data = payload.reshape(1, -1) # Vectorize the payload 104 | else: 105 | return flask.Response(response="Invalid request data type, only 'text/csv' is supported.", status=415, mimetype='text/plain') 106 | 107 | # Get predictions 108 | predictions = PredictionService.predict(data) 109 | 110 | # Convert from Numpy to CSV 111 | out = io.StringIO() 112 | pd.DataFrame({'results':predictions.flatten()}).to_csv(out, header=False, index=False) 113 | result = out.getvalue() 114 | print("Prediction Result: {}".format(result)) 115 | return flask.Response(response=result, status=200, mimetype='text/csv') 116 | 117 | 118 | if __name__ == '__main__': 119 | print("Tensorflow Version: {}".format(tf.__version__)) 120 | if len(sys.argv) < 2 or ( not sys.argv[1] in [ "serve", "train", "test"] ): 121 | raise Exception("Invalid argument: you must specify 'train' for training mode, 'serve' for predicting mode or 'test' for local testing.") 122 | 123 | train = sys.argv[1] == "train" 124 | test = sys.argv[1] == "test" 125 | 126 | if train: 127 | model.train() 128 | 129 | elif test: 130 | algo = 'TensorflowRegression' 131 | if model_cache.get(algo) is None: 132 | model_cache[algo] = load_model() 133 | req = eval(sys.argv[2]) 134 | print(model.predict(req, model_cache[algo])) 135 | 136 | else: 137 | cpu_count = multiprocessing.cpu_count() 138 | model_server_timeout = os.environ.get('MODEL_SERVER_TIMEOUT', 60) 139 | model_server_workers = int(os.environ.get('MODEL_SERVER_WORKERS', cpu_count)) 140 | start_server(model_server_timeout, model_server_workers) -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/model/assets/Dev/Dev-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "Parameters": { 3 | "EndpointInstanceCount": "1", 4 | "EndpointInstanceType": "ml.t2.large" 5 | } 6 | } 7 | 8 | -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/model/assets/Dev/deploy-model-Dev.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: 2010-09-09 2 | 3 | Description: Deploy a model to Sagemaker for Dev/QA Testing 4 | 5 | Parameters: 6 | 7 | ImageRepoName: 8 | Type: String 9 | Description: Name of the model image ECR (Docker) repo. 10 | 11 | ImageTagName: 12 | Type: String 13 | Description: Name of the model image ECR (Docker) tag. 14 | 15 | ModelName: 16 | Type: String 17 | Description: Name of the model. 18 | 19 | TrainJobId: 20 | Type: String 21 | Description: Id of the Codepipeline + SagemakerJobs. 22 | 23 | EndpointInstanceCount: 24 | Type: Number 25 | Description: Number of instances to launch for the endpoint. 26 | MinValue: 1 27 | 28 | EndpointInstanceType: 29 | Type: String 30 | Description: The ML compute instance type for the endpoint. 31 | 32 | Resources: 33 | 34 | Model: 35 | Type: AWS::SageMaker::Model 36 | Properties: 37 | ModelName: !Sub mlops-${ModelName}-${TrainJobId} 38 | PrimaryContainer: 39 | Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${ImageRepoName}:${ImageTagName} 40 | ModelDataUrl: !Sub s3://mlops-${AWS::Region}-${AWS::AccountId}/${TrainJobId}/mlops-${ModelName}-${TrainJobId}/output/model.tar.gz 41 | ExecutionRoleArn: !Sub arn:aws:iam::${AWS::AccountId}:role/MLOps 42 | 43 | EndpointConfig: 44 | Type: AWS::SageMaker::EndpointConfig 45 | Properties: 46 | ProductionVariants: 47 | - InitialInstanceCount: !Ref EndpointInstanceCount 48 | InitialVariantWeight: 1.0 49 | InstanceType: !Ref EndpointInstanceType 50 | ModelName: !GetAtt Model.ModelName 51 | VariantName: AllTraffic 52 | DependsOn: Model 53 | 54 | Endpoint: 55 | Type: AWS::SageMaker::Endpoint 56 | Properties: 57 | EndpointName: !Sub ${ModelName}-dev-endpoint 58 | EndpointConfigName: !GetAtt EndpointConfig.EndpointConfigName 59 | DependsOn: EndpointConfig -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/model/assets/Prd/Prd-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "Parameters": { 3 | "EndpointInstanceCount": "2", 4 | "EndpointInstanceType": "ml.c5.large", 5 | "EndpointMaxCapacity": "10", 6 | "ScalingTarget": "750.0" 7 | } 8 | } -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/model/assets/Prd/deploy-model-Prd.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: 2010-09-09 2 | 3 | Description: Deploy a model to Sagemaker for Production Hosting 4 | 5 | Parameters: 6 | 7 | ModelName: 8 | Type: String 9 | Description: Name of the model. 10 | 11 | TrainJobId: 12 | Type: String 13 | Description: Id of the Codepipeline + SagemakerJobs. 14 | 15 | EndpointInstanceCount: 16 | Type: Number 17 | Description: Number of instances to launch for the endpoint. 18 | MinValue: 1 19 | 20 | EndpointInstanceType: 21 | Type: String 22 | Description: The ML compute instance type for the endpoint. 23 | 24 | ModelPackageName: 25 | Type: String 26 | Description: The trained Model Package Name. 27 | 28 | EndpointMaxCapacity: 29 | Type: Number 30 | Description: Maximum number of instances to autoscale the endpoint. 31 | 32 | ScalingTarget: 33 | Type: Number 34 | Description: Target number of Invocations per Instance. 35 | 36 | Resources: 37 | 38 | Model: 39 | Type: AWS::SageMaker::Model 40 | Properties: 41 | PrimaryContainer: 42 | ModelPackageName: !Ref ModelPackageName 43 | ExecutionRoleArn: !Sub arn:aws:iam::${AWS::AccountId}:role/MLOps 44 | 45 | EndpointConfig: 46 | Type: AWS::SageMaker::EndpointConfig 47 | Properties: 48 | DataCaptureConfig: 49 | CaptureContentTypeHeader: 50 | CsvContentTypes: 51 | - "text/csv" 52 | CaptureOptions: 53 | - CaptureMode: Input 54 | - CaptureMode: Output 55 | DestinationS3Uri: !Sub s3://data-${AWS::Region}-${AWS::AccountId}/datacapture 56 | EnableCapture: True 57 | InitialSamplingPercentage: 100 58 | ProductionVariants: 59 | - InitialInstanceCount: !Ref EndpointInstanceCount 60 | InitialVariantWeight: 1.0 61 | InstanceType: !Ref EndpointInstanceType 62 | ModelName: !GetAtt Model.ModelName 63 | VariantName: AllTraffic 64 | 65 | Endpoint: 66 | Type: AWS::SageMaker::Endpoint 67 | Properties: 68 | EndpointName: !Sub ${ModelName}-prd-endpoint 69 | EndpointConfigName: !GetAtt EndpointConfig.EndpointConfigName 70 | DependsOn: EndpointConfig 71 | 72 | MonitoringSchedule: 73 | Type: AWS::SageMaker::MonitoringSchedule 74 | Properties: 75 | MonitoringScheduleConfig: 76 | MonitoringJobDefinition: 77 | MonitoringAppSpecification: 78 | ImageUri: !Sub "159807026194.dkr.ecr.${AWS::Region}.amazonaws.com/sagemaker-model-monitor-analyzer:latest" 79 | MonitoringInputs: 80 | - EndpointInput: 81 | EndpointName: !GetAtt Endpoint.EndpointName 82 | LocalPath: "/opt/ml/processing/endpointdata" 83 | MonitoringOutputConfig: 84 | MonitoringOutputs: 85 | - S3Output: 86 | LocalPath: "/opt/ml/processing/localpath" 87 | S3Uri: !Sub s3://data-${AWS::Region}-${AWS::AccountId}/reports 88 | MonitoringResources: 89 | ClusterConfig: 90 | InstanceCount: 1 91 | InstanceType: ml.m5.large 92 | VolumeSizeInGB: 50 93 | RoleArn: !Sub arn:aws:iam::${AWS::AccountId}:role/MLOps 94 | ScheduleConfig: 95 | ScheduleExpression: cron(0 * ? * * *) 96 | MonitoringScheduleName: !Sub ${ModelName}MonitoringSchedule 97 | 98 | AutoScaling: 99 | Type: AWS::ApplicationAutoScaling::ScalableTarget 100 | Properties: 101 | MaxCapacity: !Ref EndpointMaxCapacity 102 | MinCapacity: !Ref EndpointInstanceCount 103 | ResourceId: !Sub endpoint/${Endpoint.EndpointName}/variant/AllTraffic 104 | RoleARN: !Sub arn:aws:iam::${AWS::AccountId}:role/MLOps 105 | ScalableDimension: sagemaker:variant:DesiredInstanceCount 106 | ServiceNamespace: sagemaker 107 | DependsOn: Endpoint 108 | 109 | AutoScalingPolicy: 110 | Type: AWS::ApplicationAutoScaling::ScalingPolicy 111 | Properties: 112 | PolicyName: SageMakerVariantInvocationsPerInstance 113 | PolicyType: TargetTrackingScaling 114 | ResourceId: !Sub endpoint/${Endpoint.EndpointName}/variant/AllTraffic 115 | ScalableDimension: sagemaker:variant:DesiredInstanceCount 116 | ServiceNamespace: sagemaker 117 | TargetTrackingScalingPolicyConfiguration: 118 | TargetValue: !Ref ScalingTarget 119 | ScaleInCooldown: 60 120 | ScaleOutCooldown: 60 121 | PredefinedMetricSpecification: 122 | PredefinedMetricType: SageMakerVariantInvocationsPerInstance 123 | DependsOn: AutoScaling -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/model/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | phases: 3 | install: 4 | runtime-versions: 5 | python: 3.8 6 | pre_build: 7 | commands: 8 | - echo Updating Packages... 9 | - pip install --upgrade pip 10 | build: 11 | commands: 12 | - echo Build started on `date` 13 | - echo Configuring CloudFormation Parameters for $STAGE Deployment... 14 | - | 15 | sh -c """ 16 | python build.py --pipeline-name=$PIPELINE_NAME \ 17 | --image-repo-name=$IMAGE_REPO_NAME \ 18 | --image-tag=$IMAGE_TAG \ 19 | --model-name=$MODEL_NAME \ 20 | --model-package-group-name=$MODEL_GROUP \ 21 | --import-config=$CODEBUILD_SRC_DIR/assets/$STAGE/$STAGE-config.json \ 22 | --export-config=$CODEBUILD_SRC_DIR/assets/$STAGE/$STAGE-config-export.json 23 | """ 24 | - echo CloudFormation Parameters... 25 | - cat $CODEBUILD_SRC_DIR/assets/$STAGE/$STAGE-config-export.json 26 | post_build: 27 | commands: 28 | - echo Build completed on `date` 29 | artifacts: 30 | files: 31 | - assets/$STAGE/$STAGE-config-export.json 32 | - assets/$STAGE/deploy-model-$STAGE.yml 33 | discard-paths: yes -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/model/nginx.conf: -------------------------------------------------------------------------------- 1 | worker_processes 1; 2 | daemon off; # Prevent forking 3 | 4 | 5 | pid /tmp/nginx.pid; 6 | error_log /var/log/nginx/error.log; 7 | 8 | events { 9 | # defaults 10 | } 11 | 12 | http { 13 | include /etc/nginx/mime.types; 14 | default_type application/octet-stream; 15 | access_log /var/log/nginx/access.log combined; 16 | 17 | upstream gunicorn { 18 | server unix:/tmp/gunicorn.sock; 19 | } 20 | 21 | server { 22 | # SageMaker endpoint listens on port 8080 23 | listen 8080 deferred; 24 | client_max_body_size 5m; # set to `0` for unlimited 25 | 26 | keepalive_timeout 5; 27 | 28 | location ~ ^/(ping|invocations) { 29 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 30 | proxy_set_header Host $http_host; 31 | proxy_redirect off; 32 | proxy_pass http://gunicorn; 33 | } 34 | 35 | location / { 36 | return 404 "{}"; 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/model/trainingjob.json: -------------------------------------------------------------------------------- 1 | { 2 | "AlgorithmSpecification":{ 3 | "TrainingImage": ".dkr.ecr..amazonaws.com/abalone:latest", 4 | "TrainingInputMode": "File" 5 | }, 6 | "RoleArn": "arn:aws:iam:::role/MLOps", 7 | "OutputDataConfig": { 8 | "S3OutputPath": "" 9 | }, 10 | "ResourceConfig": { 11 | "InstanceCount": 1, 12 | "InstanceType": "ml.m5.xlarge", 13 | "VolumeSizeInGB": 30 14 | }, 15 | "TrainingJobName": "", 16 | "HyperParameters": { 17 | "epochs": "2000", 18 | "layers": "2", 19 | "dense_layer": "64", 20 | "batch_size": "8" 21 | }, 22 | "StoppingCondition": { 23 | "MaxRuntimeInSeconds": 360000 24 | }, 25 | "InputDataConfig": [ 26 | { 27 | "ChannelName": "training", 28 | "DataSource": { 29 | "S3DataSource": { 30 | "S3DataType": "S3Prefix", 31 | "S3Uri": "", 32 | "S3DataDistributionType": "FullyReplicated" 33 | } 34 | }, 35 | "ContentType": "text/csv", 36 | "CompressionType": "None" 37 | } 38 | ], 39 | "Tags": [] 40 | } -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/model/wsgi.py: -------------------------------------------------------------------------------- 1 | # This is just a simple wrapper for gunicorn to find your app. 2 | # If you want to change the algorithm file, simply change "predictor" above to the 3 | # new file. 4 | 5 | import app as myapp 6 | app = myapp.app -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/pipeline/EtlJobMonitor/lambda.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import io 3 | import json 4 | import os 5 | import logging 6 | from botocore.exceptions import ClientError 7 | 8 | glue = boto3.client('glue') 9 | cw = boto3.client('events') 10 | cp = boto3.client('codepipeline') 11 | 12 | logger = logging.getLogger() 13 | logger.setLevel(logging.INFO) 14 | 15 | 16 | def handler(event, context): 17 | logger.debug("## Environment Variables ##") 18 | logger.debug(os.environ) 19 | logger.debug("## Event ##") 20 | logger.debug(event) 21 | pipeline_name = os.environ['PIPELINE_NAME'] 22 | model_name = os.environ['MODEL_NAME'] 23 | result = None 24 | token = None 25 | try: 26 | response = cp.get_pipeline_state(name=pipeline_name) 27 | for stageState in response['stageStates']: 28 | if stageState['stageName'] == 'ETLApproval': 29 | for actionState in stageState['actionStates']: 30 | if actionState['actionName'] == 'ApproveETL': 31 | latestExecution = actionState['latestExecution'] 32 | executionId = stageState['latestExecution']['pipelineExecutionId'] 33 | if latestExecution['status'] != 'InProgress': 34 | raise(Exception("ETL approval is not awaiting approval: {}".format(latestExecution['status']))) 35 | token = latestExecution['token'] 36 | job_name = "abalone-preprocess-{}".format(executionId) 37 | response = glue.get_job_runs(JobName=job_name) 38 | job_run_id = response['JobRuns'][0]['Id'] 39 | response = glue.get_job_run(JobName=job_name, RunId=job_run_id) 40 | status = response['JobRun']['JobRunState'] 41 | logger.info(status) 42 | if status == "SUCCEEDED": 43 | result = { 44 | 'summary': 'Glue ETL Job completed', 45 | 'status': 'Approved' 46 | } 47 | elif status == "RUNNING": 48 | return "Glue ETL Job ({}) is in progress".format(executionId) 49 | elif status == "STARTING": 50 | return "Glue ETL Job ({}) is in progress".format(executionId) 51 | else: 52 | result = { 53 | 'summary': response['JobRun']['ErrorMessage'], 54 | 'status': 'Rejected' 55 | } 56 | except Exception as e: 57 | logger.error(e) 58 | result = { 59 | 'summary': str(e), 60 | 'status': 'Rejected' 61 | } 62 | 63 | try: 64 | response = cp.put_approval_result( 65 | pipelineName=pipeline_name, 66 | stageName='ETLApproval', 67 | actionName='ApproveETL', 68 | result=result, 69 | token=token 70 | ) 71 | except ClientError as e: 72 | error_message = e.response["Error"]["Message"] 73 | logger.error(error_message) 74 | raise Exception(error_message) 75 | 76 | try: 77 | response = cw.disable_rule(Name="etl-job-monitor-{}".format(model_name)) 78 | except ClientError as e: 79 | error_message = e.response["Error"]["Message"] 80 | logger.error(error_message) 81 | raise Exception(error_message) 82 | 83 | return "Done!" -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/pipeline/EtlLaunchJob/lambda.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import io 3 | import zipfile 4 | import json 5 | import os 6 | import logging 7 | 8 | s3 = boto3.client('s3') 9 | glue = boto3.client('glue') 10 | sm = boto3.client('sagemaker') 11 | cw = boto3.client('events') 12 | cp = boto3.client('codepipeline') 13 | 14 | logger = logging.getLogger() 15 | logger.setLevel(logging.INFO) 16 | 17 | 18 | def handler(event, context): 19 | logger.debug("## Environment Variables ##") 20 | logger.debug(os.environ) 21 | logger.debug("## Event ##") 22 | logger.debug(event) 23 | try: 24 | pipeline_name=os.environ['PIPELINE_NAME'] 25 | model_name = os.environ['MODEL_NAME'] 26 | jobId = event['CodePipeline.job']['id'] 27 | accountId = event['CodePipeline.job']['accountId'] 28 | region = os.environ.get('AWS_REGION') 29 | role = "arn:aws:iam::{}:role/MLOps".format(accountId) 30 | data_bucket = "data-{}-{}".format(region, accountId) 31 | output_bucket = "mlops-{}-{}".format(region, accountId) 32 | etlJob = None 33 | response = cp.get_pipeline_state(name=pipeline_name) 34 | for stageState in response['stageStates']: 35 | if stageState['stageName'] == 'ETL': 36 | for actionState in stageState['actionStates']: 37 | if actionState['actionName'] == 'GlueJob': 38 | executionId = stageState['latestExecution']['pipelineExecutionId'] 39 | script_location = "s3://{}/{}/code/preprocess.py".format(output_bucket, executionId) 40 | job_name = "abalone-preprocess-{}".format(executionId) 41 | logger.info("Start Glue ETL Job for jobid[{}] executionId[{}]".format(jobId, executionId)) 42 | for inputArtifacts in event["CodePipeline.job"]["data"]["inputArtifacts"]: 43 | if inputArtifacts['name'] == 'EtlSourceOutput': 44 | s3Location = inputArtifacts['location']['s3Location'] 45 | zip_bytes = s3.get_object(Bucket=s3Location['bucketName'], Key=s3Location['objectKey'])['Body'].read() 46 | with zipfile.ZipFile(io.BytesIO(zip_bytes), "r") as z: 47 | for file in z.infolist(): 48 | if file.filename == 'preprocess.py': 49 | s3.put_object(Bucket=output_bucket, Key="{}/code/{}".format(executionId, file.filename), Body=z.read(file)) 50 | if file.filename == 'etljob.json': 51 | etlJob = json.loads(z.read('etljob.json').decode('ascii')) 52 | etlJob['Name'] = job_name 53 | etlJob['Role'] = role 54 | etlJob['Command']['ScriptLocation'] = script_location 55 | glue_job_name = glue.create_job(**etlJob)['Name'] 56 | logger.info(glue_job_name) 57 | job_run_id = glue.start_job_run( 58 | JobName=job_name, 59 | Arguments={ 60 | '--S3_INPUT_BUCKET': data_bucket, 61 | '--S3_INPUT_KEY_PREFIX': 'input/raw', 62 | '--S3_OUTPUT_BUCKET': output_bucket, 63 | '--S3_OUTPUT_KEY_PREFIX': executionId+'/input' 64 | } 65 | )['JobRunId'] 66 | logger.info(job_run_id) 67 | cw.enable_rule(Name="etl-job-monitor-{}".format(model_name)) 68 | cp.put_job_success_result(jobId=jobId) 69 | except Exception as e: 70 | logger.error(e) 71 | resppnse = cp.put_job_failure_result( 72 | jobId=jobId, 73 | failureDetails={ 74 | 'type': 'ConfigurationError', 75 | 'message': str(e), 76 | 'externalExecutionId': context.aws_request_id 77 | } 78 | ) 79 | 80 | return 'Done' -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/pipeline/ModelGroup/lambda.py: -------------------------------------------------------------------------------- 1 | import io 2 | import json 3 | import os 4 | import logging 5 | import urllib3 6 | import boto3 7 | from botocore.exceptions import ClientError 8 | 9 | sm = boto3.client('sagemaker') 10 | logger = logging.getLogger() 11 | logger.setLevel(logging.INFO) 12 | http = urllib3.PoolManager() 13 | group_name = "{}PackageGroup".format(os.environ['MODEL_NAME'].capitalize()) 14 | SUCCESS = "SUCCESS" 15 | FAILED = "FAILED" 16 | 17 | 18 | def send(event, context, responseStatus, responseData, physicalResourceId=None, noEcho=False, error=None): 19 | responseUrl = event['ResponseURL'] 20 | logger.info(responseUrl) 21 | responseBody = {} 22 | responseBody['Status'] = responseStatus 23 | if error is None: 24 | responseBody['Reason'] = 'See the details in CloudWatch Log Stream: ' + context.log_stream_name + ' LogGroup: ' + context.log_group_name 25 | else: 26 | responseBody['Reason'] = error 27 | responseBody['PhysicalResourceId'] = physicalResourceId or context.log_stream_name 28 | responseBody['StackId'] = event['StackId'] 29 | responseBody['RequestId'] = event['RequestId'] 30 | responseBody['LogicalResourceId'] = event['LogicalResourceId'] 31 | responseBody['NoEcho'] = noEcho 32 | responseBody['Data'] = responseData 33 | json_responseBody = json.dumps(responseBody) 34 | logger.info("Response body:\n" + json_responseBody) 35 | headers = { 36 | 'content-type' : '', 37 | 'content-length' : str(len(json_responseBody)) 38 | } 39 | try: 40 | response = http.request('PUT',responseUrl,body=json_responseBody.encode('utf-8'),headers=headers) 41 | logger.info("Status code: " + response.reason) 42 | except Exception as e: 43 | logger.error("send(..) failed executing requests.put(..): " + str(e)) 44 | 45 | 46 | def create(event, context): 47 | logger.info("Creating model package group: {}".format(group_name)) 48 | try: 49 | response = sm.create_model_package_group( 50 | ModelPackageGroupName=group_name, 51 | ModelPackageGroupDescription='Model Package Group for Production Models.', 52 | Tags=[ 53 | { 54 | 'Key': 'Name', 55 | 'Value': group_name 56 | } 57 | ] 58 | ) 59 | except ClientError as e: 60 | error_message = e.response["Error"]["Message"] 61 | logger.error("Failed to create model package group: {}".format(error_message)) 62 | send(event, context, FAILED, {}, error=str(error_message)) 63 | package_arn = response['ModelPackageGroupArn'] 64 | send(event, context, SUCCESS, {'Arn': package_arn, 'Name': group_name}, physicalResourceId=package_arn) 65 | 66 | 67 | def update(event, context): 68 | logger.info("Received update event") 69 | send(event, context, SUCCESS, {}, physicalResourceId=event['PhysicalResourceId']) 70 | 71 | 72 | def delete(event, context): 73 | logger.info("Deleting model package group: {}".format(group_name)) 74 | try: 75 | # Get a list of the model package versions 76 | response = sm.list_model_packages( 77 | ModelPackageGroupName=group_name, 78 | ModelApprovalStatus="Approved", 79 | SortBy="CreationTime", 80 | MaxResults=100, 81 | ) 82 | # Delete model package versions 83 | for model_package in response["ModelPackageSummaryList"]: 84 | sm.delete_model_package(ModelPackageName=model_package['ModelPackageArn']) 85 | # Delete the package group 86 | sm.delete_model_package_group(ModelPackageGroupName=group_name) 87 | except ClientError as e: 88 | error_message = e.response["Error"]["Message"] 89 | logger.error("Failed to delete model package group: {}".format(error_message)) 90 | send(event, context, FAILED, {}, error=str(error_message)) 91 | send(event, context, SUCCESS, {}, physicalResourceId=event['PhysicalResourceId']) 92 | 93 | 94 | def handler(event, context): 95 | logger.debug("Boto3 Version: {}".format(boto3.__version__)) 96 | logger.debug("## Environment Variables ##") 97 | logger.debug(os.environ) 98 | logger.debug("## Event ##") 99 | logger.debug(event) 100 | if event['RequestType'] == 'Create': 101 | create(event, context) 102 | elif event['RequestType'] == 'Update': 103 | update(event, context) 104 | elif event['RequestType'] == 'Delete': 105 | delete(event, context) 106 | -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/pipeline/TrainingJobMonitor/lambda.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import io 3 | import os 4 | import logging 5 | from botocore.exceptions import ClientError 6 | 7 | sm = boto3.client('sagemaker') 8 | cw = boto3.client('events') 9 | cp = boto3.client('codepipeline') 10 | 11 | logger = logging.getLogger() 12 | logger.setLevel(logging.INFO) 13 | 14 | 15 | def handler(event, context): 16 | logger.debug("## Environment Variables ##") 17 | logger.debug(os.environ) 18 | logger.debug("## Event ##") 19 | logger.debug(event) 20 | pipeline_name = os.environ['PIPELINE_NAME'] 21 | model_name = os.environ['MODEL_NAME'] 22 | result = None 23 | token = None 24 | try: 25 | response = cp.get_pipeline_state(name=pipeline_name) 26 | for stageState in response['stageStates']: 27 | if stageState['stageName'] == 'TrainApproval': 28 | for actionState in stageState['actionStates']: 29 | if actionState['actionName'] == 'ApproveTrain': 30 | latestExecution = actionState['latestExecution'] 31 | executionId = stageState['latestExecution']['pipelineExecutionId'] 32 | if latestExecution['status'] != 'InProgress': 33 | raise(Exception("Train approval is not awaiting approval: {}".format(latestExecution['status']))) 34 | token = latestExecution['token'] 35 | if token is None: 36 | raise(Exception("Action token wasn't found. Aborting...")) 37 | response = sm.describe_training_job( 38 | TrainingJobName="mlops-{}-{}".format(model_name, executionId) 39 | ) 40 | status = response['TrainingJobStatus'] 41 | logger.info(status) 42 | if status == "Completed": 43 | result = { 44 | 'summary': 'Model trained successfully', 45 | 'status': 'Approved' 46 | } 47 | elif status == "InProgress": 48 | return "Training Job ({}) in progress".format(executionId) 49 | else: 50 | result = { 51 | 'summary': response['FailureReason'], 52 | 'status': 'Rejected' 53 | } 54 | except Exception as e: 55 | result = { 56 | 'summary': str(e), 57 | 'status': 'Rejected' 58 | } 59 | 60 | try: 61 | response = cp.put_approval_result( 62 | pipelineName=pipeline_name, 63 | stageName='TrainApproval', 64 | actionName='ApproveTrain', 65 | result=result,token=token 66 | ) 67 | except ClientError as e: 68 | error_message = e.response["Error"]["Message"] 69 | logger.error(error_message) 70 | raise Exception(error_message) 71 | 72 | try: 73 | response = cw.disable_rule(Name="training-job-monitor-{}".format(model_name)) 74 | except ClientError as e: 75 | error_message = e.response["Error"]["Message"] 76 | logger.error(error_message) 77 | raise Exception(error_message) 78 | 79 | return "Done!" -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/pipeline/TrainingLaunchJob/lambda.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import io 3 | import zipfile 4 | import json 5 | import os 6 | import logging 7 | 8 | s3 = boto3.client('s3') 9 | sm = boto3.client('sagemaker') 10 | cw = boto3.client('events') 11 | cp = boto3.client('codepipeline') 12 | logger = logging.getLogger() 13 | logger.setLevel(logging.INFO) 14 | 15 | 16 | def handler(event, context): 17 | logger.debug("## Environment Variables ##") 18 | logger.debug(os.environ) 19 | logger.debug("## Event ##") 20 | logger.debug(event) 21 | trainingJob = None 22 | pipeline_name = os.environ['PIPELINE_NAME'] 23 | model_name = os.environ['MODEL_NAME'] 24 | jobId = event['CodePipeline.job']['id'] 25 | accountId = event['CodePipeline.job']['accountId'] 26 | region = os.environ.get('AWS_REGION') 27 | pipeline_bucket = "mlops-{}-{}".format(region, accountId) 28 | try: 29 | response = cp.get_pipeline_state(name=pipeline_name) 30 | for stageState in response['stageStates']: 31 | if stageState['stageName'] == 'Train': 32 | for actionState in stageState['actionStates']: 33 | if actionState['actionName'] == 'TrainModel': 34 | executionId = stageState['latestExecution']['pipelineExecutionId'] 35 | logger.info("Start training job for 'jobid[{}]' and 'executionId[{}]'".format(jobId, executionId)) 36 | for inputArtifacts in event["CodePipeline.job"]["data"]["inputArtifacts"]: 37 | if inputArtifacts['name'] == 'ModelSourceOutput': 38 | s3Location = inputArtifacts['location']['s3Location'] 39 | zip_bytes = s3.get_object(Bucket=s3Location['bucketName'], Key=s3Location['objectKey'])['Body'].read() 40 | with zipfile.ZipFile(io.BytesIO(zip_bytes), "r") as z: 41 | trainingJob = json.loads(z.read('trainingjob.json').decode('ascii')) 42 | if trainingJob is None: 43 | raise(Exception("'trainingjob.json' not found")) 44 | trainingJob['TrainingJobName'] = "mlops-{}-{}".format(model_name, executionId) 45 | trainingJob['OutputDataConfig']['S3OutputPath'] = os.path.join('s3://', pipeline_bucket, executionId) 46 | trainingJob['InputDataConfig'][0]['DataSource']['S3DataSource']['S3Uri'] = os.path.join('s3://', pipeline_bucket, executionId, 'input/training') 47 | trainingJob['Tags'].append({'Key': 'jobid', 'Value': jobId}) 48 | logger.info(trainingJob) 49 | sm.create_training_job(**trainingJob) 50 | cw.enable_rule(Name="training-job-monitor-{}".format(model_name)) 51 | cp.put_job_success_result(jobId=jobId) 52 | except Exception as e: 53 | logger.error(e) 54 | response = cp.put_job_failure_result( 55 | jobId=jobId, 56 | failureDetails={ 57 | 'type': 'ConfigurationError', 58 | 'message': str(e), 59 | 'externalExecutionId': context.aws_request_id 60 | } 61 | ) 62 | 63 | return 'Done' -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/tests/system_test/assets/evaluateEndpoint/lambda.py: -------------------------------------------------------------------------------- 1 | import os 2 | import io 3 | import json 4 | import logging 5 | import boto3 6 | import time 7 | import botocore 8 | import numpy as np 9 | import pandas as pd 10 | from sklearn import preprocessing 11 | from sklearn.metrics import mean_squared_error 12 | from botocore.exceptions import ClientError 13 | 14 | logger = logging.getLogger() 15 | logger.setLevel(logging.INFO) 16 | s3 = boto3.client("s3") 17 | sm_client = boto3.client("sagemaker-runtime") 18 | 19 | 20 | def evaluate_model(bucket, key, endpoint_name): 21 | """ 22 | Description: 23 | ------------ 24 | Executes model predictions on the testing dataset. 25 | 26 | :bucket: (str) Pipeline S3 Bucket. 27 | :key: (str) Path to "testing" dataset. 28 | :endpoint_name: (str) Name of the 'Dev' endpoint to test. 29 | 30 | :returns: Lists of ground truth, prediction labels and response times. 31 | 32 | """ 33 | column_names = ["rings", "length", "diameter", "height", "whole weight", "shucked weight", 34 | "viscera weight", "shell weight", "sex_F", "sex_I", "sex_M"] 35 | response_times = [] 36 | predictions = [] 37 | y_test = [] 38 | obj = s3.get_object(Bucket=bucket, Key=key) 39 | test_df = pd.read_csv(io.BytesIO(obj['Body'].read()), names=column_names) 40 | y = test_df['rings'].to_numpy() 41 | X = test_df.drop(['rings'], axis=1).to_numpy() 42 | X = preprocessing.normalize(X) 43 | 44 | # Cycle through each row of the data to get a prediction 45 | for row in range(len(X)): 46 | payload = ",".join(map(str, X[row])) 47 | elapsed_time = time.time() 48 | try: 49 | response = sm_client.invoke_endpoint( 50 | EndpointName=endpoint_name, 51 | ContentType = "text/csv", 52 | Body=payload 53 | ) 54 | except ClientError as e: 55 | error_message = e.response["Error"]["Message"] 56 | logger.error(error_message) 57 | raise Exception(error_message) 58 | response_times.append(time.time() - elapsed_time) 59 | result = np.asarray(response['Body'].read().decode('utf-8').rstrip('\n')) 60 | predictions.append(float(result)) 61 | y_test.append(float(y[row])) 62 | 63 | return y_test, predictions, response_times 64 | 65 | 66 | def handler(event, context): 67 | logger.debug("## Environment Variables ##") 68 | logger.debug(os.environ) 69 | logger.debug("## Event ##") 70 | logger.debug(event) 71 | 72 | # Ensure variables passed from Model Evaluation Step 73 | if ("Bucket" in event): 74 | bucket = event["Bucket"] 75 | else: 76 | raise KeyError("S3 'Bucket' not found in Lambda event!") 77 | if ("Key" in event): 78 | key = event["Key"] 79 | else: 80 | raise KeyError("S3 'Key' not found in Lambda event!") 81 | if ("Output_Key" in event): 82 | output_key = event["Output_Key"] 83 | else: 84 | raise KeyError("S3 'Output_Uri' not found in Lambda event!") 85 | if ("Endpoint_Name" in event): 86 | endpoint_name = event["Endpoint_Name"] 87 | else: 88 | raise KeyError("'SageMaker Endpoint Name' not found in Lambda event!") 89 | 90 | # Get the evaluation results from SageMaker hosted model 91 | logger.info("Evaluating SageMaker Hosted Model ...") 92 | y, y_pred, times = evaluate_model(bucket, key, endpoint_name) 93 | 94 | # Calculate the metrics 95 | mse = mean_squared_error(y, y_pred) 96 | rmse = mean_squared_error(y, y_pred, squared=False) 97 | std = np.std(np.array(y) - np.array(y_pred)) 98 | 99 | # Save Metrics to S3 for Model Package 100 | logger.info("Root Mean Square Error: {}".format(rmse)) 101 | logger.info("Average Endpoint Response Time: {:.2f}s".format(np.mean(times))) 102 | report_dict = { 103 | "regression_metrics": { 104 | "mse": { 105 | "value": mse, 106 | "standard_deviation": std 107 | }, 108 | }, 109 | } 110 | try: 111 | s3.put_object( 112 | Bucket=bucket, 113 | Key="{}/{}".format(output_key, "evaluation.json"), 114 | Body=json.dumps(report_dict, indent=4) 115 | ) 116 | except ClientError as e: 117 | error_message = e.response["Error"]["Message"] 118 | logger.error(error_message) 119 | raise Exception(error_message) 120 | 121 | # Return results 122 | logger.info("Done!") 123 | return { 124 | "statusCode": 200, 125 | "Result": rmse, 126 | "AvgResponseTime": "{:.2f} seconds".format(np.mean(times)) 127 | } -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/tests/system_test/assets/registerModel/lambda.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import logging 4 | import boto3 5 | from botocore.exceptions import ClientError 6 | 7 | 8 | sm = boto3.client('sagemaker') 9 | logger = logging.getLogger() 10 | logger.setLevel(logging.INFO) 11 | 12 | def handler(event, context): 13 | logger.debug("Boto3 Version: {}".format(boto3.__version__)) 14 | logger.debug("## Environment Variables ##") 15 | logger.debug(os.environ) 16 | logger.debug("## Event ##") 17 | logger.debug(event) 18 | 19 | # Ensure variables passed from Model Evaluation Step 20 | if ("Model_Name" in event): 21 | model_name = event["Model_Name"] 22 | else: 23 | raise KeyError("'Model_Name' not found in Lambda event!") 24 | if ("Group_Name" in event): 25 | group_name = event['Group_Name'] 26 | else: 27 | raise KeyError("'Group_Name' not found in Lambda event!") 28 | if ("Model_Uri" in event): 29 | model_uri = event['Model_Uri'] 30 | else: 31 | raise KeyError("'Model_Uri' not found in Lambda event!") 32 | if ("Image_Uri" in event): 33 | image_uri = event['Image_Uri'] 34 | else: 35 | raise KeyError("'Image_Uri' not found in Lambda event!") 36 | if ("Job_Id" in event): 37 | job_id = event['Job_Id'] 38 | else: 39 | raise KeyError("'Job_Id' not found in Lambda event!") 40 | if ("Evaluation_Uri" in event): 41 | evaluation_uri = event['Evaluation_Uri'] 42 | else: 43 | raise KeyError("'Evluation_Uri' not found in Lambda event!") 44 | 45 | # Create request payload 46 | request = { 47 | "InferenceSpecification": { 48 | "Containers": [ 49 | { 50 | "Image": image_uri, 51 | "ModelDataUrl": model_uri 52 | } 53 | ], 54 | "SupportedContentTypes": [ 55 | "text/csv" 56 | ], 57 | "SupportedRealtimeInferenceInstanceTypes": [ 58 | "ml.t2.large", 59 | "ml.c5.large", 60 | "ml.c5.xlarge" 61 | ], 62 | "SupportedResponseMIMETypes": [ 63 | "text/csv" 64 | ], 65 | "SupportedTransformInstanceTypes": [ 66 | "ml.c5.xlarge" 67 | ] 68 | }, 69 | "MetadataProperties": { 70 | "ProjectId": str(job_id) 71 | }, 72 | "ModelApprovalStatus": "Approved", 73 | "ModelMetrics": { 74 | "ModelQuality": { 75 | "Statistics": { 76 | "ContentType": "application/json", 77 | "S3Uri": evaluation_uri 78 | } 79 | } 80 | }, 81 | "ModelPackageDescription": "Abalone Production Model", 82 | "ModelPackageGroupName": group_name 83 | } 84 | 85 | # Create the Model Package 86 | try: 87 | response = sm.create_model_package(**request) 88 | except ClientError as e: 89 | error_message = e.response["Error"]["Message"] 90 | logger.error(error_message) 91 | raise Exception(error_message) 92 | 93 | # Return results 94 | logger.info("Done!") 95 | return { 96 | "statusCode": 200, 97 | "PackageArn": response['ModelPackageArn'] 98 | } 99 | -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/tests/system_test/assets/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.19.1 2 | pandas==1.1 3 | scikit-learn==0.23 4 | -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/tests/system_test/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | env: 3 | variables: 4 | THRESHOLD: 5 | phases: 6 | install: 7 | runtime-versions: 8 | python: 3.8 9 | commands: 10 | - printenv 11 | - echo "Updating build environment ..." 12 | - python -m pip install --upgrade --force-reinstall stepfunctions==2.1.0 aws-sam-cli==1.24.1 sagemaker==2.45.0 protobuf~=3.19.0 13 | - echo "Initializing Docker daemon ..." 14 | - nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 --storage-driver=overlay& 15 | - timeout 15 sh -c "until docker info; do echo .; sleep 1; done" 16 | pre_build: 17 | commands: 18 | - echo Pre-Build started on `date` 19 | - echo "Building workflow resources ..." 20 | - | 21 | sh -c """ 22 | cd assets 23 | sam build -b deploy/ --use-container \ 24 | --manifest requirements.txt \ 25 | --template-file workflow-resources.yml 26 | """ 27 | build: 28 | commands: 29 | - echo Build started on `date` 30 | - echo "Deploying workflow resources ..." 31 | - | 32 | sh -c """ 33 | cd assets 34 | sam deploy --stack-name ${WORKFLOW_NAME} \ 35 | --template-file deploy/template.yaml \ 36 | --parameter-overrides ModelName=${MODEL_NAME} \ 37 | --capabilities CAPABILITY_NAMED_IAM \ 38 | --s3-bucket ${PIPELINE_BUCKET} \ 39 | --s3-prefix ${PIPELINE_NAME}/artifacts \ 40 | --no-fail-on-empty-changeset \ 41 | --no-confirm-changeset 42 | """ 43 | - echo "Creating System Test Workflow ..." 44 | - | 45 | sh -c """ 46 | python build.py --pipeline-name=$PIPELINE_NAME \ 47 | --image-repo-name=$IMAGE_REPO_NAME \ 48 | --image-tag=$IMAGE_TAG \ 49 | --model-name=$MODEL_NAME \ 50 | --model-package-group-name=$MODEL_GROUP \ 51 | --test-endpoint=$MODEL_NAME-dev-endpoint \ 52 | --pipeline-bucket=$PIPELINE_BUCKET 53 | """ 54 | post_build: 55 | commands: 56 | - echo Build completed on `date` 57 | artifacts: 58 | files: 59 | - input.json 60 | discard-paths: yes -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/tests/unit_test/app_test.py: -------------------------------------------------------------------------------- 1 | import json 2 | from urllib import request 3 | 4 | base_url='http://localhost:8080' 5 | headers={"Content-type": "text/csv"} 6 | 7 | # Toy request payload: `str()` 8 | payload = "6.550000000000000266e-01,5.200000000000000178e-01,1.900000000000000022e-01,\ 9 | 1.454499999999999904e+00,5.999999999999999778e-01,3.865000000000000102e-01,\ 10 | 3.829999999999999516e-01,1.000000000000000000e+00,0.000000000000000000e+00,0.000000000000000000e+00" 11 | 12 | # Encode payload to Bytes for `request() 13 | payload = payload.encode('utf-8') 14 | 15 | 16 | def predict(payload, headers): 17 | """ 18 | Description: 19 | ----------- 20 | Prediction request to the local API. 21 | 22 | :payload: (Bytes) Bytes encoded string of `abalone` features for a single observation. 23 | :headers: (dict) Dictionary specifying the request content type. 24 | """ 25 | req = request.Request("%s/invocations" % base_url, data=payload, headers=headers) 26 | resp = request.urlopen(req) 27 | print("API Response code: %d \nPrediction Response: %s" % (resp.getcode(), resp.read().decode('utf-8'))) 28 | 29 | 30 | def main(): 31 | for x in range(1, 4): 32 | print("\nStarting Test {} ...".format(x)) 33 | resp = request.urlopen("%s/ping" % base_url) 34 | print("\nPING Response code: %d" % resp.getcode()) 35 | predict(payload, headers) 36 | 37 | 38 | if __name__ == "__main__": 39 | main() -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/tests/unit_test/input/config/hyperparameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "epochs": "2", 3 | "layers": "2", 4 | "dense_layer": "64", 5 | "batch_size": "8" 6 | } -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/utils/c9_resize.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Specify the desired volume size in GiB as a command-line argument. If not specified, default to 20 GiB. 4 | VOLUME_SIZE=${1:-20} 5 | 6 | # Get the ID of the environment host Amazon EC2 instance. 7 | INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data//instance-id) 8 | 9 | # Get the ID of the Amazon EBS volume associated with the instance. 10 | VOLUME_ID=$(aws ec2 describe-instances \ 11 | --instance-id $INSTANCE_ID \ 12 | --query "Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId" \ 13 | --output text) 14 | 15 | # Resize the EBS volume. 16 | aws ec2 modify-volume --volume-id $VOLUME_ID --size $VOLUME_SIZE 17 | 18 | # Wait for the resize to finish. 19 | while [ \ 20 | "$(aws ec2 describe-volumes-modifications \ 21 | --volume-id $VOLUME_ID \ 22 | --filters Name=modification-state,Values="optimizing","completed" \ 23 | --query "length(VolumesModifications)"\ 24 | --output text)" != "1" ]; do 25 | sleep 1 26 | done 27 | 28 | if [ $(readlink -f /dev/xvda) = "/dev/xvda" ] 29 | then 30 | # Rewrite the partition table so that the partition takes up all the space that it can. 31 | sudo growpart /dev/xvda 1 32 | 33 | # Expand the size of the file system. 34 | sudo resize2fs /dev/xvda1 35 | 36 | else 37 | # Rewrite the partition table so that the partition takes up all the space that it can. 38 | sudo growpart /dev/nvme0n1 1 39 | 40 | # Expand the size of the file system. 41 | # sudo resize2fs /dev/nvme0n1p1 #(Amazon Linux 1) 42 | sudo xfs_growfs /dev/nvme0n1p1 #(Amazon Linux 2) 43 | fi -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/utils/load_sim.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import boto3 3 | import json 4 | import time 5 | import math 6 | import os 7 | from botocore.exceptions import ClientError 8 | from multiprocessing.pool import ThreadPool 9 | from datetime import datetime 10 | import numpy as np 11 | import pandas as pd 12 | from botocore.config import Config 13 | 14 | # Global Variables 15 | config = Config(retries = {'max_attempts': 10, 'mode': 'adaptive'}) 16 | sagemaker = boto3.client("sagemaker-runtime", config=config) 17 | codepipeline = boto3.client('codepipeline') 18 | s3 = boto3.client('s3') 19 | pipeline_name = 'abalone-pipeline' 20 | endpoint_name = 'abalone-prd-endpoint' 21 | pipeline_bucket = '' 22 | obj = 'input/testing/test.csv' 23 | 24 | 25 | # Helpr functions 26 | def get_env_jobid(env='prd'): 27 | """ 28 | Description: 29 | ------------ 30 | Function to return the most up to date `pipelineExecitionId` based on the 31 | environment input. 32 | 33 | :env: (str) Specifies either 'dev' or 'prd' environments. 34 | 35 | :returns: Latest CodePipeline Execution ID 36 | """ 37 | try: 38 | response = codepipeline.get_pipeline_state(name=pipeline_name) 39 | for stage in response['stageStates']: 40 | if stage['stageName'] == 'Deploy%s' % env.capitalize(): 41 | for action in stage['actionStates']: 42 | if action['actionName'] == 'Deploy%sModel' % env.capitalize(): 43 | return stage['latestExecution']['pipelineExecutionId'] 44 | except ClientError as e: 45 | error_message = e.response["Error"]["Message"] 46 | print(error_message) 47 | raise Exception(error_message) 48 | 49 | 50 | # Invoke SageMaker endpoint 51 | def predict(payload): 52 | """ 53 | Description: 54 | ------------ 55 | Invokes SageMaker endpoint with NumPy payload. 56 | 57 | :payload: (NumPy Array) Abalone datset observations as a NumPy Array. 58 | 59 | :returns: response (str) Python list containing the predicted age. 60 | """ 61 | payload = ",".join(map(str, payload)) 62 | try: 63 | request = sagemaker.invoke_endpoint( 64 | EndpointName=endpoint_name, 65 | ContentType = "text/csv", 66 | Body=payload 67 | ) 68 | except ClientError as e: 69 | error_message = e.request["Error"]["Message"] 70 | print(error_message) 71 | raise Exception(error_message) 72 | return request['Body'].read().decode('utf-8').rstrip('\n') 73 | 74 | # Execute inference test 75 | def run_test(max_threads, max_requests, dataset): 76 | """ 77 | Description: 78 | ------------ 79 | Executes the inference testing. 80 | 81 | :max_threads: (int) Integer specifying the number of concurrent threads. 82 | :max_requests: (int) Integer specifying the the number of inference requests per thread. 83 | :dataset: (NumPy) Array of test data. 84 | """ 85 | start_time = datetime.now() 86 | num_batches = math.ceil(max_requests / len(dataset)) 87 | requests = [] 88 | for i in range(num_batches): 89 | batch = dataset.copy() 90 | np.random.shuffle(batch) 91 | requests += batch.tolist() 92 | pool = ThreadPool(max_threads) 93 | result = pool.map(predict, requests) 94 | pool.close() 95 | pool.join() 96 | elapsed_time = datetime.now() - start_time 97 | print("Time elapsed (hh:mm:ss.ms) {}".format(elapsed_time)) 98 | 99 | if __name__ == "__main__": 100 | # Get latest CodePipeline Execition ID 101 | job_id=get_env_jobid() 102 | 103 | # Download test dataset 104 | s3.download_file(pipeline_bucket, os.path.join(job_id, obj), 'test.csv') 105 | test_data = pd.read_csv('test.csv', header=None) 106 | test_data = test_data.drop(test_data.columns[0], axis=1) 107 | dataset = test_data.to_numpy() 108 | 109 | print("Starting test ...") 110 | run_test(150, 1000000, dataset) -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/mlops-tutorial/utils/repository_validation.py: -------------------------------------------------------------------------------- 1 | import os 2 | import boto3 3 | from botocore.exceptions import ClientError 4 | 5 | s3 = boto3.resource('s3') 6 | ecr = boto3.client('ecr') 7 | code_commit = boto3.client('codecommit') 8 | 9 | def check_bucket(bucket): 10 | """ 11 | Description: 12 | ----------- 13 | Checks to confirm that a S3 Bucket exists. 14 | 15 | Parameters: 16 | ---------- 17 | bucket: String 18 | String specifying the name of the S3 Bucket. 19 | """ 20 | try: 21 | s3.meta.client.head_bucket(Bucket=bucket) 22 | print("Bucket: {} [".format(bucket)+u'\u2714'+"]") 23 | except ClientError as e: 24 | print("Bucket: {} [X]".format(bucket)) 25 | print("Error Reason: \n{}\n".format(e)) 26 | if bucket == os.environ['DATA_BUCKET']: 27 | print("Please check 'Training Data S3 Bucket' is not available.") 28 | else: 29 | print("Please check ' ETL Data Bucket' is not available.") 30 | 31 | 32 | def check_ecr(repo): 33 | """ 34 | Description: 35 | ----------- 36 | Checks to confirm that an Elastic Container Registry exists. 37 | 38 | Parameters: 39 | ---------- 40 | repo: String 41 | String specifying the name of the Elastic Container Registry. 42 | """ 43 | try: 44 | ecr.describe_repositories(repositoryNames=[repo]) 45 | print("Elastic Container Repository: {} [".format(repo)+u'\u2714'+"]") 46 | except ClientError as e: 47 | print("Elastic Container Repository: {} [X]".format(repo)) 48 | print("Error Reason: \n{}\n".format(e)) 49 | print("Please check 'ECR Repository' is not available") 50 | 51 | def check_codecommit(repo): 52 | """ 53 | Description: 54 | ----------- 55 | Checks to confirm that a CodeCommit Repository exists. 56 | 57 | Parameters: 58 | ---------- 59 | repo: String 60 | String specifying the name of the CodeCommit Repository 61 | """ 62 | try: 63 | code_commit.get_repository(repositoryName=repo) 64 | print("CodeCommit Repository: {} [".format(repo)+u'\u2714'+"]") 65 | except ClientError as e: 66 | print("CodeCommit Repository: {} [X]".format(repo)) 67 | print("Error Reason: \n{}\n".format(e)) 68 | print("Please check 'CodeCommit Repository' is not available") 69 | 70 | def main(): 71 | """ 72 | Description: 73 | ----------- 74 | Checks that the pipeline resources have been created correctly. 75 | """ 76 | # Run alidation checks 77 | print("Validating Data Repositories Exist ...\n") 78 | check_bucket(os.environ['DATA_BUCKET']) 79 | check_bucket(os.environ['PIPELINE_BUCKET']) 80 | check_ecr('abalone') 81 | check_codecommit('mlops') 82 | 83 | 84 | if __name__ == "__main__": 85 | main() -------------------------------------------------------------------------------- /Section-16-mlops-pipeline/preprocess-churn.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | import numpy as np 4 | import pandas as pd 5 | import datetime as dt 6 | if __name__ == "__main__": 7 | base_dir = "/opt/ml/processing" 8 | #Read Data 9 | df = pd.read_csv( 10 | f"{base_dir}/input/storedata_total.csv" 11 | ) 12 | # convert created column to datetime 13 | df["created"] = pd.to_datetime(df["created"]) 14 | #Convert firstorder and lastorder to datetime datatype 15 | df["firstorder"] = pd.to_datetime(df["firstorder"],errors='coerce') 16 | df["lastorder"] = pd.to_datetime(df["lastorder"],errors='coerce') 17 | #Drop Rows with Null Values 18 | df = df.dropna() 19 | #Create column which gives the days between the last order and the first order 20 | df['first_last_days_diff'] = (df['lastorder'] - df['firstorder']).dt.days 21 | #Create column which gives the days between the customer record was created and the first order 22 | df['created_first_days_diff'] = (df['created'] - df['firstorder']).dt.days 23 | #Drop columns 24 | df.drop(['custid', 'created','firstorder','lastorder'], axis=1, inplace=True) 25 | #Apply one hot encoding on favday and city columns 26 | df = pd.get_dummies(df, prefix=['favday', 'city'], columns=['favday', 'city']) 27 | # Split into train, validation and test datasets 28 | y = df.pop("retained") 29 | X_pre = df 30 | y_pre = y.to_numpy().reshape(len(y), 1) 31 | X = np.concatenate((y_pre, X_pre), axis=1) 32 | np.random.shuffle(X) 33 | # Split in Train, Test and Validation Datasets 34 | train, validation, test = np.split(X, [int(.7*len(X)), int(.85*len(X))]) 35 | train_rows = np.shape(train)[0] 36 | validation_rows = np.shape(validation)[0] 37 | test_rows = np.shape(test)[0] 38 | train = pd.DataFrame(train) 39 | test = pd.DataFrame(test) 40 | validation = pd.DataFrame(validation) 41 | # Convert the label column to integer 42 | train[0] = train[0].astype(int) 43 | test[0] = test[0].astype(int) 44 | validation[0] = validation[0].astype(int) 45 | # Save the Dataframes as csv files 46 | train.to_csv(f"{base_dir}/train/train.csv", header=False, index=False) 47 | validation.to_csv(f"{base_dir}/validation/validation.csv", header=False, index=False) 48 | test.to_csv(f"{base_dir}/test/test.csv", header=False, index=False) -------------------------------------------------------------------------------- /Section-4-Getting-Started-with-AWS/NLP.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-4-Getting-Started-with-AWS/NLP.png -------------------------------------------------------------------------------- /Section-4-Getting-Started-with-AWS/README.md: -------------------------------------------------------------------------------- 1 | # Section 4: Getting Started with AWS 2 | ## Preparation 3 | ### Set a unique suffix to use for the S3 bucket name: 4 | 5 | --- 6 | 7 | # Activity 2 : Create S3 Bucket using CLI 8 | 9 | ``` 10 | RANDOM_ID=$(aws secretsmanager get-random-password \ 11 | --exclude-punctuation --exclude-uppercase \ 12 | --password-length 6 --require-each-included-type \ 13 | --output text \ 14 | --query RandomPassword) 15 | 16 | ``` 17 | 18 | ### Create S3 bucket: 19 | 20 | ``` aws s3api create-bucket --bucket awsml-$RANDOM_ID``` 21 | 22 | Upload the file to AWS S3 Bucket 23 | ```aws s3 cp NLP.png s3://awsml-$RANDOM_ID ``` 24 | 25 | Display Storage class 26 | ```aws s3api list-objects-v2 --bucket awsml-$RANDOM_ID ``` 27 | 28 | ## Clean up 29 | ### Delete the file you copied to your S3 bucket: 30 | 31 | ```aws s3 rm s3://awsml-$RANDOM_ID/NLP.png``` 32 | 33 | ### Delete the S3 bucket: 34 | 35 | ```aws s3api delete-bucket --bucket awsml-$RANDOM_ID ``` 36 | 37 | 38 | --- 39 | # Activity 3 : Versioning hands On 40 | 41 | - Follow the steps as above to create S3 bucket, and practice versioning 42 | - Upload the same file (without versioning enabled) : It will overwrite 43 | - Perform Version enablement & upload again 44 | - delete the new version to get the previous version of file 45 | 46 | --- 47 | 48 | -------------------------------------------------------------------------------- /Section-5-Linux-Basics/README.md: -------------------------------------------------------------------------------- 1 | # Linux Basics 2 | 3 | ## Directory Operation: 4 | ``` 5 | pwd --> displays present working directory 6 | mkdir mydir --> create a directory 7 | cd mydir --> change directory 8 | ls --> lists contents of current directory 9 | ls -l --> list contents of current directory with detailed output 10 | ll --> list contents of current directory with detailed output 11 | cd 12 | cd .. --> to come out of a directory 13 | rm -r testdir --> to remove a directory 14 | `~` --> Home directory 15 | mkdir -p folder/subfolder/subfolder2 --> create parent and child directory 16 | ``` 17 | ## File operations: 18 | 19 | touch test.txt --> creates a empty file 20 | ls -l --> lists the contents in current directory 21 | echo "Print Output" 22 | echo "This is a text" > test.txt 23 | cat test.txt --> writes the content of file into terminal & exits 24 | more test.txt --> writes the content of file on terminal page by page ( to move to next page need to hit space bar on keyboard ) 25 | less test.txt --> open the file on terminal & can be read line by line (use arrows to scroll up & down) 26 | To comeout need to press 'q' on the key board 27 | vi test.txt --> Visual Editor 28 | This opens the file in read only mode 29 | To edit the file, press 'i' on keyboard for INSERT mode & then you can write anything 30 | once the text is entered, press 'ESC' on keyboard to return back on readonly mode 31 | press ':wq' on keyboard -- to save & come out of the file 32 | press ':q' on keyboard -- to come out of the file without saving 33 | press ':wq!' on keyboard -- to save forcefully & come out of the file 34 | press ':q!' on keyboard -- to come out of the file forcefully without saving 35 | rm test.txt --> To remove a file 36 | 37 | ## copy/rename/move operations: 38 | 39 | While performing these operations, we can always use absolute paths or relative paths accrodingly. 40 | These commands expects source & dest values 41 | 42 | cp A.txt B.txt -- makes a copy of A.txt names it to B.txt in current directory 43 | cp /home/A.txt /tmp/A.txt -- make a copy of A.txt to /tmp 44 | cp -r testdir/ newdir/ -- makes a copy of testdir & names it to newdir in current dirctory 45 | cp -r /home/testdir /tmp/testdir -- makes a copy of testdir to /tmp 46 | 47 | mv A.txt new.txt --- renames A.txt to new.txt in current path 48 | mv A.txt /tmp --- moves to A.txt to /tmp 49 | mv testdir newdir -- renames testdir to newdir in crrent path 50 | mv newdir /tmp --- moves newdir to /tmp path 51 | 52 | 53 | ## File/Directory permissions: 54 | Observe the output of ls: 55 | drwxrwxr-x 2 manifoldailearning manifoldailearning 4096 Nov 7 23:38 testdir 56 | -rw-rw-r-- 1 manifoldailearning manifoldailearning 126 Nov 7 23:37 abc.txt 57 | -rw-rw-r-- 1 manifoldailearning manifoldailearning 126 Nov 7 23:38 one.txt 58 | 59 | drwxrwxr-x or -rw-rw-r-- --> Read/Write/Execute Permission for a file or a directory in linux 60 | (1st value)manifoldailearning - owner of the file/directory 61 | (2nd value)manifoldailearning - group who owns file/directory 62 | how to update the owner & group for a file/dir 63 | 64 | chown ( change owner ) is the command to update the owners of dir/file 65 | note: note that you need to have previliges to update the permissions for a file/directory 66 | 67 | Syntax: chown owner:group filename/dirname 68 | chown murthy:development filename/dirname --> updates the owner & group of the file/directory to murthy & development 69 | 70 | ## how to update the read,write & execute permissions for a file/dir 71 | 72 | How to decode the Read/Write/Execute Permission terminology: 73 | -/d ---> denotes if it is a file or directory ( - means file / d means directory ) 74 | rwx ---> means read,write & execute permissions for super user ( root ) 75 | rw- ---> means read,write permissions for the owner of the file ( ex: manifoldailearning as above ) 76 | r-- ---> means read only permissions for others ( whoever login to the machine ) 77 | 78 | 79 | r -- Permission to read the file. 80 | w -- Permission to write (or delete) the file. 81 | x -- Permission to execute the file, or, in the case of a directory, search it. 82 | 83 | chmod ( change mode ) is the command to update the read/write/execute permissions for a file/directory 84 | r = 4, w = 2, x = 1 85 | 86 | - | rwx | rwx | rwx 87 | - | 421 | 421 | 421 88 | 89 | to update the permissions we can sum 421 90 | if super user needs to have read,write & exeute give 7 91 | if the owner need to read & write give 6 92 | if other need to have only read give 4 93 | 94 | 95 | chmod 777 file/dir -- rwx for root, rwx for owner, rwx for others 96 | chmod 764 file/dir -- rwx for root, rw for owner, r for others 97 | chmod 755 fire/dir -- rwx for root, rw for owner, rw for others 98 | chmod 400 firl/dir -- r for root, for owner/other no permissions # common reason u would see while ec2 ssh keys -------------------------------------------------------------------------------- /Section-6-CodeCommit/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-6-CodeCommit/.DS_Store -------------------------------------------------------------------------------- /Section-6-CodeCommit/CI CD/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-6-CodeCommit/CI CD/.DS_Store -------------------------------------------------------------------------------- /Section-6-CodeCommit/CI CD/CODEDEPLOY.md: -------------------------------------------------------------------------------- 1 | 2 | # Installing the CodeDeploy agent on EC2 3 | # https://docs.aws.amazon.com/codedeploy/latest/userguide/codedeploy-agent-operations-install-ubuntu.html 4 | ``` 5 | sudo yum update -y 6 | sudo yum install ruby 7 | sudo yum install wget 8 | cd /home/ec2-user 9 | wget https://aws-codedeploy-ap-south-1.s3.ap-south-1.amazonaws.com/latest/install 10 | chmod +x ./install 11 | sudo ./install auto 12 | sudo service codedeploy-agent status 13 | 14 | #If error 15 | sudo service codedeploy-agent start 16 | sudo service codedeploy-agent status 17 | ``` 18 | 19 | 20 | # create a bucket and enable versioning 21 | # https://docs.aws.amazon.com/cli/latest/reference/s3/mb.html 22 | # https://docs.aws.amazon.com/cli/latest/reference/s3api/put-bucket-versioning.html 23 | ``` 24 | aws s3 mb s3://aws-devops-manifold --region ap-south-1 --profile murthy 25 | aws s3api put-bucket-versioning --bucket aws-devops-manifold --versioning-configuration Status=Enabled --region ap-south-1 --profile murthy 26 | ``` 27 | 28 | # deploy the files into S3 29 | # https://docs.aws.amazon.com/cli/latest/reference/deploy/push.html 30 | ``` 31 | aws deploy push --application-name MyDeployDemo --s3-location s3://aws-devops-manifold/MyDeployDemo-test/app.zip --ignore-hidden-files --region ap-south-1 --profile murthy 32 | ``` -------------------------------------------------------------------------------- /Section-6-CodeCommit/CI CD/HELP.md: -------------------------------------------------------------------------------- 1 | # Help for BuildSpec 2 | 3 | https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html#build-spec-ref-example 4 | 5 | # Appspec file 6 | 7 | https://docs.aws.amazon.com/codedeploy/latest/userguide/reference-appspec-file-example.html 8 | 9 | # Help for Metadata 10 | https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-categories.html 11 | 12 | # AWS CLI Profile 13 | 14 | **Create Profile** 15 | aws configure --profile profile_name 16 | *you need access key for cli* 17 | ~/.aws/credentials - contains stored creds 18 | ~/.aws/config - contains stored profile 19 | 20 | *If Getting Error as Unable to commit* 21 | - Goto Credential Manager in Windows & Remove the Existing credentials for codecommit 22 | - Extra information - https://stackoverflow.com/questions/34517534/running-git-clone-against-aws-codecommits-gets-me-a-403-error 23 | 24 | 25 | 26 | 27 | 28 | *Display Profile Names:* 29 | aws configure list-profiles 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /Section-6-CodeCommit/CI CD/appspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.0 2 | os: linux 3 | files: 4 | - source: /index.html 5 | destination: /var/www/html/ 6 | hooks: 7 | ApplicationStop: 8 | - location: scripts/stop_server.sh 9 | timeout: 300 10 | runas: root 11 | 12 | BeforeInstall: 13 | - location: scripts/install_dependencies.sh 14 | timeout: 300 15 | runas: root 16 | 17 | AfterInstall: 18 | - location: scripts/after_install.sh 19 | timeout: 300 20 | runas: root 21 | 22 | ApplicationStart: 23 | - location: scripts/start_server.sh 24 | timeout: 300 25 | runas: root 26 | 27 | ValidateService: 28 | - location: scripts/validate_service.sh 29 | timeout: 300 30 | 31 | -------------------------------------------------------------------------------- /Section-6-CodeCommit/CI CD/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | env: 4 | variables: 5 | MY_AWS_REGION: "ap-south-1" 6 | 7 | 8 | phases: 9 | install: 10 | runtime-versions: 11 | nodejs: 12 12 | commands: 13 | - echo "Running on $MY_AWS_REGION" 14 | - echo "Print Environment Variables" 15 | - printenv 16 | - echo "Start the installation process" 17 | - echo "Installation Process Completed" 18 | pre_build: 19 | commands: 20 | - echo "This is the start of Pre-Build phase" 21 | - echo "Pre-Build phase is now completed" 22 | build: 23 | commands: 24 | - echo "we are currently on build phase" 25 | - echo "Build Activity is completed" 26 | - echo "Run Basic check whether Website is ready for use" 27 | - grep -Fq "AWS Certified DevOps Engineer" index.html 28 | - echo "Test was successful" 29 | - echo "proceed with next steps" 30 | post_build: 31 | commands: 32 | - echo "We are currently on post_build phase" 33 | - echo "post_build phase activity is completed. Moving next to Artifacts" 34 | 35 | artifacts: 36 | files: 37 | - '**/*' -------------------------------------------------------------------------------- /Section-6-CodeCommit/CI CD/code.py: -------------------------------------------------------------------------------- 1 | def add(x,y): 2 | return x+y 3 | -------------------------------------------------------------------------------- /Section-6-CodeCommit/CI CD/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Page Title 5 | 6 | 7 | 112 | 113 | 114 | 115 |
116 |

Manifold AI Learning

117 |

AWS Certified DevOps Engineer – Professional exam (DOP-C01)

118 |

Congraluations you are on Track

119 |
120 | 121 |
122 | Homepage 123 | Youtube 124 | Udemy 125 | Facebook 126 |
127 | 128 |
129 |
130 |

About Us

131 |

Manifold AI Learning ® is an online Academy with the goal to empower the students with the knowledge and skills that can be directly applied to solving the Real world problems in Data Science, Machine Learning and Artificial intelligence.

132 |

Our Courses

133 |

Join us on Youtube and start learning for free

134 |
Youtube Hands-On ML Series

135 |
Youtube Deep Learning Series

136 |
Deep Learning Tips and Tricks

137 |
Python for Data Science

138 |
Deep Learning with Pytorch

139 |
140 |
141 |

TITLE HEADING

142 |
Title description, Dec 7, 2017
143 |
Image
144 |

Some text..

145 |

Sunt in culpa qui officia deserunt mollit anim id est laborum consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco.

146 |
147 |

What will you get!!

148 |
Hands-On courses
149 |
Image
150 |

Some text..

151 |

Sunt in culpa qui officia deserunt mollit anim id est laborum consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco.

152 |
153 |
154 | 155 |
156 |

For more information, reach us at: support@manifoldailearning.in

157 |
158 | 159 | 160 | 161 | -------------------------------------------------------------------------------- /Section-6-CodeCommit/CI CD/scripts/after_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | EC2_INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) 3 | EC2_INSTANCE_TYPE=$(curl -s http://169.254.169.254/latest/meta-data/instance-type) 4 | EC2_AZ=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) 5 | sed -i "s/you are on Track/This is currently runnining on $EC2_INSANCE_TYPE with id - $EC2_INSTANCE_ID on AM - $EC2_AZ/g" /var/www/html/index.html 6 | chmod 664 /var/www/html/index.html -------------------------------------------------------------------------------- /Section-6-CodeCommit/CI CD/scripts/install_dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | yum install -y httpd 3 | 4 | -------------------------------------------------------------------------------- /Section-6-CodeCommit/CI CD/scripts/start_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | service httpd start 3 | 4 | -------------------------------------------------------------------------------- /Section-6-CodeCommit/CI CD/scripts/stop_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | isExistApp = `pgrep httpd` 3 | if [[ -n $isExistApp ]]; then 4 | service httpd stop 5 | fi 6 | 7 | -------------------------------------------------------------------------------- /Section-6-CodeCommit/CI CD/scripts/validate_service.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # verify we can access our webpage successfully 4 | curl -v --silent localhost:80 2>&1 | grep AWS -------------------------------------------------------------------------------- /Section-6-CodeCommit/deny-master-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Deny", 6 | "Action": [ 7 | "codecommit:GitPush", 8 | "codecommit:DeleteBranch", 9 | "codecommit:PutFile", 10 | "codecommit:MergeBranchesByFastForward", 11 | "codecommit:MergeBranchesBySquash", 12 | "codecommit:MergeBranchesByThreeWay", 13 | "codecommit:MergePullRequestByFastForward", 14 | "codecommit:MergePullRequestBySquash", 15 | "codecommit:MergePullRequestByThreeWay" 16 | ], 17 | "Resource": "arn:aws:codecommit:*:*:*", 18 | "Condition": { 19 | "StringEqualsIfExists": { 20 | "codecommit:References": [ 21 | "refs/heads/master" 22 | ] 23 | }, 24 | "Null": { 25 | "codecommit:References": "false" 26 | } 27 | } 28 | } 29 | ] 30 | } -------------------------------------------------------------------------------- /Section-6.1-Github/file1.py: -------------------------------------------------------------------------------- 1 | print("hello world v8") 2 | -------------------------------------------------------------------------------- /Section-6.1-Github/git4mlops.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-6.1-Github/git4mlops.png -------------------------------------------------------------------------------- /Section-6.1-Github/image-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-6.1-Github/image-1.png -------------------------------------------------------------------------------- /Section-6.1-Github/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-6.1-Github/image.png -------------------------------------------------------------------------------- /Section-7-YAML-tutorial/yaml-basics.yml: -------------------------------------------------------------------------------- 1 | # What is YAML mean? 2 | YAML: 3 | - Y: YAML 4 | - A: Ain't 5 | - M: Markup 6 | - L: Language 7 | 8 | #Lets Start writing YAML 9 | my-course: "AWS DevOps Professional Certification" 10 | version: 1.0 11 | price: 299 12 | is_public: true 13 | release_date: 2022-01-10 14 | pre_enroll: null 15 | category: 16 | - AWS 17 | - DevOps 18 | - AWS Certification 19 | course_dev: ["murthy","kavya","nirnay"] 20 | dev_details: 21 | - name: "murthy" 22 | email: "support@manifoldailearning.in" 23 | role: "Author" 24 | - name: "kavya" 25 | email: "support@manifoldailearning.in" 26 | role: "Co-Author" 27 | - name: "nirnay" 28 | email: "support@manifoldailearning.in" 29 | role: "Editor" 30 | - {name: "navishka", email: "support@manifoldailearning.in", role: "Finance"} 31 | short_description: > 32 | This is a course for 33 | AWS DevOps Professional 34 | Certification 35 | detailed_description: | 36 | This course has 37 | been created with 38 | the collaboration with 39 | Industry Experts and 40 | curated by In-house team to 41 | give best learning experience 42 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-8-CodeBuild/.DS_Store -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v1-no-artifacts/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-8-CodeBuild/build-v1-no-artifacts/.DS_Store -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v1-no-artifacts/CODEBUILD.md: -------------------------------------------------------------------------------- 1 | # Some System Settings to consider 2 | ## For CodeBuild Environment, Use the settings as below 3 | OS - Amazon Linux 2 4 | Runtime - Standard 5 | Image - aws/codebuild/amazonlinux2-aarch64-standard:2.0 6 | 7 | ## Reference Links: 8 | - https://docs.aws.amazon.com/codebuild/latest/userguide/available-runtimes.html#linux-runtimes 9 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v1-no-artifacts/CODEDEPLOY.md: -------------------------------------------------------------------------------- 1 | 2 | # Installing the CodeDeploy agent on EC2 3 | ```YAML 4 | 5 | - Create EC2 Instance with IAM Role attached 6 | - Assign the Policy `AmazonS3ReadOnlyAccess` to allow CodeDeploy agent to read the version from S3 Bucket 7 | - Add User Data as mentioned below 8 | - Assign tags to EC2 Instances 9 | - Launch the instance and run the command - `sudo service codedeploy-agent status` to validate - CodeDeploy Agent is not running in EC2 instance 10 | - Create an Application in CodeDeploy 11 | - Push app revision to S3 Bucket (create a S3 bucket with versioning if its not created) - see section - **deploy the files into S3** below 12 | - Create a Service Role for CodeDeploy and assign Codedeploy policy 13 | - Create CodeDeployment Group and assign IAM role created above 14 | - Do necessary settings and create Code Deployment 15 | - Now validate in EC2 after few seconds to see whether codeDeploy agent has been installed `sudo service codedeploy-agent status` 16 | - Run the Deployment 17 | - Verify whether the website is working (Make sure to check the security group of ec2 instance) 18 | ``` 19 | # Manual Steps for CodeDeploy Agent 20 | ``` 21 | sudo yum update -y 22 | sudo yum install -y ruby 23 | sudo yum install -y wget 24 | cd /home/ec2-user 25 | wget https://aws-codedeploy-ap-south-1.s3.ap-south-1.amazonaws.com/latest/install 26 | chmod +x ./install 27 | sudo ./install auto 28 | sudo service codedeploy-agent status 29 | 30 | #If error 31 | sudo service codedeploy-agent start 32 | sudo service codedeploy-agent status 33 | ``` 34 | 35 | 36 | # create a bucket and enable versioning 37 | ## https://docs.aws.amazon.com/cli/latest/reference/s3/mb.html 38 | ## https://docs.aws.amazon.com/cli/latest/reference/s3api/put-bucket-versioning.html 39 | ``` 40 | aws s3 mb s3://aws-manifold-devops --region ap-south-1 --profile murthy 41 | aws s3api put-bucket-versioning --bucket aws-manifold-devops --versioning-configuration Status=Enabled --region ap-south-1 42 | ``` 43 | 44 | # deploy the files into S3 45 | # https://docs.aws.amazon.com/cli/latest/reference/deploy/push.html 46 | ``` 47 | aws deploy push --application-name my-app --s3-location s3://aws-manifold-devops/ my-app/app.zip --ignore-hidden-files --region ap-south-1 48 | ``` 49 | 50 | 51 | # Installing the CodeDeploy agent on EC2 52 | # https://docs.aws.amazon.com/codedeploy/latest/userguide/codedeploy-agent-operations-install-ubuntu.html 53 | ``` 54 | sudo yum update -y 55 | sudo yum install ruby 56 | sudo yum install wget 57 | cd /home/ec2-user 58 | wget https://aws-codedeploy-ap-south-1.s3.ap-south-1.amazonaws.com/latest/install 59 | chmod +x ./install 60 | sudo ./install auto 61 | sudo service codedeploy-agent status 62 | 63 | #If error 64 | sudo service codedeploy-agent start 65 | sudo service codedeploy-agent status 66 | ``` 67 | 68 | 69 | # create a bucket and enable versioning 70 | # https://docs.aws.amazon.com/cli/latest/reference/s3/mb.html 71 | # https://docs.aws.amazon.com/cli/latest/reference/s3api/put-bucket-versioning.html 72 | ``` 73 | aws s3 mb s3://aws-devops-manifold --region ap-south-1 74 | aws s3api put-bucket-versioning --bucket aws-devops-manifold --versioning-configuration Status=Enabled --region ap-south-1 75 | ``` 76 | 77 | # deploy the files into S3 78 | # https://docs.aws.amazon.com/cli/latest/reference/deploy/push.html 79 | ``` 80 | aws deploy push --application-name MyDeployDemo --s3-location s3://aws-devops-manifold/MyDeployDemo-test/app.zip --ignore-hidden-files --region ap-south-1 81 | ``` -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v1-no-artifacts/HELP.md: -------------------------------------------------------------------------------- 1 | # Help for BuildSpec 2 | 3 | https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html#build-spec-ref-example 4 | 5 | # Appspec file 6 | 7 | https://docs.aws.amazon.com/codedeploy/latest/userguide/reference-appspec-file-example.html 8 | 9 | # Help for Metadata 10 | https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-categories.html 11 | 12 | # AWS CLI Profile 13 | 14 | **Create Profile** 15 | aws configure --profile profile_name 16 | *you need access key for cli* 17 | ~/.aws/credentials - contains stored creds 18 | ~/.aws/config - contains stored profile 19 | 20 | *If Getting Error as Unable to commit* 21 | - Goto Credential Manager in Windows & Remove the Existing credentials for codecommit 22 | - Extra information - https://stackoverflow.com/questions/34517534/running-git-clone-against-aws-codecommits-gets-me-a-403-error 23 | 24 | 25 | 26 | 27 | 28 | *Display Profile Names:* 29 | aws configure list-profiles 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v1-no-artifacts/appspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.0 2 | os: linux 3 | files: 4 | - source: /index.html 5 | destination: /var/www/html/ 6 | hooks: 7 | ApplicationStop: 8 | - location: scripts/stop_server.sh 9 | timeout: 300 10 | runas: root 11 | 12 | BeforeInstall: 13 | - location: scripts/install_dependencies.sh 14 | timeout: 300 15 | runas: root 16 | 17 | AfterInstall: 18 | - location: scripts/after_install.sh 19 | timeout: 300 20 | runas: root 21 | 22 | ApplicationStart: 23 | - location: scripts/start_server.sh 24 | timeout: 300 25 | runas: root 26 | 27 | ValidateService: 28 | - location: scripts/validate_service.sh 29 | timeout: 300 30 | 31 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v1-no-artifacts/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | env: 4 | variables: 5 | MY_AWS_REGION: "ap-south-1" 6 | 7 | 8 | phases: 9 | install: 10 | runtime-versions: 11 | nodejs: 12 12 | commands: 13 | - echo "Running on $MY_AWS_REGION" 14 | - echo "Print Environment Variables" 15 | - echo "Start the installation process" 16 | - echo "Installation Process Completed" 17 | pre_build: 18 | commands: 19 | - echo "This is the start of Pre-Build phase" 20 | - echo "Pre-Build phase is now completed" 21 | build: 22 | commands: 23 | - echo "we are currently on build phase" 24 | - echo "Build Activity is completed" 25 | - echo "Run Basic check whether Website is ready for use" 26 | - grep -Fq "AWS Certified DevOps Engineer" index.html 27 | - echo "Test was successful" 28 | - echo "proceed with next steps" 29 | post_build: 30 | commands: 31 | - echo "We are currently on post_build phase" 32 | - echo "post_build phase activity is completed. Moving to Artifacts Phase" 33 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v1-no-artifacts/scripts/after_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | EC2_INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) 3 | EC2_INSTANCE_TYPE=$(curl -s http://169.254.169.254/latest/meta-data/instance-type) 4 | EC2_AZ=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) 5 | sed -i "s/you are on Track/This is currently runnining on $EC2_INSANCE_TYPE with id - $EC2_INSTANCE_ID on AM - $EC2_AZ/g" /var/www/html/index.html 6 | chmod 664 /var/www/html/index.html -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v1-no-artifacts/scripts/install_dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | yum install -y httpd 3 | 4 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v1-no-artifacts/scripts/start_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | service httpd start 3 | 4 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v1-no-artifacts/scripts/stop_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | isExistApp = `pgrep httpd` 3 | if [[ -n $isExistApp ]]; then 4 | service httpd stop 5 | fi 6 | 7 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v1-no-artifacts/scripts/validate_service.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # verify we can access our webpage successfully 4 | curl -v --silent localhost:80 2>&1 | grep AWS -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v2-env-variables/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-8-CodeBuild/build-v2-env-variables/.DS_Store -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v2-env-variables/CODEBUILD.md: -------------------------------------------------------------------------------- 1 | # Some System Settings to consider 2 | ## For CodeBuild Environment, Use the settings as below 3 | OS - Amazon Linux 2 4 | Runtime - Standard 5 | Image - aws/codebuild/amazonlinux2-aarch64-standard:2.0 6 | 7 | ## Reference Links: 8 | - https://docs.aws.amazon.com/codebuild/latest/userguide/available-runtimes.html#linux-runtimes 9 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v2-env-variables/CODEDEPLOY.md: -------------------------------------------------------------------------------- 1 | 2 | # Installing the CodeDeploy agent on EC2 3 | ** Updated way to install with SSM ** 4 | ```YAML 5 | 6 | - Create EC2 Instance with IAM Role attached 7 | - Assign the Policy `AmazonS3ReadOnlyAccess` to allow CodeDeploy agent to read the version from S3 Bucket 8 | - Add User Data as mentioned below 9 | - Assign tags to EC2 Instances 10 | - Launch the instance and run the command - `sudo service codedeploy-agent status` to validate - CodeDeploy Agent is not running in EC2 instance 11 | - Create an Application in CodeDeploy 12 | - Push app revision to S3 Bucket (create a S3 bucket with versioning if its not created) - see section - **deploy the files into S3** below 13 | - Create a Service Role for CodeDeploy and assign Codedeploy policy 14 | - Create CodeDeployment Group and assign IAM role created above 15 | - Do necessary settings and create Code Deployment 16 | - Now validate in EC2 after few seconds to see whether codeDeploy agent has been installed `sudo service codedeploy-agent status` 17 | - Run the Deployment 18 | - Verify whether the website is working (Make sure to check the security group of ec2 instance) 19 | ``` 20 | # Manual Steps for CodeDeploy Agent 21 | ``` 22 | sudo yum update -y 23 | sudo yum install -y ruby 24 | sudo yum install -y wget 25 | cd /home/ec2-user 26 | wget https://aws-codedeploy-ap-south-1.s3.ap-south-1.amazonaws.com/latest/install 27 | chmod +x ./install 28 | sudo ./install auto 29 | sudo service codedeploy-agent status 30 | 31 | #If error 32 | sudo service codedeploy-agent start 33 | sudo service codedeploy-agent status 34 | ``` 35 | 36 | 37 | # create a bucket and enable versioning 38 | ## https://docs.aws.amazon.com/cli/latest/reference/s3/mb.html 39 | ## https://docs.aws.amazon.com/cli/latest/reference/s3api/put-bucket-versioning.html 40 | ``` 41 | aws s3 mb s3://aws-manifold-devops --region ap-south-1 --profile murthy 42 | aws s3api put-bucket-versioning --bucket aws-manifold-devops --versioning-configuration Status=Enabled --region ap-south-1 43 | ``` 44 | 45 | # deploy the files into S3 46 | # https://docs.aws.amazon.com/cli/latest/reference/deploy/push.html 47 | ``` 48 | aws deploy push --application-name my-app --s3-location s3://aws-manifold-devops/ my-app/app.zip --ignore-hidden-files --region ap-south-1 49 | ``` 50 | 51 | 52 | # Installing the CodeDeploy agent on EC2 53 | # https://docs.aws.amazon.com/codedeploy/latest/userguide/codedeploy-agent-operations-install-ubuntu.html 54 | ``` 55 | sudo yum update -y 56 | sudo yum install ruby 57 | sudo yum install wget 58 | cd /home/ec2-user 59 | wget https://aws-codedeploy-ap-south-1.s3.ap-south-1.amazonaws.com/latest/install 60 | chmod +x ./install 61 | sudo ./install auto 62 | sudo service codedeploy-agent status 63 | 64 | #If error 65 | sudo service codedeploy-agent start 66 | sudo service codedeploy-agent status 67 | ``` 68 | 69 | 70 | # create a bucket and enable versioning 71 | # https://docs.aws.amazon.com/cli/latest/reference/s3/mb.html 72 | # https://docs.aws.amazon.com/cli/latest/reference/s3api/put-bucket-versioning.html 73 | ``` 74 | aws s3 mb s3://aws-devops-manifold --region ap-south-1 75 | aws s3api put-bucket-versioning --bucket aws-devops-manifold --versioning-configuration Status=Enabled --region ap-south-1 76 | ``` 77 | 78 | # deploy the files into S3 79 | # https://docs.aws.amazon.com/cli/latest/reference/deploy/push.html 80 | ``` 81 | aws deploy push --application-name MyDeployDemo --s3-location s3://aws-devops-manifold/MyDeployDemo-test/app.zip --ignore-hidden-files --region ap-south-1 82 | ``` -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v2-env-variables/HELP.md: -------------------------------------------------------------------------------- 1 | # Help for BuildSpec 2 | 3 | https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html#build-spec-ref-example 4 | 5 | # Appspec file 6 | 7 | https://docs.aws.amazon.com/codedeploy/latest/userguide/reference-appspec-file-example.html 8 | 9 | # Help for Metadata 10 | https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-categories.html 11 | 12 | # AWS CLI Profile 13 | 14 | **Create Profile** 15 | aws configure --profile profile_name 16 | *you need access key for cli* 17 | ~/.aws/credentials - contains stored creds 18 | ~/.aws/config - contains stored profile 19 | 20 | *If Getting Error as Unable to commit* 21 | - Goto Credential Manager in Windows & Remove the Existing credentials for codecommit 22 | - Extra information - https://stackoverflow.com/questions/34517534/running-git-clone-against-aws-codecommits-gets-me-a-403-error 23 | 24 | 25 | 26 | 27 | 28 | *Display Profile Names:* 29 | aws configure list-profiles 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v2-env-variables/appspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.0 2 | os: linux 3 | files: 4 | - source: /index.html 5 | destination: /var/www/html/ 6 | hooks: 7 | ApplicationStop: 8 | - location: scripts/stop_server.sh 9 | timeout: 300 10 | runas: root 11 | 12 | BeforeInstall: 13 | - location: scripts/install_dependencies.sh 14 | timeout: 300 15 | runas: root 16 | 17 | AfterInstall: 18 | - location: scripts/after_install.sh 19 | timeout: 300 20 | runas: root 21 | 22 | ApplicationStart: 23 | - location: scripts/start_server.sh 24 | timeout: 300 25 | runas: root 26 | 27 | ValidateService: 28 | - location: scripts/validate_service.sh 29 | timeout: 300 30 | 31 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v2-env-variables/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | env: 4 | variables: 5 | MY_AWS_REGION: "ap-south-1" 6 | 7 | 8 | phases: 9 | install: 10 | runtime-versions: 11 | nodejs: 12 12 | commands: 13 | - echo "Running on $MY_AWS_REGION" 14 | - echo "hey welcome to AWS $name" 15 | - echo "My code build id is $CODEBUILD_BUILD_ID" 16 | - echo "My Golang version is $GOLANG_12_VERSIONGOLANG_12_VERSION" 17 | - echo "Print Environment Variables" 18 | - printenv 19 | - echo "Start the installation process" 20 | - echo "Installation Process Completed" 21 | 22 | pre_build: 23 | commands: 24 | - echo "This is the start of Pre-Build phase" 25 | - echo "Pre-Build phase is now completed" 26 | build: 27 | commands: 28 | - echo "we are currently on build phase" 29 | - echo "Build Activity is completed" 30 | - echo "Run Basic check whether Website is ready for use" 31 | - grep -Fq "AWS Certified DevOps Engineer" index.html 32 | - echo "Test was successful" 33 | - echo "proceed with next steps" 34 | post_build: 35 | commands: 36 | - echo "We are currently on post_build phase" 37 | - echo "post_build phase activity is completed. Moving to Artifacts Phase" 38 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v2-env-variables/scripts/after_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | EC2_INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) 3 | EC2_INSTANCE_TYPE=$(curl -s http://169.254.169.254/latest/meta-data/instance-type) 4 | EC2_AZ=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) 5 | sed -i "s/you are on Track/This is currently runnining on $EC2_INSANCE_TYPE with id - $EC2_INSTANCE_ID on AM - $EC2_AZ/g" /var/www/html/index.html 6 | chmod 664 /var/www/html/index.html -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v2-env-variables/scripts/install_dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | yum install -y httpd 3 | 4 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v2-env-variables/scripts/start_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | service httpd start 3 | 4 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v2-env-variables/scripts/stop_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | isExistApp = `pgrep httpd` 3 | if [[ -n $isExistApp ]]; then 4 | service httpd stop 5 | fi 6 | 7 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v2-env-variables/scripts/validate_service.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # verify we can access our webpage successfully 4 | curl -v --silent localhost:80 2>&1 | grep AWS -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v3-push-artifacts/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-8-CodeBuild/build-v3-push-artifacts/.DS_Store -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v3-push-artifacts/CODEBUILD.md: -------------------------------------------------------------------------------- 1 | # Some System Settings to consider 2 | ## For CodeBuild Environment, Use the settings as below 3 | OS - Amazon Linux 2 4 | Runtime - Standard 5 | Image - aws/codebuild/amazonlinux2-aarch64-standard:2.0 6 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v3-push-artifacts/CODEDEPLOY.md: -------------------------------------------------------------------------------- 1 | 2 | # Installing the CodeDeploy agent on EC2 3 | ** Updated way to install with SSM ** 4 | ```YAML 5 | 6 | - Create EC2 Instance with IAM Role attached 7 | - Assign the Policy `AmazonS3ReadOnlyAccess` to allow CodeDeploy agent to read the version from S3 Bucket 8 | - Add User Data as mentioned below 9 | - Assign tags to EC2 Instances 10 | - Launch the instance and run the command - `sudo service codedeploy-agent status` to validate - CodeDeploy Agent is not running in EC2 instance 11 | - Create an Application in CodeDeploy 12 | - Push app revision to S3 Bucket (create a S3 bucket with versioning if its not created) - see section - **deploy the files into S3** below 13 | - Create a Service Role for CodeDeploy and assign Codedeploy policy 14 | - Create CodeDeployment Group and assign IAM role created above 15 | - Do necessary settings and create Code Deployment 16 | - Now validate in EC2 after few seconds to see whether codeDeploy agent has been installed `sudo service codedeploy-agent status` 17 | - Run the Deployment 18 | - Verify whether the website is working (Make sure to check the security group of ec2 instance) 19 | ``` 20 | # Manual Steps for CodeDeploy Agent 21 | ``` 22 | sudo yum update -y 23 | sudo yum install -y ruby 24 | sudo yum install -y wget 25 | cd /home/ec2-user 26 | wget https://aws-codedeploy-ap-south-1.s3.ap-south-1.amazonaws.com/latest/install 27 | chmod +x ./install 28 | sudo ./install auto 29 | sudo service codedeploy-agent status 30 | 31 | #If error 32 | sudo service codedeploy-agent start 33 | sudo service codedeploy-agent status 34 | ``` 35 | 36 | 37 | # create a bucket and enable versioning 38 | ## https://docs.aws.amazon.com/cli/latest/reference/s3/mb.html 39 | ## https://docs.aws.amazon.com/cli/latest/reference/s3api/put-bucket-versioning.html 40 | ``` 41 | aws s3 mb s3://aws-manifold-devops --region ap-south-1 --profile murthy 42 | aws s3api put-bucket-versioning --bucket aws-manifold-devops --versioning-configuration Status=Enabled --region ap-south-1 43 | ``` 44 | 45 | # deploy the files into S3 46 | # https://docs.aws.amazon.com/cli/latest/reference/deploy/push.html 47 | ``` 48 | aws deploy push --application-name my-app --s3-location s3://aws-manifold-devops/ my-app/app.zip --ignore-hidden-files --region ap-south-1 49 | ``` 50 | 51 | 52 | # Installing the CodeDeploy agent on EC2 53 | # https://docs.aws.amazon.com/codedeploy/latest/userguide/codedeploy-agent-operations-install-ubuntu.html 54 | ``` 55 | sudo yum update -y 56 | sudo yum install ruby 57 | sudo yum install wget 58 | cd /home/ec2-user 59 | wget https://aws-codedeploy-ap-south-1.s3.ap-south-1.amazonaws.com/latest/install 60 | chmod +x ./install 61 | sudo ./install auto 62 | sudo service codedeploy-agent status 63 | 64 | #If error 65 | sudo service codedeploy-agent start 66 | sudo service codedeploy-agent status 67 | ``` 68 | 69 | 70 | # create a bucket and enable versioning 71 | # https://docs.aws.amazon.com/cli/latest/reference/s3/mb.html 72 | # https://docs.aws.amazon.com/cli/latest/reference/s3api/put-bucket-versioning.html 73 | ``` 74 | aws s3 mb s3://aws-devops-manifold --region ap-south-1 75 | aws s3api put-bucket-versioning --bucket aws-devops-manifold --versioning-configuration Status=Enabled --region ap-south-1 76 | ``` 77 | 78 | # deploy the files into S3 79 | # https://docs.aws.amazon.com/cli/latest/reference/deploy/push.html 80 | ``` 81 | aws deploy push --application-name MyDeployDemo --s3-location s3://aws-devops-manifold/MyDeployDemo-test/app.zip --ignore-hidden-files --region ap-south-1 82 | ``` -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v3-push-artifacts/HELP.md: -------------------------------------------------------------------------------- 1 | # Help for BuildSpec 2 | 3 | https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html#build-spec-ref-example 4 | 5 | # Appspec file 6 | 7 | https://docs.aws.amazon.com/codedeploy/latest/userguide/reference-appspec-file-example.html 8 | 9 | # Help for Metadata 10 | https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-categories.html 11 | 12 | # AWS CLI Profile 13 | 14 | **Create Profile** 15 | aws configure --profile profile_name 16 | *you need access key for cli* 17 | ~/.aws/credentials - contains stored creds 18 | ~/.aws/config - contains stored profile 19 | 20 | *If Getting Error as Unable to commit* 21 | - Goto Credential Manager in Windows & Remove the Existing credentials for codecommit 22 | - Extra information - https://stackoverflow.com/questions/34517534/running-git-clone-against-aws-codecommits-gets-me-a-403-error 23 | 24 | 25 | 26 | 27 | 28 | *Display Profile Names:* 29 | aws configure list-profiles 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v3-push-artifacts/appspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.0 2 | os: linux 3 | files: 4 | - source: /index.html 5 | destination: /var/www/html/ 6 | hooks: 7 | ApplicationStop: 8 | - location: scripts/stop_server.sh 9 | timeout: 300 10 | runas: root 11 | 12 | BeforeInstall: 13 | - location: scripts/install_dependencies.sh 14 | timeout: 300 15 | runas: root 16 | 17 | AfterInstall: 18 | - location: scripts/after_install.sh 19 | timeout: 300 20 | runas: root 21 | 22 | ApplicationStart: 23 | - location: scripts/start_server.sh 24 | timeout: 300 25 | runas: root 26 | 27 | ValidateService: 28 | - location: scripts/validate_service.sh 29 | timeout: 300 30 | 31 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v3-push-artifacts/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | env: 4 | variables: 5 | MY_AWS_REGION: "ap-south-1" 6 | 7 | 8 | phases: 9 | install: 10 | runtime-versions: 11 | nodejs: 12 12 | commands: 13 | - echo "Running on $MY_AWS_REGION" 14 | - echo "Print Environment Variables" 15 | - printenv 16 | - echo "Start the installation process" 17 | - echo "Installation Process Completed" 18 | - echo "hey welcome to AWS $name" 19 | - echo "My code build id is $CODEBUILD_BUILD_ID" 20 | - echo "My Golang version is $GOLANG_12_VERSIONGOLANG_12_VERSION" 21 | pre_build: 22 | commands: 23 | - echo "This is the start of Pre-Build phase" 24 | - echo "Pre-Build phase is now completed" 25 | build: 26 | commands: 27 | - echo "we are currently on build phase" 28 | - echo "Build Activity is completed" 29 | - echo "Run Basic check whether Website is ready for use" 30 | - grep -Fq "AWS Certified DevOps Engineer" index.html 31 | - echo "Test was successful" 32 | - echo "proceed with next steps" 33 | post_build: 34 | commands: 35 | - echo "We are currently on post_build phase" 36 | - echo "post_build phase activity is completed. Moving to Artifacts Phase" 37 | 38 | artifacts: 39 | files: 40 | - '**/*' -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v3-push-artifacts/scripts/after_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | EC2_INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) 3 | EC2_INSTANCE_TYPE=$(curl -s http://169.254.169.254/latest/meta-data/instance-type) 4 | EC2_AZ=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) 5 | sed -i "s/you are on Track/This is currently runnining on $EC2_INSANCE_TYPE with id - $EC2_INSTANCE_ID on AM - $EC2_AZ/g" /var/www/html/index.html 6 | chmod 664 /var/www/html/index.html -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v3-push-artifacts/scripts/install_dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | yum install -y httpd 3 | 4 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v3-push-artifacts/scripts/start_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | service httpd start 3 | 4 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v3-push-artifacts/scripts/stop_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | isExistApp = `pgrep httpd` 3 | if [[ -n $isExistApp ]]; then 4 | service httpd stop 5 | fi 6 | 7 | -------------------------------------------------------------------------------- /Section-8-CodeBuild/build-v3-push-artifacts/scripts/validate_service.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # verify we can access our webpage successfully 4 | curl -v --silent localhost:80 2>&1 | grep AWS -------------------------------------------------------------------------------- /Section-9-CodeDeploy/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-9-CodeDeploy/.DS_Store -------------------------------------------------------------------------------- /Section-9-CodeDeploy/CODEDEPLOY.md: -------------------------------------------------------------------------------- 1 | 2 | # Installing the CodeDeploy agent on EC2 3 | ```YAML 4 | 5 | - Create EC2 Instance with IAM Role attached 6 | - Assign the Policy `AmazonS3ReadOnlyAccess` to allow CodeDeploy agent to read the version from S3 Bucket 7 | - Add User Data as mentioned below 8 | - Assign tags to EC2 Instances 9 | - Launch the instance and run the command - `sudo service codedeploy-agent status` to validate - CodeDeploy Agent is not running in EC2 instance 10 | - Create an Application in CodeDeploy 11 | - Push app revision to S3 Bucket (create a S3 bucket with versioning if its not created) - see section - **deploy the files into S3** below 12 | - Create a Service Role for CodeDeploy and assign Codedeploy policy 13 | - Create CodeDeployment Group and assign IAM role created above 14 | - Do necessary settings and create Code Deployment 15 | - Run the Deployment 16 | - Verify whether the website is working (Make sure to check the security group of ec2 instance) 17 | ``` 18 | # Manual Steps for CodeDeploy Agent 19 | ``` 20 | sudo yum update -y 21 | sudo yum install -y ruby 22 | sudo yum install -y wget 23 | cd /home/ec2-user 24 | wget https://aws-codedeploy-ap-south-1.s3.ap-south-1.amazonaws.com/latest/install 25 | chmod +x ./install 26 | sudo ./install auto 27 | sudo service codedeploy-agent status 28 | 29 | #If error 30 | sudo service codedeploy-agent start 31 | sudo service codedeploy-agent status 32 | ``` 33 | 34 | 35 | # create a bucket and enable versioning 36 | ## https://docs.aws.amazon.com/cli/latest/reference/s3/mb.html 37 | ## https://docs.aws.amazon.com/cli/latest/reference/s3api/put-bucket-versioning.html 38 | ``` 39 | aws s3 mb s3://aws-manifold-devops --region ap-south-1 40 | aws s3api put-bucket-versioning --bucket aws-manifold-devops --versioning-configuration Status=Enabled --region ap-south-1 41 | ``` 42 | 43 | # deploy the files into S3 44 | # https://docs.aws.amazon.com/cli/latest/reference/deploy/push.html 45 | ``` 46 | aws deploy push --application-name demo-app --s3-location s3://aws-manifold-devops/demo-app/app.zip --ignore-hidden-files --region ap-south-1 47 | ``` 48 | 49 | 50 | # User data 51 | ``` 52 | #!bin/bash 53 | sudo yum update -y 54 | sudo yum install -y ruby wget 55 | wget https://aws-codedeploy-eu-west-1.s3.eu-west-1.amazonaws.com/latest/install 56 | chmod +x ./install 57 | sudo ./install auto 58 | sudo service codedeploy-agent status 59 | ``` 60 | -------------------------------------------------------------------------------- /Section-9-CodeDeploy/HELP.md: -------------------------------------------------------------------------------- 1 | # Help for BuildSpec 2 | 3 | https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html#build-spec-ref-example 4 | 5 | # Appspec file 6 | 7 | https://docs.aws.amazon.com/codedeploy/latest/userguide/reference-appspec-file-example.html 8 | 9 | # Help for Metadata 10 | https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-categories.html 11 | 12 | # AWS CLI Profile 13 | 14 | **Create Profile** 15 | aws configure --profile profile_name 16 | *you need access key for cli* 17 | ~/.aws/credentials - contains stored creds 18 | ~/.aws/config - contains stored profile 19 | 20 | *If Getting Error as Unable to commit* 21 | - Goto Credential Manager in Windows & Remove the Existing credentials for codecommit 22 | - Extra information - https://stackoverflow.com/questions/34517534/running-git-clone-against-aws-codecommits-gets-me-a-403-error 23 | 24 | 25 | 26 | 27 | 28 | *Display Profile Names:* 29 | aws configure list-profiles 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /Section-9-CodeDeploy/SLDC-Automation-AWS.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manifoldailearning/mlops-with-aws-datascientists/496214ea6fa49a5798b537f0279520da73f0850c/Section-9-CodeDeploy/SLDC-Automation-AWS.pdf -------------------------------------------------------------------------------- /Section-9-CodeDeploy/appspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.0 2 | os: linux 3 | files: 4 | - source: /index.html 5 | destination: /var/www/html/ 6 | hooks: 7 | ApplicationStop: 8 | - location: scripts/stop_server.sh 9 | timeout: 300 10 | runas: root 11 | 12 | BeforeInstall: 13 | - location: scripts/install_dependencies.sh 14 | timeout: 300 15 | runas: root 16 | 17 | AfterInstall: 18 | - location: scripts/after_install.sh 19 | timeout: 300 20 | runas: root 21 | 22 | ApplicationStart: 23 | - location: scripts/start_server.sh 24 | timeout: 300 25 | runas: root 26 | 27 | ValidateService: 28 | - location: scripts/validate_service.sh 29 | timeout: 300 30 | 31 | -------------------------------------------------------------------------------- /Section-9-CodeDeploy/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | env: 4 | variables: 5 | MY_AWS_REGION: "ap-south-1" 6 | 7 | 8 | phases: 9 | install: 10 | runtime-versions: 11 | nodejs: 12 12 | commands: 13 | - echo "Running on $MY_AWS_REGION" 14 | - echo "hey welcome to AWS $name" 15 | - echo "My code build id is $CODEBUILD_BUILD_ID" 16 | - echo "My Golang version is $GOLANG_12_VERSIONGOLANG_12_VERSION" 17 | - echo "Print Environment Variables" 18 | - printenv 19 | - echo "Start the installation process" 20 | - echo "Installation Process Completed" 21 | 22 | pre_build: 23 | commands: 24 | - echo "This is the start of Pre-Build phase" 25 | - echo "Pre-Build phase is now completed" 26 | build: 27 | commands: 28 | - echo "we are currently on build phase" 29 | - echo "Build Activity is completed" 30 | - echo "Run Basic check whether Website is ready for use" 31 | - grep -Fq "AWS Certified DevOps Engineer" index.html 32 | - echo "Test was successful" 33 | - echo "proceed with next steps" 34 | post_build: 35 | commands: 36 | - echo "We are currently on post_build phase" 37 | - echo "post_build phase activity is completed. Moving to Artifacts Phase" 38 | -------------------------------------------------------------------------------- /Section-9-CodeDeploy/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Page Title 5 | 6 | 7 | 112 | 113 | 114 | 115 |
116 |

Manifold AI Learning

117 |

AWS Certified DevOps Engineer – Professional exam (DOP-C01)

118 |

Congraluations you are on Track

119 |
120 | 121 |
122 | Homepage 123 | Youtube 124 | Udemy 125 | Facebook 126 |
127 | 128 |
129 |
130 |

About Us

131 |

Manifold AI Learning ® is an online Academy with the goal to empower the students with the knowledge and skills that can be directly applied to solving the Real world problems in Data Science, Machine Learning and Artificial intelligence.

132 |

Our Courses

133 |

Join us on Youtube and start learning for free

134 |
Youtube Hands-On ML Series

135 |
Youtube Deep Learning Series

136 |
Deep Learning Tips and Tricks

137 |
Python for Data Science

138 |
Deep Learning with Pytorch

139 |
140 |
141 |

TITLE HEADING

142 |
Title description, Dec 7, 2017
143 |
Image
144 |

Some text..

145 |

Sunt in culpa qui officia deserunt mollit anim id est laborum consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco.

146 |
147 |

What will you get!!

148 |
Hands-On courses
149 |
Image
150 |

Some text..

151 |

Sunt in culpa qui officia deserunt mollit anim id est laborum consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco.

152 |
153 |
154 | 155 |
156 |

For more information, reach us at: support@manifoldailearning.in

157 |
158 | 159 | 160 | 161 | -------------------------------------------------------------------------------- /Section-9-CodeDeploy/scripts/after_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | EC2_INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) 3 | EC2_INSTANCE_TYPE=$(curl -s http://169.254.169.254/latest/meta-data/instance-type) 4 | EC2_AZ=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) 5 | sed -i "s/you are on Track/This is currently runnining on $EC2_INSANCE_TYPE with id - $EC2_INSTANCE_ID on AZ - $EC2_AZ/g" /var/www/html/index.html 6 | chmod 664 /var/www/html/index.html -------------------------------------------------------------------------------- /Section-9-CodeDeploy/scripts/install_dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | yum install -y httpd 3 | 4 | -------------------------------------------------------------------------------- /Section-9-CodeDeploy/scripts/start_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | service httpd start 3 | 4 | -------------------------------------------------------------------------------- /Section-9-CodeDeploy/scripts/stop_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | isExistApp = `pgrep httpd` 3 | if [[ -n $isExistApp ]]; then 4 | service httpd stop 5 | fi 6 | 7 | -------------------------------------------------------------------------------- /Section-9-CodeDeploy/scripts/validate_service.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # verify we can access our webpage successfully 4 | curl -v --silent localhost:80 2>&1 | grep AWS --------------------------------------------------------------------------------