├── .flake8 ├── .gitignore ├── .isort.cfg ├── .pre-commit-config.yaml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README.md ├── deployment ├── README.md ├── SageMakerPipelineSourceCode │ ├── SageMakerPipelineSourceCode_stack.py │ ├── __init__.py │ └── role_policy.py ├── app.py ├── cdk.json ├── deploy.sh ├── pipeline │ └── assets │ │ ├── inference-imagebuild │ │ ├── .flake8 │ │ ├── .gitignore │ │ ├── .isort.cfg │ │ ├── .pre-commit-config.yaml │ │ ├── Dockerfile │ │ ├── buildspec.yml │ │ ├── requirements.txt │ │ └── xgboost-1.5-1-cpu-py3.json │ │ ├── modelbuild │ │ ├── .flake8 │ │ ├── .gitignore │ │ ├── .isort.cfg │ │ ├── .pre-commit-config.yaml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── buildspec.yml │ │ ├── pipelines │ │ │ ├── __init__.py │ │ │ ├── __version__.py │ │ │ ├── _utils.py │ │ │ ├── get_pipeline_definition.py │ │ │ ├── modelbuildpipeline │ │ │ │ ├── __init__.py │ │ │ │ └── pipeline.py │ │ │ └── run_pipeline.py │ │ ├── sagemaker-pipelines-project.ipynb │ │ ├── setup.cfg │ │ ├── setup.py │ │ ├── tests │ │ │ └── test_pipelines.py │ │ └── tox.ini │ │ ├── modeldeploy │ │ ├── .flake8 │ │ ├── .gitignore │ │ ├── .isort.cfg │ │ ├── .pre-commit-config.yaml │ │ ├── DigitalTwin │ │ │ ├── DigitalTwin_stack.py │ │ │ └── __init__.py │ │ ├── README.md │ │ ├── app.py │ │ ├── build.py │ │ ├── buildspec.yml │ │ ├── cdk.json │ │ ├── lambda │ │ │ └── digital_twin │ │ │ │ ├── Dockerfile │ │ │ │ ├── app.py │ │ │ │ └── requirements.txt │ │ ├── requirements.txt │ │ └── source.bat │ │ ├── processing-imagebuild │ │ ├── .flake8 │ │ ├── .gitignore │ │ ├── .isort.cfg │ │ ├── .pre-commit-config.yaml │ │ ├── Dockerfile │ │ ├── buildspec.yml │ │ ├── requirements.txt │ │ └── xgboost-1.5-1-cpu-py3.json │ │ └── training-imagebuild │ │ ├── .flake8 │ │ ├── .gitignore │ │ ├── .isort.cfg │ │ ├── .pre-commit-config.yaml │ │ ├── Dockerfile │ │ ├── buildspec.yml │ │ ├── requirements.txt │ │ └── xgboost-1.5-1-cpu-py3.json ├── requirements.txt ├── setup.py ├── source.bat └── tests │ ├── __init__.py │ └── unit │ ├── __init__.py │ └── test_SageMakerPipelineSourceCode_stack.py ├── mllib ├── __init__.py ├── digital_twin.py ├── lambda_handler.py ├── preprocess.py ├── quantile_regression.py ├── serve.py ├── simulate.py └── train.py ├── requirements.txt ├── setup.py └── tests ├── __init__.py └── test_digital_twin.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E203, E266, E501, W503, F403, F401, E741, F541, C901 3 | max-line-length = 120 4 | max-complexity = 18 5 | select = B,C,E,F,W,T4,B9 6 | exclude = .ipynb_checkpoints,.git,__pycache__,docs/source/conf.py,old,build,dist 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pdf 2 | *.xls 3 | *.xlsx 4 | *.csv 5 | *.pdf 6 | *.env 7 | *.zip 8 | *.joblib 9 | venv/ 10 | .venv/ 11 | *__pycache__* 12 | __pycache__/ 13 | .DS_Store 14 | .vscode/ 15 | .ipynb_checkpoints/ 16 | *.ipynb_checkpoints/* 17 | cdk.context.json 18 | .cdk.staging/ 19 | cdk.out/ 20 | *.egg-info 21 | *.egg 22 | deployment/pipeline/assets/inference-imagebuild/*.py 23 | deployment/pipeline/assets/training-imagebuild/*.py 24 | deployment/pipeline/assets/processing-imagebuild/*.py 25 | test.py 26 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | known_third_party=DigitalTwin,SageMakerPipelineSourceCode,aws_cdk,boto3,botocore,constructs,digital_twin,joblib,numpy,optuna,pandas,pipelines,pytest,quantile_regression,sagemaker,scikeras,scipy,setuptools,sklearn,tensorflow 3 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v2.4.0 6 | hooks: 7 | - id: trailing-whitespace 8 | - id: end-of-file-fixer 9 | - id: check-yaml 10 | - id: flake8 11 | - repo: https://github.com/asottile/seed-isort-config 12 | rev: v1.9.3 13 | hooks: 14 | - id: seed-isort-config 15 | - repo: https://github.com/pre-commit/mirrors-isort 16 | rev: v4.3.21 17 | hooks: 18 | - id: isort 19 | additional_dependencies: [toml] 20 | - repo: https://github.com/psf/black 21 | rev: 20.8b1 22 | hooks: 23 | - id: black 24 | language_version: python3 25 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 10 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 11 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 12 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 13 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 14 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | ********************** 4 | THIRD PARTY COMPONENTS 5 | ********************** 6 | This software includes third party software subject to the following copyrights: 7 | 8 | boto3 under the Apache License Version 2.0 9 | botocore under the Apache License Version 2.0 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Compressor Optimization 2 | 3 | ## Setup 4 | 5 | Once you cloned the repository create a virtual environment using 6 | 7 | ``` 8 | python3 -m venv .venv 9 | ``` 10 | 11 | Activate the environment: 12 | 13 | ``` 14 | source .venv/bin/activate 15 | ``` 16 | 17 | Next install the required libraries using: 18 | 19 | ``` 20 | pip install -r requirements.txt 21 | ``` 22 | 23 | Finally, initialize pre-commit using 24 | 25 | ``` 26 | pre-commit install 27 | ``` 28 | 29 | You can use the `experiments` folder to start your journey with Jupyter notebooks and the regular data science cycle. Once you have developed your code, model, etc. you can integrate it into the files located in `mllib`. These files will be copied into the main application and leveraged by the entire automation mechanism. 30 | 31 | The most important files in `mllib` are: 32 | 33 | * **preprocess.py**: This is the entry point for the Amazon SageMaker processing job and leverages the Docker container built and pushed to Amazon ECR. 34 | * **train.py**: This is the entry point for the Amazon SageMaker training job and leverages the Docker container built and pushed to Amazon ECR. 35 | * **serve.py**: This is the entry point for the Amazon SageMaker endpoint and leverages the Docker container built and pushed to Amazon ECR. (Note: This project deploys the models in an AWS Lambda function, i.e. this file won't be used but can be if hosting is done in Amazon SageMaker) 36 | 37 | In order to deploy your solution navigate into `deployment` folder and run 38 | 39 | ``` 40 | bash deploy.sh 41 | ``` 42 | 43 | you'll find a separate README that will give you guidance. 44 | -------------------------------------------------------------------------------- /deployment/README.md: -------------------------------------------------------------------------------- 1 | # Welcome to Predictive Maintenance with AWS 2 | 3 | The `cdk.json` file tells the CDK Toolkit how to execute your app. 4 | 5 | This project is set up like a standard Python project. The initialization process also creates 6 | a virtualenv within this project, stored under the .venv directory. To create the virtualenv 7 | it assumes that there is a `python3` executable in your path with access to the `venv` package. 8 | If for any reason the automatic creation of the virtualenv fails, you can create the virtualenv 9 | manually once the init process completes. 10 | 11 | To manually create a virtualenv on MacOS and Linux: 12 | 13 | ``` 14 | $ python3 -m venv .venv 15 | ``` 16 | 17 | After the init process completes and the virtualenv is created, you can use the following 18 | step to activate your virtualenv. 19 | 20 | ``` 21 | $ source .venv/bin/activate 22 | ``` 23 | 24 | If you are a Windows platform, you would activate the virtualenv like this: 25 | 26 | ``` 27 | % .venv\Scripts\activate.bat 28 | ``` 29 | 30 | Once the virtualenv is activated, you can install the required dependencies. 31 | 32 | ``` 33 | $ pip install -r requirements.txt 34 | ``` 35 | 36 | At this point you can now synthesize the CloudFormation template for this code. 37 | 38 | ``` 39 | $ cdk synth 40 | ``` 41 | 42 | Deploy the solution using: 43 | 44 | ``` 45 | $ cdk deploy 46 | ``` 47 | 48 | You can now begin exploring the source code, contained in the hello directory. 49 | There is also a very trivial test included that can be run like this: 50 | 51 | ``` 52 | $ pytest 53 | ``` 54 | 55 | To add additional dependencies, for example other CDK libraries, just add to 56 | your requirements.txt file and rerun the `pip install -r requirements.txt` 57 | command. 58 | 59 | ## Useful commands 60 | 61 | * `cdk ls` list all stacks in the app 62 | * `cdk synth` emits the synthesized CloudFormation template 63 | * `cdk deploy` deploy this stack to your default AWS account/region 64 | * `cdk diff` compare deployed stack with current state 65 | * `cdk docs` open CDK documentation 66 | 67 | Enjoy! 68 | -------------------------------------------------------------------------------- /deployment/SageMakerPipelineSourceCode/SageMakerPipelineSourceCode_stack.py: -------------------------------------------------------------------------------- 1 | from aws_cdk import Duration, Stack 2 | from aws_cdk import aws_codebuild as _codebuild 3 | from aws_cdk import aws_codecommit as _codecommit 4 | from aws_cdk import aws_codepipeline as _codepipeline 5 | from aws_cdk import aws_codepipeline_actions as _actions 6 | from aws_cdk import aws_ec2 as _ec2 7 | from aws_cdk import aws_ecr as _ecr 8 | from aws_cdk import aws_events as _events 9 | from aws_cdk import aws_events_targets as _targets 10 | from aws_cdk import aws_iam as _iam 11 | from aws_cdk import aws_lambda as _lambda 12 | from aws_cdk import aws_s3 as _s3 13 | from aws_cdk import aws_s3_deployment as _s3_deploy 14 | from aws_cdk import aws_sagemaker as _sagemaker 15 | from constructs import Construct 16 | 17 | from .role_policy import role_policy_model_build 18 | from .role_policy import role_policy_model_deploy 19 | from .role_policy import role_policy_sagemaker_pipeline_execution 20 | from .role_policy import role_policy_ecr 21 | 22 | class SageMakerPipelineSourceCodeStack(Stack): 23 | """SageMakerPipelineSourceCodeStack class to deploy the AWS CDK stack. 24 | 25 | Attributes: 26 | :mlops_*_policy: The IAM inline policy that gets attached 27 | to the `mlops_*_role` 28 | :mlops_*_role: The IAM role that will be created 29 | :mlops_artifacts_bucket: The Amazon S3 artifact bucket 30 | :sagemaker_project_name: The name of the Amazon SageMaker project 31 | :sagemaker_project_id: The unique Amazon SageMaker project ID 32 | :aws_account_id: The AWS account the solution gets deployed in 33 | :aws_region: The region this stack will be deployed to 34 | """ 35 | 36 | def create_iam_role(self, **kwargs) -> _iam.Role: 37 | """Create the IAM role 38 | 39 | Args: 40 | No arguments 41 | 42 | Returns: 43 | No return value 44 | """ 45 | # Create the policy document 46 | self.mlops_model_build_policy = _iam.PolicyDocument( 47 | statements=role_policy_model_build) 48 | self.mlops_model_deploy_policy = _iam.PolicyDocument( 49 | statements=role_policy_model_deploy) 50 | self.mlops_sagemaker_pipeline_policy = _iam.PolicyDocument( 51 | statements=role_policy_sagemaker_pipeline_execution) 52 | # Define the IAM role 53 | self.mlops_model_build_role = _iam.Role( 54 | self, 55 | "SageMakerMLOpsModelBuildRole", 56 | assumed_by=_iam.ServicePrincipal("codebuild.amazonaws.com"), 57 | description="The SageMakerMLOpsModelBuildRole for service interactions.", 58 | inline_policies={ 59 | "SageMakerMLOpsModelBuildPolicy": self.mlops_model_build_policy, 60 | }, 61 | ) 62 | self.mlops_model_deploy_role = _iam.Role( 63 | self, 64 | "SageMakerMLOpsModelDeployRole", 65 | assumed_by=_iam.ServicePrincipal("codebuild.amazonaws.com"), 66 | description="The SageMakerMLOpsModelDeployRole for service interactions.", 67 | inline_policies={ 68 | "SageMakerMLOpsModelDeployPolicy": self.mlops_model_deploy_policy, 69 | }, 70 | ) 71 | self.mlops_sagemaker_pipeline_role = _iam.Role( 72 | self, 73 | "SageMakerMLOpsSagemakerPipelineRole", 74 | assumed_by=_iam.ServicePrincipal("sagemaker.amazonaws.com"), 75 | description="The SageMakerPipelineRole for executing pipeline .", 76 | managed_policies=[ 77 | _iam.ManagedPolicy.from_aws_managed_policy_name("AmazonSageMakerPipelinesIntegrations") 78 | ], 79 | inline_policies={ 80 | "SageMakerMLOpsSagemkerPipelinePolicy": self.mlops_sagemaker_pipeline_policy, 81 | }, 82 | ) 83 | 84 | def create_s3_artifact_bucket(self, **kwargs) -> _s3.Bucket: 85 | """Create the Amazon S3 bucket to store all ML artifacts in 86 | 87 | Args: 88 | No arguments 89 | 90 | Returns: 91 | No return value 92 | """ 93 | # Create the Amazon S3 bucket. 94 | self.mlops_artifacts_bucket = _s3.Bucket( 95 | self, 96 | "MlOpsArtifactsBucket", 97 | bucket_name=f"sagemaker-project-{self.sagemaker_project_id}", 98 | server_access_logs_prefix="access-logging", 99 | encryption=_s3.BucketEncryption.S3_MANAGED, 100 | block_public_access=_s3.BlockPublicAccess.BLOCK_ALL, 101 | enforce_ssl=True) 102 | 103 | def create_codecommit_repository( 104 | self, 105 | construct_id: str, 106 | repository_tag: str, 107 | **kwargs) -> _codecommit.Repository: 108 | """Create an AWS CodeCommit repository 109 | 110 | Args: 111 | construct_id: The construct ID visible on the CloudFormation console for this resource 112 | repository_tag: Indicating what repository type, values `modelbuild` or `modeldeploy` 113 | 114 | Returns: 115 | repository: The AWS CodeCommit repository 116 | """ 117 | # Create AWS CodeCommit repository 118 | repository = _codecommit.Repository( 119 | self, 120 | construct_id, 121 | repository_name=f"sagemaker-{self.sagemaker_project_name}-{self.sagemaker_project_id}-{repository_tag}", 122 | description=f"{repository_tag} workflow infrastructure as code for the Project {self.sagemaker_project_name}", 123 | code=_codecommit.Code.from_zip_file( 124 | file_path=f"pipeline/assets/{repository_tag}.zip", 125 | branch="main")) 126 | 127 | return repository 128 | 129 | def _event_rule_description_mapping( 130 | self, 131 | rule_tag: str, 132 | **kwargs) -> str: 133 | """Helper function to map event rule type to a description. 134 | 135 | Args: 136 | rule_tag: Indicating what rule type, values `build` or `code` 137 | 138 | Returns: 139 | output: The description to output 140 | """ 141 | output = "No description available" 142 | mapping = { 143 | "build": "Rule to trigger a deployment when ModelBuild CodeCommit repository is updated", 144 | "code": "Rule to trigger a deployment when CodeCommit is updated with a commit", 145 | } 146 | if rule_tag in mapping: 147 | output = mapping[rule_tag] 148 | return output 149 | 150 | def create_codecommit_event_rules( 151 | self, 152 | construct_id: str, 153 | rule_tag: str, 154 | resource: _codepipeline.Pipeline, 155 | **kwargs) -> _events.Rule: 156 | """Create specific Event rules to trigger AWS CodePipeline based on push to 157 | `main` branch in the corresponding AWS CodeCommit repository. 158 | 159 | Args: 160 | construct_id: The construct ID visible on the CloudFormation console for this resource 161 | rule_tag: Indicating what rule type, values `build` or `code` 162 | resource: The AWS CodePipeline to trigger 163 | 164 | Returns: 165 | event_rule: The event rule object that was created 166 | """ 167 | # Define Event rule 168 | event_rule = _events.Rule( 169 | self, 170 | construct_id, 171 | rule_name=f"sagemaker-{self.sagemaker_project_name}-{self.sagemaker_project_id}-{rule_tag}", 172 | description=self._event_rule_description_mapping(rule_tag=rule_tag), 173 | event_pattern=_events.EventPattern( 174 | source=["aws.codecommit"], 175 | resources=[resource.pipeline_arn], 176 | detail={ 177 | "referenceType": ["branch"], 178 | "referenceName": ["main"] 179 | }, 180 | detail_type=["CodeCommit Repository State Change"]), 181 | enabled=True,) 182 | 183 | # Add target: here the AWS CodePipeline 184 | event_rule.add_target( 185 | target=_targets.CodePipeline( 186 | pipeline=resource,)) 187 | 188 | return event_rule 189 | 190 | def create_sagemaker_event_rule( 191 | self, 192 | resource: _codepipeline.Pipeline, 193 | **kwargs): 194 | """Create specific Event rules to trigger AWS CodePipeline based updated 195 | SageMaker Model registry model package. 196 | 197 | Args: 198 | resource: The AWS CodePipeline to trigger 199 | 200 | Returns: 201 | event_rule: The event rule object that was created 202 | """ 203 | # Define Event rule 204 | model_deploy_sagemaker_event_rule = _events.Rule( 205 | self, 206 | "ModelDeploySageMakerEventRule", 207 | description="Rule to trigger a deployment when SageMaker Model registry is updated with a new model package. For example, a new model package is registered with Registry", 208 | rule_name=f"sagemaker-{self.sagemaker_project_name}-{self.sagemaker_project_id}-model", 209 | event_pattern=_events.EventPattern( 210 | source=["aws.sagemaker"], 211 | detail={ 212 | "ModelPackageGroupName": [f"{self.sagemaker_project_name}-{self.sagemaker_project_id}"], 213 | }, 214 | detail_type=["SageMaker Model Package State Change"]), 215 | enabled=True,) 216 | 217 | # Add target: here the AWS CodePipeline 218 | model_deploy_sagemaker_event_rule.add_target( 219 | target=_targets.CodePipeline( 220 | pipeline=resource,)) 221 | 222 | return model_deploy_sagemaker_event_rule 223 | 224 | def create_modelbuild_pipeline( 225 | self, 226 | repository: _codecommit.Repository, 227 | **kwargs) -> _codepipeline.Pipeline: 228 | """Create an entire AWS CodePipeline with an incorporated AWS CodeBuild 229 | step. This pipeline will use `repository` as a source and execute this 230 | code in the AWS CodeBuild step. This pipeline represents the model building 231 | step. 232 | 233 | Args: 234 | repository: The AWS CodeCommit repository that will be leveraged 235 | in the pipeline 236 | 237 | Returns: 238 | model_build_pipeline: The AWS CDK CodePipeline object 239 | """ 240 | # Set a generic source artifact 241 | source_artifact = _codepipeline.Artifact() 242 | 243 | # Define the AWS CodeBuild project 244 | sagemaker_modelbuild_pipeline = _codebuild.Project( 245 | self, 246 | "SageMakerModelPipelineBuildProject", 247 | project_name=f"sagemaker-{self.sagemaker_project_name}-{self.sagemaker_project_id}-modelbuild", 248 | description="Builds the model building workflow code repository, creates the SageMaker Pipeline and executes it", 249 | role=self.mlops_model_build_role, 250 | environment=_codebuild.BuildEnvironment( 251 | build_image=_codebuild.LinuxBuildImage.from_code_build_image_id(id="aws/codebuild/amazonlinux2-x86_64-standard:4.0"), 252 | compute_type=_codebuild.ComputeType.SMALL, 253 | privileged=True 254 | ), 255 | environment_variables={ 256 | "SAGEMAKER_PROJECT_NAME": _codebuild.BuildEnvironmentVariable(value=self.sagemaker_project_name), 257 | "SAGEMAKER_PROJECT_ID": _codebuild.BuildEnvironmentVariable(value=self.sagemaker_project_id), 258 | "ARTIFACT_BUCKET": _codebuild.BuildEnvironmentVariable(value=self.mlops_artifacts_bucket.bucket_name), 259 | "SAGEMAKER_PIPELINE_NAME": _codebuild.BuildEnvironmentVariable(value=f"sagemaker-{self.sagemaker_project_name}"), 260 | "SAGEMAKER_PIPELINE_ROLE_ARN": _codebuild.BuildEnvironmentVariable(value=self.mlops_sagemaker_pipeline_role.role_arn), 261 | "AWS_REGION": _codebuild.BuildEnvironmentVariable(value=self.aws_region), 262 | }, 263 | source=_codebuild.Source.code_commit(repository=repository), 264 | timeout=Duration.hours(2)) 265 | 266 | # Defines the AWS CodePipeline 267 | model_build_pipeline = _codepipeline.Pipeline( 268 | self, 269 | "ModelBuildPipeline", 270 | pipeline_name=f"sagemaker-{self.sagemaker_project_name}-{self.sagemaker_project_id}-modelbuild", 271 | artifact_bucket=self.mlops_artifacts_bucket,) 272 | 273 | # Define actions that are executed in CodePipeline: 274 | # - `Source` leverages the AWS CodeCommit `repository` and copies that into AWS CodeBuild 275 | # - `Deploy` will run AWS CodeBuild leveragin the pulled `repository` 276 | source_action = _actions.CodeCommitSourceAction( 277 | action_name="Source", 278 | output=source_artifact, 279 | branch="main", 280 | repository=repository, 281 | code_build_clone_output=True) 282 | 283 | build_action = _actions.CodeBuildAction( 284 | action_name="Deploy", 285 | project=sagemaker_modelbuild_pipeline, 286 | input=source_artifact, 287 | outputs=[_codepipeline.Artifact()]) 288 | 289 | # Add the stages defined above to the pipeline 290 | model_build_pipeline.add_stage( 291 | stage_name="Source", 292 | actions=[source_action]) 293 | model_build_pipeline.add_stage( 294 | stage_name="Deploy", 295 | actions=[build_action]) 296 | 297 | return model_build_pipeline 298 | 299 | def create_modeldeploy_pipeline( 300 | self, 301 | repository: _codecommit.Repository, 302 | **kwargs) -> _codepipeline.Pipeline: 303 | """Create an entire AWS CodePipeline with an incorporated AWS CodeBuild 304 | step. This pipeline will use `repository` as a source and execute this 305 | code in the AWS CodeBuild step. This pipeline represents the model deployment 306 | step. 307 | 308 | Args: 309 | repository: The AWS CodeCommit repository that will be leveraged 310 | in the pipeline 311 | 312 | Returns: 313 | model_deploy_pipeline: The AWS CDK CodePipeline object 314 | """ 315 | # Set a generic source artifact 316 | source_artifact = _codepipeline.Artifact() 317 | 318 | # Define the AWS CodeBuild project 319 | model_deploy_build_project = _codebuild.Project( 320 | self, 321 | "ModelDeployBuildProject", 322 | description="Builds the Cfn template which defines the Endpoint with specified configuration", 323 | project_name=f"sagemaker-{self.sagemaker_project_name}-{self.sagemaker_project_id}-modeldeploy", 324 | role=self.mlops_model_deploy_role, 325 | environment=_codebuild.BuildEnvironment( 326 | build_image=_codebuild.LinuxBuildImage.from_code_build_image_id(id="aws/codebuild/amazonlinux2-x86_64-standard:4.0"), 327 | compute_type=_codebuild.ComputeType.SMALL, 328 | privileged=True 329 | ), 330 | environment_variables={ 331 | "SAGEMAKER_PROJECT_NAME": _codebuild.BuildEnvironmentVariable(value=self.sagemaker_project_name), 332 | "SAGEMAKER_PROJECT_ID": _codebuild.BuildEnvironmentVariable(value=self.sagemaker_project_id), 333 | "ARTIFACT_BUCKET": _codebuild.BuildEnvironmentVariable(value=self.mlops_artifacts_bucket.bucket_name), 334 | "MODEL_EXECUTION_ROLE_ARN": _codebuild.BuildEnvironmentVariable(value=self.mlops_model_deploy_role.role_arn), 335 | "SOURCE_MODEL_PACKAGE_GROUP_NAME": _codebuild.BuildEnvironmentVariable(value=f"{self.sagemaker_project_name}-{self.sagemaker_project_id}"), 336 | "AWS_REGION": _codebuild.BuildEnvironmentVariable(value=self.aws_region), 337 | "EXPORT_TEMPLATE_NAME": _codebuild.BuildEnvironmentVariable(value="template-export.yml"), 338 | "EXPORT_TEMPLATE_STAGING_CONFIG": _codebuild.BuildEnvironmentVariable(value="staging-config-export.json"), 339 | "EXPORT_TEMPLATE_PROD_CONFIG": _codebuild.BuildEnvironmentVariable(value="prod-config-export.json"), 340 | }, 341 | source=_codebuild.Source.code_commit(repository=repository), 342 | timeout=Duration.minutes(30),) 343 | 344 | # Defines the AWS CodePipeline 345 | model_deploy_pipeline = _codepipeline.Pipeline( 346 | self, 347 | "ModelDeployPipeline", 348 | pipeline_name=f"sagemaker-{self.sagemaker_project_name}-{self.sagemaker_project_id}-modeldeploy", 349 | artifact_bucket=self.mlops_artifacts_bucket,) 350 | 351 | # Define actions that are executed in CodePipeline: 352 | # - `Source` leverages the AWS CodeCommit `repository` and copies that into AWS CodeBuild 353 | # - `Deploy` will run AWS CodeBuild leveragin the pulled `repository` 354 | source_action = _actions.CodeCommitSourceAction( 355 | action_name="Source", 356 | output=source_artifact, 357 | branch="main", 358 | repository=repository, 359 | code_build_clone_output=True) 360 | 361 | build_action = _actions.CodeBuildAction( 362 | action_name="Deploy", 363 | project=model_deploy_build_project, 364 | input=source_artifact, 365 | outputs=[_codepipeline.Artifact()]) 366 | 367 | # Add the stages defined above to the pipeline 368 | model_deploy_pipeline.add_stage( 369 | stage_name="Source", 370 | actions=[source_action]) 371 | model_deploy_pipeline.add_stage( 372 | stage_name="Deploy", 373 | actions=[build_action]) 374 | 375 | return model_deploy_pipeline 376 | 377 | def create_image_type_projects( 378 | self, 379 | construct_id: str, 380 | image_type: str, 381 | model_build_pipeline: _codepipeline.Pipeline, 382 | retraining_days: int, 383 | container_image_tag: str="latest", 384 | ) -> None: 385 | """For each possible encapsulated Docker image (processing, training and inference) 386 | this method creates an end-to-end AWS CodePipeline together with a AWS CodeCommit 387 | repository and an underlying AWS CodeBuild. Each repository is a separate "microservice" 388 | oriented automated pipeline that helps build your Docker containers and push them to 389 | Amazon ECR. These images are registered in Amazon SageMaker and then made available 390 | to your Amazon SageMaker Pipeline that is run in `modelbuild` 391 | 392 | Args: 393 | construct_id: The construct ID visible on the CloudFormation console for this resource 394 | image_type: The image type you want to create, options: `processing`, `training` 395 | and `inference` 396 | model_build_pipeline: The `modelbuild` pipeline that will be triggered each time a new container is 397 | created and pushed to Amazon ECR 398 | retraining_days: The number of days these containers will get re-build. This also indicates 399 | how often your `modelbuil`, i.e. your model re-training cycle, will be 400 | container_image_tag: The Amazon ECR image tag that indicates a new container was released, default: `latest` 401 | 402 | Returns: 403 | No return 404 | """ 405 | 406 | 407 | # Create Amazon ECR repository 408 | training_ecr_repo = _ecr.Repository( 409 | self, 410 | f"DestinationEcrRepository-{image_type}", 411 | repository_name=f"sagemaker-{self.sagemaker_project_id}-{image_type}-imagebuild", 412 | image_scan_on_push = True) 413 | 414 | # Create AWS CodeCommit repository 415 | image_build_repository = _codecommit.Repository( 416 | self, 417 | f"ImageBuildCodeCommitRepository-{image_type}", 418 | repository_name=f"sagemaker-{self.sagemaker_project_name}-{self.sagemaker_project_id}-{image_type}-imagebuild", 419 | description=f"SageMaker {image_type} image-building workflow infrastructure as code for the Project {self.sagemaker_project_name}", 420 | code=_codecommit.Code.from_zip_file( 421 | file_path=f"pipeline/assets/{image_type}-imagebuild.zip", 422 | branch="main")) 423 | 424 | # Set a generic source artifact 425 | source_artifact = _codepipeline.Artifact() 426 | 427 | # Define the AWS CodeBuild project 428 | image_build_project = _codebuild.Project( 429 | self, 430 | f"ImageBuildProject-{image_type}", 431 | project_name=f"sagemaker-{self.sagemaker_project_name}-{self.sagemaker_project_id}-{image_type}-imagebuild", 432 | description=f"sagemaker-{self.sagemaker_project_name}-{self.sagemaker_project_id}-{image_type}-imagebuild", 433 | environment=_codebuild.BuildEnvironment( 434 | build_image=_codebuild.LinuxBuildImage.from_code_build_image_id(id="aws/codebuild/amazonlinux2-x86_64-standard:4.0"), 435 | compute_type=_codebuild.ComputeType.SMALL, 436 | privileged=True 437 | ), 438 | environment_variables={ 439 | "IMAGE_REPO_NAME": _codebuild.BuildEnvironmentVariable(value=training_ecr_repo.repository_name), 440 | "IMAGE_TAG": _codebuild.BuildEnvironmentVariable(value=container_image_tag), 441 | "AWS_ACCOUNT_ID": _codebuild.BuildEnvironmentVariable(value=self.aws_account_id), 442 | "AWS_REGION": _codebuild.BuildEnvironmentVariable(value=self.aws_region), 443 | }, 444 | source=_codebuild.Source.code_commit(repository=image_build_repository), 445 | timeout=Duration.hours(2),) 446 | 447 | #grant the default CodeBuild role with access to pull and push ecr images to ecr repo 448 | training_ecr_repo.grant_pull_push(image_build_project.role) 449 | image_build_project.role.attach_inline_policy(_iam.Policy( 450 | self, 451 | f"IamPolicyImageBuildPublicECR-{image_type}", 452 | document=_iam.PolicyDocument( 453 | statements=role_policy_ecr))) 454 | # Create Amazon SageMaker Image 455 | sagemaker_image = _sagemaker.CfnImage( 456 | self, 457 | f"SageMakerImage-{image_type}", 458 | image_name=f"sagemaker-{self.sagemaker_project_id}-{image_type}-imagebuild", 459 | image_role_arn = self.mlops_sagemaker_pipeline_role.role_arn, 460 | ) 461 | # Defines the AWS CodePipeline 462 | image_build_pipeline = _codepipeline.Pipeline( 463 | self, 464 | f"ImageBuildPipeline-{image_type}", 465 | pipeline_name=f"sagemaker-{self.sagemaker_project_name}-{self.sagemaker_project_id}-{image_type}-imagebuild", 466 | artifact_bucket=self.mlops_artifacts_bucket) 467 | 468 | # Define actions that are executed in CodePipeline: 469 | # - `Source` leverages the AWS CodeCommit `repository` and copies that into AWS CodeBuild 470 | # - `Deploy` will run AWS CodeBuild leveragin the pulled `repository` 471 | source_action = _actions.CodeCommitSourceAction( 472 | action_name="Source", 473 | output=source_artifact, 474 | branch="main", 475 | repository=image_build_repository, 476 | code_build_clone_output=True) 477 | 478 | build_action = _actions.CodeBuildAction( 479 | action_name="Deploy", 480 | project=image_build_project, 481 | input=source_artifact, 482 | outputs=[_codepipeline.Artifact()]) 483 | 484 | # Add the stages defined above to the pipeline 485 | image_build_pipeline.add_stage( 486 | stage_name="Source", 487 | actions=[source_action]) 488 | image_build_pipeline.add_stage( 489 | stage_name="Deploy", 490 | actions=[build_action]) 491 | 492 | # Create Event rule for image build when AWS CodeCommit is updated 493 | # on `main` branch 494 | image_build_codecommit_event_rule = _events.Rule( 495 | self, 496 | f"ImageBuildCodeCommitEventRule-{image_type}", 497 | rule_name=f"sagemaker-{self.sagemaker_project_id}-{image_type}", 498 | description="Rule to trigger an image build when CodeCommit repository is updated", 499 | event_pattern=_events.EventPattern( 500 | source=["aws.codecommit"], 501 | resources=[image_build_pipeline.pipeline_arn], 502 | detail={ 503 | "referenceType": ["branch"], 504 | "referenceName": ["main"] 505 | }, 506 | detail_type=["CodeCommit Repository State Change"]), 507 | enabled=True,) 508 | 509 | # Create Event rule for re-building the Docker image 510 | image_build_schedule_event_rule = _events.Rule( 511 | self, 512 | f"ImageBuildScheduleRule-{image_type}", 513 | rule_name=f"sagemaker-{self.sagemaker_project_id}-{image_type}-image-time", 514 | description=f"The rule to run the {image_type} pipeline on schedule", 515 | schedule=_events.Schedule.rate(Duration.days(retraining_days)), 516 | enabled=True,) 517 | 518 | # Create Event rule for modelbuild when a new version of the image is created. 519 | trigger_model_build_rule = _events.Rule( 520 | self, 521 | f"TriggerModelBuildRule-{image_type}", 522 | rule_name=f"sagemaker-{self.sagemaker_project_id}-{image_type}-image-version", 523 | description="Rule to run the model build pipeline when new versions of the image is created.", 524 | event_pattern=_events.EventPattern( 525 | source=["aws.sagemaker"], 526 | detail={ 527 | "ImageArn": [f"arn:aws:sagemaker:{self.aws_region}:{self.aws_account_id}:image/sagemaker-{self.sagemaker_project_id}-{image_type}-imagebuild"], 528 | "ImageVersionStatus": ["CREATED"], 529 | }, 530 | detail_type=["SageMaker Image Version State Change"]), 531 | enabled=True,) 532 | 533 | # Add targets to Event rules 534 | image_build_codecommit_event_rule.add_target( 535 | target=_targets.CodePipeline( 536 | pipeline=image_build_pipeline,)) 537 | 538 | image_build_schedule_event_rule.add_target( 539 | target=_targets.CodePipeline( 540 | pipeline=image_build_pipeline,)) 541 | 542 | trigger_model_build_rule.add_target( 543 | target=_targets.CodePipeline( 544 | pipeline=model_build_pipeline,)) 545 | 546 | def __init__( 547 | self, 548 | scope: Construct, 549 | construct_id: str, 550 | sagemaker_project_name: str, 551 | sagemaker_project_id: str, 552 | enable_processing_image_build_pipeline: bool, 553 | enable_training_image_build_pipeline: bool, 554 | enable_inference_image_build_pipeline: bool, 555 | aws_account_id: int, 556 | aws_region: str="eu-central-1", 557 | container_image_tag: str="latest", 558 | **kwargs, 559 | ) -> None: 560 | """Initialize the class. 561 | 562 | Args: 563 | scope: The AWS CDK app that is deployed 564 | construct_id: The construct ID visible on the CloudFormation console for this resource 565 | sagemaker_project_name: The name of the Amazon SageMaker project 566 | sagemaker_project_id: The unique Amazon SageMaker project ID 567 | enable_processing_image_build_pipeline: Indicate whether you want a `processing` image build pipeline 568 | enable_training_image_build_pipeline: Indicate whether you want a `training` image build pipeline 569 | enable_inference_image_build_pipeline: Indicate whether you want a `inference` image build pipeline 570 | aws_account_id: The AWS account the solution gets deployed in 571 | aws_region: The region this stack will be deployed to 572 | container_image_tag: The Amazon ECR image tag that indicates a new container was released, default: `latest` 573 | 574 | Returns: 575 | No return 576 | """ 577 | super().__init__(scope, construct_id, **kwargs) 578 | # Set attributes 579 | self.sagemaker_project_name = sagemaker_project_name 580 | self.sagemaker_project_id = sagemaker_project_id 581 | self.aws_account_id = aws_account_id 582 | self.aws_region = aws_region 583 | 584 | # Create IAM role wiht IAM policy and set attribute 585 | self.create_iam_role() 586 | 587 | # Create MLOps S3 artifact bucket 588 | mlops_artifacts_bucket = self.create_s3_artifact_bucket() 589 | 590 | # Create build repository 591 | model_build_repository = self.create_codecommit_repository( 592 | construct_id="ModelBuildCodeCommitRepository", 593 | repository_tag="modelbuild") 594 | 595 | # Create deploy repository 596 | model_deploy_repository = self.create_codecommit_repository( 597 | construct_id="ModelDeployCodeCommitRepository", 598 | repository_tag="modeldeploy") 599 | 600 | # Create the modelbuild pipeline 601 | sagemaker_modelbuild_pipeline = self.create_modelbuild_pipeline(repository=model_build_repository) 602 | 603 | # Create the modeldeploy pipeline 604 | sagemaker_modeldeploy_pipeline = self.create_modeldeploy_pipeline(repository=model_deploy_repository) 605 | 606 | # Create event rule for model build 607 | model_build_codecommit_event_rule = self.create_codecommit_event_rules( 608 | construct_id="ModelBuildCodeCommitEventRule", 609 | rule_tag="build", 610 | resource=sagemaker_modelbuild_pipeline) 611 | 612 | # Create event rule for model deploy 613 | model_deploy_codecommit_event_rule = self.create_codecommit_event_rules( 614 | construct_id="ModelDeployCodeCommitEventRule", 615 | rule_tag="code", 616 | resource=sagemaker_modeldeploy_pipeline) 617 | 618 | # Create the rule that triggers model deployment through 619 | # Amazon SageMaker Registry 620 | model_deploy_sagemaker_event_rule = self.create_sagemaker_event_rule(resource=sagemaker_modeldeploy_pipeline) 621 | 622 | # Check which automated image building AWS CodePipelines, including 623 | # AWS CodeCommit and AWS CodeBuild should be created. Options are: 624 | # - 'processing' 625 | # - 'training' 626 | # - 'inference' 627 | if enable_processing_image_build_pipeline: 628 | processing_stack = self.create_image_type_projects( 629 | "ProcessingImageStack", 630 | image_type="processing", 631 | model_build_pipeline=sagemaker_modelbuild_pipeline, 632 | retraining_days=90, 633 | container_image_tag=container_image_tag) 634 | if enable_training_image_build_pipeline: 635 | training_stack = self.create_image_type_projects( 636 | "TrainingImageStack", 637 | image_type="training", 638 | model_build_pipeline=sagemaker_modelbuild_pipeline, 639 | retraining_days=30, 640 | container_image_tag=container_image_tag) 641 | if enable_inference_image_build_pipeline: 642 | inference_stack = self.create_image_type_projects( 643 | "InferenceImageStack", 644 | image_type="inference", 645 | model_build_pipeline=sagemaker_modelbuild_pipeline, 646 | retraining_days=120, 647 | container_image_tag=container_image_tag) 648 | -------------------------------------------------------------------------------- /deployment/SageMakerPipelineSourceCode/__init__.py: -------------------------------------------------------------------------------- 1 | # Initalize 2 | -------------------------------------------------------------------------------- /deployment/SageMakerPipelineSourceCode/role_policy.py: -------------------------------------------------------------------------------- 1 | from aws_cdk import aws_iam as _iam 2 | 3 | role_policy_ecr = [ 4 | _iam.PolicyStatement( 5 | actions=[ 6 | "ecr:BatchGetImage", 7 | "ecr:Describe*", 8 | "ecr:GetDownloadUrlForLayer", 9 | 10 | ], 11 | resources=[ 12 | "arn:aws:ecr:*:*:repository/sagemaker-*", 13 | "arn:aws:ecr:*:*:repository/cdk-*", 14 | ], 15 | 16 | ), 17 | _iam.PolicyStatement( 18 | actions=[ 19 | "sagemaker:CreateImageVersion", 20 | "sagemaker:DescribeImageVersion", 21 | ], 22 | resources=[ 23 | "arn:aws:sagemaker:*:*:image/sagemaker-*-imagebuild", 24 | ], 25 | ), 26 | ] 27 | 28 | role_policy_model_build = [ 29 | _iam.PolicyStatement( 30 | actions=[ 31 | "iam:GetRole", 32 | "iam:CreateRole", 33 | "iam:PassRole", 34 | "sts:AssumeRole", 35 | ], 36 | resources=[ 37 | "arn:aws:iam::*:role/lambda-update-manifest-role", 38 | "arn:aws:iam::*:role/EnergyOptimization-SageMakerMLOpsSagemakerPipeline*", 39 | ], 40 | ), 41 | _iam.PolicyStatement( 42 | actions=[ 43 | "sagemaker:DescribeImageVersion", 44 | "sagemaker:DescribePipeline", 45 | "sagemaker:AddTags", 46 | "sagemaker:ListTags", 47 | "sagemaker:CreatePipeline", 48 | "sagemaker:StartPipelineExecution", 49 | "sagemaker:DescribePipelineExecution", 50 | "sagemaker:UpdatePipeline", 51 | "sagemaker:ListPipelineExecutionSteps" 52 | ], 53 | resources=[ 54 | "arn:aws:sagemaker:*:*:pipeline/enopt-project-cdk-*", 55 | "arn:aws:sagemaker:*:*:image/sagemaker-*-imagebuild*", 56 | "arn:aws:sagemaker:*:*:image-version/*/*", 57 | ], 58 | ), 59 | _iam.PolicyStatement( 60 | actions=[ 61 | "lambda:CreateFunction", 62 | "lambda:InvokeFunction", 63 | "lambda:UpdateFunctionCode", 64 | ], 65 | resources=[ 66 | "arn:aws:lambda:*:*:function:lambda-update-manifest", 67 | ], 68 | ), 69 | ] 70 | role_policy_sagemaker_pipeline_execution = [ 71 | _iam.PolicyStatement( 72 | actions=[ 73 | "iam:GetRole", 74 | "iam:CreateRole", 75 | "iam:PassRole", 76 | "sts:AssumeRole", 77 | ], 78 | resources=[ 79 | "arn:aws:iam::*:role/lambda-update-manifest-role", 80 | "arn:aws:iam::*:role/EnergyOptimization-SageMakerMLOpsSagemakerPipeline*", 81 | ], 82 | ), 83 | _iam.PolicyStatement( 84 | actions=[ 85 | "lambda:InvokeFunction", 86 | ], 87 | resources=[ 88 | "arn:aws:lambda:*:*:function:lambda-update-manifest", 89 | ] 90 | ), 91 | _iam.PolicyStatement( 92 | actions=[ 93 | "ecr:GetAuthorizationToken", 94 | ], 95 | resources=[ 96 | "*", 97 | ], 98 | ), 99 | _iam.PolicyStatement( 100 | actions=[ 101 | "sagemaker:CreateProcessingJob", 102 | "sagemaker:StopProcessingJob", 103 | "sagemaker:DescribeProcessingJob", 104 | "sagemaker:CreateTrainingJob", 105 | "sagemaker:StopTrainingJob", 106 | "sagemaker:DescribeTrainingJob", 107 | "sagemaker:CreateArtifact", 108 | "sagemaker:AddTags", 109 | "sagemaker:DescribeModelPackage", 110 | "sagemaker:CreateModelPackage", 111 | "sagemaker:DescribeModelPackageGroup", 112 | "sagemaker:CreateModelPackageGroup", 113 | "sagemaker:DescribeImageVersion", 114 | ], 115 | resources=[ 116 | "arn:aws:sagemaker:*:*:processing-job/pipelines-*", 117 | "arn:aws:sagemaker:*:*:training-job/pipelines-*", 118 | "arn:aws:sagemaker:*:*:model-package/*", 119 | "arn:aws:sagemaker:*:*:model-package-group/*", 120 | "arn:aws:sagemaker:*:*:image-version/*/*", 121 | "arn:aws:sagemaker:*:*:image/sagemaker-*-imagebuild*" 122 | ], 123 | ), 124 | _iam.PolicyStatement( 125 | actions=[ 126 | "s3:GetObject*", 127 | "s3:GetBucket*", 128 | "s3:List*", 129 | "s3:DeleteObject*", 130 | "s3:PutObject", 131 | "s3:PutObjectLegalHold", 132 | "s3:PutObjectRetention", 133 | "s3:PutObjectTagging", 134 | "s3:PutObjectVersionTagging", 135 | ], 136 | resources=[ 137 | "arn:aws:s3:::*sagemaker*", 138 | ], 139 | ), 140 | _iam.PolicyStatement( 141 | actions=[ 142 | "ecr:BatchGetImage", 143 | "ecr:Describe*", 144 | "ecr:GetDownloadUrlForLayer", 145 | ], 146 | resources=[ 147 | "arn:aws:ecr:*:*:repository/sagemaker-*", 148 | "arn:aws:ecr:*:*:repository/cdk-*", 149 | ], 150 | ), 151 | _iam.PolicyStatement( 152 | actions=[ 153 | "logs:CreateLogGroup", 154 | "logs:CreateLogStream", 155 | "logs:PutLogEvents" 156 | ], 157 | resources=[ 158 | "arn:aws:logs:*:*:log-group:*", 159 | "arn:aws:logs:*:*:log-group:*:log-stream:*", 160 | ], 161 | ), 162 | 163 | ] 164 | 165 | 166 | role_policy_model_deploy = [ 167 | _iam.PolicyStatement( 168 | actions=["cloudformation:DescribeStacks"], 169 | resources=[ 170 | "arn:aws:cloudformation:*:*:stack/CDKToolkit/*", 171 | ], 172 | ), 173 | _iam.PolicyStatement( 174 | actions=[ 175 | # "iam:GetRole", 176 | #"iam:CreateRole", 177 | # "iam:DeleteRole", 178 | # "iam:AttachRolePolicy", 179 | # "iam:PutRolePolicy", 180 | # "iam:CreatePolicy", 181 | # "iam:DetachRolePolicy", 182 | # "iam:DeleteRolePolicy", 183 | # "iam:GetRolePolicy", 184 | "iam:PassRole", 185 | "sts:AssumeRole", 186 | "ssm:GetParameter", 187 | ], 188 | resources=[ 189 | "arn:aws:iam::*:role/*deploy-role*", 190 | "arn:aws:iam::*:role/*file-publishing*", 191 | "arn:aws:iam::*:role/*image-publishing-role*", 192 | "arn:aws:ssm:*:*:parameter/cdk-bootstrap*", 193 | ], 194 | ), 195 | _iam.PolicyStatement( 196 | actions=[ 197 | "sagemaker:ListTags", 198 | "sagemaker:DescribeModelPackage", 199 | "sagemaker:ListModelPackages", 200 | "sagemaker:DescribeImageVersion", 201 | "sagemaker:AddTags", 202 | ], 203 | resources=[ 204 | "arn:aws:sagemaker:*:*:image-version/sagemaker-*-imagebuild/*", 205 | "arn:aws:sagemaker:*:*:model-package/enopt-project*", 206 | ], 207 | ), 208 | _iam.PolicyStatement( 209 | actions=[ 210 | "lambda:CreateFunction", 211 | "lambda:InvokeFunction", 212 | "lambda:UpdateFunctionCode", 213 | ], 214 | resources=[ 215 | "arn:aws:lambda:*:*:function:lambda-update-manifest", 216 | ], 217 | ), 218 | _iam.PolicyStatement( 219 | actions=[ 220 | "ecr:DescribeImages", 221 | "ecr:DescribeRepositories", 222 | "ecr:CompleteLayerUpload", 223 | "ecr:UploadLayerPart", 224 | "ecr:InitiateLayerUpload", 225 | "ecr:BatchCheckLayerAvailability", 226 | "ecr:PutImage" 227 | ], 228 | resources=[ 229 | "arn:aws:ecr:*:*:repository/sagemaker-*", 230 | "arn:aws:ecr:*:*:repository/cdk-*", 231 | ], 232 | ), 233 | _iam.PolicyStatement( 234 | actions=["ecr:GetAuthorizationToken"], 235 | resources=["*"], 236 | ), 237 | ] 238 | -------------------------------------------------------------------------------- /deployment/app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import random 5 | import string 6 | import aws_cdk as _cdk 7 | import cdk_nag as cdknag 8 | import boto3 9 | from SageMakerPipelineSourceCode.SageMakerPipelineSourceCode_stack import \ 10 | SageMakerPipelineSourceCodeStack 11 | 12 | 13 | def get_random_string(length): 14 | # choose from all lowercase letter 15 | letters = string.ascii_lowercase 16 | result_str = "".join(random.choice(letters) for i in range(length)) 17 | return result_str 18 | 19 | app = _cdk.App() 20 | 21 | aws_region = boto3.session.Session().region_name 22 | aws_account_id = boto3.client('sts').get_caller_identity().get('Account') 23 | project_id = get_random_string(12) 24 | 25 | stack = SageMakerPipelineSourceCodeStack( 26 | app, 27 | "EnergyOptimization", 28 | sagemaker_project_name="enopt-project-cdk", 29 | sagemaker_project_id=f"p-{project_id}", 30 | enable_processing_image_build_pipeline=True, 31 | enable_training_image_build_pipeline=True, 32 | enable_inference_image_build_pipeline=False, 33 | aws_account_id=aws_account_id, 34 | aws_region=aws_region, 35 | container_image_tag="latest") 36 | _cdk.Aspects.of(app).add(cdknag.AwsSolutionsChecks()) 37 | 38 | 39 | cdknag.NagSuppressions.add_stack_suppressions( 40 | stack, 41 | [ 42 | cdknag.NagPackSuppression( 43 | id="AwsSolutions-IAM4", 44 | reason="Use AWS managed poclicies AWSLambdaBasicExecutionRole", 45 | ) 46 | ], 47 | ) 48 | 49 | cdknag.NagSuppressions.add_stack_suppressions( 50 | stack, 51 | [ 52 | cdknag.NagPackSuppression( 53 | id="AwsSolutions-IAM5", 54 | reason="Use AWS managed poclicies CodeBuild Project with defaults from cdk", 55 | ) 56 | ], 57 | ) 58 | 59 | cdknag.NagSuppressions.add_stack_suppressions( 60 | stack, 61 | [ 62 | cdknag.NagPackSuppression( 63 | id="AwsSolutions-CB4", 64 | reason="S3 and Sagemaker Jobs are encrpyted", 65 | ) 66 | ], 67 | ) 68 | app.synth() 69 | -------------------------------------------------------------------------------- /deployment/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "python3 app.py", 3 | "watch": { 4 | "include": [ 5 | "**" 6 | ], 7 | "exclude": [ 8 | "README.md", 9 | "cdk*.json", 10 | "requirements*.txt", 11 | "source.bat", 12 | "**/__init__.py", 13 | "python/__pycache__", 14 | "*.ipynb_checkpoints*", 15 | "tests" 16 | ] 17 | }, 18 | "context": { 19 | "@aws-cdk/core:stackRelativeExports": true, 20 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 21 | "@aws-cdk/core:target-partitions": [ 22 | "aws", 23 | "aws-cn" 24 | ], 25 | "aws-cdk:enableDiffNoFail": true, 26 | "@aws-cdk/aws-ecr-assets:dockerIgnoreSupport": true, 27 | "@aws-cdk/aws-secretsmanager:parseOwnedSecretName": true, 28 | "@aws-cdk/aws-kms:defaultKeyPolicies": true 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /deployment/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | create_virtual_env () { 4 | # This function creates a virtual environment and activates it 5 | echo "Update Python and create virtual environment..." 6 | python3 -m ensurepip --upgrade 7 | python3 -m pip install --upgrade pip 8 | python3 -m pip install --upgrade virtualenv 9 | python3 -m venv .venv 10 | 11 | echo "Source virtual environment..." 12 | source .venv/bin/activate 13 | } 14 | 15 | 16 | install_cdk () { 17 | # This function installs CDK, the requirements file and bootstraps the environment 18 | echo "Install CDK and Python requirements..." 19 | npm install -g aws-cdk 20 | pip install -r requirements.txt 21 | 22 | cd .. 23 | } 24 | 25 | copy_inference_files () { 26 | # Copy the files mandatory for inference imagebuild 27 | echo "Copy Python files to inference imagebuild repo" 28 | cp mllib/digital_twin.py deployment/pipeline/assets/inference-imagebuild/ 29 | cp mllib/quantile_regression.py deployment/pipeline/assets/inference-imagebuild/ 30 | cp mllib/serve.py deployment/pipeline/assets/inference-imagebuild/ 31 | } 32 | 33 | copy_processing_files () { 34 | # Copy the files mandatory for processing imagebuild 35 | echo "Copy Python files to processing imagebuild repo" 36 | cp mllib/digital_twin.py deployment/pipeline/assets/processing-imagebuild/ 37 | cp mllib/quantile_regression.py deployment/pipeline/assets/processing-imagebuild/ 38 | } 39 | 40 | copy_training_files () { 41 | # Copy the files mandatory for training imagebuild 42 | echo "Copy Python files to training imagebuild repo" 43 | cp mllib/digital_twin.py deployment/pipeline/assets/training-imagebuild/ 44 | cp mllib/quantile_regression.py deployment/pipeline/assets/training-imagebuild/ 45 | cp mllib/train.py deployment/pipeline/assets/training-imagebuild/ 46 | } 47 | 48 | copy_lambda_deploy_files () { 49 | # Copy the files mandatory for lambda model deploy 50 | echo "Copy Python files to lambda model deploy repo" 51 | # digital_twin = Simulation Lambda 52 | cp mllib/digital_twin.py deployment/pipeline/assets/modeldeploy/lambda/digital_twin/ 53 | cp mllib/quantile_regression.py deployment/pipeline/assets/modeldeploy/lambda/digital_twin/ 54 | } 55 | 56 | copy_pipeline_files () { 57 | # Copy the files mandatory for model building 58 | echo "Copy Python files to model building repo" 59 | cp mllib/preprocess.py deployment/pipeline/assets/modelbuild/pipelines/modelbuildpipeline/ 60 | cp mllib/simulate.py deployment/pipeline/assets/modelbuild/pipelines/modelbuildpipeline/ 61 | cp mllib/lambda_handler.py deployment/pipeline/assets/modelbuild/pipelines/modelbuildpipeline/ 62 | #cp mllib/digital_twin.py deployment/pipeline/assets/modelbuild/pipelines/modelbuildpipeline/ 63 | #cp mllib/quantile_regression.py deployment/pipeline/assets/modelbuild/pipelines/modelbuildpipeline/ 64 | } 65 | 66 | cleanup_repository () { 67 | # Remove unwanted files from repository structure 68 | find . -type f -name ".DS_Store" -delete 69 | find . -type f -name ".ipynb_checkpoints" -delete 70 | find . -type f -name "__pycache__" -delete 71 | find . -type f -name "*.egg-info" -delete 72 | } 73 | 74 | zip_files() { 75 | # This function zips the files, moves them up one level and goes back 3 levels 76 | # Note: In order to land where you started submit 3 levels to cd into... 77 | echo "ZIP files in $1" 78 | cd $1 79 | zip -r $2 * 80 | mv $2 ../ 81 | cd ../../../ 82 | } 83 | 84 | cleanup_files () { 85 | # Remove all copied files to make repo clean again 86 | rm pipeline/assets/inference-imagebuild/digital_twin.py 87 | rm pipeline/assets/inference-imagebuild/quantile_regression.py 88 | rm pipeline/assets/inference-imagebuild/serve.py 89 | rm pipeline/assets/processing-imagebuild/digital_twin.py 90 | rm pipeline/assets/processing-imagebuild/quantile_regression.py 91 | rm pipeline/assets/training-imagebuild/digital_twin.py 92 | rm pipeline/assets/training-imagebuild/quantile_regression.py 93 | rm pipeline/assets/training-imagebuild/train.py 94 | rm pipeline/assets/modeldeploy/lambda/digital_twin/digital_twin.py 95 | rm pipeline/assets/modeldeploy/lambda/digital_twin/quantile_regression.py 96 | rm pipeline/assets/modelbuild/pipelines/modelbuildpipeline/preprocess.py 97 | rm pipeline/assets/modelbuild/pipelines/modelbuildpipeline/simulate.py 98 | rm pipeline/assets/modelbuild/pipelines/modelbuildpipeline/lambda_handler.py 99 | 100 | rm pipeline/assets/inference-imagebuild.zip 101 | rm pipeline/assets/modelbuild.zip 102 | rm pipeline/assets/processing-imagebuild.zip 103 | rm pipeline/assets/training-imagebuild.zip 104 | rm pipeline/assets/modeldeploy.zip 105 | rm -rf cdk.out/asset* 106 | } 107 | 108 | create_virtual_env 109 | install_cdk 110 | copy_inference_files 111 | copy_processing_files 112 | copy_training_files 113 | copy_lambda_deploy_files 114 | copy_pipeline_files 115 | cleanup_repository 116 | 117 | cd deployment 118 | 119 | zip_files "pipeline/assets/inference-imagebuild" "inference-imagebuild.zip" 120 | zip_files "pipeline/assets/modelbuild" "modelbuild.zip" 121 | zip_files "pipeline/assets/processing-imagebuild" "processing-imagebuild.zip" 122 | zip_files "pipeline/assets/training-imagebuild" "training-imagebuild.zip" 123 | zip_files "pipeline/assets/modeldeploy" "modeldeploy.zip" 124 | 125 | echo "Bootstrap account" 126 | cdk bootstrap 127 | 128 | echo "Deploy CDK Stack..." 129 | cdk deploy --require-approval never 130 | 131 | cleanup_files 132 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/inference-imagebuild/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E203, E266, E501, W503, F403, F401, E741, F541, C901 3 | max-line-length = 120 4 | max-complexity = 18 5 | select = B,C,E,F,W,T4,B9 6 | exclude = .ipynb_checkpoints,.git,__pycache__,docs/source/conf.py,old,build,dist 7 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/inference-imagebuild/.gitignore: -------------------------------------------------------------------------------- 1 | *.pdf 2 | *.xls 3 | *.xlsx 4 | *.pdf 5 | *.env 6 | *.zip 7 | venv/ 8 | .venv/ 9 | *__pycache__* 10 | __pycache__/ 11 | .DS_Store 12 | .vscode/ 13 | .ipynb_checkpoints/ 14 | *.ipynb_checkpoints/* 15 | cdk.context.json 16 | .cdk.staging/ 17 | cdk.out/ 18 | *.egg-info 19 | *.egg 20 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/inference-imagebuild/.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | known_third_party=SageMakerPipelineConstructs,SageMakerPipelineSourceCode,aws_cdk,boto3,botocore,constructs,data_pipeline,digital_twin,joblib,numpy,pandas,pipelines,pytest,quantile_regression,river,sagemaker,scipy,setuptools,sklearn,tqdm 3 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/inference-imagebuild/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v2.4.0 6 | hooks: 7 | - id: trailing-whitespace 8 | - id: end-of-file-fixer 9 | - id: flake8 10 | - repo: https://github.com/asottile/seed-isort-config 11 | rev: v1.9.3 12 | hooks: 13 | - id: seed-isort-config 14 | - repo: https://github.com/pre-commit/mirrors-isort 15 | rev: v4.3.21 16 | hooks: 17 | - id: isort 18 | additional_dependencies: [toml] 19 | - repo: https://github.com/psf/black 20 | rev: 20.8b1 21 | hooks: 22 | - id: black 23 | language_version: python3 24 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/inference-imagebuild/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG ECR_ENDPOINT 2 | FROM $ECR_ENDPOINT/sagemaker-xgboost:1.5-1-cpu-py3 3 | 4 | COPY requirements.txt requirements.txt 5 | COPY digital_twin.py /opt/ml/code/digital_twin.py 6 | COPY quantile_regression.py /opt/ml/code/quantile_regression.py 7 | COPY serve.py /opt/ml/code/serve.py 8 | 9 | RUN python3 -m pip install -r requirements.txt 10 | 11 | ENV SAGEMAKER_PROGRAM serve.py 12 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/inference-imagebuild/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | phases: 4 | pre_build: 5 | commands: 6 | - echo Logging in to Amazon ECR... 7 | # Extracting endpoint information using jq based on region and sed to strip off the quotes at both ends of the string. 8 | - ECR_ENDPOINT=`jq --arg region "$AWS_REGION" '.[$region]' xgboost-1.5-1-cpu-py3.json | sed 's/"//g'` 9 | # Using cut to remove the 12 digit account number and the period from ECR_ENDPOINT to retrieve repository of the base image. 10 | - BASE_IMAGE_REPOSITORY=`echo $ECR_ENDPOINT | cut -c14-` 11 | - aws --region $AWS_REGION ecr get-login-password | docker login --username AWS --password-stdin $ECR_ENDPOINT 12 | build: 13 | commands: 14 | - echo Build started on `date` 15 | - echo Building the Docker image... 16 | - docker build --build-arg ECR_ENDPOINT=$ECR_ENDPOINT -t $IMAGE_REPO_NAME:$IMAGE_TAG . 17 | - docker tag $IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_ID.$BASE_IMAGE_REPOSITORY/$IMAGE_REPO_NAME:$IMAGE_TAG 18 | post_build: 19 | commands: 20 | - if [ $CODEBUILD_BUILD_SUCCEEDING = 0 ]; then exit 1; fi 21 | - echo Build completed on `date` 22 | - echo Logging in to Amazon ECR... 23 | - aws --region $AWS_REGION ecr get-login-password | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.$BASE_IMAGE_REPOSITORY 24 | - echo Pushing the Docker image... 25 | - docker push $AWS_ACCOUNT_ID.$BASE_IMAGE_REPOSITORY/$IMAGE_REPO_NAME:$IMAGE_TAG 26 | - aws --region $AWS_REGION sagemaker create-image-version --image-name $IMAGE_REPO_NAME --base-image $AWS_ACCOUNT_ID.$BASE_IMAGE_REPOSITORY/$IMAGE_REPO_NAME:$IMAGE_TAG 27 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/inference-imagebuild/requirements.txt: -------------------------------------------------------------------------------- 1 | pip==22.1.2 2 | boto3==1.24.22 3 | shap==0.41.0 4 | imbalanced-learn==0.9.1 5 | werkzeug==2.0.3 6 | pandas==1.2.4 7 | numpy==1.22.0 8 | xgboost==1.5.2 9 | scikit-learn==1.1.1 10 | optuna==2.10.1 11 | scikeras==0.8.0 12 | tensorflow==2.9.3 13 | keras==2.9.0 14 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/inference-imagebuild/xgboost-1.5-1-cpu-py3.json: -------------------------------------------------------------------------------- 1 | { 2 | "af-south-1": "510948584623.dkr.ecr.af-south-1.amazonaws.com", 3 | "ap-east-1": "651117190479.dkr.ecr.ap-east-1.amazonaws.com", 4 | "ap-northeast-1": "354813040037.dkr.ecr.ap-northeast-1.amazonaws.com", 5 | "ap-northeast-2": "366743142698.dkr.ecr.ap-northeast-2.amazonaws.com", 6 | "ap-south-1": "720646828776.dkr.ecr.ap-south-1.amazonaws.com", 7 | "ap-southeast-1": "121021644041.dkr.ecr.ap-southeast-1.amazonaws.com", 8 | "ap-southeast-2": "783357654285.dkr.ecr.ap-southeast-2.amazonaws.com", 9 | "ca-central-1": "341280168497.dkr.ecr.ca-central-1.amazonaws.com", 10 | "cn-north-1": "450853457545.dkr.ecr.cn-north-1.amazonaws.com.cn", 11 | "cn-northwest-1": "451049120500.dkr.ecr.cn-northwest-1.amazonaws.com.cn", 12 | "eu-central-1": "492215442770.dkr.ecr.eu-central-1.amazonaws.com", 13 | "eu-north-1": "662702820516.dkr.ecr.eu-north-1.amazonaws.com", 14 | "eu-west-1": "141502667606.dkr.ecr.eu-west-1.amazonaws.com", 15 | "eu-west-2": "764974769150.dkr.ecr.eu-west-2.amazonaws.com", 16 | "eu-west-3": "659782779980.dkr.ecr.eu-west-3.amazonaws.com", 17 | "eu-south-1": "978288397137.dkr.ecr.eu-south-1.amazonaws.com", 18 | "me-south-1": "801668240914.dkr.ecr.me-south-1.amazonaws.com", 19 | "sa-east-1": "737474898029.dkr.ecr.sa-east-1.amazonaws.com", 20 | "us-east-1": "683313688378.dkr.ecr.us-east-1.amazonaws.com", 21 | "us-east-2": "257758044811.dkr.ecr.us-east-2.amazonaws.com", 22 | "us-west-1": "746614075791.dkr.ecr.us-west-1.amazonaws.com", 23 | "us-west-2": "246618743249.dkr.ecr.us-west-2.amazonaws.com" 24 | } 25 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E203, E266, E501, W503, F403, F401, E741, F541, C901 3 | max-line-length = 120 4 | max-complexity = 18 5 | select = B,C,E,F,W,T4,B9 6 | exclude = .ipynb_checkpoints,.git,__pycache__,docs/source/conf.py,old,build,dist 7 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/.gitignore: -------------------------------------------------------------------------------- 1 | *.pdf 2 | *.xls 3 | *.xlsx 4 | *.pdf 5 | *.env 6 | *.zip 7 | venv/ 8 | .venv/ 9 | *__pycache__* 10 | __pycache__/ 11 | .DS_Store 12 | .vscode/ 13 | .ipynb_checkpoints/ 14 | *.ipynb_checkpoints/* 15 | cdk.context.json 16 | .cdk.staging/ 17 | cdk.out/ 18 | *.egg-info 19 | *.egg 20 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | known_third_party=SageMakerPipelineConstructs,SageMakerPipelineSourceCode,aws_cdk,boto3,botocore,constructs,data_pipeline,digital_twin,joblib,numpy,pandas,pipelines,pytest,quantile_regression,river,sagemaker,scipy,setuptools,sklearn,tqdm 3 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v2.4.0 6 | hooks: 7 | - id: trailing-whitespace 8 | - id: end-of-file-fixer 9 | - id: flake8 10 | - repo: https://github.com/asottile/seed-isort-config 11 | rev: v1.9.3 12 | hooks: 13 | - id: seed-isort-config 14 | - repo: https://github.com/pre-commit/mirrors-isort 15 | rev: v4.3.21 16 | hooks: 17 | - id: isort 18 | additional_dependencies: [toml] 19 | - repo: https://github.com/psf/black 20 | rev: 20.8b1 21 | hooks: 22 | - id: black 23 | language_version: python3 24 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 10 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 11 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 12 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 13 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 14 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/README.md: -------------------------------------------------------------------------------- 1 | ## Overview of this Project and its components 2 | 3 | This project provides a starting point for creating a CICD system for machine learning. There are three primary components to this project: 4 | 5 | * Image generation 6 | * Model building 7 | * Model deployment 8 | 9 | Each of the above components is made up of: 10 | 11 | * A CodeCommit repository (like this one) 12 | * A CodeBuild project 13 | * A CodePipeline 14 | * Various SageMaker components 15 | 16 | This repository is for the project's **Model building** component. It was created as part of a SageMaker Project. 17 | 18 | In the placeholder code provided in this repository, we are solving the modelbuildpipeline age prediction problem using the modelbuildpipeline dataset. 19 | ## Layout of the model building repository 20 | 21 | ``` 22 | |-- codebuild-buildspec.yml 23 | |-- CONTRIBUTING.md 24 | |-- pipelines 25 | | |-- modelbuildpipeline 26 | | | |-- evaluate.py 27 | | | |-- __init__.py 28 | | | |-- pipeline.py 29 | | | |-- preprocess.py 30 | | |-- get_pipeline_definition.py 31 | | |-- __init__.py 32 | | |-- run_pipeline.py 33 | | |-- _utils.py 34 | | |-- __version__.py 35 | |-- README.md 36 | |-- sagemaker-pipelines-project.ipynb 37 | |-- setup.cfg 38 | |-- setup.py 39 | |-- tests 40 | | |-- test_pipelines.py 41 | |-- tox.ini 42 | ``` 43 | 44 | This section provides an overview of how the code is organized and the purpose of important files. In particular, `pipelines/pipelines.py` contains the core of the business logic for this problem. It has the code to express the ML steps involved in generating an ML model. You will also find the code that supports preprocessing and evaluation steps in `preprocess.py` and `evaluate.py` files respectively. 45 | 46 | Once you understand the code structure described below, you can inspect the code and can start customizing it for your own business case. This is only sample code--you own this repository and can adapt it for your business use case. Feel free to modify the files, commit your changes and watch as the CICD system reacts to them. 47 | 48 | You can also use the `sagemaker-pipelines-project.ipynb` notebook to experiment from SageMaker Studio before you are ready to checkin your code. 49 | 50 | ``` 51 | |-- codebuild-buildspec.yml 52 | ``` 53 | Your codebuild execution instructions. This file contains the instructions needed to create or update a SageMaker Model Building pipeline and start an execution of it. You will see that this file has the fields definined for naming the Pipeline, ModelPackageGroup etc. You can customize them as required. 54 | 55 | ``` 56 | |-- pipelines 57 | | |-- modelbuildpipeline 58 | | | |-- evaluate.py 59 | | | |-- __init__.py 60 | | | |-- pipeline.py 61 | | | |-- preprocess.py 62 | ``` 63 | Your pipeline artifacts, which includes a pipeline module defining the required `get_pipeline` method that returns an instance of a SageMaker pipeline, a preprocessing script that is used in feature engineering, and a model evaluation script to measure the Mean Squared Error of the model that's trained by the pipeline. This is the core business logic, and if you want to create your own folder, you can do so, and implement the `get_pipeline` interface as illustrated here. 64 | 65 | ``` 66 | |-- pipelines 67 | | |-- get_pipeline_definition.py 68 | | |-- __init__.py 69 | | |-- run_pipeline.py 70 | | |-- _utils.py 71 | | |-- __version__.py 72 | ``` 73 | Utility modules for getting the pipeline definition JSON and running pipelines (you do not typically need to modify these): 74 | 75 | ``` 76 | |-- setup.cfg 77 | |-- setup.py 78 | ``` 79 | Standard Python package artifacts. 80 | 81 | ``` 82 | |-- tests 83 | | |-- test_pipelines.py 84 | ``` 85 | A stubbed testing module for testing your pipeline as you develop. 86 | 87 | ``` 88 | `-- tox.ini 89 | ``` 90 | The `tox` testing framework configuration. 91 | ## Dataset for the Example modelbuildpipeline Pipeline 92 | 93 | The dataset used is the [UCI Machine Learning modelbuildpipeline Dataset](https://archive.ics.uci.edu/ml/datasets/modelbuildpipeline) [1]. The aim for this task is to determine the age of an modelbuildpipeline (a kind of shellfish) from its physical measurements. At the core, it's a regression problem. 94 | 95 | The dataset contains several features - length (longest shell measurement), diameter (diameter perpendicular to length), height (height with meat in the shell), whole_weight (weight of whole modelbuildpipeline), shucked_weight (weight of meat), viscera_weight (gut weight after bleeding), shell_weight (weight after being dried), sex ('M', 'F', 'I' where 'I' is Infant), as well as rings (integer). 96 | 97 | The number of rings turns out to be a good approximation for age (age is rings + 1.5). However, to obtain this number requires cutting the shell through the cone, staining the section, and counting the number of rings through a microscope -- a time-consuming task. However, the other physical measurements are easier to determine. We use the dataset to build a predictive model of the variable rings through these other physical measurements. 98 | 99 | We'll upload the data to a bucket we own. But first we gather some constants we can use later throughout the notebook. 100 | 101 | [1] Dua, D. and Graff, C. (2019). [UCI Machine Learning Repository](http://archive.ics.uci.edu/ml). Irvine, CA: University of California, School of Information and Computer Science. 102 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | phases: 4 | install: 5 | runtime-versions: 6 | python: 3.9 7 | commands: 8 | - pip install --upgrade --force-reinstall . "awscli>1.20.30" 9 | 10 | build: 11 | commands: 12 | - export PYTHONUNBUFFERED=TRUE 13 | - export SAGEMAKER_PROJECT_NAME_ID="${SAGEMAKER_PROJECT_NAME}-${SAGEMAKER_PROJECT_ID}" 14 | - | 15 | run-pipeline --module-name pipelines.modelbuildpipeline.pipeline \ 16 | --role-arn $SAGEMAKER_PIPELINE_ROLE_ARN \ 17 | --tags "[{\"Key\":\"sagemaker:project-name\", \"Value\":\"${SAGEMAKER_PROJECT_NAME}\"}, {\"Key\":\"sagemaker:project-id\", \"Value\":\"${SAGEMAKER_PROJECT_ID}\"}]" \ 18 | --kwargs "{\"region\":\"${AWS_REGION}\",\"role\":\"${SAGEMAKER_PIPELINE_ROLE_ARN}\",\"default_bucket\":\"${ARTIFACT_BUCKET}\",\"pipeline_name\":\"${SAGEMAKER_PROJECT_NAME_ID}\",\"model_package_group_name\":\"${SAGEMAKER_PROJECT_NAME_ID}\",\"base_job_prefix\":\"${SAGEMAKER_PROJECT_NAME_ID}\",\"project_id\":\"${SAGEMAKER_PROJECT_ID}\"}" 19 | - echo "Create/Update of the SageMaker Pipeline and execution completed." 20 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/pipelines/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-mlops-template-with-aws-lambda-deployment/e3ce6da486482afd11cc8fc596551eaece5f0836/deployment/pipeline/assets/modelbuild/pipelines/__init__.py -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/pipelines/__version__.py: -------------------------------------------------------------------------------- 1 | """Metadata for the pipelines package.""" 2 | 3 | __title__ = "pipelines" 4 | __description__ = "pipelines - template package" 5 | __version__ = "0.0.1" 6 | __author__ = "Michael Wallner" 7 | __author_email__ = "" 8 | __license__ = "Apache 2.0" 9 | __url__ = "aws.amazon.com" 10 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/pipelines/_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | """Provides utilities for SageMaker Pipeline CLI.""" 14 | from __future__ import absolute_import 15 | 16 | import ast 17 | 18 | 19 | def get_pipeline_driver(module_name, passed_args=None): 20 | """Gets the driver for generating your pipeline definition. 21 | 22 | Pipeline modules must define a get_pipeline() module-level method. 23 | 24 | Args: 25 | module_name: The module name of your pipeline. 26 | passed_args: Optional passed arguments that your pipeline may be templated by. 27 | 28 | Returns: 29 | The SageMaker Workflow pipeline. 30 | """ 31 | _imports = __import__(module_name, fromlist=["get_pipeline"]) 32 | kwargs = convert_struct(passed_args) 33 | return _imports.get_pipeline(**kwargs) 34 | 35 | 36 | def convert_struct(str_struct=None): 37 | return ast.literal_eval(str_struct) if str_struct else {} 38 | 39 | 40 | def get_pipeline_custom_tags(module_name, args, tags): 41 | """Gets the custom tags for pipeline 42 | 43 | Returns: 44 | Custom tags to be added to the pipeline 45 | """ 46 | try: 47 | _imports = __import__(module_name, fromlist=["get_pipeline_custom_tags"]) 48 | kwargs = convert_struct(args) 49 | return _imports.get_pipeline_custom_tags( 50 | tags, kwargs["region"], kwargs["sagemaker_project_arn"] 51 | ) 52 | except Exception as e: 53 | print(f"Error getting project tags: {e}") 54 | return tags 55 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/pipelines/get_pipeline_definition.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | """A CLI to get pipeline definitions from pipeline modules.""" 14 | from __future__ import absolute_import 15 | 16 | import argparse 17 | import sys 18 | 19 | from pipelines._utils import get_pipeline_driver 20 | 21 | 22 | def main(): # pragma: no cover 23 | """The main harness that gets the pipeline definition JSON. 24 | 25 | Prints the json to stdout or saves to file. 26 | """ 27 | parser = argparse.ArgumentParser( 28 | "Gets the pipeline definition for the pipeline script." 29 | ) 30 | 31 | parser.add_argument( 32 | "-n", 33 | "--module-name", 34 | dest="module_name", 35 | type=str, 36 | help="The module name of the pipeline to import.", 37 | ) 38 | parser.add_argument( 39 | "-f", 40 | "--file-name", 41 | dest="file_name", 42 | type=str, 43 | default=None, 44 | help="The file to output the pipeline definition json to.", 45 | ) 46 | parser.add_argument( 47 | "-kwargs", 48 | "--kwargs", 49 | dest="kwargs", 50 | default=None, 51 | help="Dict string of keyword arguments for the pipeline generation (if supported)", 52 | ) 53 | args = parser.parse_args() 54 | 55 | if args.module_name is None: 56 | parser.print_help() 57 | sys.exit(2) 58 | 59 | try: 60 | pipeline = get_pipeline_driver(args.module_name, args.kwargs) 61 | content = pipeline.definition() 62 | if args.file_name: 63 | with open(args.file_name, "w") as f: 64 | f.write(content) 65 | else: 66 | print(content) 67 | except Exception as e: # pylint: disable=W0703 68 | print(f"Exception: {e}") 69 | sys.exit(1) 70 | 71 | 72 | if __name__ == "__main__": 73 | main() 74 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/pipelines/modelbuildpipeline/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-mlops-template-with-aws-lambda-deployment/e3ce6da486482afd11cc8fc596551eaece5f0836/deployment/pipeline/assets/modelbuild/pipelines/modelbuildpipeline/__init__.py -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/pipelines/modelbuildpipeline/pipeline.py: -------------------------------------------------------------------------------- 1 | """Example workflow pipeline script for RUL pipeline. 2 | 3 | . -RegisterModel 4 | . 5 | Process-> Train -> Evaluate -> Condition . 6 | . 7 | . -(stop) 8 | 9 | Implements a get_pipeline(**kwargs) method. 10 | """ 11 | import json 12 | import logging 13 | import os 14 | 15 | import boto3 16 | import sagemaker 17 | from botocore.exceptions import ClientError 18 | from sagemaker.estimator import Estimator 19 | from sagemaker.inputs import TrainingInput 20 | from sagemaker.lambda_helper import Lambda 21 | from sagemaker.model_metrics import MetricsSource, ModelMetrics 22 | from sagemaker.processing import (ProcessingInput, ProcessingOutput, 23 | ScriptProcessor) 24 | from sagemaker.sklearn.processing import SKLearnProcessor 25 | from sagemaker.workflow.condition_step import ConditionStep 26 | from sagemaker.workflow.conditions import ConditionLessThanOrEqualTo 27 | from sagemaker.workflow.functions import JsonGet 28 | from sagemaker.workflow.lambda_step import (LambdaOutput, LambdaOutputTypeEnum, 29 | LambdaStep) 30 | from sagemaker.workflow.parameters import ParameterInteger, ParameterString 31 | from sagemaker.workflow.pipeline import Pipeline 32 | from sagemaker.workflow.properties import PropertyFile 33 | from sagemaker.workflow.step_collections import RegisterModel 34 | from sagemaker.workflow.steps import ProcessingStep, TrainingStep 35 | from sagemaker.workflow.pipeline_context import PipelineSession 36 | 37 | BASE_DIR = os.path.dirname(os.path.realpath(__file__)) 38 | 39 | logger = logging.getLogger(__name__) 40 | 41 | 42 | def create_lambda_role(role_name): 43 | """Create a role for your lambda function. 44 | 45 | Args: 46 | role_name: the name of the newly created role 47 | 48 | Returns: 49 | role arn 50 | """ 51 | iam = boto3.client("iam") 52 | try: 53 | response = iam.create_role( 54 | RoleName=role_name, 55 | AssumeRolePolicyDocument=json.dumps( 56 | { 57 | "Version": "2012-10-17", 58 | "Statement": [ 59 | { 60 | "Effect": "Allow", 61 | "Principal": {"Service": "lambda.amazonaws.com"}, 62 | "Action": "sts:AssumeRole", 63 | } 64 | ], 65 | } 66 | ), 67 | Description="Role for Lambda to call SageMaker functions", 68 | ) 69 | 70 | role_arn = response["Role"]["Arn"] 71 | 72 | response = iam.attach_role_policy( 73 | RoleName=role_name, 74 | PolicyArn="arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", 75 | ) 76 | 77 | response = iam.create_policy( 78 | PolicyName='LambdaS3Access', 79 | PolicyDocument=json.dumps({ 80 | "Version": "2012-10-17", 81 | "Statement": [ 82 | { 83 | "Action": [ 84 | "s3:DeleteObject*", 85 | "s3:PutObject", 86 | "s3:Abort*" 87 | ], 88 | "Resource": [ 89 | "arn:aws:s3:::sagemaker-*", 90 | ], 91 | "Effect": "Allow" 92 | } 93 | ] 94 | }), 95 | Description='Enable write access to S3', 96 | ) 97 | 98 | arn = response["Policy"]["Arn"] 99 | 100 | response = iam.attach_role_policy( 101 | PolicyArn=arn, 102 | RoleName=role_name, 103 | ) 104 | 105 | return role_arn 106 | 107 | except iam.exceptions.EntityAlreadyExistsException: 108 | print(f"Using ARN from existing role: {role_name}") 109 | response = iam.get_role(RoleName=role_name) 110 | return response["Role"]["Arn"] 111 | 112 | 113 | def get_pipeline_session(region, default_bucket): 114 | 115 | """Gets the sagemaker pipeline session based on the region. 116 | Args: 117 | region: the aws region to start the session 118 | default_bucket: the bucket to use for storing the artifacts 119 | 120 | Returns: 121 | `sagemaker.workflow.pipeline_context.PipelineSession instance 122 | """ 123 | boto_session = boto3.Session(region_name=region) 124 | 125 | sagemaker_client = boto_session.client("sagemaker") 126 | return sagemaker.workflow.pipeline_context.PipelineSession( 127 | boto_session=boto_session, 128 | sagemaker_client=sagemaker_client, 129 | default_bucket=default_bucket, 130 | ) 131 | 132 | 133 | def resolve_ecr_uri_from_image_versions(sagemaker_session, image_versions, image_name): 134 | """Gets ECR URI from image versions 135 | Args: 136 | sagemaker_session: boto3 session for sagemaker client 137 | image_versions: list of the image versions 138 | image_name: Name of the image 139 | 140 | Returns: 141 | ECR URI of the image version 142 | """ 143 | 144 | # Fetch image details to get the Base Image URI 145 | for image_version in image_versions: 146 | if image_version["ImageVersionStatus"] == "CREATED": 147 | image_arn = image_version["ImageVersionArn"] 148 | version = image_version["Version"] 149 | logger.info(f"Identified the latest image version: {image_arn}") 150 | response = sagemaker_session.sagemaker_client.describe_image_version( 151 | ImageName=image_name, Version=version 152 | ) 153 | return response["ContainerImage"] 154 | return None 155 | 156 | 157 | def resolve_ecr_uri(sagemaker_session, image_arn): 158 | """Gets the ECR URI from the image name 159 | 160 | Args: 161 | sagemaker_session: boto3 session for sagemaker client 162 | image_name: name of the image 163 | 164 | Returns: 165 | ECR URI of the latest image version 166 | """ 167 | 168 | # Fetching image name from image_arn (^arn:aws(-[\w]+)*:sagemaker:.+:[0-9]{12}:image/[a-z0-9]([-.]?[a-z0-9])*$) 169 | image_name = image_arn.partition("image/")[2] 170 | try: 171 | # Fetch the image versions 172 | next_token = "" 173 | while True: 174 | response = sagemaker_session.sagemaker_client.list_image_versions( 175 | ImageName=image_name, 176 | MaxResults=100, 177 | SortBy="VERSION", 178 | SortOrder="DESCENDING", 179 | NextToken=next_token, 180 | ) 181 | ecr_uri = resolve_ecr_uri_from_image_versions( 182 | sagemaker_session, response["ImageVersions"], image_name 183 | ) 184 | if "NextToken" in response: 185 | next_token = response["NextToken"] 186 | 187 | if ecr_uri is not None: 188 | return ecr_uri 189 | 190 | # Return error if no versions of the image found 191 | error_message = f"No image version found for image name: {image_name}" 192 | logger.error(error_message) 193 | raise Exception(error_message) 194 | 195 | except ( 196 | ClientError, 197 | sagemaker_session.sagemaker_client.exceptions.ResourceNotFound, 198 | ) as e: 199 | error_message = e.response["Error"]["Message"] 200 | logger.error(error_message) 201 | raise Exception(error_message) 202 | 203 | 204 | def get_pipeline( 205 | region, 206 | role=None, 207 | default_bucket="sagemaker-p-project-123456789042", 208 | model_package_group_name="EnergyMgtPackageGroup", 209 | pipeline_name="EnergyMgtPipeline", 210 | base_job_prefix="EnergyMgt", 211 | project_id="p-abcdefghijkl", 212 | ): 213 | """Gets a SageMaker ML Pipeline instance working with on RUL data. 214 | 215 | Args: 216 | region: AWS region to create and run the pipeline. 217 | role: IAM role to create and run steps and pipeline. 218 | default_bucket: the bucket to use for storing the artifacts 219 | 220 | Returns: 221 | an instance of a pipeline 222 | """ 223 | pipeline_session = get_pipeline_session(region, default_bucket) 224 | lambda_role = create_lambda_role("lambda-update-manifest-role") 225 | if role is None: 226 | role = sagemaker.session.get_execution_role(pipeline_session) 227 | 228 | # parameters for pipeline execution 229 | processing_instance_count = ParameterInteger( 230 | name="ProcessingInstanceCount", default_value=1 231 | ) 232 | processing_instance_type = ParameterString( 233 | name="ProcessingInstanceType", default_value="ml.m5.xlarge" 234 | ) 235 | training_instance_type = ParameterString( 236 | name="TrainingInstanceType", default_value="ml.m5.4xlarge" 237 | ) 238 | inference_instance_type = ParameterString( 239 | name="InferenceInstanceType", default_value="ml.m5.4xlarge" 240 | ) 241 | model_approval_status = ParameterString( 242 | name="ModelApprovalStatus", default_value="PendingManualApproval" 243 | ) 244 | processing_image_name = "sagemaker-{0}-processing-imagebuild".format(project_id) 245 | training_image_name = "sagemaker-{0}-training-imagebuild".format(project_id) 246 | inference_image_name = "sagemaker-{0}-training-imagebuild".format(project_id) 247 | 248 | # processing step for feature engineering 249 | try: 250 | processing_image_uri = ( 251 | pipeline_session.sagemaker_client.describe_image_version( 252 | ImageName=processing_image_name 253 | )["ContainerImage"] 254 | ) 255 | except (pipeline_session.sagemaker_client.exceptions.ResourceNotFound): 256 | processing_image_uri = sagemaker.image_uris.retrieve( 257 | framework="xgboost", 258 | region=region, 259 | version="1.3-1", 260 | py_version="py3", 261 | instance_type=processing_instance_type, 262 | ) 263 | script_processor = ScriptProcessor( 264 | image_uri=processing_image_uri, 265 | instance_type=processing_instance_type, 266 | instance_count=processing_instance_count, 267 | base_job_name=f"{base_job_prefix}/EnergyMgt-preprocess", 268 | command=["python3"], 269 | sagemaker_session=pipeline_session, 270 | role=role, 271 | volume_size_in_gb=100, 272 | network_config=sagemaker.network.NetworkConfig(enable_network_isolation=True) 273 | ) 274 | processing_args = script_processor.run( 275 | outputs=[ 276 | ProcessingOutput(output_name="train", source="/opt/ml/processing/train"), 277 | ], 278 | code=os.path.join(BASE_DIR, "preprocess.py"), 279 | arguments=["--bucket-name", default_bucket], 280 | ) 281 | step_process = ProcessingStep( 282 | name="PreprocessEnergyMgtData", 283 | step_args=processing_args 284 | ) 285 | 286 | # training step for generating model artifacts 287 | model_path = ( 288 | f"s3://{pipeline_session.default_bucket()}/{base_job_prefix}/EnergyMgtTrain" 289 | ) 290 | 291 | try: 292 | training_image_uri = pipeline_session.sagemaker_client.describe_image_version( 293 | ImageName=training_image_name 294 | )["ContainerImage"] 295 | except (pipeline_session.sagemaker_client.exceptions.ResourceNotFound): 296 | training_image_uri = sagemaker.image_uris.retrieve( 297 | framework="xgboost", 298 | region=region, 299 | version="1.3-1", 300 | py_version="py3", 301 | instance_type=training_instance_type, 302 | ) 303 | 304 | xgb_train = Estimator( 305 | image_uri=training_image_uri, 306 | instance_type=training_instance_type, 307 | instance_count=1, 308 | output_path=model_path, 309 | base_job_name=f"{base_job_prefix}/EnergyMgt-train", 310 | sagemaker_session=pipeline_session, 311 | role=role, 312 | enable_sagemaker_metrics=True, 313 | metric_definitions=[ 314 | {'Name': 'keras_validation_c1c2:mae', 'Regex': 'Validation_c1c2_keras_mae=(.*?);'}, 315 | {'Name': 'keras_validation_c1c2c4:mae', 'Regex': 'Validation_c1c2c4_keras_mae=(.*?);'}, 316 | {'Name': 'keras_validation_c1c4:mae', 'Regex': 'Validation_c1c4_keras_mae=(.*?);'}, 317 | {'Name': 'keras_validation_c1c3c4:mae', 'Regex': 'Validation_c1c3c4_keras_mae=(.*?);'}, 318 | {'Name': 'keras_validation_c1c2c3c4:mae', 'Regex': 'Validation_c1c2c3c4_keras_mae=(.*?);'}, 319 | {'Name': 'keras_validation_c1c3:mae', 'Regex': 'Validation_c1c3_keras_mae=(.*?);'}, 320 | {'Name': 'keras_validation_c1c2c3:mae', 'Regex': 'Validation_c1c2c3_keras_mae=(.*?);'}, 321 | {'Name': 'keras_validation_c3c4:mae', 'Regex': 'Validation_c3c4_keras_mae=(.*?);'}, 322 | {'Name': 'keras_validation_c2c3c4:mae', 'Regex': 'Validation_c2c3c4_keras_mae=(.*?);'}, 323 | {'Name': 'keras_validation_c2c4:mae', 'Regex': 'Validation_c2c4_keras_mae=(.*?);'}, 324 | {'Name': 'keras_validation_c2c3:mae', 'Regex': 'Validation_c2c3_keras_mae=(.*?);'}, 325 | {'Name': 'keras_train_c1c2:mae', 'Regex': 'Train_c1c2_keras_mae=(.*?);'}, 326 | {'Name': 'keras_train_c1c2c4:mae', 'Regex': 'Train_c1c2c4_keras_mae=(.*?);'}, 327 | {'Name': 'keras_train_c1c4:mae', 'Regex': 'Train_c1c4_keras_mae=(.*?);'}, 328 | {'Name': 'keras_train_c1c3c4:mae', 'Regex': 'Train_c1c3c4_keras_mae=(.*?);'}, 329 | {'Name': 'keras_train_c1c2c3c4:mae', 'Regex': 'Train_c1c2c3c4_keras_mae=(.*?);'}, 330 | {'Name': 'keras_train_c1c3:mae', 'Regex': 'Train_c1c3_keras_mae=(.*?);'}, 331 | {'Name': 'keras_train_c1c2c3:mae', 'Regex': 'Train_c1c2c3_keras_mae=(.*?);'}, 332 | {'Name': 'keras_train_c3c4:mae', 'Regex': 'Train_c3c4_keras_mae=(.*?);'}, 333 | {'Name': 'keras_train_c2c3c4:mae', 'Regex': 'Train_c2c3c4_keras_mae=(.*?);'}, 334 | {'Name': 'keras_train_c2c4:mae', 'Regex': 'Train_c2c4_keras_mae=(.*?);'}, 335 | {'Name': 'keras_train_c2c3:mae', 'Regex': 'Train_c2c3_keras_mae=(.*?);'}, 336 | ], 337 | enable_network_isolation=True, 338 | disable_profiler=True 339 | 340 | ) 341 | 342 | xgb_train.set_hyperparameters( 343 | quantile=0.5, test_size=0.2, random_state=42, degree=4, remove_outliers=False, 344 | verbose=1, dual_model=False, optimizer="adam", optimizer__learning_rate=0.001, 345 | epochs=2000 346 | ) 347 | train_args = xgb_train.fit({ 348 | "train": TrainingInput( 349 | s3_data=step_process.properties.ProcessingOutputConfig.Outputs[ 350 | "train" 351 | ].S3Output.S3Uri, 352 | content_type="text/csv", 353 | ), 354 | }) 355 | step_train = TrainingStep( 356 | name="TrainEnergyMgtModel", 357 | step_args=train_args 358 | ) 359 | 360 | try: 361 | inference_image_uri = pipeline_session.sagemaker_client.describe_image_version( 362 | ImageName=inference_image_name 363 | )["ContainerImage"] 364 | except (pipeline_session.sagemaker_client.exceptions.ResourceNotFound): 365 | inference_image_uri = sagemaker.image_uris.retrieve( 366 | framework="xgboost", 367 | region=region, 368 | version="1.3-1", 369 | py_version="py3", 370 | instance_type=inference_instance_type, 371 | ) 372 | 373 | simulate_processor = ScriptProcessor( 374 | image_uri=processing_image_uri, 375 | instance_type="ml.c5.9xlarge", 376 | instance_count=processing_instance_count, 377 | base_job_name=f"{base_job_prefix}/EnergyMgt-simulate", 378 | command=["python3"], 379 | sagemaker_session=pipeline_session, 380 | role=role, 381 | volume_size_in_gb=10, 382 | network_config=sagemaker.network.NetworkConfig(enable_network_isolation=True) 383 | 384 | ) 385 | inference_args = simulate_processor.run( 386 | inputs = [ 387 | ProcessingInput( 388 | source=step_train.properties.ModelArtifacts.S3ModelArtifacts, 389 | destination='/opt/ml/processing/input', 390 | input_name="model.tar.gz") 391 | ], 392 | outputs=[ 393 | ProcessingOutput(output_name="train", source="/opt/ml/processing/train"), 394 | ], 395 | code=os.path.join(BASE_DIR, "simulate.py"), 396 | ) 397 | simulate_step = ProcessingStep( 398 | name="SimulateEnergyMgtData", 399 | step_args=inference_args, 400 | depends_on=[step_train], 401 | ) 402 | 403 | # Lambda helper class can be used to create the Lambda function 404 | func = Lambda( 405 | function_name="lambda-update-manifest", 406 | execution_role_arn=lambda_role, 407 | script=os.path.join(BASE_DIR, "lambda_handler.py"), 408 | handler="lambda_handler.lambda_handler", 409 | ) 410 | 411 | output_param_1 = LambdaOutput(output_name="statusCode", output_type=LambdaOutputTypeEnum.String) 412 | output_param_2 = LambdaOutput(output_name="body", output_type=LambdaOutputTypeEnum.String) 413 | 414 | step_put_manifest = LambdaStep( 415 | name="UpdateManifestLambda", 416 | lambda_func=func, 417 | inputs={ 418 | "TRAINING": step_process.properties.ProcessingOutputConfig.Outputs["train"].S3Output.S3Uri, 419 | "SIMULATION": simulate_step.properties.ProcessingOutputConfig.Outputs["train"].S3Output.S3Uri, 420 | "BUCKET": default_bucket, 421 | }, 422 | outputs=[output_param_1, output_param_2], 423 | depends_on=[simulate_step], 424 | ) 425 | 426 | step_register = RegisterModel( 427 | name="RegisterEnergyMgtModel", 428 | estimator=xgb_train, 429 | image_uri=inference_image_uri, 430 | model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts, 431 | content_types=["text/csv"], 432 | response_types=["text/csv"], 433 | inference_instances=["ml.t2.medium", "ml.m5.large"], 434 | transform_instances=["ml.m5.large"], 435 | model_package_group_name=model_package_group_name, 436 | approval_status=model_approval_status, 437 | depends_on=[step_put_manifest], 438 | ) 439 | 440 | # pipeline instance 441 | pipeline = Pipeline( 442 | name=pipeline_name, 443 | parameters=[ 444 | processing_instance_type, 445 | processing_instance_count, 446 | training_instance_type, 447 | model_approval_status, 448 | ], 449 | steps=[step_process, step_train, simulate_step, step_put_manifest, step_register], 450 | sagemaker_session=pipeline_session, 451 | ) 452 | return pipeline 453 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/pipelines/run_pipeline.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | """A CLI to create or update and run pipelines.""" 14 | from __future__ import absolute_import 15 | 16 | import argparse 17 | import json 18 | import sys 19 | 20 | from pipelines._utils import convert_struct, get_pipeline_driver 21 | 22 | 23 | def main(): # pragma: no cover 24 | """The main harness that creates or updates and runs the pipeline. 25 | 26 | Creates or updates the pipeline and runs it. 27 | """ 28 | parser = argparse.ArgumentParser( 29 | "Creates or updates and runs the pipeline for the pipeline script." 30 | ) 31 | 32 | parser.add_argument( 33 | "-n", 34 | "--module-name", 35 | dest="module_name", 36 | type=str, 37 | help="The module name of the pipeline to import.", 38 | ) 39 | parser.add_argument( 40 | "-kwargs", 41 | "--kwargs", 42 | dest="kwargs", 43 | default=None, 44 | help="Dict string of keyword arguments for the pipeline generation (if supported)", 45 | ) 46 | parser.add_argument( 47 | "-role-arn", 48 | "--role-arn", 49 | dest="role_arn", 50 | type=str, 51 | help="The role arn for the pipeline service execution role.", 52 | ) 53 | parser.add_argument( 54 | "-description", 55 | "--description", 56 | dest="description", 57 | type=str, 58 | default=None, 59 | help="The description of the pipeline.", 60 | ) 61 | parser.add_argument( 62 | "-tags", 63 | "--tags", 64 | dest="tags", 65 | default=None, 66 | help="""List of dict strings of '[{"Key": "string", "Value": "string"}, ..]'""", 67 | ) 68 | args = parser.parse_args() 69 | 70 | if args.module_name is None or args.role_arn is None: 71 | parser.print_help() 72 | sys.exit(2) 73 | tags = convert_struct(args.tags) 74 | 75 | try: 76 | pipeline = get_pipeline_driver(args.module_name, args.kwargs) 77 | print( 78 | "###### Creating/updating a SageMaker Pipeline with the following definition:" 79 | ) 80 | parsed = json.loads(pipeline.definition()) 81 | print(json.dumps(parsed, indent=2, sort_keys=True)) 82 | 83 | upsert_response = pipeline.upsert( 84 | role_arn=args.role_arn, description=args.description, tags=tags 85 | ) 86 | print("\n###### Created/Updated SageMaker Pipeline: Response received:") 87 | print(upsert_response) 88 | 89 | execution = pipeline.start() 90 | print(f"\n###### Execution started with PipelineExecutionArn: {execution.arn}") 91 | 92 | print("Waiting for the execution to finish...") 93 | execution.wait(delay=60, max_attempts=200) 94 | print("\n#####Execution completed. Execution step details:") 95 | 96 | print(execution.list_steps()) 97 | # Todo print the status? 98 | except Exception as e: # pylint: disable=W0703 99 | print(f"Exception: {e}") 100 | sys.exit(1) 101 | 102 | 103 | if __name__ == "__main__": 104 | main() 105 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/sagemaker-pipelines-project.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Orchestrating Jobs, Model Registration, and Continuous Deployment with Amazon SageMaker\n", 8 | "\n", 9 | "Amazon SageMaker offers Machine Learning application developers and Machine Learning operations engineers the ability to orchestrate SageMaker jobs and author reproducible Machine Learning pipelines, deploy custom-build models for inference in real-time with low latency or offline inferences with Batch Transform, and track lineage of artifacts. You can institute sound operational practices in deploying and monitoring production workflows, deployment of model artifacts, and track artifact lineage through a simple interface, adhering to safety and best-practice paradigms for Machine Learning application development.\n", 10 | "\n", 11 | "The SageMaker Pipelines service supports a SageMaker Machine Learning Pipeline Domain Specific Language (DSL), which is a declarative Json specification. This DSL defines a Directed Acyclic Graph (DAG) of pipeline parameters and SageMaker job steps. The SageMaker Python Software Developer Kit (SDK) streamlines the generation of the pipeline DSL using constructs that are already familiar to engineers and scientists alike.\n", 12 | "\n", 13 | "The SageMaker Model Registry is where trained models are stored, versioned, and managed. Data Scientists and Machine Learning Engineers can compare model versions, approve models for deployment, and deploy models from different AWS accounts, all from a single Model Registry. SageMaker enables customers to follow the best practices with ML Ops and getting started right. Customers are able to standup a full ML Ops end-to-end system with a single API call." 14 | ] 15 | }, 16 | { 17 | "cell_type": "markdown", 18 | "metadata": {}, 19 | "source": [ 20 | "## SageMaker Pipelines\n", 21 | "\n", 22 | "Amazon SageMaker Pipelines support the following activites:\n", 23 | "\n", 24 | "* Pipelines - A Directed Acyclic Graph of steps and conditions to orchestrate SageMaker jobs and resource creation.\n", 25 | "* Processing Job steps - A simplified, managed experience on SageMaker to run data processing workloads, such as feature engineering, data validation, model evaluation, and model interpretation.\n", 26 | "* Training Job steps - An iterative process that teaches a model to make predictions by presenting examples from a training dataset.\n", 27 | "* Conditional step execution - Provides conditional execution of branches in a pipeline.\n", 28 | "* Registering Models - Creates a model package resource in the Model Registry that can be used to create deployable models in Amazon SageMaker.\n", 29 | "* Creating Model steps - Create a model for use in transform steps or later publication as an endpoint.\n", 30 | "* Parameterized Pipeline executions - Allows pipeline executions to vary by supplied parameters.\n", 31 | "* Transform Job steps - A batch transform to preprocess datasets to remove noise or bias that interferes with training or inference from your dataset, get inferences from large datasets, and run inference when you don't need a persistent endpoint." 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "metadata": {}, 37 | "source": [ 38 | "## Layout of the SageMaker ImageBuild ModelBuild ModelDeploy Project Template\n", 39 | "\n", 40 | "The template provides a starting point for bringing your SageMaker Pipeline development to production.\n", 41 | "\n", 42 | "```\n", 43 | "|-- codebuild-buildspec.yml\n", 44 | "|-- pipelines\n", 45 | "| |-- modelbuildpipeline\n", 46 | "| | |-- __init__.py\n", 47 | "| | |-- pipeline.py\n", 48 | "| | |-- preprocess.py\n", 49 | "| |-- get_pipeline_definition.py\n", 50 | "| |-- __init__.py\n", 51 | "| |-- run_pipeline.py\n", 52 | "| |-- _utils.py\n", 53 | "| |-- __version__.py\n", 54 | "|-- README.md\n", 55 | "|-- sagemaker-pipelines-project.ipynb\n", 56 | "|-- setup.cfg\n", 57 | "|-- setup.py\n", 58 | "|-- tests\n", 59 | "| |-- test_pipelines.py\n", 60 | "|-- tox.ini\n", 61 | "```" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "A description of some of the artifacts is provided below:\n", 69 | "

\n", 70 | "Your codebuild execution instructions:\n", 71 | "```\n", 72 | "|-- codebuild-buildspec.yml\n", 73 | "```\n", 74 | "

\n", 75 | "Your pipeline artifacts, which includes a pipeline module defining the required `get_pipeline` method that returns an instance of a SageMaker pipeline, a preprocessing script that is used in feature engineering, and a model evaluation script to measure the Mean Squared Error of the model that's trained by the pipeline:\n", 76 | "\n", 77 | "```\n", 78 | "|-- pipelines\n", 79 | "| |-- modelbuildpipeline\n", 80 | "| | |-- __init__.py\n", 81 | "| | |-- pipeline.py\n", 82 | "| | `-- preprocess.py\n", 83 | "\n", 84 | "```\n", 85 | "

\n", 86 | "Utility modules for getting pipeline definition jsons and running pipelines:\n", 87 | "\n", 88 | "```\n", 89 | "|-- pipelines\n", 90 | "| |-- get_pipeline_definition.py\n", 91 | "| |-- __init__.py\n", 92 | "| |-- run_pipeline.py\n", 93 | "| |-- _utils.py\n", 94 | "| `-- __version__.py\n", 95 | "```\n", 96 | "

\n", 97 | "Python package artifacts:\n", 98 | "```\n", 99 | "|-- setup.cfg\n", 100 | "|-- setup.py\n", 101 | "```\n", 102 | "

\n", 103 | "A stubbed testing module for testing your pipeline as you develop:\n", 104 | "```\n", 105 | "|-- tests\n", 106 | "| `-- test_pipelines.py\n", 107 | "```\n", 108 | "

\n", 109 | "The `tox` testing framework configuration:\n", 110 | "```\n", 111 | "`-- tox.ini\n", 112 | "```" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "### A SageMaker Pipeline\n", 120 | "\n", 121 | "The pipeline that we create follows a typical Machine Learning Application pattern of pre-processing, training, evaluation, and conditional model registration and publication, if the quality of the model is sufficient.\n", 122 | "\n", 123 | "Also, we are allowed to create the image for processing/ training/ inference containers, depending on which the SDK utilizes the Image URI to perform the model building.\n", 124 | "\n", 125 | "![A typical ML Application pipeline](img/pipeline-full.png)\n", 126 | "\n", 127 | "### Getting some constants\n", 128 | "\n", 129 | "We get some constants from the local execution environment." 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "import boto3\n", 139 | "import sagemaker\n", 140 | "\n", 141 | "PROJECT_ID = \"TO BE FILLED\" # p-randomstring: your project ID\n", 142 | "PROJECT_NAME = \"TO BE FILLED\" # Your project name\n", 143 | "\n", 144 | "region = boto3.Session().region_name\n", 145 | "role = sagemaker.get_execution_role()\n", 146 | "default_bucket = \"TO BE FILLED\" # the S3 bucket you want to use!\n", 147 | "\n", 148 | "# Change these to reflect your project/business name or if you want to separate ModelPackageGroup/Pipeline from the rest of your team\n", 149 | "model_package_group_name = f\"{PROJECT_NAME}-{PROJECT_ID}\"\n", 150 | "pipeline_name = f\"{PROJECT_NAME}-{PROJECT_ID}\"" 151 | ] 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "metadata": {}, 156 | "source": [ 157 | "### Get the pipeline instance\n", 158 | "\n", 159 | "Here we get the pipeline instance from your pipeline module so that we can work with it." 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "execution_count": null, 165 | "metadata": {}, 166 | "outputs": [], 167 | "source": [ 168 | "from pipelines.modelbuildpipeline.pipeline import get_pipeline\n", 169 | "\n", 170 | "\n", 171 | "pipeline = get_pipeline(\n", 172 | " region=region,\n", 173 | " role=role,\n", 174 | " default_bucket=default_bucket,\n", 175 | " model_package_group_name=model_package_group_name,\n", 176 | " pipeline_name=pipeline_name,\n", 177 | " base_job_prefix=\"pipeline\",\n", 178 | " project_id=PROJECT_ID\n", 179 | ")" 180 | ] 181 | }, 182 | { 183 | "cell_type": "markdown", 184 | "metadata": {}, 185 | "source": [ 186 | "### Submit the pipeline to SageMaker and start execution\n", 187 | "\n", 188 | "Let's submit our pipeline definition to the workflow service. The role passed in will be used by the workflow service to create all the jobs defined in the steps." 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": null, 194 | "metadata": {}, 195 | "outputs": [], 196 | "source": [ 197 | "pipeline.upsert(role_arn=role)" 198 | ] 199 | }, 200 | { 201 | "cell_type": "markdown", 202 | "metadata": {}, 203 | "source": [ 204 | "We'll start the pipeline, accepting all the default parameters.\n", 205 | "\n", 206 | "Values can also be passed into these pipeline parameters on starting of the pipeline, and will be covered later. " 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": null, 212 | "metadata": {}, 213 | "outputs": [], 214 | "source": [ 215 | "execution = pipeline.start()" 216 | ] 217 | }, 218 | { 219 | "cell_type": "markdown", 220 | "metadata": {}, 221 | "source": [ 222 | "### Pipeline Operations: examining and waiting for pipeline execution\n", 223 | "\n", 224 | "Now we describe execution instance and list the steps in the execution to find out more about the execution." 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": null, 230 | "metadata": {}, 231 | "outputs": [], 232 | "source": [ 233 | "execution.describe()" 234 | ] 235 | }, 236 | { 237 | "cell_type": "markdown", 238 | "metadata": {}, 239 | "source": [ 240 | "We can wait for the execution by invoking `wait()` on the execution:" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": null, 246 | "metadata": {}, 247 | "outputs": [], 248 | "source": [ 249 | "execution.wait()" 250 | ] 251 | }, 252 | { 253 | "cell_type": "markdown", 254 | "metadata": {}, 255 | "source": [ 256 | "We can list the execution steps to check out the status and artifacts:" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": null, 262 | "metadata": {}, 263 | "outputs": [], 264 | "source": [ 265 | "execution.list_steps()" 266 | ] 267 | }, 268 | { 269 | "cell_type": "markdown", 270 | "metadata": {}, 271 | "source": [ 272 | "### Parameterized Executions\n", 273 | "\n", 274 | "We can run additional executions of the pipeline specifying different pipeline parameters. The parameters argument is a dictionary whose names are the parameter names, and whose values are the primitive values to use as overrides of the defaults.\n", 275 | "\n", 276 | "Of particular note, based on the performance of the model, we may want to kick off another pipeline execution, but this time on a compute-optimized instance type and set the model approval status automatically be \"Approved\". This means that the model package version generated by the `RegisterModel` step will automatically be ready for deployment through CI/CD pipelines, such as with SageMaker Projects." 277 | ] 278 | }, 279 | { 280 | "cell_type": "code", 281 | "execution_count": null, 282 | "metadata": {}, 283 | "outputs": [], 284 | "source": [ 285 | "execution = pipeline.start(\n", 286 | " parameters=dict(\n", 287 | " ProcessingInstanceType=\"ml.c5.xlarge\",\n", 288 | " ModelApprovalStatus=\"Approved\",\n", 289 | " )\n", 290 | ")" 291 | ] 292 | }, 293 | { 294 | "cell_type": "code", 295 | "execution_count": null, 296 | "metadata": {}, 297 | "outputs": [], 298 | "source": [ 299 | "execution.wait()" 300 | ] 301 | }, 302 | { 303 | "cell_type": "code", 304 | "execution_count": null, 305 | "metadata": {}, 306 | "outputs": [], 307 | "source": [ 308 | "execution.list_steps()" 309 | ] 310 | } 311 | ], 312 | "metadata": { 313 | "instance_type": "ml.t3.medium", 314 | "kernelspec": { 315 | "display_name": "Python 3 (Data Science)", 316 | "language": "python", 317 | "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/datascience-1.0" 318 | }, 319 | "language_info": { 320 | "codemirror_mode": { 321 | "name": "ipython", 322 | "version": 3 323 | }, 324 | "file_extension": ".py", 325 | "mimetype": "text/x-python", 326 | "name": "python", 327 | "nbconvert_exporter": "python", 328 | "pygments_lexer": "ipython3", 329 | "version": "3.7.10" 330 | } 331 | }, 332 | "nbformat": 4, 333 | "nbformat_minor": 4 334 | } 335 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/setup.cfg: -------------------------------------------------------------------------------- 1 | [tool:pytest] 2 | addopts = 3 | -vv 4 | testpaths = tests 5 | 6 | [aliases] 7 | test=pytest 8 | 9 | [metadata] 10 | description-file = README.md 11 | license_file = LICENSE 12 | 13 | [wheel] 14 | universal = 1 15 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import setuptools 4 | from pipelines import __version__ as version 5 | 6 | with open("README.md", "r") as f: 7 | readme = f.read() 8 | 9 | 10 | required_packages = [ 11 | "pip==22.1.2", 12 | "boto3==1.24.22", 13 | "shap==0.41.0", 14 | "imbalanced-learn==0.9.1", 15 | "werkzeug==2.0.3", 16 | "pandas==1.2.4", 17 | "numpy==1.22.0", 18 | "xgboost==1.5.2", 19 | "scikit-learn==1.1.1", 20 | "sagemaker", 21 | "pre-commit", 22 | "pytest==6.2.5", 23 | "optuna==2.10.1", 24 | "scikeras==0.8.0", 25 | "tensorflow-cpu==2.9.3", 26 | "keras==2.9.0", 27 | ] 28 | extras = { 29 | "test": [ 30 | "black", 31 | "coverage", 32 | "flake8", 33 | "mock", 34 | "pydocstyle", 35 | "pytest", 36 | "pytest-cov", 37 | "sagemaker", 38 | "tox", 39 | ] 40 | } 41 | setuptools.setup( 42 | name=version.__title__, 43 | description=version.__description__, 44 | version=version.__version__, 45 | author=version.__author__, 46 | author_email=version.__author_email__, 47 | long_description=readme, 48 | long_description_content_type="text/markdown", 49 | url=version.__url__, 50 | license=version.__license__, 51 | packages=setuptools.find_packages(), 52 | include_package_data=True, 53 | python_requires=">=3.6", 54 | install_requires=required_packages, 55 | extras_require=extras, 56 | entry_points={ 57 | "console_scripts": [ 58 | "get-pipeline-definition=pipelines.get_pipeline_definition:main", 59 | "run-pipeline=pipelines.run_pipeline:main", 60 | ] 61 | }, 62 | classifiers=[ 63 | "Development Status :: 3 - Alpha", 64 | "Intended Audience :: Developers", 65 | "Natural Language :: English", 66 | "Programming Language :: Python", 67 | "Programming Language :: Python :: 3", 68 | "Programming Language :: Python :: 3.6", 69 | "Programming Language :: Python :: 3.7", 70 | "Programming Language :: Python :: 3.8", 71 | ], 72 | ) 73 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/tests/test_pipelines.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | # @pytest.mark.xfail 5 | @pytest.mark.skip(reason="no way of currently testing this") 6 | def test_that_you_wrote_tests(): 7 | assert False, "No tests written" 8 | 9 | @pytest.mark.skip(reason="no way of currently testing this") 10 | def test_pipelines_importable(): 11 | import pipelines # noqa: F401 12 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modelbuild/tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | envlist = black-format,flake8,pydocstyle,py{36,37,38} 8 | 9 | [flake8] 10 | max-line-length = 120 11 | exclude = 12 | build/ 13 | .git 14 | __pycache__ 15 | .tox 16 | venv/ 17 | 18 | max-complexity = 10 19 | 20 | ignore = 21 | C901, 22 | E203, # whitespace before ':': Black disagrees with and explicitly violates this. 23 | FI10, 24 | FI12, 25 | FI13, 26 | FI14, 27 | FI15, 28 | FI16, 29 | FI17, 30 | FI18, # __future__ import "annotations" missing -> check only Python 3.7 compatible 31 | FI50, 32 | FI51, 33 | FI52, 34 | FI53, 35 | FI54, 36 | FI55, 37 | FI56, 38 | FI57, 39 | W503 40 | 41 | require-code = True 42 | 43 | [testenv] 44 | commands = 45 | pytest --cov=pipelines --cov-append {posargs} 46 | coverage report --fail-under=0 47 | deps = .[test] 48 | depends = 49 | {py36,py37,py38}: clean 50 | 51 | [testenv:flake8] 52 | skipdist = true 53 | skip_install = true 54 | deps = flake8 55 | commands = flake8 56 | 57 | [testenv:black-format] 58 | deps = black 59 | commands = 60 | black -l 100 ./ 61 | 62 | [testenv:black-check] 63 | deps = black 64 | commands = 65 | black -l 100 --check ./ 66 | 67 | [testenv:clean] 68 | skip_install = true 69 | deps = coverage 70 | commands = coverage erase 71 | 72 | [testenv:pydocstyle] 73 | deps = pydocstyle 74 | commands = 75 | pydocstyle pipelines 76 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E203, E266, E501, W503, F403, F401, E741, F541, C901 3 | max-line-length = 120 4 | max-complexity = 18 5 | select = B,C,E,F,W,T4,B9 6 | exclude = .ipynb_checkpoints,.git,__pycache__,docs/source/conf.py,old,build,dist 7 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/.gitignore: -------------------------------------------------------------------------------- 1 | *.pdf 2 | *.xls 3 | *.xlsx 4 | *.pdf 5 | *.env 6 | *.zip 7 | venv/ 8 | .venv/ 9 | *__pycache__* 10 | __pycache__/ 11 | .DS_Store 12 | .vscode/ 13 | .ipynb_checkpoints/ 14 | *.ipynb_checkpoints/* 15 | cdk.context.json 16 | .cdk.staging/ 17 | cdk.out/ 18 | *.egg-info 19 | *.egg 20 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | known_third_party=SageMakerPipelineConstructs,SageMakerPipelineSourceCode,aws_cdk,boto3,botocore,constructs,data_pipeline,digital_twin,joblib,numpy,pandas,pipelines,pytest,quantile_regression,river,sagemaker,scipy,setuptools,sklearn,tqdm 3 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v2.4.0 6 | hooks: 7 | - id: trailing-whitespace 8 | - id: end-of-file-fixer 9 | - id: flake8 10 | - repo: https://github.com/asottile/seed-isort-config 11 | rev: v1.9.3 12 | hooks: 13 | - id: seed-isort-config 14 | - repo: https://github.com/pre-commit/mirrors-isort 15 | rev: v4.3.21 16 | hooks: 17 | - id: isort 18 | additional_dependencies: [toml] 19 | - repo: https://github.com/psf/black 20 | rev: 20.8b1 21 | hooks: 22 | - id: black 23 | language_version: python3 24 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/DigitalTwin/DigitalTwin_stack.py: -------------------------------------------------------------------------------- 1 | from aws_cdk import Duration, Stack 2 | from aws_cdk import aws_iam as _iam 3 | from aws_cdk import aws_lambda as _lambda 4 | from aws_cdk import aws_lambda_event_sources as _event_sources 5 | from aws_cdk import aws_s3 as _s3 6 | from aws_cdk import Size 7 | from constructs import Construct 8 | 9 | class DigitalTwinStack(Stack): 10 | 11 | def __init__(self, scope: Construct, id: str, **kwargs) -> None: 12 | super().__init__(scope, id, **kwargs) 13 | 14 | # Defines an AWS Lambda resource 15 | _lambda.DockerImageFunction( 16 | self, 'DigitalTwin', 17 | code=_lambda.DockerImageCode.from_image_asset("lambda/digital_twin"), 18 | memory_size=1024, 19 | ephemeral_storage_size=Size.mebibytes(1024), 20 | timeout=Duration.seconds(120), 21 | ) -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/DigitalTwin/__init__.py: -------------------------------------------------------------------------------- 1 | # Initialize. 2 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/README.md: -------------------------------------------------------------------------------- 1 | ## MLOps for SageMaker Endpoint Deployment 2 | 3 | This is a sample code repository for demonstrating how you can organize your code for deploying an realtime inference Endpoint infrastructure. This code repository is created as part of creating a Project in SageMaker. 4 | 5 | This code repository has the code to find the latest approved ModelPackage for the associated ModelPackageGroup and automaticaly deploy it to the Endpoint on detecting a change (`build.py`). This code repository also defines the CloudFormation template which defines the Endpoints as infrastructure. It also has configuration files associated with `staging` and `prod` stages. 6 | 7 | Upon triggering a deployment, the CodePipeline pipeline will deploy 2 Endpoints - `staging` and `prod`. After the first deployment is completed, the CodePipeline waits for a manual approval step for promotion to the prod stage. You will need to go to CodePipeline AWS Managed Console to complete this step. 8 | 9 | You own this code and you can modify this template to change as you need it, add additional tests for your custom validation. 10 | 11 | A description of some of the artifacts is provided below: 12 | 13 | 14 | ## Layout of the SageMaker ModelBuild Project Template 15 | 16 | `buildspec.yml` 17 | - this file is used by the CodePipeline's Build stage to build a CloudFormation template. 18 | 19 | `build.py` 20 | - this python file contains code to get the latest approve package arn and exports staging and configuration files. This is invoked from the Build stage. 21 | 22 | `endpoint-config-template.yml` 23 | - this CloudFormation template file is packaged by the build step in the CodePipeline and is deployed in different stages. 24 | 25 | `staging-config.json` 26 | - this configuration file is used to customize `staging` stage in the pipeline. You can configure the instance type, instance count here. 27 | 28 | `prod-config.json` 29 | - this configuration file is used to customize `prod` stage in the pipeline. You can configure the instance type, instance count here. 30 | 31 | `test\buildspec.yml` 32 | - this file is used by the CodePipeline's `staging` stage to run the test code of the following python file 33 | 34 | `test\test.py` 35 | - this python file contains code to describe and invoke the staging endpoint. You can customize to add more tests here. 36 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import aws_cdk as cdk 4 | import cdk_nag as cdknag 5 | from DigitalTwin.DigitalTwin_stack import DigitalTwinStack 6 | 7 | app = cdk.App() 8 | 9 | stack = DigitalTwinStack( 10 | app, 11 | "DigitalTwinStack") 12 | cdk.Aspects.of(app).add(cdknag.AwsSolutionsChecks()) 13 | cdknag.NagSuppressions.add_stack_suppressions( 14 | stack, 15 | [ 16 | cdknag.NagPackSuppression( 17 | id="AwsSolutions-IAM4", 18 | reason="Use AWS managed poclicies AWSLambdaBasicExecutionRole", 19 | ) 20 | ], 21 | ) 22 | app.synth() 23 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/build.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import logging 4 | import os 5 | 6 | import boto3 7 | from botocore.exceptions import ClientError 8 | 9 | logger = logging.getLogger(__name__) 10 | sm_client = boto3.client("sagemaker") 11 | s3 = boto3.client("s3") 12 | paginator = sm_client.get_paginator('list_training_jobs') 13 | 14 | def get_approved_package(model_package_group_name): 15 | """Gets the latest approved model package for a model package group. 16 | 17 | Args: 18 | model_package_group_name: The model package group name. 19 | 20 | Returns: 21 | The SageMaker Model Package Creation Time. 22 | """ 23 | try: 24 | # Get the latest approved model package 25 | response = sm_client.list_model_packages( 26 | ModelPackageGroupName=model_package_group_name, 27 | ModelApprovalStatus="Approved", 28 | SortBy="CreationTime", 29 | MaxResults=100, 30 | ) 31 | approved_packages = response["ModelPackageSummaryList"] 32 | 33 | # Fetch more packages if none returned with continuation token 34 | while len(approved_packages) == 0 and "NextToken" in response: 35 | logger.debug( 36 | "Getting more packages for token: {}".format(response["NextToken"]) 37 | ) 38 | response = sm_client.list_model_packages( 39 | ModelPackageGroupName=model_package_group_name, 40 | ModelApprovalStatus="Approved", 41 | SortBy="CreationTime", 42 | MaxResults=100, 43 | NextToken=response["NextToken"], 44 | ) 45 | approved_packages.extend(response["ModelPackageSummaryList"]) 46 | 47 | # Return error if no packages found 48 | if len(approved_packages) == 0: 49 | error_message = f"No approved ModelPackage found for ModelPackageGroup: {model_package_group_name}" 50 | logger.error(error_message) 51 | raise Exception(error_message) 52 | 53 | desc = sm_client.describe_model_package( 54 | ModelPackageName=approved_packages[0]["ModelPackageArn"] 55 | ) 56 | print("Initial ARN: ", approved_packages[0]["ModelPackageArn"]) 57 | model = desc["InferenceSpecification"]["Containers"][0]["ModelDataUrl"] 58 | dttm = desc["LastModifiedTime"] 59 | for package in approved_packages: 60 | desc = sm_client.describe_model_package( 61 | ModelPackageName=package["ModelPackageArn"] 62 | ) 63 | if desc["LastModifiedTime"] > dttm: 64 | print("Updated ARN: ", approved_packages[0]["ModelPackageArn"]) 65 | model = desc["InferenceSpecification"]["Containers"][0]["ModelDataUrl"] 66 | 67 | return model 68 | except ClientError as e: 69 | error_message = e.response["Error"]["Message"] 70 | logger.error(error_message) 71 | raise Exception(error_message) 72 | 73 | def download_model_tar_from_s3(model_package_group_name): 74 | s3_url = get_approved_package(model_package_group_name=model_package_group_name) 75 | with open('model.tar.gz', 'wb') as f: 76 | s3.download_fileobj(s3_url.split("/")[2], "/".join(s3_url.split("/")[3:]), f) 77 | return "Success" 78 | 79 | if __name__ == "__main__": 80 | parser = argparse.ArgumentParser() 81 | parser.add_argument( 82 | "--log-level", type=str, default=os.environ.get("LOGLEVEL", "INFO").upper() 83 | ) 84 | parser.add_argument("--model-package-group-name", type=str, required=True) 85 | args, _ = parser.parse_known_args() 86 | 87 | # Configure logging to output the line number and message 88 | log_format = "%(levelname)s: [%(filename)s:%(lineno)s] %(message)s" 89 | logging.basicConfig(format=log_format, level=args.log_level) 90 | 91 | status = download_model_tar_from_s3(model_package_group_name=args.model_package_group_name) 92 | logging.info(f"Status = {status}") 93 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | env: 4 | shell: bash 5 | 6 | phases: 7 | install: 8 | runtime_versions: 9 | python: 3.8 10 | commands: 11 | - python3 -m ensurepip --upgrade 12 | - python3 -m pip install --upgrade pip 13 | - python3 -m pip install --upgrade virtualenv 14 | - python3 -m venv .venv 15 | - source .venv/bin/activate 16 | - npm install -g aws-cdk@2.26.0 17 | - pip install -r requirements.txt 18 | - cdk bootstrap 19 | build: 20 | commands: 21 | - python build.py --model-package-group-name "$SOURCE_MODEL_PACKAGE_GROUP_NAME" 22 | - tar -xf model.tar.gz 23 | - cp model.joblib lambda/digital_twin 24 | - rm model.tar.gz 25 | - rm model.joblib 26 | - cdk deploy --require-approval never 27 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "python3 app.py", 3 | "watch": { 4 | "include": [ 5 | "**" 6 | ], 7 | "exclude": [ 8 | "README.md", 9 | "cdk*.json", 10 | "requirements*.txt", 11 | "source.bat", 12 | "**/__init__.py", 13 | "python/__pycache__", 14 | "*.ipynb_checkpoints*", 15 | "tests" 16 | ] 17 | }, 18 | "context": { 19 | "@aws-cdk/core:stackRelativeExports": true, 20 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 21 | "@aws-cdk/core:target-partitions": [ 22 | "aws", 23 | "aws-cn" 24 | ], 25 | "aws-cdk:enableDiffNoFail": true, 26 | "@aws-cdk/aws-secretsmanager:parseOwnedSecretName": true, 27 | "@aws-cdk/aws-kms:defaultKeyPolicies": true 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/lambda/digital_twin/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/lambda/python:3.9 2 | 3 | COPY app.py requirements.txt digital_twin.py quantile_regression.py model.joblib ./ 4 | 5 | RUN python3.9 -m pip install -r requirements.txt -t . 6 | 7 | CMD ["app.lambda_handler"] 8 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/lambda/digital_twin/app.py: -------------------------------------------------------------------------------- 1 | """This is the main app that runs in your AWS Lambda function.""" 2 | 3 | import base64 4 | import io 5 | import json 6 | import logging 7 | import os 8 | # Import libraries 9 | import tempfile 10 | 11 | import boto3 12 | import botocore 13 | import joblib 14 | import numpy as np 15 | import pandas as pd 16 | from digital_twin import DigitalTwin 17 | from quantile_regression import QuantileRegression 18 | 19 | import optuna 20 | 21 | features = [ 22 | "flow", 23 | "pressure", 24 | ] 25 | target = "power" 26 | config_col = "config" 27 | 28 | twin = joblib.load("model.joblib") 29 | 30 | 31 | def lambda_handler(event, context): 32 | """Main entry point for the AWS Lambda function. 33 | 34 | :event: Must contain 'body' and 'key' where 35 | 'body' = the observed number from your machine 36 | 'key' = the model location in your S3 bucket 37 | :context: The AWS Lambda context sent to the function. 38 | 39 | :return: The dict with status code and output_body 40 | 41 | """ 42 | boundary = { 43 | "c1c2": [180, 390], 44 | "c1c2c4": [180, 390], 45 | "c1c4": [180, 390], 46 | "c1c3c4": [180, 460], 47 | "c1c2c3c4": [180, 460], 48 | "c1c3": [180, 390], 49 | "c1c2c3": [180, 460], 50 | "c3c4": [180, 390], 51 | "c2c3c4": [180, 460], 52 | "c2c4": [180, 390], 53 | "c2c3": [180, 390], 54 | } 55 | 56 | clf = [ 57 | "c1c2", 58 | "c1c2c4", 59 | "c1c4", 60 | "c1c3c4", 61 | "c1c2c3c4", 62 | "c1c3", 63 | "c1c2c3", 64 | "c3c4", 65 | "c2c3c4", 66 | "c2c4", 67 | "c2c3", 68 | ] 69 | flow = json.loads(event["flow"]) 70 | press = json.loads(event["pressure"]) 71 | n = int(json.loads(event["simulations"])) 72 | no_of_trials = int(json.loads(event["no_of_trials"])) 73 | train_error_weight = float(json.loads(event["train_error_weight"])) 74 | 75 | results = twin.run_monte_carlo( 76 | clf=clf, 77 | flow=flow, 78 | press=press, 79 | n=n, 80 | no_of_trials=no_of_trials, 81 | boundary=boundary, 82 | train_error_weight=train_error_weight, 83 | algorithm="keras_regression" 84 | ) 85 | 86 | # Return your response 87 | results = [{list(x.keys())[0]: float(x[list(x.keys())[0]]/n)} for x in results] 88 | 89 | logging.info("Return...") 90 | return { 91 | "statusCode": 200, 92 | "body": json.dumps(results), 93 | } 94 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/lambda/digital_twin/requirements.txt: -------------------------------------------------------------------------------- 1 | pip==22.1.2 2 | boto3==1.24.22 3 | shap==0.41.0 4 | imbalanced-learn==0.9.1 5 | werkzeug==2.0.3 6 | pandas==1.2.4 7 | numpy==1.22.0 8 | xgboost==1.5.2 9 | scikit-learn==1.1.1 10 | optuna==2.10.1 11 | scikeras==0.8.0 12 | tensorflow-cpu==2.9.3 13 | keras==2.9.0 14 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/requirements.txt: -------------------------------------------------------------------------------- 1 | aws-cdk-lib==2.37.0 2 | constructs>=10.0.0,<11.0.0 3 | pytest==6.2.5 4 | boto3 5 | wheel 6 | cdk-nag==2.14.6 -------------------------------------------------------------------------------- /deployment/pipeline/assets/modeldeploy/source.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem The sole purpose of this script is to make the command 4 | rem 5 | rem source .venv/bin/activate 6 | rem 7 | rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. 8 | rem On Windows, this command just runs this batch file (the argument is ignored). 9 | rem 10 | rem Now we don't need to document a Windows command for activating a virtualenv. 11 | 12 | echo Executing .venv\Scripts\activate.bat for you 13 | .venv\Scripts\activate.bat 14 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/processing-imagebuild/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E203, E266, E501, W503, F403, F401, E741, F541, C901 3 | max-line-length = 120 4 | max-complexity = 18 5 | select = B,C,E,F,W,T4,B9 6 | exclude = .ipynb_checkpoints,.git,__pycache__,docs/source/conf.py,old,build,dist 7 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/processing-imagebuild/.gitignore: -------------------------------------------------------------------------------- 1 | *.pdf 2 | *.xls 3 | *.xlsx 4 | *.pdf 5 | *.env 6 | *.zip 7 | venv/ 8 | .venv/ 9 | *__pycache__* 10 | __pycache__/ 11 | .DS_Store 12 | .vscode/ 13 | .ipynb_checkpoints/ 14 | *.ipynb_checkpoints/* 15 | cdk.context.json 16 | .cdk.staging/ 17 | cdk.out/ 18 | *.egg-info 19 | *.egg 20 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/processing-imagebuild/.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | known_third_party=SageMakerPipelineConstructs,SageMakerPipelineSourceCode,aws_cdk,boto3,botocore,constructs,data_pipeline,digital_twin,joblib,numpy,pandas,pipelines,pytest,quantile_regression,river,sagemaker,scipy,setuptools,sklearn,tqdm 3 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/processing-imagebuild/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v2.4.0 6 | hooks: 7 | - id: trailing-whitespace 8 | - id: end-of-file-fixer 9 | - id: flake8 10 | - repo: https://github.com/asottile/seed-isort-config 11 | rev: v1.9.3 12 | hooks: 13 | - id: seed-isort-config 14 | - repo: https://github.com/pre-commit/mirrors-isort 15 | rev: v4.3.21 16 | hooks: 17 | - id: isort 18 | additional_dependencies: [toml] 19 | - repo: https://github.com/psf/black 20 | rev: 20.8b1 21 | hooks: 22 | - id: black 23 | language_version: python3 24 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/processing-imagebuild/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG ECR_ENDPOINT 2 | FROM $ECR_ENDPOINT/sagemaker-xgboost:1.5-1-cpu-py3 3 | 4 | COPY requirements.txt requirements.txt 5 | COPY digital_twin.py /opt/ml/code/digital_twin.py 6 | COPY quantile_regression.py /opt/ml/code/quantile_regression.py 7 | 8 | RUN python3 -m pip install -r requirements.txt 9 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/processing-imagebuild/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | phases: 4 | pre_build: 5 | commands: 6 | - echo Logging in to Amazon ECR... 7 | # Extracting endpoint information using jq based on region and sed to strip off the quotes at both ends of the string. 8 | - ECR_ENDPOINT=`jq --arg region "$AWS_REGION" '.[$region]' xgboost-1.5-1-cpu-py3.json | sed 's/"//g'` 9 | # Using cut to remove the 12 digit account number and the period from ECR_ENDPOINT to retrieve repository of the base image. 10 | - BASE_IMAGE_REPOSITORY=`echo $ECR_ENDPOINT | cut -c14-` 11 | - aws --region $AWS_REGION ecr get-login-password | docker login --username AWS --password-stdin $ECR_ENDPOINT 12 | build: 13 | commands: 14 | - echo Build started on `date` 15 | - echo Building the Docker image... 16 | - docker build --build-arg ECR_ENDPOINT=$ECR_ENDPOINT -t $IMAGE_REPO_NAME:$IMAGE_TAG . 17 | - docker tag $IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_ID.$BASE_IMAGE_REPOSITORY/$IMAGE_REPO_NAME:$IMAGE_TAG 18 | post_build: 19 | commands: 20 | - if [ $CODEBUILD_BUILD_SUCCEEDING = 0 ]; then exit 1; fi 21 | - echo Build completed on `date` 22 | - echo Logging in to Amazon ECR... 23 | - aws --region $AWS_REGION ecr get-login-password | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.$BASE_IMAGE_REPOSITORY 24 | - echo Pushing the Docker image... 25 | - docker push $AWS_ACCOUNT_ID.$BASE_IMAGE_REPOSITORY/$IMAGE_REPO_NAME:$IMAGE_TAG 26 | - aws --region $AWS_REGION sagemaker create-image-version --image-name $IMAGE_REPO_NAME --base-image $AWS_ACCOUNT_ID.$BASE_IMAGE_REPOSITORY/$IMAGE_REPO_NAME:$IMAGE_TAG 27 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/processing-imagebuild/requirements.txt: -------------------------------------------------------------------------------- 1 | pip==22.1.2 2 | boto3==1.24.22 3 | shap==0.41.0 4 | imbalanced-learn==0.9.1 5 | werkzeug==2.0.3 6 | pandas==1.2.4 7 | numpy==1.22.0 8 | xgboost==1.5.2 9 | scikit-learn==1.1.1 10 | optuna==2.10.1 11 | scikeras==0.8.0 12 | tensorflow==2.9.3 13 | keras==2.9.0 14 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/processing-imagebuild/xgboost-1.5-1-cpu-py3.json: -------------------------------------------------------------------------------- 1 | { 2 | "af-south-1": "510948584623.dkr.ecr.af-south-1.amazonaws.com", 3 | "ap-east-1": "651117190479.dkr.ecr.ap-east-1.amazonaws.com", 4 | "ap-northeast-1": "354813040037.dkr.ecr.ap-northeast-1.amazonaws.com", 5 | "ap-northeast-2": "366743142698.dkr.ecr.ap-northeast-2.amazonaws.com", 6 | "ap-south-1": "720646828776.dkr.ecr.ap-south-1.amazonaws.com", 7 | "ap-southeast-1": "121021644041.dkr.ecr.ap-southeast-1.amazonaws.com", 8 | "ap-southeast-2": "783357654285.dkr.ecr.ap-southeast-2.amazonaws.com", 9 | "ca-central-1": "341280168497.dkr.ecr.ca-central-1.amazonaws.com", 10 | "cn-north-1": "450853457545.dkr.ecr.cn-north-1.amazonaws.com.cn", 11 | "cn-northwest-1": "451049120500.dkr.ecr.cn-northwest-1.amazonaws.com.cn", 12 | "eu-central-1": "492215442770.dkr.ecr.eu-central-1.amazonaws.com", 13 | "eu-north-1": "662702820516.dkr.ecr.eu-north-1.amazonaws.com", 14 | "eu-west-1": "141502667606.dkr.ecr.eu-west-1.amazonaws.com", 15 | "eu-west-2": "764974769150.dkr.ecr.eu-west-2.amazonaws.com", 16 | "eu-west-3": "659782779980.dkr.ecr.eu-west-3.amazonaws.com", 17 | "eu-south-1": "978288397137.dkr.ecr.eu-south-1.amazonaws.com", 18 | "me-south-1": "801668240914.dkr.ecr.me-south-1.amazonaws.com", 19 | "sa-east-1": "737474898029.dkr.ecr.sa-east-1.amazonaws.com", 20 | "us-east-1": "683313688378.dkr.ecr.us-east-1.amazonaws.com", 21 | "us-east-2": "257758044811.dkr.ecr.us-east-2.amazonaws.com", 22 | "us-west-1": "746614075791.dkr.ecr.us-west-1.amazonaws.com", 23 | "us-west-2": "246618743249.dkr.ecr.us-west-2.amazonaws.com" 24 | } 25 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/training-imagebuild/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E203, E266, E501, W503, F403, F401, E741, F541, C901 3 | max-line-length = 120 4 | max-complexity = 18 5 | select = B,C,E,F,W,T4,B9 6 | exclude = .ipynb_checkpoints,.git,__pycache__,docs/source/conf.py,old,build,dist 7 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/training-imagebuild/.gitignore: -------------------------------------------------------------------------------- 1 | *.pdf 2 | *.xls 3 | *.xlsx 4 | *.pdf 5 | *.env 6 | *.zip 7 | venv/ 8 | .venv/ 9 | *__pycache__* 10 | __pycache__/ 11 | .DS_Store 12 | .vscode/ 13 | .ipynb_checkpoints/ 14 | *.ipynb_checkpoints/* 15 | cdk.context.json 16 | .cdk.staging/ 17 | cdk.out/ 18 | *.egg-info 19 | *.egg 20 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/training-imagebuild/.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | known_third_party=SageMakerPipelineConstructs,SageMakerPipelineSourceCode,aws_cdk,boto3,botocore,constructs,data_pipeline,digital_twin,joblib,numpy,pandas,pipelines,pytest,quantile_regression,river,sagemaker,scipy,setuptools,sklearn,tqdm 3 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/training-imagebuild/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v2.4.0 6 | hooks: 7 | - id: trailing-whitespace 8 | - id: end-of-file-fixer 9 | - id: flake8 10 | - repo: https://github.com/asottile/seed-isort-config 11 | rev: v1.9.3 12 | hooks: 13 | - id: seed-isort-config 14 | - repo: https://github.com/pre-commit/mirrors-isort 15 | rev: v4.3.21 16 | hooks: 17 | - id: isort 18 | additional_dependencies: [toml] 19 | - repo: https://github.com/psf/black 20 | rev: 20.8b1 21 | hooks: 22 | - id: black 23 | language_version: python3 24 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/training-imagebuild/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG ECR_ENDPOINT 2 | FROM $ECR_ENDPOINT/sagemaker-xgboost:1.5-1-cpu-py3 3 | 4 | COPY requirements.txt requirements.txt 5 | COPY digital_twin.py /opt/ml/code/digital_twin.py 6 | COPY quantile_regression.py /opt/ml/code/quantile_regression.py 7 | COPY train.py /opt/ml/code/train.py 8 | 9 | RUN python3 -m pip install -r requirements.txt 10 | 11 | ENV SAGEMAKER_PROGRAM train.py 12 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/training-imagebuild/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | phases: 4 | pre_build: 5 | commands: 6 | - echo Logging in to Amazon ECR... 7 | # Extracting endpoint information using jq based on region and sed to strip off the quotes at both ends of the string. 8 | - ECR_ENDPOINT=`jq --arg region "$AWS_REGION" '.[$region]' xgboost-1.5-1-cpu-py3.json | sed 's/"//g'` 9 | # Using cut to remove the 12 digit account number and the period from ECR_ENDPOINT to retrieve repository of the base image. 10 | - BASE_IMAGE_REPOSITORY=`echo $ECR_ENDPOINT | cut -c14-` 11 | - aws --region $AWS_REGION ecr get-login-password | docker login --username AWS --password-stdin $ECR_ENDPOINT 12 | build: 13 | commands: 14 | - echo Build started on `date` 15 | - echo Building the Docker image... 16 | - docker build --build-arg ECR_ENDPOINT=$ECR_ENDPOINT -t $IMAGE_REPO_NAME:$IMAGE_TAG . 17 | - docker tag $IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_ID.$BASE_IMAGE_REPOSITORY/$IMAGE_REPO_NAME:$IMAGE_TAG 18 | post_build: 19 | commands: 20 | - if [ $CODEBUILD_BUILD_SUCCEEDING = 0 ]; then exit 1; fi 21 | - echo Build completed on `date` 22 | - echo Logging in to Amazon ECR... 23 | - aws --region $AWS_REGION ecr get-login-password | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.$BASE_IMAGE_REPOSITORY 24 | - echo Pushing the Docker image... 25 | - docker push $AWS_ACCOUNT_ID.$BASE_IMAGE_REPOSITORY/$IMAGE_REPO_NAME:$IMAGE_TAG 26 | - aws --region $AWS_REGION sagemaker create-image-version --image-name $IMAGE_REPO_NAME --base-image $AWS_ACCOUNT_ID.$BASE_IMAGE_REPOSITORY/$IMAGE_REPO_NAME:$IMAGE_TAG 27 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/training-imagebuild/requirements.txt: -------------------------------------------------------------------------------- 1 | pip==22.1.2 2 | boto3==1.24.22 3 | shap==0.41.0 4 | imbalanced-learn==0.9.1 5 | werkzeug==2.0.3 6 | pandas==1.2.4 7 | numpy==1.22.0 8 | xgboost==1.5.2 9 | scikit-learn==1.1.1 10 | optuna==2.10.1 11 | scikeras==0.8.0 12 | tensorflow==2.9.3 13 | keras==2.9.0 14 | -------------------------------------------------------------------------------- /deployment/pipeline/assets/training-imagebuild/xgboost-1.5-1-cpu-py3.json: -------------------------------------------------------------------------------- 1 | { 2 | "af-south-1": "510948584623.dkr.ecr.af-south-1.amazonaws.com", 3 | "ap-east-1": "651117190479.dkr.ecr.ap-east-1.amazonaws.com", 4 | "ap-northeast-1": "354813040037.dkr.ecr.ap-northeast-1.amazonaws.com", 5 | "ap-northeast-2": "366743142698.dkr.ecr.ap-northeast-2.amazonaws.com", 6 | "ap-south-1": "720646828776.dkr.ecr.ap-south-1.amazonaws.com", 7 | "ap-southeast-1": "121021644041.dkr.ecr.ap-southeast-1.amazonaws.com", 8 | "ap-southeast-2": "783357654285.dkr.ecr.ap-southeast-2.amazonaws.com", 9 | "ca-central-1": "341280168497.dkr.ecr.ca-central-1.amazonaws.com", 10 | "cn-north-1": "450853457545.dkr.ecr.cn-north-1.amazonaws.com.cn", 11 | "cn-northwest-1": "451049120500.dkr.ecr.cn-northwest-1.amazonaws.com.cn", 12 | "eu-central-1": "492215442770.dkr.ecr.eu-central-1.amazonaws.com", 13 | "eu-north-1": "662702820516.dkr.ecr.eu-north-1.amazonaws.com", 14 | "eu-west-1": "141502667606.dkr.ecr.eu-west-1.amazonaws.com", 15 | "eu-west-2": "764974769150.dkr.ecr.eu-west-2.amazonaws.com", 16 | "eu-west-3": "659782779980.dkr.ecr.eu-west-3.amazonaws.com", 17 | "eu-south-1": "978288397137.dkr.ecr.eu-south-1.amazonaws.com", 18 | "me-south-1": "801668240914.dkr.ecr.me-south-1.amazonaws.com", 19 | "sa-east-1": "737474898029.dkr.ecr.sa-east-1.amazonaws.com", 20 | "us-east-1": "683313688378.dkr.ecr.us-east-1.amazonaws.com", 21 | "us-east-2": "257758044811.dkr.ecr.us-east-2.amazonaws.com", 22 | "us-west-1": "746614075791.dkr.ecr.us-west-1.amazonaws.com", 23 | "us-west-2": "246618743249.dkr.ecr.us-west-2.amazonaws.com" 24 | } 25 | -------------------------------------------------------------------------------- /deployment/requirements.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | cdk-nag -------------------------------------------------------------------------------- /deployment/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | setuptools.setup( 4 | name="SageMakerPipelineSourceCode", 5 | version="0.0.1", 6 | description="This is the stack that deploys the core code for the SM pipeline.", 7 | long_description_content_type="text/markdown", 8 | author="Michael Wallner", 9 | package_dir={"": "SageMakerPipelineSourceCode"}, 10 | packages=setuptools.find_packages(where="SageMakerPipelineSourceCode"), 11 | install_requires=[ 12 | "aws-cdk-lib==2.11.0", 13 | "constructs>=10.0.0,<11.0.0", 14 | "pytest==6.2.5", 15 | "boto3", 16 | ], 17 | python_requires=">=3.6", 18 | ) 19 | -------------------------------------------------------------------------------- /deployment/source.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem The sole purpose of this script is to make the command 4 | rem 5 | rem source .venv/bin/activate 6 | rem 7 | rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. 8 | rem On Windows, this command just runs this batch file (the argument is ignored). 9 | rem 10 | rem Now we don't need to document a Windows command for activating a virtualenv. 11 | 12 | echo Executing .venv\Scripts\activate.bat for you 13 | .venv\Scripts\activate.bat 14 | -------------------------------------------------------------------------------- /deployment/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Initialize 2 | -------------------------------------------------------------------------------- /deployment/tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # Initialize 2 | -------------------------------------------------------------------------------- /deployment/tests/unit/test_SageMakerPipelineSourceCode_stack.py: -------------------------------------------------------------------------------- 1 | import aws_cdk as core 2 | import aws_cdk.assertions as assertions 3 | from SageMakerPipelineSourceCode.SageMakerPipelineSourceCode_stack import \ 4 | SageMakerPipelineSourceCodeStack 5 | 6 | 7 | def test_code_commit_repos_created(): 8 | app = core.App() 9 | stack = SageMakerPipelineSourceCodeStack(app, 10 | "SageMakerPipelineSourceCode", 11 | sagemaker_project_name="predima-compressor-temp", 12 | sagemaker_project_id="p-da34uwob29zf", 13 | enable_processing_image_build_pipeline=True, 14 | enable_training_image_build_pipeline=True, 15 | enable_inference_image_build_pipeline=False, 16 | aws_account_id=42, 17 | aws_region="eu-central-1", 18 | container_image_tag="latest") 19 | template = assertions.Template.from_stack(stack) 20 | 21 | template.resource_count_is("AWS::CodeCommit::Repository", 4) 22 | 23 | 24 | def test_code_pipelines_created(): 25 | app = core.App() 26 | stack = SageMakerPipelineSourceCodeStack(app, 27 | "SageMakerPipelineSourceCode", 28 | sagemaker_project_name="predima-compressor-temp", 29 | sagemaker_project_id="p-da34uwob29zf", 30 | enable_processing_image_build_pipeline=True, 31 | enable_training_image_build_pipeline=True, 32 | enable_inference_image_build_pipeline=False, 33 | aws_account_id=42, 34 | aws_region="eu-central-1", 35 | container_image_tag="latest") 36 | template = assertions.Template.from_stack(stack) 37 | 38 | template.resource_count_is("AWS::CodePipeline::Pipeline", 4) 39 | 40 | def test_code_build_projects_created(): 41 | app = core.App() 42 | stack = SageMakerPipelineSourceCodeStack(app, 43 | "SageMakerPipelineSourceCode", 44 | sagemaker_project_name="predima-compressor-temp", 45 | sagemaker_project_id="p-da34uwob29zf", 46 | enable_processing_image_build_pipeline=True, 47 | enable_training_image_build_pipeline=True, 48 | enable_inference_image_build_pipeline=False, 49 | aws_account_id=42, 50 | aws_region="eu-central-1", 51 | container_image_tag="latest") 52 | template = assertions.Template.from_stack(stack) 53 | 54 | template.resource_count_is("AWS::CodeBuild::Project", 4) 55 | 56 | def test_ecr_repositories_created(): 57 | app = core.App() 58 | stack = SageMakerPipelineSourceCodeStack(app, 59 | "SageMakerPipelineSourceCode", 60 | sagemaker_project_name="predima-compressor-temp", 61 | sagemaker_project_id="p-da34uwob29zf", 62 | enable_processing_image_build_pipeline=True, 63 | enable_training_image_build_pipeline=True, 64 | enable_inference_image_build_pipeline=False, 65 | aws_account_id=42, 66 | aws_region="eu-central-1", 67 | container_image_tag="latest") 68 | template = assertions.Template.from_stack(stack) 69 | 70 | template.resource_count_is("AWS::ECR::Repository", 2) 71 | 72 | 73 | def test_sagemaker_images_created(): 74 | app = core.App() 75 | stack = SageMakerPipelineSourceCodeStack(app, 76 | "SageMakerPipelineSourceCode", 77 | sagemaker_project_name="predima-compressor-temp", 78 | sagemaker_project_id="p-da34uwob29zf", 79 | enable_processing_image_build_pipeline=True, 80 | enable_training_image_build_pipeline=True, 81 | enable_inference_image_build_pipeline=False, 82 | aws_account_id=42, 83 | aws_region="eu-central-1", 84 | container_image_tag="latest") 85 | template = assertions.Template.from_stack(stack) 86 | 87 | template.resource_count_is("AWS::SageMaker::Image", 2) 88 | 89 | def test_s3_bucket_created(): 90 | app = core.App() 91 | stack = SageMakerPipelineSourceCodeStack(app, 92 | "SageMakerPipelineSourceCode", 93 | sagemaker_project_name="predima-compressor-temp", 94 | sagemaker_project_id="p-da34uwob29zf", 95 | enable_processing_image_build_pipeline=True, 96 | enable_training_image_build_pipeline=True, 97 | enable_inference_image_build_pipeline=False, 98 | aws_account_id=42, 99 | aws_region="eu-central-1", 100 | container_image_tag="latest") 101 | template = assertions.Template.from_stack(stack) 102 | 103 | template.resource_count_is("AWS::S3::Bucket", 1) 104 | 105 | def test_event_rules_created(): 106 | app = core.App() 107 | stack = SageMakerPipelineSourceCodeStack(app, 108 | "SageMakerPipelineSourceCode", 109 | sagemaker_project_name="predima-compressor-temp", 110 | sagemaker_project_id="p-da34uwob29zf", 111 | enable_processing_image_build_pipeline=True, 112 | enable_training_image_build_pipeline=True, 113 | enable_inference_image_build_pipeline=False, 114 | aws_account_id=42, 115 | aws_region="eu-central-1", 116 | container_image_tag="latest") 117 | template = assertions.Template.from_stack(stack) 118 | 119 | template.resource_count_is("AWS::Events::Rule", 13) 120 | 121 | def test_iam_policies_created(): 122 | app = core.App() 123 | stack = SageMakerPipelineSourceCodeStack(app, 124 | "SageMakerPipelineSourceCode", 125 | sagemaker_project_name="predima-compressor-temp", 126 | sagemaker_project_id="p-da34uwob29zf", 127 | enable_processing_image_build_pipeline=True, 128 | enable_training_image_build_pipeline=True, 129 | enable_inference_image_build_pipeline=False, 130 | aws_account_id=42, 131 | aws_region="eu-central-1", 132 | container_image_tag="latest") 133 | template = assertions.Template.from_stack(stack) 134 | 135 | template.resource_count_is("AWS::IAM::Policy", 15) 136 | 137 | 138 | def test_iam_roles_created(): 139 | app = core.App() 140 | stack = SageMakerPipelineSourceCodeStack(app, 141 | "SageMakerPipelineSourceCode", 142 | sagemaker_project_name="predima-compressor-temp", 143 | sagemaker_project_id="p-da34uwob29zf", 144 | enable_processing_image_build_pipeline=True, 145 | enable_training_image_build_pipeline=True, 146 | enable_inference_image_build_pipeline=False, 147 | aws_account_id=42, 148 | aws_region="eu-central-1", 149 | container_image_tag="latest") 150 | template = assertions.Template.from_stack(stack) 151 | 152 | template.resource_count_is("AWS::IAM::Role", 15) 153 | -------------------------------------------------------------------------------- /mllib/__init__.py: -------------------------------------------------------------------------------- 1 | # Initialize 2 | -------------------------------------------------------------------------------- /mllib/digital_twin.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import random 4 | 5 | import numpy as np 6 | import pandas as pd 7 | from quantile_regression import QuantileRegression 8 | from scikeras.wrappers import KerasRegressor 9 | from sklearn.compose import ColumnTransformer 10 | from sklearn.ensemble import IsolationForest 11 | from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score 12 | from sklearn.model_selection import train_test_split 13 | from sklearn.pipeline import Pipeline 14 | from sklearn.preprocessing import (MinMaxScaler, OneHotEncoder, 15 | PolynomialFeatures, RobustScaler) 16 | from tensorflow import keras 17 | from tensorflow.keras import layers 18 | 19 | 20 | class DigitalTwin: 21 | """Digital Twin class that learns behaviour of your assets. 22 | The :class:`DigitalTwin` leverages :class:`QuantileRegression` 23 | class to learn machine power consumption. 24 | Parameters 25 | ---------- 26 | configurations : str 27 | The different setups the machine assets can have, e.g. 28 | looking at compressors it could be ["C1", "C2", "C1C2"] 29 | quantile : float, default=0.5 30 | The quantile that the model tries to predict. It must be strictly 31 | between 0 and 1. If 0.5 (default), the model predicts the 50% 32 | quantile, i.e. the median. 33 | Attributes 34 | ---------- 35 | configurations : array of shape (n_configurations,) 36 | The different setups the machine assets can have 37 | quantile : float, default=0.5 38 | The quantile that the model tries to predict. It must be strictly 39 | between 0 and 1. If 0.5 (default), the model predicts the 50% 40 | quantile, i.e. the median. 41 | models : array of shape (n_configurations,) 42 | Trained model per configuration. 43 | test_mae_errors : array of shape (n_configurations,) 44 | Test MAE per configuration's trained model. 45 | train_mae_errors : array of shape (n_features,) 46 | Train MAE per configuration's trained model. 47 | Examples 48 | -------- 49 | >>> twin = DigitalTwin( 50 | >>> configurations=df["configuration_column"].unique().tolist() 51 | >>> ).train(df=df, features=["x", "y"], target="z", config_col="configuration_column") 52 | """ 53 | 54 | def __init__( 55 | self, 56 | configurations: list, 57 | features: list, 58 | target: str, 59 | config_col: str, 60 | quantile: float = 0.5, 61 | ): 62 | """Initialize the class. 63 | 64 | Args: 65 | :configurations: the different asset configurations 66 | :quantile: the quantile of the quantile forrest, default: 0.5 67 | 68 | """ 69 | self.configurations = configurations 70 | self.quantile = quantile 71 | self.models = {} 72 | self.test_mae_errors = {} 73 | self.train_mae_errors = {} 74 | self.features = features 75 | self.target = target 76 | self.config_col = config_col 77 | 78 | def _quantile_regression( 79 | self, 80 | df: pd.DataFrame, 81 | test_size: float = 0.2, 82 | random_state: int = 42, 83 | degree: int = 2, 84 | verbose: bool = True, 85 | remove_outliers: bool = False, 86 | ) -> QuantileRegression: 87 | """Train the digital twin based on all possible configurations. 88 | Parameters 89 | ---------- 90 | df : {array-like, DataFrame} of shape (n_samples, n_features+n_target+n_configuration) 91 | Training DataFrame having features, target and configuration. 92 | features : {array-like} string array with the feature column names 93 | target : {string} the target column 94 | config_col : {string} configuration column name in DataFrame 95 | test_size : {float} ratio of train-test-split 96 | random_state : {float} a random state for reproducability 97 | degree : {float} polynomial degree you want to add as additional features 98 | verbose : {bool} whether to print training evaluation or not 99 | Returns 100 | ------- 101 | self : object 102 | Returns self. 103 | """ 104 | for config in self.configurations: 105 | temp = df[df[self.config_col] == config] 106 | X = temp[self.features] 107 | y = temp[self.target] 108 | 109 | X_train, X_test, y_train, y_test = train_test_split( 110 | X, y, test_size=test_size, random_state=random_state 111 | ) 112 | 113 | estimators = [ 114 | ("scale", RobustScaler()), 115 | ("poly", PolynomialFeatures(degree=degree)), 116 | ("regressor", QuantileRegression(quantile=self.quantile)), 117 | ] 118 | 119 | pipeline = Pipeline(estimators) 120 | 121 | model = pipeline.fit(X_train, y_train) 122 | 123 | self.models[config] = model 124 | 125 | test_pred = model.predict(X_test) 126 | train_pred = model.predict(X_train) 127 | 128 | self.test_mae_errors[config] = mean_absolute_error(test_pred, y_test) 129 | self.train_mae_errors[config] = mean_absolute_error(train_pred, y_train) 130 | 131 | if verbose: 132 | print(config) 133 | print("------------") 134 | print("In Sample Errors") 135 | print(f"Combination {config} yields: Mean = {np.mean(y_train)}") 136 | print(f"Combination {config} yields: Stddev = {np.std(y_train)}") 137 | print( 138 | f"Combination {config} yields: RMSE = {np.sqrt(mean_squared_error(train_pred, y_train))}" 139 | ) 140 | print( 141 | f"Combination {config} yields: MAE = {self.train_mae_errors[config]}" 142 | ) 143 | 144 | print("Out of Sample Errors") 145 | print(f"Combination {config} yields: Mean = {np.mean(y_test)}") 146 | print(f"Combination {config} yields: Stddev = {np.std(y_test)}") 147 | print( 148 | f"Combination {config} yields: RMSE = {np.sqrt(mean_squared_error(test_pred, y_test))}" 149 | ) 150 | print( 151 | f"Combination {config} yields: MAE = {self.test_mae_errors[config]}" 152 | ) 153 | print("------------") 154 | 155 | model = pipeline.fit(X, y) 156 | self.models[config] = model 157 | 158 | return self 159 | 160 | def _baseline_model(self): 161 | """Baseline Keras model 162 | Parameters 163 | ---------- 164 | Returns 165 | ------- 166 | model : keras.Sequential() 167 | Returns the model. 168 | """ 169 | # create model 170 | model = keras.Sequential() 171 | model.add(layers.Dense(256, input_dim=26, kernel_initializer='normal', activation='relu')) 172 | model.add(layers.Dense(32, kernel_initializer='normal', activation='relu')) 173 | model.add(layers.Dense(64, kernel_initializer='normal', activation='relu')) 174 | model.add(layers.Dense(16, kernel_initializer='normal', activation='relu')) 175 | model.add(layers.Dense(8, kernel_initializer='normal', activation='relu')) 176 | model.add(layers.Dense(1, kernel_initializer='normal')) 177 | # Compile model 178 | model.compile(loss='mean_absolute_error', optimizer='adam') 179 | return model 180 | 181 | def _keras_regression( 182 | self, 183 | df: pd.DataFrame, 184 | test_size: float = 0.2, 185 | random_state: int = 42, 186 | degree: int = 2, 187 | optimizer: str="adam", 188 | optimizer__learning_rate: float=0.001, 189 | epochs: int=1000, 190 | verbose: int=0): 191 | 192 | regression = KerasRegressor( 193 | model=self._baseline_model(), 194 | optimizer=optimizer, 195 | optimizer__learning_rate=optimizer__learning_rate, 196 | epochs=epochs, 197 | verbose=verbose, 198 | ) 199 | 200 | X = df[self.features+[self.config_col]].reset_index(drop=True) 201 | y = df[self.target].reset_index(drop=True) 202 | 203 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=random_state, stratify=X[self.config_col]) 204 | 205 | numeric_features = self.features 206 | numeric_steps = [ 207 | ("scale", MinMaxScaler()), 208 | ("poly", PolynomialFeatures(degree=degree)), 209 | ] 210 | numeric_transformer = Pipeline(steps=numeric_steps) 211 | 212 | categorical_features = [self.config_col] 213 | categorical_steps = [ 214 | ('onehot', OneHotEncoder(handle_unknown='ignore')), 215 | ] 216 | categorical_transformer = Pipeline(steps=categorical_steps) 217 | 218 | preprocessor = ColumnTransformer( 219 | transformers=[ 220 | ('num', numeric_transformer, numeric_features), 221 | ('cat', categorical_transformer, categorical_features)]) 222 | 223 | self.keras_reg = Pipeline( 224 | steps=[ 225 | ('preprocessor', preprocessor), 226 | ("regressor", regression), 227 | ]) 228 | 229 | self.keras_reg = self.keras_reg.fit(X_train, y_train) 230 | 231 | test_pred = self.keras_reg.predict(X_test) 232 | train_pred = self.keras_reg.predict(X_train) 233 | 234 | test_mae_errors = mean_absolute_error(test_pred, y_test) 235 | train_mae_errors = mean_absolute_error(train_pred, y_train) 236 | 237 | for config in self.configurations: 238 | test_index = X_test.config == config 239 | temp = X_test[test_index] 240 | test_pred = self.keras_reg.predict(temp) 241 | oos = mean_absolute_error(test_pred, y_test[test_index]) 242 | 243 | train_index = X_train.config == config 244 | temp = X_train[train_index] 245 | train_pred = self.keras_reg.predict(temp) 246 | ins = mean_absolute_error(train_pred, y_train[train_index]) 247 | 248 | self.test_mae_errors[config] = oos 249 | self.train_mae_errors[config] = ins 250 | if verbose: 251 | print(config) 252 | print("------------") 253 | print("In Sample Errors") 254 | print(f"Combination {config} yields: Mean = {np.mean(y_train[train_index])}") 255 | print(f"Combination {config} yields: Stddev = {np.std(y_train[train_index])}") 256 | print( 257 | f"Combination {config} yields: RMSE = {np.sqrt(mean_squared_error(train_pred, y_train[train_index]))}" 258 | ) 259 | print( 260 | f"Combination {config} yields: MAE = {self.train_mae_errors[config]}" 261 | ) 262 | 263 | print("Out of Sample Errors") 264 | print(f"Combination {config} yields: Mean = {np.mean(y_test)}") 265 | print(f"Combination {config} yields: Stddev = {np.std(y_test)}") 266 | print( 267 | f"Combination {config} yields: RMSE = {np.sqrt(mean_squared_error(test_pred, y_test[test_index]))}" 268 | ) 269 | print( 270 | f"Combination {config} yields: MAE = {self.test_mae_errors[config]}" 271 | ) 272 | print("------------") 273 | 274 | return self 275 | 276 | def train( 277 | self, 278 | df: pd.DataFrame, 279 | test_size: float = 0.2, 280 | random_state: int = 42, 281 | degree: int = 2, 282 | verbose: bool = True, 283 | optimizer: str="adam", 284 | optimizer__learning_rate: float=0.001, 285 | epochs: int=1000, 286 | algorithm: str="keras_regression", 287 | remove_outliers: bool = False, 288 | ) -> QuantileRegression: 289 | """Train the digital twin based on all possible configurations. 290 | Parameters 291 | ---------- 292 | df : {array-like, DataFrame} of shape (n_samples, n_features+n_target+n_configuration) 293 | Training DataFrame having features, target and configuration. 294 | features : {array-like} string array with the feature column names 295 | target : {string} the target column 296 | config_col : {string} configuration column name in DataFrame 297 | test_size : {float} ratio of train-test-split 298 | random_state : {float} a random state for reproducability 299 | degree : {float} polynomial degree you want to add as additional features 300 | verbose : {bool} whether to print training evaluation or not 301 | Returns 302 | ------- 303 | self : object 304 | Returns self. 305 | """ 306 | if remove_outliers: 307 | df = self.remove_outliers(df=df) 308 | if algorithm == "quantile_regression": 309 | return self._quantile_regression( 310 | df=df, 311 | test_size=test_size, 312 | random_state=random_state, 313 | degree=degree, 314 | verbose=verbose,) 315 | else: 316 | return self._keras_regression( 317 | df=df, 318 | test_size=test_size, 319 | random_state=random_state, 320 | degree=degree, 321 | optimizer=optimizer, 322 | optimizer__learning_rate=optimizer__learning_rate, 323 | epochs=epochs, 324 | verbose=verbose,) 325 | 326 | return self 327 | 328 | def remove_outliers( 329 | self, 330 | df: pd.DataFrame, 331 | n_estimators: int = 200, 332 | max_samples: float = 0.75, 333 | contamination: float = 0.01, 334 | max_features: float = 0.75, 335 | n_jobs: int = -1, 336 | random_state: int = 42, 337 | ) -> pd.DataFrame: 338 | """Remove outliers of DataFrame 339 | Parameters 340 | ---------- 341 | df : {array-like, DataFrame} of shape (n_samples, n_features+n_target) 342 | Training DataFrame having features, target and configuration. 343 | n_estimators : int, default=100 344 | The number of base estimators in the ensemble. 345 | max_samples : "auto", int or float, default="auto" 346 | The number of samples to draw from X to train each base estimator. 347 | - If int, then draw `max_samples` samples. 348 | - If float, then draw `max_samples * X.shape[0]` samples. 349 | - If "auto", then `max_samples=min(256, n_samples)`. 350 | If max_samples is larger than the number of samples provided, 351 | all samples will be used for all trees (no sampling). 352 | contamination : 'auto' or float, default='auto' 353 | The amount of contamination of the data set, i.e. the proportion 354 | of outliers in the data set. Used when fitting to define the threshold 355 | on the scores of the samples. 356 | - If 'auto', the threshold is determined as in the 357 | original paper. 358 | - If float, the contamination should be in the range (0, 0.5]. 359 | .. versionchanged:: 0.22 360 | The default value of ``contamination`` changed from 0.1 361 | to ``'auto'``. 362 | max_features : int or float, default=1.0 363 | The number of features to draw from X to train each base estimator. 364 | - If int, then draw `max_features` features. 365 | - If float, then draw `max_features * X.shape[1]` features. 366 | n_jobs : int, default=None 367 | The number of jobs to run in parallel for both :meth:`fit` and 368 | :meth:`predict`. ``None`` means 1 unless in a 369 | :obj:`joblib.parallel_backend` context. ``-1`` means using all 370 | processors. See :term:`Glossary ` for more details. 371 | random_state : int, RandomState instance or None, default=None 372 | Controls the pseudo-randomness of the selection of the feature 373 | and split values for each branching step and each tree in the forest. 374 | Pass an int for reproducible results across multiple function calls. 375 | See :term:`Glossary `. 376 | Returns 377 | ------- 378 | self : pandas.core.series.Series 379 | Returns pandas series with historic power consumption. 380 | """ 381 | X = df[self.features + [self.target]].reset_index(drop=True) 382 | self.iso = IsolationForest( 383 | n_estimators=n_estimators, 384 | max_samples=max_samples, 385 | contamination=contamination, 386 | max_features=max_features, 387 | n_jobs=n_jobs, 388 | random_state=random_state, 389 | ).fit(X) 390 | df["contamination"] = self.iso.predict(df[self.features + [self.target]]) 391 | df = df[df["contamination"] == 1].reset_index(drop=True) 392 | if "dttm" in df.columns: 393 | df.index = df.dttm 394 | return df 395 | 396 | def predict_efficiency(self, df: pd.DataFrame, algorithm: str="quantile_regression") -> pd.Series: 397 | """Predict historical asset efficiency 398 | Parameters 399 | ---------- 400 | df : {array-like, DataFrame} of shape (n_samples, n_features+n_target+n_configuration) 401 | Training DataFrame having features, target and configuration. 402 | Returns 403 | ------- 404 | self : pandas.core.series.Series 405 | Returns pandas series with historic power consumption. 406 | """ 407 | if algorithm == "quantile_regression": 408 | temp = df.copy("deep") 409 | temp["model_prediction"] = 0 410 | uconfig = df[self.config_col].unique() 411 | for config in uconfig: 412 | tmp = temp[temp[self.config_col] == config] 413 | pred = self.models[config].predict(tmp[self.features]) 414 | temp.loc[temp[self.config_col] == config, "model_prediction"] = pred 415 | return temp["model_prediction"] 416 | else: 417 | return self.keras_reg.predict(df) 418 | 419 | def create_staging_table(self, load, staging_rules): 420 | """Create staging table. 421 | Parameters 422 | ---------- 423 | load : {array-like, Series} target load column 424 | staging_rules : {array-like} string array needs to be of size (2, ) 425 | Returns 426 | ------- 427 | self : list 428 | Returns power consumption. 429 | """ 430 | staging = np.zeros_like(load) 431 | for switch_point in staging_rules[1]: 432 | staging += np.where(load > switch_point, 1, 0) 433 | 434 | return [staging_rules[0][s] for s in staging.astype("int")] 435 | 436 | def simulate_conditions( 437 | self, load_conditions, staging_rules, features, target, config_col="config" 438 | ): 439 | """Simulate load conditions. 440 | Parameters 441 | ---------- 442 | load_conditions : {array-like, DataFrame} the last x weeks load conditions 443 | staging_rules : {array-like} string array needs to be of size (2, ) 444 | features : {array-like} string array with the feature column names 445 | target : {string} the target column 446 | config_col : {string} configuration column name in DataFrame 447 | Returns 448 | ------- 449 | self : float 450 | Returns power consumption. 451 | """ 452 | load_conditions[config_col] = self.create_staging_table( 453 | load=load_conditions[target], staging_rules=staging_rules 454 | ) 455 | 456 | grouped = load_conditions.groupby(config_col) 457 | 458 | power_consumption = 0 459 | 460 | for name, group in grouped: 461 | 462 | power_predictions = ( 463 | self.models[name].predict(group[features]) + self.test_mae_errors[name] 464 | ) 465 | 466 | power_consumption += np.nansum(power_predictions) 467 | 468 | return power_consumption 469 | 470 | def range_subset(self, range1, range2): 471 | """Whether range1 is in range2 472 | Parameters 473 | ---------- 474 | range1: {range-like, range} of shape (n_samples,) 475 | range2: {range-like, range} of shape (n_samples,) 476 | Training DataFrame having features, target and configuration. 477 | Returns 478 | ------- 479 | One of True or False : bool 480 | Returns whether range1 is in range2 or not 481 | """ 482 | if not range1: 483 | return True # empty range is subset of anything 484 | if not range2: 485 | return False # non-empty range can't be subset of empty range 486 | if len(range1) > 1 and range1.step % range2.step: 487 | return False # must have a single value or integer multiple step 488 | return range1.start in range2 and range1[-1] in range2 489 | 490 | def objective_function_quantile( 491 | self, clf, flow, press, n=100, boundary={}, train_error_weight=1.0 492 | ): 493 | """Function to be minimize -> total energy consumption per compressor setting given shopfloor constraints 494 | Parameters 495 | ---------- 496 | clf: {list} of shape (n_compressor_settings,) 497 | flow: {list} of shape (2,) 498 | press: {list} of shape (2,) 499 | n: {integer} number of random entries to be generated 500 | boundary: {dict} contstraints per compressor setting 501 | train_error_weight: {float} factor to add/subtract/eliminate training error, e.g. 502 | 1.0: add MAE error from test set 503 | 0.0: ignore error 504 | -1.0: subtract MAE error from test set 505 | Returns 506 | ------- 507 | power_consumption : dict 508 | Returns summed power consumption per compressor setting 509 | """ 510 | power_consumption = {} 511 | 512 | # Generate random entries in range 513 | flow_vec = np.random.randint(low=flow[0], high=flow[1], size=n) 514 | press_vec = np.random.randint(low=press[0], high=press[1], size=n) 515 | 516 | # For each selected setting option 517 | for name in clf: 518 | # If compressor setting has boundary check and either skip or ignore 519 | if name in boundary: 520 | bounds = boundary[name] 521 | if not self.range_subset( 522 | range1=range(flow[0], flow[1]), range2=range(bounds[0], bounds[1]) 523 | ): 524 | continue 525 | power_predictions = 0 526 | # Predict and sum overall power consumption 527 | power_predictions = self.models[name].predict( 528 | pd.DataFrame( 529 | data=np.transpose([flow_vec, press_vec]), columns=self.features 530 | ) 531 | ) + (train_error_weight * self.test_mae_errors[name]) 532 | power_consumption[name] = np.nansum(power_predictions) 533 | 534 | return power_consumption 535 | 536 | def objective_function_keras( 537 | self, clf, flow, press, n=100, boundary={}, train_error_weight=1.0 538 | ): 539 | """Function to be minimize -> total energy consumption per compressor setting given shopfloor constraints 540 | Parameters 541 | ---------- 542 | clf: {list} of shape (n_compressor_settings,) 543 | flow: {list} of shape (2,) 544 | press: {list} of shape (2,) 545 | n: {integer} number of random entries to be generated 546 | boundary: {dict} contstraints per compressor setting 547 | train_error_weight: {float} factor to add/subtract/eliminate training error, e.g. 548 | 1.0: add MAE error from test set 549 | 0.0: ignore error 550 | -1.0: subtract MAE error from test set 551 | Returns 552 | ------- 553 | power_consumption : dict 554 | Returns summed power consumption per compressor setting 555 | """ 556 | power_consumption = {} 557 | 558 | # Generate random entries in range 559 | flow_vec = np.random.randint(low=flow[0], high=flow[1], size=n) 560 | press_vec = np.random.randint(low=press[0], high=press[1], size=n) 561 | 562 | # For each selected setting option 563 | for name in clf: 564 | # If compressor setting has boundary check and either skip or ignore 565 | if name in boundary: 566 | bounds = boundary[name] 567 | if not self.range_subset( 568 | range1=range(flow[0], flow[1]), range2=range(bounds[0], bounds[1]) 569 | ): 570 | continue 571 | power_predictions = 0 572 | # Predict and sum overall power consumption 573 | confs = np.repeat(a=name, repeats=n) 574 | D = self.keras_reg["preprocessor"].transform( 575 | pd.DataFrame( 576 | data=np.transpose([flow_vec, press_vec, confs]), columns=self.features+[self.config_col] 577 | ) 578 | ) 579 | power_predictions = self.keras_reg["regressor"].model(D) 580 | power_consumption[name] = np.nansum(power_predictions) 581 | 582 | return power_consumption 583 | 584 | def make_trial( 585 | self, 586 | clf, 587 | flow, 588 | press, 589 | n=100, 590 | no_of_trials=1000, 591 | boundary={}, 592 | train_error_weight=1.0, 593 | algorithm="quantile_regression", 594 | ): 595 | """Function to be minimize -> total energy consumption per compressor setting given shopfloor constraints 596 | Parameters 597 | ---------- 598 | clf: {list} of shape (n_compressor_settings,) 599 | flow: {list} of shape (2,) 600 | press: {list} of shape (2,) 601 | n: {integer} number of random entries to be generated 602 | no_of_trials: {integer} number of trials to run 603 | boundary: {dict} contstraints per compressor setting 604 | train_error_weight: {float} factor to add/subtract/eliminate training error, e.g. 605 | 1.0: add MAE error from test set 606 | 0.0: ignore error 607 | -1.0: subtract MAE error from test set 608 | Returns 609 | ------- 610 | power_consumption : dict 611 | Returns summed power consumption per compressor setting 612 | """ 613 | trials = [] 614 | for t in range(no_of_trials): 615 | energy = 0 616 | if algorithm == "quantile_regression": 617 | energy = self.objective_function_quantile( 618 | clf=clf, 619 | flow=flow, 620 | press=press, 621 | n=n, 622 | boundary=boundary, 623 | train_error_weight=train_error_weight, 624 | ) 625 | else: 626 | energy = self.objective_function_keras( 627 | clf=clf, 628 | flow=flow, 629 | press=press, 630 | n=n, 631 | boundary=boundary, 632 | train_error_weight=train_error_weight, 633 | ) 634 | 635 | # Setting is not compliant 636 | if len(energy) == 0: 637 | return {} 638 | key = min(energy, key=energy.get) 639 | trials.append({key: energy[key]}) 640 | return trials 641 | 642 | def run_monte_carlo( 643 | self, 644 | clf, 645 | flow, 646 | press, 647 | n=100, 648 | no_of_trials=1000, 649 | boundary={}, 650 | train_error_weight=1.0, 651 | algorithm="quantile_regression", 652 | ): 653 | """Run Monte Carlo simulation based on constraints 654 | Parameters 655 | ---------- 656 | clf: {list} of shape (n_compressor_settings,) 657 | flow: {list} of shape (2,) 658 | press: {list} of shape (2,) 659 | n: {integer} number of random entries to be generated 660 | no_of_trials: {integer} number of trials to run 661 | boundary: {dict} contstraints per compressor setting 662 | train_error_weight: {float} factor to add/subtract/eliminate training error, e.g. 663 | 1.0: add MAE error from test set 664 | 0.0: ignore error 665 | -1.0: subtract MAE error from test set 666 | Returns 667 | ------- 668 | power_consumption : dict 669 | Returns summed power consumption per compressor setting 670 | """ 671 | results = [] 672 | 673 | while len(clf) > 0: 674 | counts = {} 675 | minima = {} 676 | trials = self.make_trial( 677 | clf=clf, 678 | flow=flow, 679 | press=press, 680 | n=n, 681 | no_of_trials=no_of_trials, 682 | boundary=boundary, 683 | train_error_weight=train_error_weight, 684 | algorithm=algorithm, 685 | ) 686 | if len(trials) > 0: 687 | for t in trials: 688 | key = min(t, key=t.get) 689 | if key not in counts: 690 | counts[key] = 1 691 | minima[key] = round(t[key], 0) 692 | else: 693 | counts[key] += 1 694 | minima[key] = round(min(minima[key], t[key]), 0) 695 | asset = max(counts, key=counts.get) 696 | results.append({asset: minima[asset]}) 697 | clf.remove(asset) 698 | else: 699 | clf = [] 700 | 701 | return results 702 | -------------------------------------------------------------------------------- /mllib/lambda_handler.py: -------------------------------------------------------------------------------- 1 | """This Lambda function updates codebuild with the latest model artifact 2 | and then triggers the codepipeline. 3 | """ 4 | 5 | import json 6 | 7 | import boto3 8 | 9 | s3 = boto3.client("s3") 10 | 11 | 12 | def put_manifest(uri, bucket, key): 13 | """Get the codebuild project name based on key 14 | 15 | Args: 16 | key: the key searched for in your codebuild projects 17 | 18 | Returns: 19 | project name or empty string 20 | """ 21 | manifest = { 22 | "fileLocations": [ 23 | { 24 | "URIs": [ 25 | uri 26 | ] 27 | } 28 | ], 29 | "globalUploadSettings": { 30 | "format": "CSV", 31 | "delimiter": ",", 32 | "textqualifier": "'", 33 | "containsHeader": "true" 34 | } 35 | } 36 | 37 | s3.put_object( 38 | Body=json.dumps(manifest), 39 | Bucket=bucket, 40 | Key=key) 41 | 42 | 43 | def lambda_handler(event, context): 44 | """Your Lambda entry function 45 | 46 | Args: 47 | event: the event sent to the lambda 48 | context: the context passed into lambda 49 | 50 | Returns: 51 | dictionary with status code 52 | """ 53 | training = event["TRAINING"] 54 | simulation = event["SIMULATION"] 55 | bucket = event["BUCKET"] 56 | 57 | put_manifest( 58 | uri=f"{training}/dataset.csv", 59 | bucket=bucket, 60 | key="quicksight/resampled_data.json") 61 | 62 | put_manifest( 63 | uri=f"{simulation}/simulation.csv", 64 | bucket=bucket, 65 | key="quicksight/simulation.json") 66 | return { 67 | "statusCode": 200, 68 | "body": json.dumps("Manifest files in place!"), 69 | } 70 | -------------------------------------------------------------------------------- /mllib/preprocess.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import pathlib 4 | import sys 5 | from datetime import datetime, timedelta 6 | 7 | import boto3 8 | import numpy as np 9 | import pandas as pd 10 | 11 | logger = logging.getLogger() 12 | logger.setLevel(logging.INFO) 13 | logger.addHandler(logging.StreamHandler()) 14 | 15 | stats = { 16 | "example_flow_mean": { 17 | "c1c2": 283.27, 18 | "c1c2c3": 473.01, 19 | "c1c2c3c4": 476.51, 20 | "c1c2c4": 358.86, 21 | "c1c3": 258.11, 22 | "c1c3c4": 450.12, 23 | "c1c4": 258.61, 24 | "c2c3": 254.56, 25 | "c2c3c4": 449.53, 26 | "c2c4": 258.27, 27 | "c3c4": 294.25 28 | }, 29 | "example_flow_std": { 30 | "c1c2": 48.69, 31 | "c1c2c3": 76.38, 32 | "c1c2c3c4": 90.91, 33 | "c1c2c4": 65.88, 34 | "c1c3": 47.17, 35 | "c1c3c4": 83.56, 36 | "c1c4": 49.13, 37 | "c2c3": 52.34, 38 | "c2c3c4": 73.12, 39 | "c2c4": 42.48, 40 | "c3c4": 56.63 41 | }, 42 | "example_pressure_mean": { 43 | "c1c2": 70.48, 44 | "c1c2c3": 70.9, 45 | "c1c2c3c4": 70.88, 46 | "c1c2c4": 70.99, 47 | "c1c3": 70.8, 48 | "c1c3c4": 70.81, 49 | "c1c4": 70.34, 50 | "c2c3": 70.41, 51 | "c2c3c4": 71.02, 52 | "c2c4": 70.82, 53 | "c3c4": 70.57 54 | }, 55 | "example_pressure_std": { 56 | "c1c2": 0.43, 57 | "c1c2c3": 0.59, 58 | "c1c2c3c4": 0.75, 59 | "c1c2c4": 0.6, 60 | "c1c3": 0.79, 61 | "c1c3c4": 0.69, 62 | "c1c4": 0.43, 63 | "c2c3": 0.52, 64 | "c2c3c4": 0.58, 65 | "c2c4": 0.67, 66 | "c3c4": 0.77 67 | }, 68 | "example_power_mean": { 69 | "c1c2": 61.74, 70 | "c1c2c3": 64.07, 71 | "c1c2c3c4": 62.91, 72 | "c1c2c4": 61.33, 73 | "c1c3": 63.36, 74 | "c1c3c4": 59.55, 75 | "c1c4": 54.36, 76 | "c2c3": 61.97, 77 | "c2c3c4": 59.89, 78 | "c2c4": 56.04, 79 | "c3c4": 54.68 80 | }, 81 | "example_power_std": { 82 | "c1c2": 0.91, 83 | "c1c2c3": 0.56, 84 | "c1c2c3c4": 0.97, 85 | "c1c2c4": 1.17, 86 | "c1c3": 1.28, 87 | "c1c3c4": 1.53, 88 | "c1c4": 1.14, 89 | "c2c3": 1.21, 90 | "c2c3c4": 1.34, 91 | "c2c4": 1.32, 92 | "c3c4": 1.01 93 | }, 94 | } 95 | 96 | def create_data_points(stats, setting, size, next_start="2020-01-01"): 97 | flow = np.random.normal(loc=stats["example_flow_mean"][setting], scale=stats["example_flow_std"][setting], size=size).reshape(-1, 1) 98 | pressure = np.random.normal(loc=stats["example_pressure_mean"][setting], scale=stats["example_pressure_std"][setting], size=size).reshape(-1, 1) 99 | power = np.random.normal(loc=stats["example_power_mean"][setting], scale=stats["example_power_std"][setting], size=size).reshape(-1, 1) 100 | dttm = pd.date_range(start=next_start, freq="H", periods=size) 101 | return [dttm, flow, pressure, power, np.repeat(a=setting, repeats=size)] 102 | 103 | if __name__ == "__main__": 104 | logger.debug("Starting preprocessing.") 105 | parser = argparse.ArgumentParser() 106 | parser.add_argument("--bucket-name", type=str, required=True) 107 | args = parser.parse_args() 108 | 109 | base_dir = "/opt/ml/processing" 110 | pathlib.Path(f"{base_dir}/data").mkdir(parents=True, exist_ok=True) 111 | 112 | low = 12 113 | high = 48 114 | compressor_settings = ["c1c2", "c1c3", "c1c4", "c2c3", "c2c4", "c3c4", "c1c2c3", "c1c2c4", "c1c3c4", "c2c3c4", "c1c2c3c4"] 115 | setting = np.random.choice(compressor_settings) 116 | size = int(np.random.uniform(low=low, high=high)) 117 | 118 | data = [pd.DataFrame(data=create_data_points(stats=stats, setting=setting, size=size)).transpose()] 119 | next_start = data[0].iloc[-1, 0] + timedelta(minutes=1) 120 | 121 | while next_start < datetime.now(): 122 | setting = np.random.choice(compressor_settings) 123 | size = int(np.random.uniform(low=low, high=high)) 124 | data.append(pd.DataFrame(data=create_data_points(stats=stats, setting=setting, size=size, next_start=next_start)).transpose()) 125 | next_start = data[-1].iloc[-1, 0] + timedelta(minutes=1) 126 | 127 | df = pd.concat(objs=data) 128 | df.columns = ["dttm", "flow", "pressure", "power", "config"] 129 | df.index = df.dttm 130 | df = df.drop("dttm", axis=1) 131 | df[["flow", "pressure", "power"]] = df[["flow", "pressure", "power"]].astype(float) 132 | 133 | # Convert time_in_seconds to a dttm and sort by this column 134 | df = df.sort_values(by="dttm", ascending=True) 135 | 136 | # Save new dataset to S3 137 | logger.info("Writing out datasets to %s.", base_dir) 138 | df.to_csv( 139 | f"{base_dir}/train/dataset.csv", 140 | index=True, 141 | ) 142 | -------------------------------------------------------------------------------- /mllib/quantile_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from scipy.optimize import minimize 4 | from sklearn.base import BaseEstimator, RegressorMixin 5 | from sklearn.utils.validation import check_array, check_is_fitted, check_X_y 6 | 7 | 8 | class QuantileRegression(BaseEstimator, RegressorMixin): 9 | """Linear regression model that predicts conditional quantiles. 10 | The linear :class:`QuantileRegression` optimizes the pinball loss for a 11 | desired `quantile` and is robust to outliers. 12 | Parameters 13 | ---------- 14 | quantile : float, default=0.5 15 | The quantile that the model tries to predict. It must be strictly 16 | between 0 and 1. If 0.5 (default), the model predicts the 50% 17 | quantile, i.e. the median. 18 | Attributes 19 | ---------- 20 | coef_ : array of shape (n_features,) 21 | Estimated coefficients for the features. 22 | intercept_: float value 23 | Estimated intercept. 24 | Examples 25 | -------- 26 | >>> import numpy as np 27 | >>> n_samples, n_features = 10, 2 28 | >>> rng = np.random.RandomState(0) 29 | >>> y = rng.randn(n_samples) 30 | >>> X = rng.randn(n_samples, n_features) 31 | >>> reg = QuantileRegression(quantile=0.8).fit(X, y) 32 | >>> np.mean(y <= reg.predict(X)) 33 | 0.8 34 | """ 35 | 36 | def __init__(self, quantile=0.5): 37 | """Initialize the class. 38 | 39 | Args: 40 | :quantile: the quantile of the quantile forrest, default: 0.5 41 | 42 | """ 43 | self.quantile = quantile 44 | 45 | def quantile_loss(self, coefs, X, y): 46 | """Define the quantile loss function for optimization 47 | Parameters 48 | ---------- 49 | X : {array-like, sparse matrix} of shape (n_samples, n_features) 50 | Training data. 51 | y : array-like of shape (n_samples,) 52 | Target values. 53 | coefs : array-like of shape (n_features+1,) 54 | Regression coefficients. 55 | Returns 56 | ------- 57 | mean : array-like of shape (n_features+1,) 58 | Returns the calculated mean. 59 | """ 60 | 61 | preds = X @ coefs[:-1] + coefs[-1] 62 | mean = np.nanmean( 63 | (preds >= y) * (1 - self.quantile) * (preds - y) 64 | + (preds < y) * self.quantile * (y - preds) 65 | ) 66 | 67 | return mean 68 | 69 | def fit(self, X, y): 70 | """Fit the model according to the given training data. 71 | Parameters 72 | ---------- 73 | X : {array-like, sparse matrix} of shape (n_samples, n_features) 74 | Training data. 75 | y : array-like of shape (n_samples,) 76 | Target values. 77 | Returns 78 | ------- 79 | self : object 80 | Returns self. 81 | """ 82 | # scikit-learn check X and y 83 | X, y = check_X_y(X, y) 84 | 85 | d = X.shape[1] 86 | x0 = np.repeat(0.0, d + 1) # starting vector 87 | 88 | # Run optimization 89 | *self.coef_, self.intercept_ = minimize( 90 | self.quantile_loss, x0=x0, args=(X, y) 91 | ).x # the heavy lifting 92 | 93 | return self 94 | 95 | def predict(self, X): 96 | """Predict with the trained model. 97 | Parameters 98 | ---------- 99 | X : {array-like, sparse matrix} of shape (n_samples, n_features) 100 | Training data. 101 | Returns 102 | ------- 103 | self : object 104 | Returns self. 105 | """ 106 | # scikit-learn check if model is fitted 107 | check_is_fitted(self) 108 | 109 | # scikit-learn check X 110 | X = check_array(X) 111 | 112 | # Return X in |R^(nxn) times self.coef_ in |R^n + self.intercept_ 113 | return X @ self.coef_ + self.intercept_ 114 | -------------------------------------------------------------------------------- /mllib/serve.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import argparse 4 | import logging 5 | import os 6 | from io import StringIO 7 | 8 | import joblib 9 | import numpy as np 10 | import pandas as pd 11 | from digital_twin import DigitalTwin 12 | from quantile_regression import QuantileRegression 13 | 14 | features = [ 15 | "flow", 16 | "pressure", 17 | ] 18 | target = "power" 19 | config_col = "config" 20 | 21 | logger = logging.getLogger() 22 | logger.setLevel(logging.INFO) 23 | logger.addHandler(logging.StreamHandler()) 24 | 25 | 26 | def input_fn(input_data, content_type="text/csv"): 27 | """Parse input data payload. 28 | 29 | Args: 30 | input_data (pandas.core.frame.DataFrame): A pandas.core.frame.DataFrame. 31 | content_type (str): A string expected to be 'text/csv'. 32 | 33 | Returns: 34 | df: pandas.core.frame.DataFrame 35 | """ 36 | try: 37 | if "text/csv" in content_type: 38 | df = pd.read_csv(StringIO(input_data), header=None) 39 | return df 40 | elif "application/json" in content_type: 41 | df = pd.read_json(StringIO(input_data.decode("utf-8"))) 42 | return df 43 | else: 44 | df = pd.read_csv(StringIO(input_data.decode("utf-8")), header=None) 45 | return df 46 | except ValueError as e: 47 | raise logger.error(f"ValueError {e}") 48 | 49 | 50 | def output_fn(prediction, accept="text/csv"): 51 | """Format prediction output. 52 | 53 | Args: 54 | prediction (pandas.core.frame.DataFrame): A DataFrame with predictions. 55 | accept (str): A string expected to be 'text/csv'. 56 | 57 | Returns: 58 | df: str (in CSV format) 59 | """ 60 | return prediction.to_csv(index=False) 61 | 62 | 63 | def predict_fn(input_data, model): 64 | """Preprocess input data. 65 | 66 | Args: 67 | input_data (pandas.core.frame.DataFrame): A pandas.core.frame.DataFrame. 68 | model: A regression model 69 | 70 | Returns: 71 | output: pandas.core.frame.DataFrame 72 | """ 73 | # Read your model and config file 74 | output = model.predict_efficiency(df=input_data, algorithm="keras_regression") 75 | return output 76 | 77 | 78 | def model_fn(model_dir): 79 | """Deserialize fitted model. 80 | 81 | This simple function takes the path of the model, loads it, 82 | deserializes it and returns it for prediction. 83 | 84 | Args: 85 | model_dir (str): A string that indicates where the model is located. 86 | 87 | Returns: 88 | model: 89 | """ 90 | # Load the model and deserialize 91 | model = joblib.load(os.path.join(model_dir, "model.joblib")) 92 | return model 93 | -------------------------------------------------------------------------------- /mllib/simulate.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import logging 4 | import multiprocessing as mp 5 | import pathlib 6 | # Make library accessible 7 | import sys 8 | import tarfile 9 | from datetime import datetime 10 | 11 | import joblib 12 | import numpy as np 13 | import pandas as pd 14 | 15 | sys.path.append("/opt/ml/code/") 16 | 17 | 18 | logger = logging.getLogger() 19 | logger.setLevel(logging.INFO) 20 | logger.addHandler(logging.StreamHandler()) 21 | 22 | clf = [ 23 | "c1c2", 24 | "c1c2c4", 25 | "c1c4", 26 | "c1c3c4", 27 | "c1c2c3c4", 28 | "c1c3", 29 | "c1c2c3", 30 | "c3c4", 31 | "c2c3c4", 32 | "c2c4", 33 | "c2c3", 34 | ] 35 | 36 | def parallel_func(data): 37 | results = {} 38 | flow = data[:2] 39 | press = data[2:] 40 | clf = ["c1c2","c1c2c4","c1c4","c1c3c4","c1c2c3c4","c1c3","c1c2c3","c3c4","c2c3c4","c2c4","c2c3"] 41 | boundary = { 42 | "c1c2": [180, 390], 43 | "c1c2c4": [180, 390], 44 | "c1c4": [180, 390], 45 | "c1c3c4": [180, 460], 46 | "c1c2c3c4": [180, 460], 47 | "c1c3": [180, 390], 48 | "c1c2c3": [180, 460], 49 | "c3c4": [180, 390], 50 | "c2c3c4": [180, 460], 51 | "c2c4": [180, 390], 52 | "c2c3": [180, 390], 53 | } 54 | n = 100 55 | no_of_trials = 10 56 | train_error_weight = 1.0 57 | res = twin.run_monte_carlo( 58 | clf=clf, 59 | flow=flow, 60 | press=press, 61 | n=n, 62 | no_of_trials=no_of_trials, 63 | boundary=boundary, 64 | train_error_weight=train_error_weight, 65 | algorithm="keras_regression") 66 | for r in res: 67 | key = list(r.keys())[0] 68 | results[key] = r[key] 69 | results["flow_low"] = flow[0] 70 | results["flow_high"] = flow[1] 71 | results["pressure_low"] = press[0] 72 | results["pressure_high"] = press[1] 73 | return results 74 | 75 | def extract_key(x): 76 | vec = list(json.loads(x.to_json()).values()) 77 | vec = [y if y is not None else np.nan for y in vec] 78 | argmin = np.nanargmin(vec) 79 | key = list(x.keys())[argmin] 80 | return key 81 | 82 | def extract_value(x): 83 | vec = list(json.loads(x.to_json()).values()) 84 | vec = [y if y is not None else np.nan for y in vec] 85 | argmin = np.nanmin(vec) 86 | return argmin 87 | 88 | if __name__ == "__main__": 89 | logger.debug("Starting simulation.") 90 | 91 | base_dir = "/opt/ml/processing" 92 | pathlib.Path(f"{base_dir}/data").mkdir(parents=True, exist_ok=True) 93 | 94 | 95 | tar = tarfile.open("/opt/ml/processing/input/model.tar.gz") 96 | tar.extractall() 97 | 98 | twin = joblib.load("model.joblib") 99 | 100 | low = 180 101 | high = 450 102 | flows = [] 103 | for i in range(low, high, 20): 104 | for j in range(low, high, 20): 105 | flows.append([i, j]) 106 | 107 | flows = pd.DataFrame(data=flows, columns=["flow_low", "flow_high"]).sort_values(by=["flow_low", "flow_high"]) 108 | flows["filter"] = flows.apply(lambda x: True if x.iloc[0] < x.iloc[1] else False, axis=1) 109 | flows = flows[flows["filter"] == True].reset_index(drop=True).drop("filter", axis=1) 110 | 111 | low = 50 112 | high = 75 113 | pressures = [] 114 | for i in range(low, high, 5): 115 | for j in range(low, high, 5): 116 | pressures.append([i, j]) 117 | 118 | pressures = pd.DataFrame(data=pressures, columns=["pressure_low", "pressure_high"]).sort_values(by=["pressure_low", "pressure_high"]) 119 | pressures["filter"] = pressures.apply(lambda x: True if x.iloc[0] < x.iloc[1] else False, axis=1) 120 | pressures = pressures[pressures["filter"] == True].reset_index(drop=True).drop("filter", axis=1) 121 | 122 | data = pd.DataFrame() 123 | for i in range(0, pressures.shape[0]): 124 | tmp = flows.copy("deep") 125 | tmp[pressures.columns.tolist()] = pressures.iloc[i, :] 126 | data = pd.concat(objs=[data, tmp], axis=0) 127 | data = data.reset_index(drop=True) 128 | 129 | pool = mp.Pool(mp.cpu_count()) 130 | predictions = [pool.apply(parallel_func, args=(dat, )) for dat in np.array(data)] 131 | pool.close() 132 | 133 | df = pd.DataFrame(predictions) 134 | df.loc[:, clf] = df.loc[:, clf].apply(lambda x: np.round(x/100, 0), axis=1) 135 | df["minimum_key"] = df.loc[:, clf].apply(lambda x: extract_key(x=x), axis=1) 136 | df["minimum_value"] = df.loc[:, clf].apply(lambda x: extract_value(x=x), axis=1) 137 | 138 | # Save new dataset to S3 139 | logger.info("Writing out datasets to %s.", base_dir) 140 | df.to_csv( 141 | f"{base_dir}/train/simulation.csv", 142 | index=False, 143 | ) 144 | -------------------------------------------------------------------------------- /mllib/train.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import argparse 4 | import logging 5 | import os 6 | from io import StringIO 7 | 8 | import joblib 9 | import numpy as np 10 | import pandas as pd 11 | from digital_twin import DigitalTwin 12 | from quantile_regression import QuantileRegression 13 | 14 | logger = logging.getLogger() 15 | logger.setLevel(logging.INFO) 16 | logger.addHandler(logging.StreamHandler()) 17 | 18 | 19 | features = [ 20 | "flow", 21 | "pressure", 22 | ] 23 | target = "power" 24 | config_col = "config" 25 | 26 | if __name__ == "__main__": 27 | parser = argparse.ArgumentParser() 28 | # Hyperparameters are described here 29 | # In this simple example only 4 Hyperparamters are permitted 30 | parser.add_argument("--quantile", type=float, default=0.5) 31 | parser.add_argument("--test_size", type=float, default=0.2) 32 | parser.add_argument("--random_state", type=int, default=42) 33 | parser.add_argument("--degree", type=int, default=3) 34 | parser.add_argument("--verbose", type=int, default=0) 35 | parser.add_argument("--dual_model", type=bool, default=False) 36 | parser.add_argument("--remove_outliers", type=bool, default=False) 37 | parser.add_argument("--optimizer", type=str, default="adam") 38 | parser.add_argument("--optimizer__learning_rate", type=float, default=0.001) 39 | parser.add_argument("--epochs", type=int, default=2000) 40 | 41 | # SageMaker specific arguments. Defaults are set in the environment variables. 42 | parser.add_argument("--model_dir", type=str, default=os.environ.get("SM_MODEL_DIR")) 43 | parser.add_argument("--train", type=str, default=os.environ["SM_CHANNEL_TRAIN"]) 44 | args = parser.parse_args() 45 | 46 | # Take the set of files and read them all into a single pandas dataframe 47 | train_files = [os.path.join(args.train, file) for file in os.listdir(args.train)] 48 | if len(train_files) == 0: 49 | raise ValueError( 50 | ( 51 | "There are no files in {}.\n" 52 | + "This usually indicates that the channel ({}) was incorrectly specified,\n" 53 | + "the data specification in S3 was incorrectly specified or the role specified\n" 54 | + "does not have permission to access the data." 55 | ).format(args.train, "train") 56 | ) 57 | 58 | # Read DataFrames into array and concatenate them into one DF 59 | train_data = [pd.read_csv(file) for file in train_files] 60 | train_data = pd.concat(train_data) 61 | 62 | configurations = train_data[config_col].unique().tolist() 63 | # Set train and validation 64 | twin = DigitalTwin( 65 | configurations=configurations, 66 | features=features, 67 | target=target, 68 | config_col=config_col, 69 | quantile=args.quantile, 70 | ) 71 | 72 | if args.dual_model: 73 | 74 | twin = twin.train( 75 | df=train_data, 76 | test_size=args.test_size, 77 | random_state=args.random_state, 78 | degree=2, 79 | verbose=args.verbose, 80 | remove_outliers=args.remove_outliers, 81 | algorithm="quantile_regression", 82 | ) 83 | 84 | for configs in configurations: 85 | print(f"Train_{configs}_quantile_mae={twin.train_mae_errors[configs]};") 86 | print(f"Validation_{configs}_quantile_mae={twin.test_mae_errors[configs]};") 87 | 88 | twin = twin.train( 89 | df=train_data, 90 | test_size=args.test_size, 91 | random_state=args.random_state, 92 | degree=args.degree, 93 | verbose=args.verbose, 94 | optimizer=args.optimizer, 95 | optimizer__learning_rate=args.optimizer__learning_rate, 96 | epochs=args.epochs, 97 | algorithm="keras_regression", 98 | remove_outliers=args.remove_outliers, 99 | ) 100 | 101 | for configs in configurations: 102 | print(f"Train_{configs}_keras_mae={twin.train_mae_errors[configs]};") 103 | print(f"Validation_{configs}_keras_mae={twin.test_mae_errors[configs]};") 104 | 105 | # Save the model and config_data to the model_dir so that it can be loaded by model_fn 106 | joblib.dump(twin, os.path.join(args.model_dir, "model.joblib")) 107 | 108 | # Print Success 109 | logger.info("Saved model!") 110 | 111 | 112 | def input_fn(input_data, content_type="text/csv"): 113 | """Parse input data payload. 114 | 115 | Args: 116 | input_data (pandas.core.frame.DataFrame): A pandas.core.frame.DataFrame. 117 | content_type (str): A string expected to be 'text/csv'. 118 | 119 | Returns: 120 | df: pandas.core.frame.DataFrame 121 | """ 122 | try: 123 | if "text/csv" in content_type: 124 | df = pd.read_csv(StringIO(input_data), header=None) 125 | return df 126 | elif "application/json" in content_type: 127 | df = pd.read_json(StringIO(input_data.decode("utf-8"))) 128 | return df 129 | else: 130 | df = pd.read_csv(StringIO(input_data.decode("utf-8")), header=None) 131 | return df 132 | except ValueError as e: 133 | raise logger.error(f"ValueError {e}") 134 | 135 | 136 | def output_fn(prediction, accept="text/csv"): 137 | """Format prediction output. 138 | 139 | Args: 140 | prediction (pandas.core.frame.DataFrame): A DataFrame with predictions. 141 | accept (str): A string expected to be 'text/csv'. 142 | 143 | Returns: 144 | df: str (in CSV format) 145 | """ 146 | return prediction.to_csv(index=False) 147 | 148 | 149 | def predict_fn(input_data, model): 150 | """Preprocess input data. 151 | 152 | Args: 153 | input_data (pandas.core.frame.DataFrame): A pandas.core.frame.DataFrame. 154 | model: A regression model 155 | 156 | Returns: 157 | output: pandas.core.frame.DataFrame 158 | """ 159 | # Read your model and config file 160 | output = model.predict_efficiency(df=input_data, algorithm="keras_regression") 161 | return output 162 | 163 | 164 | def model_fn(model_dir): 165 | """Deserialize fitted model. 166 | 167 | This simple function takes the path of the model, loads it, 168 | deserializes it and returns it for prediction. 169 | 170 | Args: 171 | model_dir (str): A string that indicates where the model is located. 172 | 173 | Returns: 174 | model: 175 | """ 176 | # Load the model and deserialize 177 | model = joblib.load(os.path.join(model_dir, "model.joblib")) 178 | return model 179 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | setuptools.setup( 4 | name="MLLib", 5 | version="0.0.1", 6 | description="This repository serves as the one-stop-shop for the mlops template.", 7 | long_description_content_type="text/markdown", 8 | author="Michael Wallner", 9 | install_requires=[ 10 | "pre-commit", 11 | "scikit-learn", 12 | "imblearn", 13 | "pandas", 14 | "numpy", 15 | "pytest==6.2.5", 16 | "boto3", 17 | ], 18 | python_requires=">=3.6", 19 | ) 20 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Initialize 2 | -------------------------------------------------------------------------------- /tests/test_digital_twin.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import sys 4 | 5 | import numpy as np 6 | import pandas as pd 7 | import pytest 8 | from digital_twin import DigitalTwin 9 | from pandas._testing import assert_frame_equal 10 | --------------------------------------------------------------------------------