├── .gitignore ├── LICENSE ├── README.md ├── app.py ├── cdk.json ├── docs ├── architecture.png ├── cloudwatch_logs.png ├── metadata_table_entry.png ├── x-ray_1.png └── x-ray_2.png ├── infrastructure ├── __init__.py └── cdk_image_analyzer_stack.py ├── requirements.txt ├── setup.py ├── source.bat └── src ├── __init__.py ├── processing_lambda.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | package-lock.json 3 | __pycache__ 4 | .pytest_cache 5 | .env 6 | *.egg-info 7 | 8 | # CDK asset staging directory 9 | .cdk.staging 10 | cdk.out 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Maurice Borgmeier 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CDK Image Analyzer Demo 2 | 3 | This is a demo CDK application that's used to showcase some AWS services as well as a serverless processing workflow. 4 | 5 | Things this demo shows 6 | 7 | - AWS X-Ray 8 | - S3 Event Notifications 9 | - Amazon Rekognition 10 | - boto3 clients vs. resources 11 | - AWS CDK 12 | 13 | ## Architecture 14 | 15 | ![architecture](docs/architecture.png) 16 | 17 | ## Deployment 18 | 19 | 1. Have the CDK and the AWS SDK installed 20 | 2. Configure your AWS credentials 21 | 3. Clone the repo 22 | 4. Nagivate to the repo 23 | 5. Create a new virtual environment: `python3 -m venv .env` 24 | 6. Activate the virtual environment: `source .env\bin\activate` 25 | 7. Install the dependencies: `pip install -r requirements.txt` 26 | 8. Deploy the CDK app: `cdk deploy` 27 | 28 | ## Opportunities for demonstration 29 | 30 | ### 1) What it does 31 | 32 | 1. Deploy the app (see instructions above) 33 | 2. Log in to the AWS Console and open the S3 Console 34 | 3. Locate the Input bucket, the name will be something like `cdk-image-analyzer-inputbucket...` 35 | 4. Upload an image to the input bucket 36 | 5. Navigate to DynamoDB and open the metadata table, it will be called something like `cdk-image-analyzer-metadatatable...` 37 | 6. You should see a single item in the table, click on it an show the labels that have been generated by Rekognition. 38 | 39 | ![dynamodb](docs/metadata_table_entry.png) 40 | 41 | ### 2) How it works 42 | 43 | 1. Open the lambda function in the lambda console, it will be called something like `cdk-image-analyzer-processinglambda...` 44 | 2. Point out the event handler and explain, how data is extracted from the S3 event. 45 | 3. Point out the different ways boto3 is used - Rekognition Client and Table Resource 46 | 4. Note the import of the X-Ray SDK and the `patch_all()` method 47 | 5. Now upload another image and show the result in DynamoDB 48 | 6. Open CloudWatch logs and take a look at the logs of the lambda function, point out the response from Rekognition 49 | 50 | ![cloudwatch](docs/cloudwatch_logs.png) 51 | 52 | ### 3) X-Ray 53 | 54 | 1. After you've done 1) and 2), there should be some traces in AWS X-Ray 55 | 2. Open the X-Ray service and click on the service map 56 | 3. Wait a couple of seconds, it's not that fast on first load, maybe adjust the time range in the top right corner 57 | 58 | **Screenshots** 59 | 60 | ![x-ray](docs/x-ray_1.png) 61 | ![x-ray](docs/x-ray_2.png) 62 | 63 | ### 4) CDK App 64 | 65 | 1. Open the CDK App and explain the separation of the business logic (`src` directory) from the infrastructure 66 | 2. Explain how the CDK makes permissions management a lot easier using `grants` and point out the additional IAM statement to allow access to Rekognition. 67 | 3. Run `cdk synth` to show the output that gets passed to CloudFormation 68 | 4. Change something on the infrastructure, e.g. add an S3 bucket and run `cdk diff` to show what changed 69 | 5. Run `cdk deploy` to update the architecture 70 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from aws_cdk import core 4 | 5 | from infrastructure.cdk_image_analyzer_stack import CdkImageAnalyzerStack 6 | 7 | 8 | app = core.App() 9 | CdkImageAnalyzerStack(app, "cdk-image-analyzer") 10 | 11 | app.synth() 12 | -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "python3 app.py", 3 | "context": { 4 | "@aws-cdk/core:enableStackNameDuplicates": "true", 5 | "aws-cdk:enableDiffNoFail": "true" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /docs/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AWS-Devops-Projects/CDK-Demo/58406871903cabc1a9aa9da069b19fc5e33cc1c2/docs/architecture.png -------------------------------------------------------------------------------- /docs/cloudwatch_logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AWS-Devops-Projects/CDK-Demo/58406871903cabc1a9aa9da069b19fc5e33cc1c2/docs/cloudwatch_logs.png -------------------------------------------------------------------------------- /docs/metadata_table_entry.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AWS-Devops-Projects/CDK-Demo/58406871903cabc1a9aa9da069b19fc5e33cc1c2/docs/metadata_table_entry.png -------------------------------------------------------------------------------- /docs/x-ray_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AWS-Devops-Projects/CDK-Demo/58406871903cabc1a9aa9da069b19fc5e33cc1c2/docs/x-ray_1.png -------------------------------------------------------------------------------- /docs/x-ray_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AWS-Devops-Projects/CDK-Demo/58406871903cabc1a9aa9da069b19fc5e33cc1c2/docs/x-ray_2.png -------------------------------------------------------------------------------- /infrastructure/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AWS-Devops-Projects/CDK-Demo/58406871903cabc1a9aa9da069b19fc5e33cc1c2/infrastructure/__init__.py -------------------------------------------------------------------------------- /infrastructure/cdk_image_analyzer_stack.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import aws_cdk.aws_dynamodb as dynamodb 4 | import aws_cdk.aws_iam as iam 5 | import aws_cdk.aws_lambda as _lambda 6 | import aws_cdk.aws_lambda_event_sources as lambda_event_sources 7 | import aws_cdk.aws_s3 as s3 8 | from aws_cdk import core 9 | 10 | from lambda_bundler import build_lambda_package, build_layer_package 11 | 12 | class CdkImageAnalyzerStack(core.Stack): 13 | 14 | def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: 15 | super().__init__(scope, id, **kwargs) 16 | 17 | input_bucket = s3.Bucket( 18 | self, 19 | "input-bucket" 20 | ) 21 | 22 | metadata_table = dynamodb.Table( 23 | self, 24 | "metadata-table", 25 | billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST, 26 | partition_key=dynamodb.Attribute(name="PK",type=dynamodb.AttributeType.STRING) 27 | ) 28 | 29 | dependency_layer_path = build_layer_package( 30 | requirement_files=[os.path.join(os.path.dirname(__file__), "..", "src", "requirements.txt")] 31 | ) 32 | 33 | dependency_layer = _lambda.LayerVersion( 34 | self, 35 | "dependency-layer", 36 | code=_lambda.Code.from_asset(path=dependency_layer_path), 37 | compatible_runtimes=[_lambda.Runtime.PYTHON_3_7, _lambda.Runtime.PYTHON_3_8] 38 | ) 39 | 40 | lambda_package_path = build_lambda_package( 41 | code_directories=[os.path.join(os.path.dirname(__file__), "..", "src")], 42 | ) 43 | 44 | processing_lambda = _lambda.Function( 45 | self, 46 | "processing-lambda", 47 | code=_lambda.Code.from_asset(path=lambda_package_path), 48 | handler="src.processing_lambda.lambda_handler", 49 | runtime=_lambda.Runtime.PYTHON_3_8, 50 | timeout=core.Duration.seconds(30), 51 | environment={ 52 | "METADATA_TABLE_NAME": metadata_table.table_name 53 | }, 54 | layers=[dependency_layer], 55 | description="Triggers object recognition on an S3 object and stores the metadata", 56 | tracing=_lambda.Tracing.ACTIVE 57 | ) 58 | 59 | # Allow Lambda to talk to Rekognition 60 | processing_lambda.add_to_role_policy( 61 | iam.PolicyStatement( 62 | actions=["rekognition:DetectLabels"], 63 | resources=["*"] 64 | ) 65 | ) 66 | 67 | # Create the S3 Event Trigger for Lambda 68 | processing_lambda.add_event_source( 69 | lambda_event_sources.S3EventSource( 70 | input_bucket, 71 | events=[s3.EventType.OBJECT_CREATED] 72 | ) 73 | ) 74 | 75 | input_bucket.grant_read(processing_lambda) 76 | metadata_table.grant_read_write_data(processing_lambda) 77 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | CDK_VERSION="1.51.0" 4 | 5 | with open("README.md") as fp: 6 | long_description = fp.read() 7 | 8 | 9 | setuptools.setup( 10 | name="cdk_image_analyzer", 11 | version="0.0.1", 12 | 13 | description="A CDK demo app", 14 | long_description=long_description, 15 | long_description_content_type="text/markdown", 16 | 17 | author="Maurice Borgmeier", 18 | 19 | package_dir={"": "infrastructure"}, 20 | packages=setuptools.find_packages(where="cdk_image_analyzer"), 21 | 22 | install_requires=[ 23 | f"aws-cdk.aws-dynamodb=={CDK_VERSION}", 24 | f"aws-cdk.aws-lambda=={CDK_VERSION}", 25 | f"aws-cdk.aws-lambda-event-sources=={CDK_VERSION}", 26 | f"aws-cdk.aws-s3=={CDK_VERSION}", 27 | f"aws-cdk.core=={CDK_VERSION}", 28 | "lambda-bundler==0.1.0", 29 | ], 30 | 31 | python_requires=">=3.6", 32 | 33 | classifiers=[ 34 | "Development Status :: 4 - Beta", 35 | 36 | "Intended Audience :: Developers", 37 | 38 | "License :: OSI Approved :: Apache Software License", 39 | 40 | "Programming Language :: JavaScript", 41 | "Programming Language :: Python :: 3 :: Only", 42 | "Programming Language :: Python :: 3.6", 43 | "Programming Language :: Python :: 3.7", 44 | "Programming Language :: Python :: 3.8", 45 | 46 | "Topic :: Software Development :: Code Generators", 47 | "Topic :: Utilities", 48 | 49 | "Typing :: Typed", 50 | ], 51 | ) 52 | -------------------------------------------------------------------------------- /source.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem The sole purpose of this script is to make the command 4 | rem 5 | rem source .env/bin/activate 6 | rem 7 | rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. 8 | rem On Windows, this command just runs this batch file (the argument is ignored). 9 | rem 10 | rem Now we don't need to document a Windows command for activating a virtualenv. 11 | 12 | echo Executing .env\Scripts\activate.bat for you 13 | .env\Scripts\activate.bat 14 | -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AWS-Devops-Projects/CDK-Demo/58406871903cabc1a9aa9da069b19fc5e33cc1c2/src/__init__.py -------------------------------------------------------------------------------- /src/processing_lambda.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import typing 4 | 5 | from decimal import Decimal 6 | from urllib.parse import unquote 7 | 8 | import boto3 9 | from aws_xray_sdk.core import patch_all 10 | 11 | ENV_METADATA_TABLE_NAME = "METADATA_TABLE_NAME" 12 | 13 | patch_all() 14 | 15 | def parse_s3_event(event) -> typing.Iterable[typing.Tuple[str, str]]: 16 | for record in event["Records"]: 17 | bucket_name = record["s3"]["bucket"]["name"] 18 | object_key = record["s3"]["object"]["key"] 19 | object_key = unquote(object_key) 20 | yield bucket_name, object_key 21 | 22 | def lambda_handler(event: dict, context: "LambdaContext") -> None: 23 | 24 | print(json.dumps(event)) 25 | 26 | # Initialize the boto3 client to talk to rekognition 27 | rekognition_client = boto3.client("rekognition") 28 | 29 | # Initialize a boto3 resource as an abstraction of the dynamo db table 30 | table = boto3.resource("dynamodb").Table( 31 | os.environ.get(ENV_METADATA_TABLE_NAME) 32 | ) 33 | 34 | # Parse the s3 event, see above 35 | for bucket_name, object_key in parse_s3_event(event): 36 | 37 | response = rekognition_client.detect_labels( 38 | Image={ 39 | "S3Object": { 40 | "Bucket": bucket_name, 41 | "Name": object_key 42 | } 43 | } 44 | ) 45 | 46 | print("Rekognition response:", json.dumps(response)) 47 | 48 | # Flatten the response structure 49 | labels = [] 50 | for item in response["Labels"]: 51 | labels.append({ 52 | "Name": item["Name"], 53 | "Confidence": Decimal(str(item["Confidence"])), 54 | "Parents": item["Parents"] 55 | 56 | }) 57 | 58 | # Store the item in the metadata table 59 | table.put_item( 60 | Item={ 61 | "PK": object_key, 62 | "Labels": labels 63 | } 64 | ) 65 | -------------------------------------------------------------------------------- /src/requirements.txt: -------------------------------------------------------------------------------- 1 | aws-xray-sdk==2.6.0 2 | boto3==1.14 --------------------------------------------------------------------------------