├── LICENSE ├── README.md ├── access_cloudtrail_with_athena ├── README.md ├── consolidated.ddl ├── consolidated.sql ├── raw_partitioned.ddl ├── raw_partitioned.sql ├── raw_unpartitioned.ddl ├── raw_unpartitioned.sql ├── single_file.ddl └── single_file.sql ├── athena_performance_comparison ├── .gitignore ├── README.md ├── cloudformation │ ├── glue_job.yml │ ├── provisioned.yml │ ├── serverless.yml │ ├── tables-gz.yml │ └── tables.yml ├── generator │ └── clickstream_generator.py └── glue │ └── transform.py ├── aurora_data_api ├── .gitignore ├── README.md ├── cloudformation.yml ├── example_notifications.json ├── init_database.py └── lambda.py ├── cdk_cross_stack_references ├── 1-broken │ ├── .gitignore │ ├── .npmignore │ ├── bin │ │ └── app.ts │ ├── cdk.json │ ├── jest.config.js │ ├── lib │ │ ├── stack-1.ts │ │ └── stack-2.ts │ ├── package.json │ └── tsconfig.json ├── 2-better │ ├── .gitignore │ ├── .npmignore │ ├── bin │ │ └── app.ts │ ├── cdk.json │ ├── jest.config.js │ ├── lib │ │ ├── stack-1.ts │ │ └── stack-2.ts │ ├── package.json │ └── tsconfig.json ├── 3-best │ ├── .gitignore │ ├── .npmignore │ ├── bin │ │ └── app.ts │ ├── cdk.json │ ├── jest.config.js │ ├── lib │ │ ├── stack-1.ts │ │ └── stack-2.ts │ ├── package.json │ └── tsconfig.json └── README.md ├── cloudtrail_aggregation ├── README.md ├── stage-1 │ ├── README.md │ ├── cloudformation.yml │ ├── concurrent_lambda.py │ ├── lambda.py │ ├── populate_queue.py │ └── table.ddl ├── stage-2 │ ├── Dockerfile │ ├── README.md │ ├── cloudformation-lci.yml │ ├── cloudformation.yml │ ├── lambda.py │ └── table.ddl └── stage-3 │ ├── README.md │ ├── airflow_dag.py │ ├── cloudformation.yml │ ├── ctas.ddl │ ├── lambda.py │ ├── policy.json │ └── table.ddl ├── cloudtrail_to_elasticsearch ├── .gitignore ├── Makefile.pip ├── Makefile.poetry ├── README.md ├── cloudformation.yml ├── cloudtrail_to_elasticsearch │ ├── __init__.py │ ├── bulk_upload.py │ ├── es_helper.py │ ├── lambda_handler.py │ ├── processor.py │ └── s3_helper.py ├── pyproject.toml ├── requirements.txt └── tests │ ├── resources │ ├── event_with_request_parameters-transformed.json │ ├── event_with_request_parameters.json │ ├── s3_test_event.json │ ├── simple_event-transformed.json │ └── simple_event.json │ ├── test_lambda_handler.py │ └── test_transforms.py ├── cloudtrail_to_kinesis ├── .gitignore ├── Makefile ├── README.md ├── cloudformation.yml └── lambda │ ├── __main__.py │ ├── file_processor.py │ ├── index.py │ └── kinesis_writer.py ├── dms_firehose ├── README.md ├── dms.yml ├── firehose.yml ├── glue_tables.yml └── postgres-dms-firehose.png ├── ecs_task_monitor ├── README.md └── cloudformation.yml ├── firehose_iceberg ├── README.md ├── firehose-iceberg.yml └── firehose-parquet.yml ├── infrastructure-tools-comparison-2 ├── CDK-2 │ ├── .gitignore │ ├── .npmignore │ ├── README.md │ ├── bin │ │ └── app.ts │ ├── cdk.json │ ├── lib │ │ ├── stack.ts │ │ └── standard_queue.ts │ ├── package.json │ └── tsconfig.json ├── CDK │ ├── .gitignore │ ├── .npmignore │ ├── README.md │ ├── bin │ │ └── app.ts │ ├── cdk.json │ ├── lib │ │ ├── stack.ts │ │ └── standard_queue.ts │ ├── package.json │ └── tsconfig.json ├── CloudFormation │ ├── README.md │ └── template.yml ├── README.md └── Terraform │ ├── .gitignore │ ├── README.md │ ├── main.tf │ └── modules │ └── create_sqs │ ├── README.md │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── infrastructure-tools-comparison ├── CDK │ ├── .gitignore │ ├── .npmignore │ ├── README.md │ ├── bin │ │ └── users_and_groups.ts │ ├── cdk.json │ ├── lib │ │ └── stack.ts │ ├── package.json │ └── tsconfig.json ├── CFNDSL │ ├── .gitignore │ ├── README.md │ └── script.rb ├── CloudFormation │ ├── README.md │ └── template.yml ├── README.md └── Terraform │ ├── .gitignore │ ├── README.md │ └── main.tf ├── lambda_build_in_docker ├── .gitignore ├── LICENSE ├── README.md ├── cloudformation.yml ├── docker │ ├── Dockerfile │ └── Makefile ├── requirements.txt └── src │ └── index.py ├── lambda_container_images ├── README.md ├── cloudformation │ ├── cloudformation-ecr.yml │ └── cloudformation-lambda.yml ├── lambda │ ├── .gitignore │ ├── Dockerfile │ ├── requirements.txt │ └── src │ │ └── handler.py └── terraform │ ├── .gitignore │ ├── ecr │ ├── main.tf │ ├── outputs.tf │ └── variables.tf │ └── lambda │ ├── main.tf │ └── variables.tf ├── lambda_four_ways ├── README.md ├── cloudformation.yml ├── go │ ├── .gitignore │ ├── Makefile │ ├── go.mod │ ├── go.sum │ ├── lambda.go │ └── types.go ├── java │ ├── .gitignore │ ├── Makefile │ ├── pom.xml │ └── src │ │ ├── assembly │ │ └── deployment.xml │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── chariotsolutions │ │ │ └── example │ │ │ └── DynamoLambda.java │ │ └── resources │ │ └── logback.xml ├── javascript │ ├── .gitignore │ ├── Makefile │ ├── lambda.mjs │ ├── package-lock.json │ └── package.json └── python │ ├── .gitignore │ ├── Makefile │ └── lambda.py ├── rds-flyway-migrations ├── README.md ├── cloudformation.yml ├── deploy.sh └── flyway │ └── migrations │ ├── V2023.02.08.001__config-db-options.sql │ ├── V2023.02.08.002__create-user-table.sql │ └── V2023.02.08.003__create-role-tables.sql ├── sandbox-policies ├── BasicUserPolicy.iam ├── PreventAdminRoleDeletion.scp ├── README.md └── SandboxGuardrail.scp ├── springboot-iam-auth ├── .gitignore ├── README.md ├── pom.xml └── src │ ├── cloudformation │ └── rds_iam.yml │ └── main │ ├── java │ └── com │ │ └── chariotsolutions │ │ └── example │ │ └── springboot │ │ ├── Application.java │ │ ├── Runner.java │ │ └── datasource │ │ └── IAMAuthDataSource.java │ └── resources │ ├── application.properties │ └── logback.xml ├── springboot-secman-auth ├── .gitignore ├── README.md ├── pom.xml └── src │ ├── cloudformation │ └── rds_secretsmanager.yml │ └── main │ ├── java │ └── com │ │ └── chariotsolutions │ │ └── example │ │ └── springboot │ │ ├── Application.java │ │ └── Runner.java │ └── resources │ ├── application.properties │ └── logback.xml ├── two_buckets_and_a_lambda ├── .gitignore ├── README.md ├── cloudformation │ ├── cloudformation.yml │ ├── deploy.sh │ └── undeploy.sh ├── docs │ ├── webapp-architecture.png │ └── webapp-ui.png ├── lambda │ ├── credentials.py │ ├── processor.py │ └── signed_url.py ├── static │ ├── index.html │ └── js │ │ ├── credentials.js │ │ └── signed-url.js └── terraform │ ├── .gitignore │ ├── apigw.tf │ ├── buckets.tf │ ├── lambdas.tf │ ├── main.tf │ ├── modules │ └── lambda │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ ├── outputs.tf │ └── variables.tf ├── untagged_ec2_cleanup ├── README.md └── cloudformation.yml ├── websocket_to_kinesis ├── .gitignore ├── Dockerfile.client ├── Dockerfile.server ├── README.md ├── client.py ├── cloudformation.yml └── server.py └── xray_glue ├── README.md ├── data ├── addToCart │ └── 2021 │ │ └── 08 │ │ └── 03 │ │ ├── 0befd9c8-64bf-4069-974c-477084abda61.json │ │ ├── 3db3213d-2820-41fa-930f-3626e2bfc5a7.json │ │ ├── c1776299-39ea-458d-82d5-f880aa1d2806.json │ │ └── f0a003b2-cecb-41ae-9c27-98efe0f6ce7c.json ├── checkoutComplete │ └── 2021 │ │ └── 08 │ │ └── 03 │ │ ├── 27b16c94-537e-471a-8b2e-db5a254f9810.json │ │ └── cdde8067-180a-43d2-b9ee-11684357b80f.json ├── checkoutStarted │ └── 2021 │ │ └── 08 │ │ └── 03 │ │ ├── 39c34051-4878-48bd-90a5-efa74e7c24d0.json │ │ └── 8f315535-813d-4f1e-aabd-115224b9eec3.json └── updateItemQuantity │ └── 2021 │ └── 08 │ └── 03 │ └── b8b8db59-f661-4326-9b90-a97e6d56aca9.json ├── jobs ├── example-1.py ├── example-2.py ├── example-3.py └── example-4.py └── scripts ├── deploy.sh ├── jobs.yml └── undeploy.sh /LICENSE: -------------------------------------------------------------------------------- 1 | MIT No-Attribution License 2 | 3 | Copyright (c) 2019-2025 Chariot Solutions 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so. 6 | 7 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This repository contains examples of AWS-related topics, typically examples from 2 | Chariot's [blog](https://chariotsolutions.com/blog/). 3 | 4 | Each sub-directory is a distinct project, with a README that describes the project 5 | in more detail: 6 | 7 | All Chariot-developed content is licensed using the [MIT No Attribution](https://github.com/aws/mit-0) 8 | license. This means that you can use the content without restriction, and do not even need to mention 9 | Chariot. We have attached the license text to files that might be copied in their entirety; you might 10 | find it useful to import those files as-is in order to establish provenance. 11 | 12 | Beware that we have imported some third-party files that are independently licensed. Those will always 13 | have their original license headers. 14 | -------------------------------------------------------------------------------- /access_cloudtrail_with_athena/README.md: -------------------------------------------------------------------------------- 1 | This directory contains example Athena table definitions and queries for CloudTrail logs. 2 | It exists to support [this blog post](https://chariotsolutions.com/blog/post/rightsizing-data-for-athena/). 3 | 4 | The table definitions are based on the definition provided [here](https://docs.aws.amazon.com/athena/latest/ug/cloudtrail-logs.html), 5 | with the addition of partitioning information. There are four definitions: 6 | 7 | * `raw_unpartitioned.ddl` 8 | 9 | The base table definition for uploaded CloudTrail log files, without any partitions. 10 | Queries against this definition will scan the entire dataset. 11 | 12 | * `raw_partitioned.ddl` 13 | 14 | The based CloudTrail table definition using partition projection for account ID, 15 | region, and date. 16 | 17 | * `consolidated.ddl` 18 | 19 | A table definition that is applied to a "consolidated" dataset, in which all of the 20 | event records are combined into files based on date. The table definition partitions 21 | by date, following the uploaded files. 22 | 23 | * `single_file.ddl` 24 | 25 | A table definition that's applied to a dataset consisting of a single file with all 26 | of the event records. There isn't any partitioning. 27 | 28 | To use these table definitions you will need to edit the S3 location to correspond to 29 | your CloudTrail data store. For the `raw_partitioned` definition you'll also have to 30 | provide a list of the account IDs used for partitioning. 31 | 32 | Each of the table definitions has a corresponding SQL query that counts the number of 33 | records by account, region, and date. These queries use partitioning columns where 34 | available, otherwise they extract the information from the parsed data. 35 | -------------------------------------------------------------------------------- /access_cloudtrail_with_athena/consolidated.ddl: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE `cloudtrail_consolidated_by_date` ( 2 | `eventversion` string COMMENT 'from deserializer', 3 | `useridentity` struct,sessionissuer:struct>> COMMENT 'from deserializer', 4 | `eventtime` string COMMENT 'from deserializer', 5 | `eventsource` string COMMENT 'from deserializer', 6 | `eventname` string COMMENT 'from deserializer', 7 | `awsregion` string COMMENT 'from deserializer', 8 | `sourceipaddress` string COMMENT 'from deserializer', 9 | `useragent` string COMMENT 'from deserializer', 10 | `errorcode` string COMMENT 'from deserializer', 11 | `errormessage` string COMMENT 'from deserializer', 12 | `requestparameters` string COMMENT 'from deserializer', 13 | `responseelements` string COMMENT 'from deserializer', 14 | `additionaleventdata` string COMMENT 'from deserializer', 15 | `requestid` string COMMENT 'from deserializer', 16 | `eventid` string COMMENT 'from deserializer', 17 | `resources` array> COMMENT 'from deserializer', 18 | `eventtype` string COMMENT 'from deserializer', 19 | `apiversion` string COMMENT 'from deserializer', 20 | `readonly` string COMMENT 'from deserializer', 21 | `recipientaccountid` string COMMENT 'from deserializer', 22 | `serviceeventdetails` string COMMENT 'from deserializer', 23 | `sharedeventid` string COMMENT 'from deserializer', 24 | `vpcendpointid` string COMMENT 'from deserializer') 25 | PARTITIONED BY ( 26 | `ingest_date` string) 27 | ROW FORMAT SERDE 28 | 'org.apache.hive.hcatalog.data.JsonSerDe' 29 | STORED AS INPUTFORMAT 30 | 'org.apache.hadoop.mapred.TextInputFormat' 31 | OUTPUTFORMAT 32 | 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' 33 | LOCATION 34 | 's3://BUCKET/PREFIX/' 35 | TBLPROPERTIES ( 36 | 'projection.enabled'='true', 37 | 'projection.ingest_date.format'='yyyy/MM/dd', 38 | 'projection.ingest_date.range'='2020/01/01,NOW', 39 | 'projection.ingest_date.type'='date', 40 | 'storage.location.template'='s3://BUCKET/PREFIX/${ingest_date}', 41 | ) 42 | -------------------------------------------------------------------------------- /access_cloudtrail_with_athena/consolidated.sql: -------------------------------------------------------------------------------- 1 | select count(*) 2 | from cloudtrail_consolidated_by_date 3 | where useridentity.accountid = '123456789012' 4 | and ingest_date between '2020/10/01' and '2020/10/31' 5 | and awsregion = 'us-east-1' 6 | -------------------------------------------------------------------------------- /access_cloudtrail_with_athena/raw_partitioned.ddl: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE `cloudtrail_raw_partitioned` ( 2 | `eventversion` string COMMENT 'from deserializer', 3 | `useridentity` struct,sessionissuer:struct>> COMMENT 'from deserializer', 4 | `eventtime` string COMMENT 'from deserializer', 5 | `eventsource` string COMMENT 'from deserializer', 6 | `eventname` string COMMENT 'from deserializer', 7 | `awsregion` string COMMENT 'from deserializer', 8 | `sourceipaddress` string COMMENT 'from deserializer', 9 | `useragent` string COMMENT 'from deserializer', 10 | `errorcode` string COMMENT 'from deserializer', 11 | `errormessage` string COMMENT 'from deserializer', 12 | `requestparameters` string COMMENT 'from deserializer', 13 | `responseelements` string COMMENT 'from deserializer', 14 | `additionaleventdata` string COMMENT 'from deserializer', 15 | `requestid` string COMMENT 'from deserializer', 16 | `eventid` string COMMENT 'from deserializer', 17 | `resources` array> COMMENT 'from deserializer', 18 | `eventtype` string COMMENT 'from deserializer', 19 | `apiversion` string COMMENT 'from deserializer', 20 | `readonly` string COMMENT 'from deserializer', 21 | `recipientaccountid` string COMMENT 'from deserializer', 22 | `serviceeventdetails` string COMMENT 'from deserializer', 23 | `sharedeventid` string COMMENT 'from deserializer', 24 | `vpcendpointid` string COMMENT 'from deserializer') 25 | PARTITIONED BY ( 26 | `account_id` string, 27 | `region` string, 28 | `ingest_date` string) 29 | ROW FORMAT SERDE 30 | 'com.amazon.emr.hive.serde.CloudTrailSerde' 31 | STORED AS INPUTFORMAT 32 | 'com.amazon.emr.cloudtrail.CloudTrailInputFormat' 33 | OUTPUTFORMAT 34 | 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' 35 | LOCATION 36 | 's3://BUCKET/PREFIX/' 37 | TBLPROPERTIES ( 38 | 'projection.account_id.type'='enum', 39 | 'projection.account_id.values'='123456789012,...', 40 | 'projection.enabled'='true', 41 | 'projection.ingest_date.format'='yyyy/MM/dd', 42 | 'projection.ingest_date.range'='2020/01/01,NOW', 43 | 'projection.ingest_date.type'='date', 44 | 'projection.region.type'='enum', 45 | 'projection.region.values'='ap-northeast-1,ap-northeast-2,ap-northeast-3,ap-south-1,ap-southeast-1,ap-southeast-2,ca-central-1,eu-central-1,eu-north-1,eu-west-1,eu-west-2,eu-west-3,sa-east-1,us-east-1,us-east-2,us-west-1,us-west-2', 46 | 'storage.location.template'='s3://BUCKET/PREFIX/${account_id}/CloudTrail/${region}/${ingest_date}', 47 | ) 48 | -------------------------------------------------------------------------------- /access_cloudtrail_with_athena/raw_partitioned.sql: -------------------------------------------------------------------------------- 1 | select count(*) 2 | from cloudtrail_raw_partitioned 3 | where account_id = '123456789012' 4 | and ingest_date between '2020/10/01' and '2020/10/31' 5 | and region = 'us-east-1' 6 | -------------------------------------------------------------------------------- /access_cloudtrail_with_athena/raw_unpartitioned.ddl: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE `cloudtrail_raw_unpartitioned`( 2 | `eventversion` string COMMENT 'from deserializer', 3 | `useridentity` struct,sessionissuer:struct>> COMMENT 'from deserializer', 4 | `eventtime` string COMMENT 'from deserializer', 5 | `eventsource` string COMMENT 'from deserializer', 6 | `eventname` string COMMENT 'from deserializer', 7 | `awsregion` string COMMENT 'from deserializer', 8 | `sourceipaddress` string COMMENT 'from deserializer', 9 | `useragent` string COMMENT 'from deserializer', 10 | `errorcode` string COMMENT 'from deserializer', 11 | `errormessage` string COMMENT 'from deserializer', 12 | `requestparameters` string COMMENT 'from deserializer', 13 | `responseelements` string COMMENT 'from deserializer', 14 | `additionaleventdata` string COMMENT 'from deserializer', 15 | `requestid` string COMMENT 'from deserializer', 16 | `eventid` string COMMENT 'from deserializer', 17 | `resources` array> COMMENT 'from deserializer', 18 | `eventtype` string COMMENT 'from deserializer', 19 | `apiversion` string COMMENT 'from deserializer', 20 | `readonly` string COMMENT 'from deserializer', 21 | `recipientaccountid` string COMMENT 'from deserializer', 22 | `serviceeventdetails` string COMMENT 'from deserializer', 23 | `sharedeventid` string COMMENT 'from deserializer', 24 | `vpcendpointid` string COMMENT 'from deserializer') 25 | ROW FORMAT SERDE 26 | 'com.amazon.emr.hive.serde.CloudTrailSerde' 27 | STORED AS INPUTFORMAT 28 | 'com.amazon.emr.cloudtrail.CloudTrailInputFormat' 29 | OUTPUTFORMAT 30 | 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' 31 | LOCATION 32 | 's3://BUCKET/PREFIX/' 33 | 34 | -------------------------------------------------------------------------------- /access_cloudtrail_with_athena/raw_unpartitioned.sql: -------------------------------------------------------------------------------- 1 | select count(*) 2 | from cloudtrail_raw_unpartitioned 3 | where useridentity.accountid = '123456789012' 4 | and substr(eventtime, 1, 7) = '2020-10' 5 | and awsregion = 'us-east-1' 6 | -------------------------------------------------------------------------------- /access_cloudtrail_with_athena/single_file.ddl: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE `cloudtrail_single_file`( 2 | `eventversion` string COMMENT 'from deserializer', 3 | `useridentity` struct,sessionissuer:struct>> COMMENT 'from deserializer', 4 | `eventtime` string COMMENT 'from deserializer', 5 | `eventsource` string COMMENT 'from deserializer', 6 | `eventname` string COMMENT 'from deserializer', 7 | `awsregion` string COMMENT 'from deserializer', 8 | `sourceipaddress` string COMMENT 'from deserializer', 9 | `useragent` string COMMENT 'from deserializer', 10 | `errorcode` string COMMENT 'from deserializer', 11 | `errormessage` string COMMENT 'from deserializer', 12 | `requestparameters` string COMMENT 'from deserializer', 13 | `responseelements` string COMMENT 'from deserializer', 14 | `additionaleventdata` string COMMENT 'from deserializer', 15 | `requestid` string COMMENT 'from deserializer', 16 | `eventid` string COMMENT 'from deserializer', 17 | `resources` array> COMMENT 'from deserializer', 18 | `eventtype` string COMMENT 'from deserializer', 19 | `apiversion` string COMMENT 'from deserializer', 20 | `readonly` string COMMENT 'from deserializer', 21 | `recipientaccountid` string COMMENT 'from deserializer', 22 | `serviceeventdetails` string COMMENT 'from deserializer', 23 | `sharedeventid` string COMMENT 'from deserializer', 24 | `vpcendpointid` string COMMENT 'from deserializer') 25 | ROW FORMAT SERDE 26 | 'org.apache.hive.hcatalog.data.JsonSerDe' 27 | STORED AS INPUTFORMAT 28 | 'org.apache.hadoop.mapred.TextInputFormat' 29 | OUTPUTFORMAT 30 | 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' 31 | LOCATION 32 | 's3://BUCKET/PREFIX/' 33 | -------------------------------------------------------------------------------- /access_cloudtrail_with_athena/single_file.sql: -------------------------------------------------------------------------------- 1 | select count(*) 2 | from cloudtrail_single_file 3 | where useridentity.accountid = '123456789012' 4 | and substr(eventtime, 1, 7) = '2020-10' 5 | and awsregion = 'us-east-1' 6 | -------------------------------------------------------------------------------- /athena_performance_comparison/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | 3 | bin/ 4 | include/ 5 | lib/ 6 | lib64 7 | pyvenv.cfg 8 | share/ 9 | -------------------------------------------------------------------------------- /aurora_data_api/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | -------------------------------------------------------------------------------- /aurora_data_api/README.md: -------------------------------------------------------------------------------- 1 | # Aurora Serverless Data API Example 2 | 3 | This is an implementation of the Lambda described [in this blog post](https://chariotsolutions.com/blog/post/serverless-databasing-with-the-aurora-data-api/). 4 | 5 | It uses the Data API to update a user's email preferences in response to an 6 | "unsubscribe" notification from an email provider (using the message format of 7 | the [Twilio SendGrid](https://docs.sendgrid.com/for-developers/tracking-events/event#events) 8 | webhook). 9 | 10 | 11 | ## Deploying 12 | 13 | Deploying this Lambda is a multi-step process: 14 | 15 | 1. Create a CloudFormation stack from the template `cloudformation.yml`. This 16 | stack includes an HTTP Gateway and Aurora Serverless database cluster, in 17 | addition to a "dummy" Lambda. 18 | 19 | The stack has two parameters: 20 | 21 | * `VpcId`: VPC where the database will be deployed. 22 | * `SubnetIds`: subnets where the database will be deployed. Provide at least two. 23 | 24 | And it provides the following outputs: 25 | 26 | * `APIGatewayUrl`: the endpoint called by the webhook. 27 | * `DatabaseSecretArn`: the ARN of the secret holding the database connection 28 | information. This is needed to run the `init_database.py` script. 29 | * `DatabaseInstanceArn`: the ARN of the Aurora Serverless cluster. This is 30 | also needed to run the `init_database.py` script. 31 | 32 | *Beware:* you will incur AWS usage charges for this example. They should be 33 | minimal (on the order of a few cents), but don't forget to delete the stack 34 | when you're done. 35 | 36 | 2. Run the script `init_database.py` to create the database table and add 37 | sample users. You must provide the two ARNs from the stack outputs. The 38 | script creates three default users, corresponding to the records in the 39 | sample event. If you want to create more users (for example, to try 40 | your own events), pass their email addresses as command-line arguments: 41 | 42 | ``` 43 | ./init_database.py SECRET_ARN DATABASE_ARN me@example.com you@example.com ... 44 | ``` 45 | 46 | You will need to have Python 3 installed to run this script, along with the 47 | `boto3` library and its dependencies. If you've already installed the AWS 48 | command-line program (`awscli`), these will be present. 49 | 50 | Note that this script also uses the Data API to perform its actions. 51 | 52 | 3. Use the Console to replace the source code of the Lambda with the contents 53 | of the file `lambda.py`. 54 | 55 | 56 | ## Running 57 | 58 | To see the current email status of your users, run the following query in the 59 | [Console Query Editor](https://console.aws.amazon.com/rds/home#query-editor:): 60 | 61 | ``` 62 | select * from EMAIL_PREFERENCES 63 | ``` 64 | 65 | After running `init_database.py`, you should see some number of users, all of 66 | which have a `MARKETING_OPT_IN` value of `true`. 67 | 68 | Now, use `curl` to upload the sample event: 69 | 70 | ``` 71 | curl -X POST -d @example_notifications.json API_GATEWAY_ENDPOINT 72 | ``` 73 | 74 | After this, you can re-run the query, and you should see that `user2@example.com` 75 | and `user3@example.com` have a `MARKETING_OPT_IN` value of `false`. 76 | -------------------------------------------------------------------------------- /aurora_data_api/example_notifications.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "email": "user1@example.com", 4 | "timestamp": 1513299569, 5 | "smtp-id": "<14c5d75ce93.dfd.64b469@ismtpd-555>", 6 | "event": "dropped", 7 | "category": "cat marketing", 8 | "sg_event_id": "sg_event_id", 9 | "sg_message_id": "sg_message_id", 10 | "reason": "Bounced Address", 11 | "status": "5.0.0" 12 | }, 13 | { 14 | "email": "user2@example.com", 15 | "timestamp": 1513299569, 16 | "smtp-id": "<14c5d75ce93.dfd.64b469@ismtpd-555>", 17 | "event": "spamreport", 18 | "category": "cat marketing", 19 | "sg_event_id": "sg_event_id", 20 | "sg_message_id": "sg_message_id" 21 | }, 22 | { 23 | "email": "user3@example.com", 24 | "timestamp": 1513299569, 25 | "smtp-id": "<14c5d75ce93.dfd.64b469@ismtpd-555>", 26 | "event": "unsubscribe", 27 | "category": "cat marketing", 28 | "sg_event_id": "sg_event_id", 29 | "sg_message_id": "sg_message_id" 30 | } 31 | ] 32 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/1-broken/.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | !jest.config.js 3 | *.d.ts 4 | node_modules 5 | 6 | # CDK asset staging directory 7 | .cdk.staging 8 | cdk.out 9 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/1-broken/.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/1-broken/bin/app.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import 'source-map-support/register'; 3 | import * as cdk from 'aws-cdk-lib'; 4 | 5 | import { Stack_1, Stack_1_Props } from '../lib/stack-1'; 6 | import { Stack_2, Stack_2_Props } from '../lib/stack-2'; 7 | 8 | 9 | const env = { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION } 10 | 11 | const app = new cdk.App(); 12 | 13 | const stack_1 = new Stack_1(app, "Stack1", { 14 | env: env, 15 | taskDefinitionName: "example-1-broken", 16 | }); 17 | 18 | const stack_2 = new Stack_2(app, "Stack2", { 19 | env: env, 20 | taskDefinition: stack_1.taskDefinition 21 | }); 22 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/1-broken/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node --prefer-ts-exts bin/app.ts", 3 | "watch": { 4 | "include": [ 5 | "**" 6 | ], 7 | "exclude": [ 8 | "README.md", 9 | "cdk*.json", 10 | "**/*.d.ts", 11 | "**/*.js", 12 | "tsconfig.json", 13 | "package*.json", 14 | "yarn.lock", 15 | "node_modules", 16 | "test" 17 | ] 18 | }, 19 | "context": { 20 | "@aws-cdk/aws-iam:oidcRejectUnauthorizedConnections": true, 21 | "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, 22 | "@aws-cdk/core:stackRelativeExports": true, 23 | "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, 24 | "@aws-cdk/aws-lambda:recognizeVersionProps": true, 25 | "@aws-cdk/aws-lambda:recognizeLayerVersion": true, 26 | "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, 27 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 28 | "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, 29 | "@aws-cdk/core:checkSecretUsage": true, 30 | "@aws-cdk/aws-iam:minimizePolicies": true, 31 | "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, 32 | "@aws-cdk/core:validateSnapshotRemovalPolicy": true, 33 | "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, 34 | "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, 35 | "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, 36 | "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, 37 | "@aws-cdk/core:enablePartitionLiterals": true, 38 | "@aws-cdk/core:target-partitions": [ 39 | "aws", 40 | "aws-cn" 41 | ] 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/1-broken/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | testEnvironment: 'node', 3 | roots: ['/test'], 4 | testMatch: ['**/*.test.ts'], 5 | transform: { 6 | '^.+\\.tsx?$': 'ts-jest' 7 | } 8 | }; 9 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/1-broken/lib/stack-1.ts: -------------------------------------------------------------------------------- 1 | import * as cdk from 'aws-cdk-lib'; 2 | import * as ecs from "aws-cdk-lib/aws-ecs"; 3 | import * as logs from "aws-cdk-lib/aws-logs"; 4 | import { Construct } from 'constructs'; 5 | 6 | 7 | export interface Stack_1_Props extends cdk.StackProps { 8 | 9 | /** The name of the task definition. */ 10 | readonly taskDefinitionName: string; 11 | } 12 | 13 | 14 | export class Stack_1 extends cdk.Stack { 15 | 16 | public readonly taskDefinition: ecs.TaskDefinition; 17 | 18 | 19 | constructor(scope: Construct, id: string, props: Stack_1_Props) { 20 | super(scope, id, props); 21 | 22 | const logGroup = new logs.LogGroup(this, "Logs", { 23 | logGroupName: "/ecs/" + props.taskDefinitionName, 24 | removalPolicy: cdk.RemovalPolicy.DESTROY, 25 | }); 26 | 27 | this.taskDefinition = new ecs.FargateTaskDefinition(this, 'TaskDefinition', { 28 | family: props.taskDefinitionName, 29 | cpu: 256, 30 | memoryLimitMiB: 512, 31 | }); 32 | 33 | const mainContainer = this.taskDefinition.addContainer("MainContainer", { 34 | image: ecs.ContainerImage.fromRegistry("hello-world:latest"), 35 | logging: ecs.LogDriver.awsLogs({ 36 | logGroup: logGroup, 37 | streamPrefix: "main-", 38 | }), 39 | }); 40 | } 41 | } 42 | 43 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/1-broken/lib/stack-2.ts: -------------------------------------------------------------------------------- 1 | import * as cdk from 'aws-cdk-lib'; 2 | import * as ecs from "aws-cdk-lib/aws-ecs"; 3 | import * as events from "aws-cdk-lib/aws-events"; 4 | import * as lambda from "aws-cdk-lib/aws-lambda"; 5 | import * as targets from "aws-cdk-lib/aws-events-targets"; 6 | import { Construct } from 'constructs'; 7 | 8 | 9 | export interface Stack_2_Props extends cdk.StackProps { 10 | 11 | /** The task definition that triggers the Lambda. */ 12 | readonly taskDefinition: ecs.TaskDefinition; 13 | } 14 | 15 | 16 | export class Stack_2 extends cdk.Stack { 17 | constructor(scope: Construct, id: string, props: Stack_2_Props) { 18 | super(scope, id, props); 19 | 20 | const handler = new lambda.Function(this, 'TaskListener', { 21 | runtime: lambda.Runtime.PYTHON_3_9, 22 | handler: 'index.handler', 23 | code: lambda.Code.fromInline( 24 | "import json\n" + 25 | "\n" + 26 | "def handler(event, context):\n" + 27 | " print(json.dumps(event))\n" 28 | ), 29 | }); 30 | 31 | 32 | const rule = new events.Rule(this, "TaskListenerTrigger", { 33 | description: 34 | "Triggers the " + handler.functionName + " on ECS task completion", 35 | eventPattern: { 36 | source: ["aws.ecs"], 37 | detailType: ["ECS Task State Change"], 38 | detail: { 39 | lastStatus: ["STOPPED"], 40 | taskDefinitionArn: [props.taskDefinition.taskDefinitionArn], 41 | }, 42 | }, 43 | }); 44 | 45 | rule.addTarget(new targets.LambdaFunction(handler, {})); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/1-broken/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "1-broken", 3 | "version": "0.0.0", 4 | "bin": { 5 | "1-broken": "bin/app.ts" 6 | }, 7 | "scripts": { 8 | "build": "tsc", 9 | "watch": "tsc -w", 10 | "test": "jest", 11 | "cdk": "cdk" 12 | }, 13 | "devDependencies": { 14 | "@types/jest": "^29.5.14", 15 | "@types/node": "22.7.9", 16 | "jest": "^29.7.0", 17 | "ts-jest": "^29.2.5", 18 | "aws-cdk": "2.177.0", 19 | "ts-node": "^10.9.2", 20 | "typescript": "~5.6.3" 21 | }, 22 | "dependencies": { 23 | "aws-cdk-lib": "2.177.0", 24 | "constructs": "^10.0.0" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/1-broken/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2018", 4 | "module": "commonjs", 5 | "lib": [ 6 | "es2018" 7 | ], 8 | "declaration": true, 9 | "strict": true, 10 | "noImplicitAny": true, 11 | "strictNullChecks": true, 12 | "noImplicitThis": true, 13 | "alwaysStrict": true, 14 | "noUnusedLocals": false, 15 | "noUnusedParameters": false, 16 | "noImplicitReturns": true, 17 | "noFallthroughCasesInSwitch": false, 18 | "inlineSourceMap": true, 19 | "inlineSources": true, 20 | "experimentalDecorators": true, 21 | "strictPropertyInitialization": false, 22 | "typeRoots": [ 23 | "./node_modules/@types" 24 | ] 25 | }, 26 | "exclude": [ 27 | "node_modules", 28 | "cdk.out" 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/2-better/.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | !jest.config.js 3 | *.d.ts 4 | node_modules 5 | 6 | # CDK asset staging directory 7 | .cdk.staging 8 | cdk.out 9 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/2-better/.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/2-better/bin/app.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import 'source-map-support/register'; 3 | import * as cdk from 'aws-cdk-lib'; 4 | 5 | import { Stack_1, Stack_1_Props } from '../lib/stack-1'; 6 | import { Stack_2, Stack_2_Props } from '../lib/stack-2'; 7 | 8 | 9 | const env = { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION } 10 | 11 | const taskDefinitionArnExportPath = "/cdk/parameters/2-better/taskDefinitionArn"; 12 | 13 | const app = new cdk.App(); 14 | 15 | const stack_1 = new Stack_1(app, "Stack1", { 16 | env: env, 17 | taskDefinitionName: "example-2-better", 18 | taskDefinitionArnExportPath: taskDefinitionArnExportPath, 19 | }); 20 | 21 | const stack_2 = new Stack_2(app, "Stack2", { 22 | env: env, 23 | taskDefinitionArnExportPath: taskDefinitionArnExportPath, 24 | }); 25 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/2-better/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node --prefer-ts-exts bin/app.ts", 3 | "watch": { 4 | "include": [ 5 | "**" 6 | ], 7 | "exclude": [ 8 | "README.md", 9 | "cdk*.json", 10 | "**/*.d.ts", 11 | "**/*.js", 12 | "tsconfig.json", 13 | "package*.json", 14 | "yarn.lock", 15 | "node_modules", 16 | "test" 17 | ] 18 | }, 19 | "context": { 20 | "@aws-cdk/aws-iam:oidcRejectUnauthorizedConnections": true, 21 | "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, 22 | "@aws-cdk/core:stackRelativeExports": true, 23 | "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, 24 | "@aws-cdk/aws-lambda:recognizeVersionProps": true, 25 | "@aws-cdk/aws-lambda:recognizeLayerVersion": true, 26 | "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, 27 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 28 | "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, 29 | "@aws-cdk/core:checkSecretUsage": true, 30 | "@aws-cdk/aws-iam:minimizePolicies": true, 31 | "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, 32 | "@aws-cdk/core:validateSnapshotRemovalPolicy": true, 33 | "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, 34 | "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, 35 | "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, 36 | "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, 37 | "@aws-cdk/core:enablePartitionLiterals": true, 38 | "@aws-cdk/core:target-partitions": [ 39 | "aws", 40 | "aws-cn" 41 | ] 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/2-better/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | testEnvironment: 'node', 3 | roots: ['/test'], 4 | testMatch: ['**/*.test.ts'], 5 | transform: { 6 | '^.+\\.tsx?$': 'ts-jest' 7 | } 8 | }; 9 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/2-better/lib/stack-1.ts: -------------------------------------------------------------------------------- 1 | import * as cdk from 'aws-cdk-lib'; 2 | import * as ecs from "aws-cdk-lib/aws-ecs"; 3 | import * as logs from "aws-cdk-lib/aws-logs"; 4 | import * as ssm from "aws-cdk-lib/aws-ssm"; 5 | import { Construct } from 'constructs'; 6 | 7 | 8 | export interface Stack_1_Props extends cdk.StackProps { 9 | 10 | /** The name of the task definition. */ 11 | readonly taskDefinitionName: string; 12 | 13 | /** Where we save the ARN for the task definition, so that it can be consumed later */ 14 | readonly taskDefinitionArnExportPath: string; 15 | } 16 | 17 | 18 | export class Stack_1 extends cdk.Stack { 19 | 20 | constructor(scope: Construct, id: string, props: Stack_1_Props) { 21 | super(scope, id, props); 22 | 23 | const logGroup = new logs.LogGroup(this, "Logs", { 24 | logGroupName: "/ecs/" + props.taskDefinitionName, 25 | removalPolicy: cdk.RemovalPolicy.DESTROY, 26 | }); 27 | 28 | const taskDefinition = new ecs.FargateTaskDefinition(this, 'TaskDefinition', { 29 | family: props.taskDefinitionName, 30 | cpu: 256, 31 | memoryLimitMiB: 512, 32 | }); 33 | 34 | const mainContainer = taskDefinition.addContainer("MainContainer", { 35 | image: ecs.ContainerImage.fromRegistry("hello-world:latest"), 36 | logging: ecs.LogDriver.awsLogs({ 37 | logGroup: logGroup, 38 | streamPrefix: "main-", 39 | }), 40 | }); 41 | 42 | const ssmParameter = new ssm.StringParameter(this, 'TaskDefinitionArnParameter', { 43 | parameterName: props.taskDefinitionArnExportPath, 44 | stringValue: taskDefinition.taskDefinitionArn, 45 | }); 46 | } 47 | } 48 | 49 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/2-better/lib/stack-2.ts: -------------------------------------------------------------------------------- 1 | import * as cdk from 'aws-cdk-lib'; 2 | import * as ecs from "aws-cdk-lib/aws-ecs"; 3 | import * as events from "aws-cdk-lib/aws-events"; 4 | import * as lambda from "aws-cdk-lib/aws-lambda"; 5 | import * as ssm from "aws-cdk-lib/aws-ssm"; 6 | import * as targets from "aws-cdk-lib/aws-events-targets"; 7 | import { Construct } from 'constructs'; 8 | 9 | 10 | export interface Stack_2_Props extends cdk.StackProps { 11 | 12 | /** Where we retrieve the ARN for the task definition. */ 13 | readonly taskDefinitionArnExportPath: string; 14 | } 15 | 16 | 17 | export class Stack_2 extends cdk.Stack { 18 | 19 | constructor(scope: Construct, id: string, props: Stack_2_Props) { 20 | super(scope, id, props); 21 | 22 | const handler = new lambda.Function(this, 'TaskListener', { 23 | runtime: lambda.Runtime.PYTHON_3_9, 24 | handler: 'index.handler', 25 | code: lambda.Code.fromInline( 26 | "import json\n" + 27 | "\n" + 28 | "def handler(event, context):\n" + 29 | " print(json.dumps(event))\n" 30 | ), 31 | }); 32 | 33 | const taskDefinitionArn = ssm.StringParameter.valueForStringParameter(this, props.taskDefinitionArnExportPath); 34 | 35 | const rule = new events.Rule(this, "TaskListenerTrigger", { 36 | description: 37 | "Triggers the " + handler.functionName + " on ECS task completion", 38 | eventPattern: { 39 | source: ["aws.ecs"], 40 | detailType: ["ECS Task State Change"], 41 | detail: { 42 | lastStatus: ["STOPPED"], 43 | taskDefinitionArn: [taskDefinitionArn], 44 | }, 45 | }, 46 | }); 47 | 48 | rule.addTarget(new targets.LambdaFunction(handler, {})); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/2-better/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "2-better", 3 | "version": "0.0.0", 4 | "bin": { 5 | "2-better": "bin/app.ts" 6 | }, 7 | "scripts": { 8 | "build": "tsc", 9 | "watch": "tsc -w", 10 | "test": "jest", 11 | "cdk": "cdk" 12 | }, 13 | "devDependencies": { 14 | "@types/jest": "^29.5.14", 15 | "@types/node": "22.7.9", 16 | "jest": "^29.7.0", 17 | "ts-jest": "^29.2.5", 18 | "aws-cdk": "2.177.0", 19 | "ts-node": "^10.9.2", 20 | "typescript": "~5.6.3" 21 | }, 22 | "dependencies": { 23 | "aws-cdk-lib": "2.177.0", 24 | "constructs": "^10.0.0" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/2-better/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2018", 4 | "module": "commonjs", 5 | "lib": [ 6 | "es2018" 7 | ], 8 | "declaration": true, 9 | "strict": true, 10 | "noImplicitAny": true, 11 | "strictNullChecks": true, 12 | "noImplicitThis": true, 13 | "alwaysStrict": true, 14 | "noUnusedLocals": false, 15 | "noUnusedParameters": false, 16 | "noImplicitReturns": true, 17 | "noFallthroughCasesInSwitch": false, 18 | "inlineSourceMap": true, 19 | "inlineSources": true, 20 | "experimentalDecorators": true, 21 | "strictPropertyInitialization": false, 22 | "typeRoots": [ 23 | "./node_modules/@types" 24 | ] 25 | }, 26 | "exclude": [ 27 | "node_modules", 28 | "cdk.out" 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/3-best/.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | !jest.config.js 3 | *.d.ts 4 | node_modules 5 | 6 | # CDK asset staging directory 7 | .cdk.staging 8 | cdk.out 9 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/3-best/.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/3-best/bin/app.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import 'source-map-support/register'; 3 | import * as cdk from 'aws-cdk-lib'; 4 | 5 | import { Stack_1, Stack_1_Props } from '../lib/stack-1'; 6 | import { Stack_2, Stack_2_Props } from '../lib/stack-2'; 7 | 8 | 9 | const env = { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION } 10 | 11 | const app = new cdk.App(); 12 | 13 | const stack_1 = new Stack_1(app, "Stack1", { 14 | env: env, 15 | taskDefinitionName: "example-3-best", 16 | }); 17 | 18 | const stack_2 = new Stack_2(app, "Stack2", { 19 | env: env, 20 | taskDefinitionFamilyName: stack_1.taskDefinitionFamilyName, 21 | }); 22 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/3-best/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node --prefer-ts-exts bin/app.ts", 3 | "watch": { 4 | "include": [ 5 | "**" 6 | ], 7 | "exclude": [ 8 | "README.md", 9 | "cdk*.json", 10 | "**/*.d.ts", 11 | "**/*.js", 12 | "tsconfig.json", 13 | "package*.json", 14 | "yarn.lock", 15 | "node_modules", 16 | "test" 17 | ] 18 | }, 19 | "context": { 20 | "@aws-cdk/aws-iam:oidcRejectUnauthorizedConnections": true, 21 | "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, 22 | "@aws-cdk/core:stackRelativeExports": true, 23 | "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, 24 | "@aws-cdk/aws-lambda:recognizeVersionProps": true, 25 | "@aws-cdk/aws-lambda:recognizeLayerVersion": true, 26 | "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, 27 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 28 | "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, 29 | "@aws-cdk/core:checkSecretUsage": true, 30 | "@aws-cdk/aws-iam:minimizePolicies": true, 31 | "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, 32 | "@aws-cdk/core:validateSnapshotRemovalPolicy": true, 33 | "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, 34 | "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, 35 | "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, 36 | "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, 37 | "@aws-cdk/core:enablePartitionLiterals": true, 38 | "@aws-cdk/core:target-partitions": [ 39 | "aws", 40 | "aws-cn" 41 | ] 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/3-best/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | testEnvironment: 'node', 3 | roots: ['/test'], 4 | testMatch: ['**/*.test.ts'], 5 | transform: { 6 | '^.+\\.tsx?$': 'ts-jest' 7 | } 8 | }; 9 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/3-best/lib/stack-1.ts: -------------------------------------------------------------------------------- 1 | import * as cdk from 'aws-cdk-lib'; 2 | import * as ecs from "aws-cdk-lib/aws-ecs"; 3 | import * as logs from "aws-cdk-lib/aws-logs"; 4 | import { Construct } from 'constructs'; 5 | 6 | 7 | export interface Stack_1_Props extends cdk.StackProps { 8 | 9 | /** The name of the task definition. */ 10 | readonly taskDefinitionName: string; 11 | } 12 | 13 | 14 | export class Stack_1 extends cdk.Stack { 15 | 16 | public readonly taskDefinitionFamilyName: string; 17 | 18 | constructor(scope: Construct, id: string, props: Stack_1_Props) { 19 | super(scope, id, props); 20 | 21 | // we're just using the provided name here, but it's good practice to expose 22 | // the variable separately so that we have the ability to change it 23 | this.taskDefinitionFamilyName = props.taskDefinitionName; 24 | 25 | // note: this is an example stack so we don't want to preserve the log group 26 | // when the stack is deleted 27 | const logGroup = new logs.LogGroup(this, "Logs", { 28 | logGroupName: "/ecs/" + props.taskDefinitionName, 29 | removalPolicy: cdk.RemovalPolicy.DESTROY, 30 | }); 31 | 32 | const taskDefinition = new ecs.FargateTaskDefinition(this, 'TaskDefinition', { 33 | family: this.taskDefinitionFamilyName, 34 | cpu: 256, 35 | memoryLimitMiB: 1024, 36 | }); 37 | 38 | const mainContainer = taskDefinition.addContainer("MainContainer", { 39 | image: ecs.ContainerImage.fromRegistry("hello-world:latest"), 40 | logging: ecs.LogDriver.awsLogs({ 41 | logGroup: logGroup, 42 | streamPrefix: "main-", 43 | }), 44 | }); 45 | } 46 | } 47 | 48 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/3-best/lib/stack-2.ts: -------------------------------------------------------------------------------- 1 | import * as cdk from 'aws-cdk-lib'; 2 | import * as ecs from "aws-cdk-lib/aws-ecs"; 3 | import * as events from "aws-cdk-lib/aws-events"; 4 | import * as lambda from "aws-cdk-lib/aws-lambda"; 5 | import * as targets from "aws-cdk-lib/aws-events-targets"; 6 | import { Construct } from 'constructs'; 7 | 8 | 9 | export interface Stack_2_Props extends cdk.StackProps { 10 | 11 | /** The family name of the task definition that triggers the Lambda. */ 12 | readonly taskDefinitionFamilyName: string; 13 | } 14 | 15 | 16 | export class Stack_2 extends cdk.Stack { 17 | 18 | constructor(scope: Construct, id: string, props: Stack_2_Props) { 19 | super(scope, id, props); 20 | 21 | const handler = new lambda.Function(this, 'TaskListener', { 22 | runtime: lambda.Runtime.PYTHON_3_9, 23 | handler: 'index.handler', 24 | code: lambda.Code.fromInline( 25 | "import json\n" + 26 | "\n" + 27 | "def handler(event, context):\n" + 28 | " print(json.dumps(event))\n" 29 | ), 30 | }); 31 | 32 | 33 | const taskDefinitionPrefix = "arn:aws:ecs:" + 34 | props.env!.region! + ":" + props.env!.account! + 35 | ":task-definition/" + props.taskDefinitionFamilyName; 36 | 37 | const rule = new events.Rule(this, "TaskListenerTrigger", { 38 | description: 39 | "Triggers the " + handler.functionName + " on ECS task completion", 40 | eventPattern: { 41 | source: ["aws.ecs"], 42 | detailType: ["ECS Task State Change"], 43 | detail: { 44 | lastStatus: ["STOPPED"], 45 | taskDefinitionArn: [{ 46 | prefix: taskDefinitionPrefix, 47 | }], 48 | }, 49 | }, 50 | }); 51 | 52 | rule.addTarget(new targets.LambdaFunction(handler, {})); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/3-best/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "3-best", 3 | "version": "0.0.0", 4 | "bin": { 5 | "3-best": "bin/app.ts" 6 | }, 7 | "scripts": { 8 | "build": "tsc", 9 | "watch": "tsc -w", 10 | "test": "jest", 11 | "cdk": "cdk" 12 | }, 13 | "devDependencies": { 14 | "@types/jest": "^29.5.14", 15 | "@types/node": "22.7.9", 16 | "jest": "^29.7.0", 17 | "ts-jest": "^29.2.5", 18 | "aws-cdk": "2.177.0", 19 | "ts-node": "^10.9.2", 20 | "typescript": "~5.6.3" 21 | }, 22 | "dependencies": { 23 | "aws-cdk-lib": "2.177.0", 24 | "constructs": "^10.0.0" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /cdk_cross_stack_references/3-best/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2018", 4 | "module": "commonjs", 5 | "lib": [ 6 | "es2018" 7 | ], 8 | "declaration": true, 9 | "strict": true, 10 | "noImplicitAny": true, 11 | "strictNullChecks": true, 12 | "noImplicitThis": true, 13 | "alwaysStrict": true, 14 | "noUnusedLocals": false, 15 | "noUnusedParameters": false, 16 | "noImplicitReturns": true, 17 | "noFallthroughCasesInSwitch": false, 18 | "inlineSourceMap": true, 19 | "inlineSources": true, 20 | "experimentalDecorators": true, 21 | "strictPropertyInitialization": false, 22 | "typeRoots": [ 23 | "./node_modules/@types" 24 | ] 25 | }, 26 | "exclude": [ 27 | "node_modules", 28 | "cdk.out" 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /cloudtrail_aggregation/README.md: -------------------------------------------------------------------------------- 1 | # CloudTrail Aggregation 2 | 3 | This directory contains example programs for the blog post [Aggregating Files In Your Data Lake](https://chariotsolutions.com/blog/post/aggregating-files-in-your-data-lake-part-1/) 4 | and follow-on posts. 5 | 6 | It consists of the following pieces: 7 | 8 | * [Stage 1](stage-1): a Lambda that aggregates raw CloudTrail log files into compressed NDJSON 9 | (newline-delimited JSON) files, a day at a time. 10 | 11 | * [Stage 2](stage-2): a Lambda that aggregates a month's worth of CloudTrail logs from the files 12 | produced by stage 1, producing Parquet output. There's also a [Docker version](stage-2-docker) 13 | in addition to the simple Lambda. 14 | 15 | * [Stage 3](stage-3): uses Athena to aggregate a month's worth of CloudTrail logs, again using 16 | the files produced by stage 1, and again producing Parquet output. This transform is implemented 17 | using both Airflow and Lambda to invoke an Athena CTAS query to build each partition. 18 | -------------------------------------------------------------------------------- /cloudtrail_aggregation/stage-1/populate_queue.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # 3 | # MIT No Attribution 4 | # 5 | # Copyright Chariot Solutions 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 8 | # software and associated documentation files (the "Software"), to deal in the Software 9 | # without restriction, including without limitation the rights to use, copy, modify, 10 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 11 | # permit persons to whom the Software is furnished to do so. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 14 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 15 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 16 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 17 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 18 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | # 20 | ################################################################################ 21 | 22 | #!/usr/bin/env python3 23 | 24 | import boto3 25 | import json 26 | 27 | from datetime import date, timedelta 28 | 29 | QUEUE_URL = "https://sqs.us-east-1.amazonaws.com/123456789012/cloudtrail-aggregation-trigger" 30 | 31 | client = boto3.client('sqs') 32 | 33 | dd = date(2023, 12, 1) 34 | while dd <= date(2023, 12, 31): 35 | msg = json.dumps({"month": dd.month, "day": dd.day, "year": dd.year}) 36 | client.send_message(QueueUrl=QUEUE_URL, MessageBody=msg) 37 | dd += timedelta(days=1) 38 | -------------------------------------------------------------------------------- /cloudtrail_aggregation/stage-1/table.ddl: -------------------------------------------------------------------------------- 1 | -- column definitions from CloudTrail generated DDL, with addition of partitioning 2 | -- and a SerDe that translates nested objects to stringified JSON 3 | 4 | CREATE EXTERNAL TABLE `cloudtrail_daily` ( 5 | eventVersion STRING, 6 | userIdentity STRUCT< 7 | type: STRING, 8 | principalId: STRING, 9 | arn: STRING, 10 | accountId: STRING, 11 | invokedBy: STRING, 12 | accessKeyId: STRING, 13 | userName: STRING, 14 | sessionContext: STRUCT< 15 | attributes: STRUCT< 16 | mfaAuthenticated: STRING, 17 | creationDate: STRING>, 18 | sessionIssuer: STRUCT< 19 | type: STRING, 20 | principalId: STRING, 21 | arn: STRING, 22 | accountId: STRING, 23 | username: STRING>, 24 | ec2RoleDelivery: STRING, 25 | webIdFederationData: MAP>>, 26 | eventTime STRING, 27 | eventSource STRING, 28 | eventName STRING, 29 | awsRegion STRING, 30 | sourceIpAddress STRING, 31 | userAgent STRING, 32 | errorCode STRING, 33 | errorMessage STRING, 34 | requestParameters STRING, 35 | responseElements STRING, 36 | additionalEventData STRING, 37 | requestId STRING, 38 | eventId STRING, 39 | resources ARRAY>, 43 | eventType STRING, 44 | apiVersion STRING, 45 | readOnly STRING, 46 | recipientAccountId STRING, 47 | serviceEventDetails STRING, 48 | sharedEventID STRING, 49 | vpcEndpointId STRING, 50 | tlsDetails STRUCT< 51 | tlsVersion: STRING, 52 | cipherSuite: STRING, 53 | clientProvidedHostHeader: STRING> 54 | ) 55 | PARTITIONED BY ( 56 | year string, 57 | month string, 58 | day string 59 | ) 60 | ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe' 61 | WITH SERDEPROPERTIES ( 62 | 'ignore.malformed.json' = 'true' 63 | ) 64 | LOCATION 's3://MY_BUCKET/cloudtrail_daily/' 65 | TBLPROPERTIES ( 66 | 'classification' = 'cloudtrail', 67 | 'projection.enabled' = 'true', 68 | 'storage.location.template' = 's3://MY_BUCKET/cloudtrail_daily/${year}/${month}/${day}/', 69 | 'projection.year.type' = 'integer', 70 | 'projection.year.range' = '2013,2038', 71 | 'projection.year.digits' = '4', 72 | 'projection.month.type' = 'integer', 73 | 'projection.month.range' = '1,12', 74 | 'projection.month.digits' = '2', 75 | 'projection.day.type' = 'integer', 76 | 'projection.day.range' = '1,31', 77 | 'projection.day.digits' = '2' 78 | ); 79 | -------------------------------------------------------------------------------- /cloudtrail_aggregation/stage-2/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazon/aws-lambda-python:3.11 2 | 3 | RUN pip install boto3 pyarrow 4 | 5 | COPY lambda.py ${LAMBDA_TASK_ROOT} 6 | 7 | CMD [ "lambda.lambda_handler" ] 8 | -------------------------------------------------------------------------------- /cloudtrail_aggregation/stage-2/table.ddl: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE cloudtrail_monthly ( 2 | event_id string, 3 | request_id string, 4 | shared_event_id string, 5 | event_time timestamp, 6 | event_name string, 7 | event_source string, 8 | event_version string, 9 | aws_region string, 10 | source_ip_address string, 11 | recipient_account_id string, 12 | user_identity string, 13 | request_parameters string, 14 | response_elements string, 15 | additional_event_data string, 16 | resources string 17 | ) 18 | PARTITIONED BY ( 19 | year string, 20 | month string 21 | ) 22 | ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' 23 | STORED AS 24 | INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' 25 | OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' 26 | LOCATION 's3://MY_BUCKET/cloudtrail_monthly/' 27 | TBLPROPERTIES ( 28 | 'classification' = 'parquet', 29 | 'projection.enabled' = 'true', 30 | 'storage.location.template' = 's3://MY_BUCKET/cloudtrail_monthly/${year}/${month}/', 31 | 'projection.year.type' = 'integer', 32 | 'projection.year.range' = '2019,2029', 33 | 'projection.year.digits' = '4', 34 | 'projection.month.type' = 'integer', 35 | 'projection.month.range' = '1,12', 36 | 'projection.month.digits' = '2' 37 | ) 38 | -------------------------------------------------------------------------------- /cloudtrail_aggregation/stage-3/ctas.ddl: -------------------------------------------------------------------------------- 1 | -- replace MY_BUCKET by your actual bucket name 2 | -- replace MONTH and YEAR by the actual month and year that you want to transform 3 | -- if you don't like the name "cloudtrail_athena" or want to use a different prefix, feel free to change that too 4 | -- remember to drop this table once you've verified the transformed data 5 | 6 | CREATE TABLE cloudtrail_athena_temp_YEAR_MONTH 7 | WITH ( 8 | format = 'parquet', 9 | bucketed_by = ARRAY['event_id'], 10 | bucket_count = 4, 11 | external_location = 's3://MY_BUCKET/cloudtrail_athena/YEAR/MONTH 12 | write_compression = 'SNAPPY' 13 | ) AS 14 | select eventid as event_id, 15 | requestid as request_id, 16 | cast(from_iso8601_timestamp(eventtime) as timestamp) as event_time, 17 | eventsource as event_source, 18 | eventname as event_name, 19 | awsregion as aws_region, 20 | sourceipaddress as source_ip_address, 21 | recipientaccountid as recipient_account_id, 22 | json_format(cast (useridentity as JSON)) as user_identity, 23 | useridentity.invokedby as invoked_by, 24 | useridentity.principalid as principal_id, 25 | json_format(cast (resources as JSON)) as resources 26 | from default.cloudtrail_daily 27 | where year = 'YEAR' 28 | and month = 'MONTH 29 | 30 | 31 | -------------------------------------------------------------------------------- /cloudtrail_aggregation/stage-3/policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "AthenaQuery", 6 | "Effect": "Allow", 7 | "Action": [ 8 | "athena:GetWorkGroup", 9 | "athena:StartQueryExecution", 10 | "athena:GetQueryExecution", 11 | "athena:GetQueryResults" 12 | ], 13 | "Resource": [ 14 | "arn:aws:athena:AWS_REGION:AWS_ACCOUNT_NUMBER:workgroup/ATHENA_WORKGROUP" 15 | ] 16 | }, 17 | { 18 | "Sid": "AthenaBucket", 19 | "Effect": "Allow", 20 | "Action": [ 21 | "s3:AbortMultipartUpload", 22 | "s3:GetBucketLocation", 23 | "s3:GetObject", 24 | "s3:ListBucket", 25 | "s3:ListMultipartUploadParts", 26 | "s3:PutObject" 27 | ], 28 | "Resource": [ 29 | "arn:aws:s3:::ATHENA_WORKGROUP_BUCKET", 30 | "arn:aws:s3:::ATHENA_WORKGROUP_BUCKET/ATHENA_WORKGROUP_PREFIX/*" 31 | ] 32 | }, 33 | { 34 | "Sid": "ReadSourceBucket", 35 | "Effect": "Allow", 36 | "Action": [ 37 | "s3:GetBucketLocation", 38 | "s3:GetObject", 39 | "s3:ListBucket" 40 | ], 41 | "Resource": [ 42 | "arn:aws:s3:::SRC_BUCKET", 43 | "arn:aws:s3:::SRC_BUCKET/SRC_PREFIX/*" 44 | ] 45 | }, 46 | { 47 | "Sid": "WriteDestBucket", 48 | "Effect": "Allow", 49 | "Action": [ 50 | "s3:AbortMultipartUpload", 51 | "s3:GetBucketLocation", 52 | "s3:GetObject", 53 | "s3:DeleteObject", 54 | "s3:ListBucket", 55 | "s3:ListMultipartUploadParts", 56 | "s3:PutObject" 57 | ], 58 | "Resource": [ 59 | "arn:aws:s3:::DST_BUCKET", 60 | "arn:aws:s3:::DST_BUCKET/DST_PREFIX/*" 61 | ] 62 | }, 63 | { 64 | "Sid": "Glue", 65 | "Effect": "Allow", 66 | "Action": [ 67 | "glue:CreateTable", 68 | "glue:DeletePartition", 69 | "glue:DeleteTable", 70 | "glue:GetDatabase", 71 | "glue:GetDatabases", 72 | "glue:GetPartition", 73 | "glue:GetPartitions", 74 | "glue:GetTable", 75 | "glue:GetTables" 76 | ], 77 | "Resource": [ 78 | "arn:aws:glue:AWS_REGION:AWS_ACCOUNT_NUMBER:catalog", 79 | "arn:aws:glue:AWS_REGION:AWS_ACCOUNT_NUMBER:database/GLUE_DATABASE", 80 | "arn:aws:glue:AWS_REGION:AWS_ACCOUNT_NUMBER:table/GLUE_DATABASE/*" 81 | ] 82 | } 83 | ] 84 | } 85 | -------------------------------------------------------------------------------- /cloudtrail_aggregation/stage-3/table.ddl: -------------------------------------------------------------------------------- 1 | -- replace MY_BUCKET by your actual bucket name 2 | -- if you don't like the name "cloudtrail_athena" or want to use a different prefix, feel free to change that too 3 | -- if you decide to change the name of the table, you will also need to change the S3 location and location template 4 | 5 | CREATE EXTERNAL TABLE cloudtrail_athena ( 6 | event_id STRING, 7 | request_id STRING, 8 | event_time TIMESTAMP, 9 | event_source STRING, 10 | event_name STRING, 11 | aws_region STRING, 12 | source_ip_address STRING, 13 | recipient_account_id STRING, 14 | user_identity STRING, 15 | invoked_by STRING, 16 | principal_id STRING, 17 | resources STRING 18 | ) 19 | PARTITIONED BY ( 20 | year string, 21 | month string 22 | ) 23 | ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' 24 | STORED AS PARQUET 25 | LOCATION 's3://MY_BUCKET/cloudtrail_athena/' 26 | TBLPROPERTIES ( 27 | 'classification' = 'parquet', 28 | 'projection.enabled' = 'true', 29 | 'storage.location.template' = 's3://MY_BUCKET/cloudtrail_athena/${year}/${month}/', 30 | 'projection.year.type' = 'integer', 31 | 'projection.year.range' = '2019,2029', 32 | 'projection.year.digits' = '4', 33 | 'projection.month.type' = 'integer', 34 | 'projection.month.range' = '1,12', 35 | 'projection.month.digits' = '2' 36 | ) 37 | 38 | -------------------------------------------------------------------------------- /cloudtrail_to_elasticsearch/.gitignore: -------------------------------------------------------------------------------- 1 | *.zip 2 | *.pyc 3 | dist/ 4 | build/ 5 | -------------------------------------------------------------------------------- /cloudtrail_to_elasticsearch/Makefile.pip: -------------------------------------------------------------------------------- 1 | .PHONY: default deploy package init clean 2 | 3 | LAMBDA_NAME ?= CloudTrail_to_OpenSearch 4 | 5 | SRC_DIR := $(PWD)/cloudtrail_to_elasticsearch 6 | BUILD_DIR := $(PWD)/build 7 | DEPLOY_DIR := $(PWD) 8 | ARTIFACT := lambda.zip 9 | 10 | default: package 11 | 12 | deploy: package 13 | aws lambda update-function-code --function-name $(LAMBDA_NAME) --zip-file fileb://$(DEPLOY_DIR)/$(ARTIFACT) 14 | 15 | package: init 16 | rm -f ${DEPLOY_DIR}/${ARTIFACT} 17 | cp -r $(SRC_DIR) ${BUILD_DIR} 18 | cd $(BUILD_DIR) ; zip -r ${DEPLOY_DIR}/${ARTIFACT} . -x '*.pyc' 19 | 20 | test: init quicktest 21 | 22 | quicktest: 23 | PYTHONPATH="${PWD}:$(BUILD_DIR)" python -m unittest discover -s tests 24 | 25 | init: 26 | mkdir -p ${DEPLOY_DIR} 27 | mkdir -p ${BUILD_DIR} 28 | pip install --upgrade -r requirements.txt -t $(BUILD_DIR) 29 | 30 | clean: 31 | rm -f $(DEPLOY_DIR)/$(ARTIFACT) 32 | rm -rf ${BUILD_DIR} 33 | rm -rf */__pycache__ 34 | -------------------------------------------------------------------------------- /cloudtrail_to_elasticsearch/Makefile.poetry: -------------------------------------------------------------------------------- 1 | .PHONY: default deploy package test quicktest init clean 2 | 3 | LAMBDA_NAME ?= CloudTrail_to_OpenSearch 4 | 5 | POETRY_DIST_DIR := $(PWD)/dist 6 | BUILD_DIR := $(PWD)/build 7 | DEPLOY_DIR := $(PWD) 8 | ARTIFACT := lambda.zip 9 | 10 | default: package 11 | 12 | deploy: package 13 | aws lambda update-function-code --function-name $(LAMBDA_NAME) --zip-file fileb://$(DEPLOY_DIR)/$(ARTIFACT) 14 | 15 | package: test 16 | poetry build 17 | mkdir -p $(BUILD_DIR) 18 | poetry run pip install --upgrade -t $(BUILD_DIR) $(POETRY_DIST_DIR)/*.whl 19 | cd $(BUILD_DIR) ; zip -r $(DEPLOY_DIR)/$(ARTIFACT) . -x '*.pyc' 20 | 21 | test: init quicktest 22 | 23 | quicktest: 24 | poetry run python -m unittest discover -s tests 25 | 26 | init: 27 | poetry install 28 | 29 | clean: 30 | rm -f $(DEPLOY_DIR)/$(ARTIFACT) 31 | rm -rf ${POETRY_DIST_DIR} 32 | rm -rf ${BUILD_DIR} 33 | rm -rf */__pycache__ 34 | -------------------------------------------------------------------------------- /cloudtrail_to_elasticsearch/cloudtrail_to_elasticsearch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chariotsolutions/aws-examples/cd636134fe14eb7314114d97290b5bd6124c25dc/cloudtrail_to_elasticsearch/cloudtrail_to_elasticsearch/__init__.py -------------------------------------------------------------------------------- /cloudtrail_to_elasticsearch/cloudtrail_to_elasticsearch/lambda_handler.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # 3 | # MIT No Attribution 4 | # 5 | # Copyright Chariot Solutions 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 8 | # software and associated documentation files (the "Software"), to deal in the Software 9 | # without restriction, including without limitation the rights to use, copy, modify, 10 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 11 | # permit persons to whom the Software is furnished to do so. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 14 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 15 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 16 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 17 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 18 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | # 20 | ################################################################################ 21 | 22 | """ Lambda function to upload CloudTrail events to Elasticsearch. This module 23 | decomposes the event and calls the processor module to do all the work. 24 | """ 25 | 26 | 27 | import json 28 | import cloudtrail_to_elasticsearch.processor 29 | 30 | px = cloudtrail_to_elasticsearch.processor.create() 31 | 32 | def handle(event, context): 33 | for wrapper_record in event.get('Records', []): 34 | message = json.loads(wrapper_record['body']) 35 | for record in message.get('Records', []): 36 | eventName = record['eventName'] 37 | bucket = record['s3']['bucket']['name'] 38 | key = record['s3']['object']['key'] 39 | try: 40 | print(f"processing s3://{bucket}/{key}") 41 | px.process_from_s3(bucket, key) 42 | except Exception as ex: 43 | print(f"failed to process file: {ex}") 44 | -------------------------------------------------------------------------------- /cloudtrail_to_elasticsearch/cloudtrail_to_elasticsearch/s3_helper.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # 3 | # MIT No Attribution 4 | # 5 | # Copyright Chariot Solutions 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 8 | # software and associated documentation files (the "Software"), to deal in the Software 9 | # without restriction, including without limitation the rights to use, copy, modify, 10 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 11 | # permit persons to whom the Software is furnished to do so. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 14 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 15 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 16 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 17 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 18 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | # 20 | ################################################################################ 21 | 22 | import boto3 23 | import gzip 24 | 25 | 26 | class S3Helper: 27 | """ Provides functions for interacting with S3. This class allows isolated unit 28 | testing of the operational modules. 29 | """ 30 | 31 | def retrieve(self, bucket, key, gzipped=True): 32 | """ Retrieves the contents of an S3 object, optionally un-GZipping it. 33 | """ 34 | object = boto3.resource('s3').Object(bucket, key) 35 | body = object.get()['Body'] 36 | try: 37 | raw = body.read() 38 | if gzipped: 39 | return gzip.decompress(raw) 40 | else: 41 | return raw 42 | finally: 43 | body.close() 44 | 45 | 46 | def iterate_bucket(self, bucket, prefix, fn): 47 | """ Executes the provided function(bucket, key) for every key 48 | in the specified bucket with the specified prefix. 49 | """ 50 | paginator = boto3.client('s3').get_paginator('list_objects') 51 | for page in paginator.paginate(Bucket=bucket, Prefix=prefix): 52 | for obj in page['Contents']: 53 | key = obj['Key'] 54 | fn(bucket, key) 55 | 56 | -------------------------------------------------------------------------------- /cloudtrail_to_elasticsearch/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "cloudtrail_to_elasticsearch" 3 | version = "0.1.0" 4 | description = "Transforms CloudTrail event logs that are stored on S3 and uploads them to an Elasticsearch cluster" 5 | authors = ["Keith Gregory "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.9" 9 | requests = "~2.32.0" 10 | aws-requests-auth = "~0.4.3" 11 | 12 | [tool.poetry.dev-dependencies] 13 | boto3 = "^1.26.158" 14 | 15 | [build-system] 16 | requires = ["poetry-core>=1.0.0"] 17 | build-backend = "poetry.core.masonry.api" 18 | -------------------------------------------------------------------------------- /cloudtrail_to_elasticsearch/requirements.txt: -------------------------------------------------------------------------------- 1 | # boto3 2 | requests 3 | aws-requests-auth 4 | -------------------------------------------------------------------------------- /cloudtrail_to_elasticsearch/tests/resources/event_with_request_parameters-transformed.json: -------------------------------------------------------------------------------- 1 | { 2 | "eventVersion": "1.05", 3 | "userIdentity": { 4 | "type": "AssumedRole", 5 | "principalId": "AROA3LUQPVEKAAOC4652P:RDSAuroraServeless", 6 | "arn": "arn:aws:sts::123456789012:assumed-role/AWSServiceRoleForRDS/RDSAuroraServeless", 7 | "accountId": "123456789012", 8 | "accessKeyId": "ASIA3L5KUQPV5QOWIAA6", 9 | "sessionContext": { 10 | "sessionIssuer": { 11 | "type": "Role", 12 | "principalId": "AROA3LUQPVEKAAOC4652P", 13 | "arn": "arn:aws:iam::123456789012:role/aws-service-role/rds.amazonaws.com/AWSServiceRoleForRDS", 14 | "accountId": "123456789012", 15 | "userName": "AWSServiceRoleForRDS" 16 | }, 17 | "webIdFederationData": {}, 18 | "attributes": { 19 | "mfaAuthenticated": "false", 20 | "creationDate": "2020-09-13T21:45:55Z" 21 | } 22 | }, 23 | "invokedBy": "rds.amazonaws.com" 24 | }, 25 | "eventTime": "2020-09-14T01:27:53Z", 26 | "eventSource": "logs.amazonaws.com", 27 | "eventName": "DescribeLogStreams", 28 | "awsRegion": "us-east-1", 29 | "sourceIPAddress": "rds.amazonaws.com", 30 | "userAgent": "rds.amazonaws.com", 31 | "requestParameters_raw": "{\"logGroupName\": \"some_group\", \"logStreamNamePrefix\": \"some_stream\"}", 32 | "requestParameters_flattened": { 33 | "logGroupName": [ 34 | "some_group" 35 | ], 36 | "logStreamNamePrefix": [ 37 | "some_stream" 38 | ] 39 | }, 40 | "requestID": "ac7a380c-60db-4d0e-884f-71b5bda83606", 41 | "eventID": "8333c255-11f2-49ef-9980-76b5dc1fb2da", 42 | "eventType": "AwsApiCall", 43 | "apiVersion": "20140328", 44 | "recipientAccountId": "123456789012" 45 | } 46 | -------------------------------------------------------------------------------- /cloudtrail_to_elasticsearch/tests/resources/event_with_request_parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "eventVersion": "1.05", 3 | "userIdentity": { 4 | "type": "AssumedRole", 5 | "principalId": "AROA3LUQPVEKAAOC4652P:RDSAuroraServeless", 6 | "arn": "arn:aws:sts::123456789012:assumed-role/AWSServiceRoleForRDS/RDSAuroraServeless", 7 | "accountId": "123456789012", 8 | "accessKeyId": "ASIA3L5KUQPV5QOWIAA6", 9 | "sessionContext": { 10 | "sessionIssuer": { 11 | "type": "Role", 12 | "principalId": "AROA3LUQPVEKAAOC4652P", 13 | "arn": "arn:aws:iam::123456789012:role/aws-service-role/rds.amazonaws.com/AWSServiceRoleForRDS", 14 | "accountId": "123456789012", 15 | "userName": "AWSServiceRoleForRDS" 16 | }, 17 | "webIdFederationData": {}, 18 | "attributes": { 19 | "mfaAuthenticated": "false", 20 | "creationDate": "2020-09-13T21:45:55Z" 21 | } 22 | }, 23 | "invokedBy": "rds.amazonaws.com" 24 | }, 25 | "eventTime": "2020-09-14T01:27:53Z", 26 | "eventSource": "logs.amazonaws.com", 27 | "eventName": "DescribeLogStreams", 28 | "awsRegion": "us-east-1", 29 | "sourceIPAddress": "rds.amazonaws.com", 30 | "userAgent": "rds.amazonaws.com", 31 | "requestParameters": { 32 | "logGroupName": "some_group", 33 | "logStreamNamePrefix": "some_stream" 34 | }, 35 | "responseElements": null, 36 | "requestID": "ac7a380c-60db-4d0e-884f-71b5bda83606", 37 | "eventID": "8333c255-11f2-49ef-9980-76b5dc1fb2da", 38 | "eventType": "AwsApiCall", 39 | "apiVersion": "20140328", 40 | "recipientAccountId": "123456789012" 41 | } 42 | -------------------------------------------------------------------------------- /cloudtrail_to_elasticsearch/tests/resources/s3_test_event.json: -------------------------------------------------------------------------------- 1 | { 2 | "Records": [ 3 | { 4 | "eventVersion": "2.0", 5 | "eventSource": "aws:s3", 6 | "awsRegion": "us-west-2", 7 | "eventTime": "1970-01-01T00:00:00.000Z", 8 | "eventName": "ObjectCreated:Put", 9 | "userIdentity": { 10 | "principalId": "EXAMPLE" 11 | }, 12 | "requestParameters": { 13 | "sourceIPAddress": "127.0.0.1" 14 | }, 15 | "responseElements": { 16 | "x-amz-request-id": "EXAMPLE123456789", 17 | "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH" 18 | }, 19 | "s3": { 20 | "s3SchemaVersion": "1.0", 21 | "configurationId": "testConfigRule", 22 | "bucket": { 23 | "name": "my-s3-bucket", 24 | "ownerIdentity": { 25 | "principalId": "EXAMPLE" 26 | }, 27 | "arn": "arn:aws:s3:::example-bucket" 28 | }, 29 | "object": { 30 | "key": "HappyFace.jpg", 31 | "size": 1024, 32 | "eTag": "0123456789abcdef0123456789abcdef", 33 | "sequencer": "0A1B2C3D4E5F678901" 34 | } 35 | } 36 | } 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /cloudtrail_to_elasticsearch/tests/resources/simple_event-transformed.json: -------------------------------------------------------------------------------- 1 | { 2 | "eventVersion": "1.08", 3 | "userIdentity": { 4 | "type": "AssumedRole", 5 | "principalId": "AROAIQ5YG2D5ASTSBJPRE:TrustedAdvisor_123456789012_62d91a74-cffb-4d27-865d-58d51417a1b8", 6 | "arn": "arn:aws:sts::123456789012:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_123456789012_62d91a74-cffb-4d27-865d-58d51417a1b8", 7 | "accountId": "123456789012", 8 | "accessKeyId": "ASIA25XXSJ5QZ54GMEF6", 9 | "invokedBy": "trustedadvisor.amazonaws.com" 10 | }, 11 | "eventTime": "2021-01-27T01:35:02Z", 12 | "eventSource": "rds.amazonaws.com", 13 | "eventName": "DescribeAccountAttributes", 14 | "awsRegion": "ap-southeast-2", 15 | "sourceIPAddress": "trustedadvisor.amazonaws.com", 16 | "userAgent": "trustedadvisor.amazonaws.com", 17 | "requestID": "879fd683-df19-4d30-9a6b-288d80fb8587", 18 | "eventID": "569e7727-2a83-4b14-8190-70e92dc16ad1", 19 | "readOnly": true, 20 | "eventType": "AwsApiCall", 21 | "managementEvent": true, 22 | "eventCategory": "Management", 23 | "recipientAccountId": "123456789012" 24 | } 25 | -------------------------------------------------------------------------------- /cloudtrail_to_elasticsearch/tests/resources/simple_event.json: -------------------------------------------------------------------------------- 1 | { 2 | "eventVersion": "1.08", 3 | "userIdentity": { 4 | "type": "AssumedRole", 5 | "principalId": "AROAIQ5YG2D5ASTSBJPRE:TrustedAdvisor_123456789012_62d91a74-cffb-4d27-865d-58d51417a1b8", 6 | "arn": "arn:aws:sts::123456789012:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_123456789012_62d91a74-cffb-4d27-865d-58d51417a1b8", 7 | "accountId": "123456789012", 8 | "accessKeyId": "ASIA25XXSJ5QZ54GMEF6", 9 | "invokedBy": "trustedadvisor.amazonaws.com" 10 | }, 11 | "eventTime": "2021-01-27T01:35:02Z", 12 | "eventSource": "rds.amazonaws.com", 13 | "eventName": "DescribeAccountAttributes", 14 | "awsRegion": "ap-southeast-2", 15 | "sourceIPAddress": "trustedadvisor.amazonaws.com", 16 | "userAgent": "trustedadvisor.amazonaws.com", 17 | "requestParameters": null, 18 | "responseElements": null, 19 | "requestID": "879fd683-df19-4d30-9a6b-288d80fb8587", 20 | "eventID": "569e7727-2a83-4b14-8190-70e92dc16ad1", 21 | "readOnly": true, 22 | "eventType": "AwsApiCall", 23 | "managementEvent": true, 24 | "eventCategory": "Management", 25 | "recipientAccountId": "123456789012" 26 | } 27 | -------------------------------------------------------------------------------- /cloudtrail_to_elasticsearch/tests/test_lambda_handler.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | import unittest 5 | 6 | from unittest.mock import Mock 7 | 8 | 9 | class TestLambdaHandler(unittest.TestCase): 10 | 11 | def setUp(self): 12 | os.environ['ES_HOSTNAME'] = "localhost:9200" 13 | os.environ['AWS_ACCESS_KEY_ID'] = "AKIADONTTHINKSO12345" 14 | os.environ['AWS_SECRET_ACCESS_KEY'] = "NONEOFYOURBUSINESS1234567891234455555555" 15 | os.environ['AWS_REGION'] = "us-east-1" 16 | 17 | def tearDown(self): 18 | if 'cloudtrail_to_elasticsearch.lambda_handler' in sys.modules: 19 | del sys.modules['cloudtrail_to_elasticsearch.lambda_handler'] 20 | 21 | 22 | def test_import(self): 23 | """ This is a basic test to ensure that everything imports without problem. I 24 | use it as he first test for a Lambda handler, although later tests will 25 | generally do the same thing. 26 | """ 27 | import cloudtrail_to_elasticsearch.lambda_handler 28 | 29 | 30 | def test_event_handling(self): 31 | """ Takes a sample S3 event (in this case, from the AWS docs) and verifies that 32 | it's deconstructed as passed to a mock handler. 33 | """ 34 | import cloudtrail_to_elasticsearch.lambda_handler 35 | cloudtrail_to_elasticsearch.lambda_handler.px = mock = Mock() 36 | with open("tests/resources/s3_test_event.json") as f: 37 | event = json.load(f) 38 | cloudtrail_to_elasticsearch.lambda_handler.handle(event, None) 39 | mock.process_from_s3.assert_called_once_with("my-s3-bucket", "HappyFace.jpg") 40 | -------------------------------------------------------------------------------- /cloudtrail_to_elasticsearch/tests/test_transforms.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | import unittest 5 | 6 | 7 | # module under test 8 | from cloudtrail_to_elasticsearch import processor 9 | 10 | 11 | class TestTransforms(unittest.TestCase): 12 | 13 | def test_filename_to_index_name(self): 14 | filename = "123456789012/CloudTrail/us-east-1/2020/03/30/219289705433_CloudTrail_us-east-1_20200330T2110Z_T7Y7JlIYAD5Ej3HX.json.gz" 15 | expected_index_name = "cloudtrail-2020-03" 16 | self.assertEqual(expected_index_name, processor.index_name(filename), "index name extracted from filename") 17 | 18 | 19 | def test_bogus_filename_to_index_name(self): 20 | filename = "123456789012/CloudTrail/us-east-1/03/30/219289705433_CloudTrail_us-east-1_0330T2110Z_T7Y7JlIYAD5Ej3HX.json.gz" 21 | self.assertIsNone(processor.index_name(filename), "unparseable filename") 22 | 23 | 24 | def test_event_without_flattening(self): 25 | """ Removes the request_parameters and response_elements attributes from the source, 26 | since they don't have any values. 27 | """ 28 | with open("tests/resources/simple_event.json") as f: 29 | orig = json.load(f) 30 | with open("tests/resources/simple_event-transformed.json") as f: 31 | expected = json.load(f) 32 | self.assertEqual(expected, processor.transform_events([orig])[0], "simple event, no flattening") 33 | 34 | 35 | def test_event_with_request_parameters(self): 36 | """ Replaces request_parameters with request_parameters_raw, a JSONified string representation, 37 | and request_parameters_flattened, a key->list representation. 38 | """ 39 | with open("tests/resources/event_with_request_parameters.json") as f: 40 | orig = json.load(f) 41 | with open("tests/resources/event_with_request_parameters-transformed.json") as f: 42 | expected = json.load(f) 43 | self.assertEqual(expected, processor.transform_events([orig])[0], "simple event, no flattening") 44 | -------------------------------------------------------------------------------- /cloudtrail_to_kinesis/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | *.zip 3 | -------------------------------------------------------------------------------- /cloudtrail_to_kinesis/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: default deploy package init clean 2 | 3 | LAMBDA_NAME ?= CloudTrail_Event_Processor 4 | 5 | SRC_DIR := $(PWD)/lambda 6 | BUILD_DIR := $(SRC_DIR) 7 | DEPLOY_DIR := $(SRC_DIR) 8 | ARTIFACT := lambda.zip 9 | 10 | default: package 11 | 12 | deploy: package 13 | aws lambda update-function-code --function-name $(LAMBDA_NAME) --zip-file fileb://$(DEPLOY_DIR)/$(ARTIFACT) 14 | 15 | package: init 16 | rm -f ${DEPLOY_DIR}/${ARTIFACT} 17 | cd $(BUILD_DIR) ; zip ${DEPLOY_DIR}/${ARTIFACT} *.py 18 | 19 | clean: 20 | rm -f $(DEPLOY_DIR)/$(ARTIFACT) 21 | rm -rf */__pycache__ 22 | -------------------------------------------------------------------------------- /cloudtrail_to_kinesis/README.md: -------------------------------------------------------------------------------- 1 | Lambda to deconstruct CloudTrail files and write the individual records to a Kinesis Data Stream. 2 | 3 | To build and deploy: 4 | 5 | 1. Run [cloudformation.yml](cloudformation.yml) to create the basic infrastructure. 6 | 7 | This template creates the following resources: 8 | 9 | * An SQS queue and dead-letter queue, to receive S3 notifications. 10 | * A Lambda and related resources (execution role, log group, trigger) to handle those notifications. 11 | * A Kinesis Data Stream to receive events. By default this has 1 shard and retains records for 24 hours. 12 | 13 | You must provide the name and prefix of the CloudTrail bucket, using parameters `CloudTrailBucket` and 14 | `CloudTrailBucketPrefix`. The latter is used to restrict access to files in the source bucket; it defaults 15 | to blank, meaning that the Lambda can access any files in the bucket. If provided, it must include a 16 | trailing slash (eg: `AWSLogs/`). 17 | 18 | There are also parameters used to name each of the created resources, and for configuration of the Kinesis 19 | stream. These all have defaults. 20 | 21 | 2. Deploy the Lambda 22 | 23 | The CloudFormation template creates a Lambda with a dummy handler. You can either manually create a ZIP 24 | file from the contents of the `lambda` directory, or use _make_: 25 | 26 | ``` 27 | make deploy 28 | ``` 29 | 30 | If you changed the name of the Lambda in step 1, you'll need to tell _make_ the actual name: 31 | 32 | ``` 33 | make deploy LAMBDA_NAME=Your_Lambda_Name 34 | ``` 35 | 36 | 3. Update the CloudTrail bucket's notification policy to send events to the notification queue. 37 | 38 | This is a manual process, because CloudFormation can only attach a notification when it creates the 39 | bucket. In the case of CloudTrail, you will have already created this bucket when you set up the 40 | trail. 41 | 42 | In the Console, open the bucket, go to the Properties tab, scroll down to "Event notifications", 43 | and click "Create event notification". Enter a name for the notification, the prefix where your 44 | CloudTrail log files are stored, select "All object create events", and set the destination as 45 | the SQS queue created by CloudFormation. 46 | 47 | 48 | You can also run the transformation locally, passing either an S3 URL or a local filename along with 49 | the name of the Kinesis stream. You must have the `boto3` library installed. 50 | 51 | ``` 52 | python lambda s3://com-example-cloudtrail/AWSLogs/o-unx82b27e/123456789012/CloudTrail/us-east-1/2024/09/05/123456789012_CloudTrail_us-east-1_20240905T1725Z_Rr5pMUQHMIcl1qD4.json.gz cloudtrail_events 53 | ``` 54 | 55 | ``` 56 | python lambda /tmp/123456789012_CloudTrail_us-east-1_20240905T0230Z_Hvjj6jOPNs4RVsVB.json.gz cloudtrail_events 57 | ``` 58 | -------------------------------------------------------------------------------- /cloudtrail_to_kinesis/lambda/__main__.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # 3 | # MIT No Attribution 4 | # 5 | # Copyright Chariot Solutions 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 8 | # software and associated documentation files (the "Software"), to deal in the Software 9 | # without restriction, including without limitation the rights to use, copy, modify, 10 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 11 | # permit persons to whom the Software is furnished to do so. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 14 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 15 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 16 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 17 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 18 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | # 20 | ################################################################################ 21 | 22 | """ Entry point for processing a file locally. 23 | 24 | python . FILE_NAME KINESIS_STREAM 25 | 26 | FILE_NAME can be either an S3 URL or a local filename. 27 | KINESIS_STREAM is the name of a Kinesis Data Stream. 28 | 29 | To run, you must have boto3 available on your PYTHONPATH, and have all 30 | nevessary AWS permissions. 31 | """ 32 | 33 | 34 | import boto3 35 | import logging 36 | import re 37 | import sys 38 | 39 | from file_processor import FileProcessor 40 | 41 | if len(sys.argv) != 3: 42 | print(__doc__) 43 | sys.exit(1) 44 | 45 | logging.basicConfig(level=logging.INFO) 46 | 47 | fp = FileProcessor(boto3.client('s3'), boto3.client('kinesis')) 48 | 49 | m = re.match(r"s3:\/\/(.*?)\/(.*)", sys.argv[1]) 50 | if m: 51 | fp.process(s3_bucket=m.group(1), s3_key=m.group(2), stream_name=sys.argv[2]) 52 | else: 53 | with open(sys.argv[1], 'rb') as f: 54 | data = f.read() 55 | fp.process(data=data, stream_name=sys.argv[2]) 56 | -------------------------------------------------------------------------------- /cloudtrail_to_kinesis/lambda/file_processor.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # 3 | # MIT No Attribution 4 | # 5 | # Copyright Chariot Solutions 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 8 | # software and associated documentation files (the "Software"), to deal in the Software 9 | # without restriction, including without limitation the rights to use, copy, modify, 10 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 11 | # permit persons to whom the Software is furnished to do so. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 14 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 15 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 16 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 17 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 18 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | # 20 | ################################################################################ 21 | 22 | import boto3 23 | import gzip 24 | import json 25 | import logging 26 | import time 27 | 28 | from kinesis_writer import KinesisWriter 29 | 30 | 31 | logger = logging.getLogger(__name__) 32 | 33 | class FileProcessor: 34 | 35 | FIELDS_TO_STRINGIFY = [ 36 | "additionalEventData", "addendum", "edgeDeviceDetails", "insightDetails", 37 | "requestParameters", "responseElements", "resources", "serviceEventDetails", 38 | "tlsDetails", "userIdentity" 39 | ] 40 | 41 | 42 | def __init__(self, s3_client=None, kinesis_client=None): 43 | self.s3_client = s3_client 44 | self.kinesis_client = kinesis_client 45 | 46 | 47 | def extract_records(self, s3_bucket=None, s3_key=None, data=None): 48 | if not data: 49 | s3_result = self.s3_client.get_object(Bucket=s3_bucket, Key=s3_key) 50 | data = s3_result['Body'].read() 51 | if data.startswith(b'\x1f\x8b'): 52 | data = gzip.decompress(data) 53 | event = json.loads(data) 54 | records = event['Records'] 55 | for rec in records: 56 | for field_name in FileProcessor.FIELDS_TO_STRINGIFY: 57 | field_value = rec.get(field_name) 58 | if field_value: 59 | rec[field_name] = json.dumps(field_value) 60 | return records 61 | 62 | 63 | def process(self, s3_bucket=None, s3_key=None, data=None, stream_name=None): 64 | writer = KinesisWriter(self.kinesis_client, stream_name) 65 | recs = self.extract_records(s3_bucket=s3_bucket, s3_key=s3_key, data=data) 66 | for rec in recs: 67 | writer.enqueue(json.dumps(rec), rec.get('eventID')) 68 | while writer.flush(): 69 | time.sleep(0.25) 70 | logger.info(f"wrote {len(recs)} records") 71 | -------------------------------------------------------------------------------- /cloudtrail_to_kinesis/lambda/index.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # 3 | # MIT No Attribution 4 | # 5 | # Copyright Chariot Solutions 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 8 | # software and associated documentation files (the "Software"), to deal in the Software 9 | # without restriction, including without limitation the rights to use, copy, modify, 10 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 11 | # permit persons to whom the Software is furnished to do so. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 14 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 15 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 16 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 17 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 18 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | # 20 | ################################################################################ 21 | 22 | import boto3 23 | import gzip 24 | import json 25 | import os 26 | import time 27 | import sys 28 | 29 | from file_processor import FileProcessor 30 | 31 | 32 | kinesis_stream = os.environ['KINESIS_STREAM'] 33 | fp = FileProcessor(boto3.client('s3'), boto3.client('kinesis')) 34 | 35 | 36 | def lambda_handler(event, context): 37 | for wrapper_record in event.get('Records', []): 38 | message = json.loads(wrapper_record['body']) 39 | for record in message.get('Records', []): 40 | eventName = record['eventName'] 41 | bucket = record['s3']['bucket']['name'] 42 | key = record['s3']['object']['key'] 43 | logger.info(f"processing s3://{bucket}/{key}") 44 | try: 45 | fp.process(s3_bucket=bucket, s3_key=key, stream_name=kinesis_stream) 46 | except Exception as ex: 47 | print(f"failed to process file: {ex}") 48 | -------------------------------------------------------------------------------- /dms_firehose/README.md: -------------------------------------------------------------------------------- 1 | CloudFormation templates to set up a data pipeline from a Postgres database server to Iceberg 2 | tables in a data lake, using Amazon Database Migration Service and AWS Data Firehose. Used as a 3 | data source for [this blog post]( https://chariotsolutions.com/blog/post/dms_firehose_iceberg/). 4 | 5 | ![Architecture diagram for pipeline from Postgres to data lake, using AWS DMS and Firehose](postgres-dms-firehose.png) 6 | 7 | Prerequisites: 8 | 9 | * Install and configure the `pglogical` extension in your Postgres database server 10 | (see [this AWS doc](https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Source.PostgreSQL.html) or the blog post 11 | linked above for more information. 12 | * Create a Secrets Manager secret to hold connection information, if you don't already have one. 13 | * Create a Kinesis Data stream to receive captured records. For best performance, configure with four (4) shards. 14 | * Create an S3 bucket to serve as your data lake. 15 | 16 | There are three templates, which should be applied in the order shown: 17 | 18 | * [dms.yml](dms.yml): creates the DMS replication instance, endpoints, and replication task. _Do not_ start the 19 | task until all stacks have been created. 20 | * [glue.yml](glue.yml): creates a dedicated Glue database and tables to hold TPC-C data. 21 | * [firehose.yml](firehose.yml): creates a Firehose (with transformation Lambda) that extracts records from the 22 | Kinesis stream and writes them to the appropriate Iceberg table. 23 | 24 | You'll see that there are several parameters that are shared between stacks, such as the ARN of the Kinesis stream. 25 | 26 | Once you have everything in place, you can run the [BenchBase](https://github.com/cmu-db/benchbase) TPC-C benchmark. 27 | 28 | This will require a Java development environment with Maven; if you do not have such an environment, I recommend 29 | spinning up an `md5.large` EC2 instance and installing the necessary packages. 30 | 31 | To run the benchmark you'll need to change the configuration file to include your database credentials. I also 32 | increased the scale factor to 2, and the number of terminals to 10. 33 | -------------------------------------------------------------------------------- /dms_firehose/postgres-dms-firehose.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chariotsolutions/aws-examples/cd636134fe14eb7314114d97290b5bd6124c25dc/dms_firehose/postgres-dms-firehose.png -------------------------------------------------------------------------------- /firehose_iceberg/README.md: -------------------------------------------------------------------------------- 1 | This project contains CloudFormation templates for the blog post [Populating Iceberg Tables 2 | with Amazon Data Firehose](https://chariotsolutions.com/blog/post/firehose_iceberg/). 3 | 4 | Both templates create a Kinesis Firehose fed from a pre-existing Kinesis Data Stream, along with a 5 | Glue table to receive data from that Firehose. The first, [firehose-parquet.yml](firehose-parquet.yml), 6 | writes the records as Parquet; the second, [firehose-iceberg.yml](firehose-iceberg.yml), writes to 7 | an Iceberg table. 8 | 9 | Both templates take the same set of parameters: 10 | 11 | 12 | * `KinesisStreamName`: the stream where source events are written; defaults to `cloudtrail_events`. 13 | * `S3Bucket`: the bucket where destination files will be written; no default. 14 | * `GlueDatabaseName`: name of the Glue database where the table definition is created; defaults to 15 | `default`. 16 | * `GlueTableName`: the name of the destionation table; no default. 17 | * `FirehoseErrorPrefix`: a prefix in the destination bucket where any Firehose errors will be 18 | written. 19 | * `FirehoseBufferingInterval`: the number of seconds that Firehose waits to build batches of events; 20 | default varies depending on whether output is Parquet or Iceberg. 21 | * `FirehoseFilesizeMB`: the target file size for files produced by Firehose; default varies depending 22 | on whether output is Parquet or Iceberg. 23 | 24 | Notes: 25 | 26 | * The Glue table name is used as the prefix for the table's data files. 27 | 28 | * The Kinesis stream must hold individual CloudTrail events, _not_ notifications S3 bucket notifications. 29 | See [cloudtrail_to_kinesis](../cloudtrail_to_kinesis) to configure such a stream. 30 | 31 | * The Iceberg template creates a simple transformation Lambda, which lowercases top-level field names 32 | from the incoming data (for consistency with the Parquet version). If you change this Lambda (for 33 | example, to snake-case the names), you must also change the `GluePrimaryKey` parameter. 34 | 35 | * The Iceberg template does not configure table optimizations. As-of this writing, the only optimization 36 | that CloudFormation supports is compaction, and creating that optimization in the same template as the 37 | table causes CloudFormation to fail. However, this emplate does create (and output) a role that Glue can 38 | use to perform optimizations. 39 | 40 | * Destroying the CloudFormation stack does _not_ delete data stored in S3; you must do this manually. 41 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CDK-2/.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | *.d.ts 3 | node_modules 4 | 5 | # CDK asset staging directory 6 | .cdk.staging 7 | cdk.out 8 | 9 | # for example code we'll expect this to be regenerated 10 | package-lock.json 11 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CDK-2/.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CDK-2/README.md: -------------------------------------------------------------------------------- 1 | This variant of the project uses the AWS Cloud Development Kit (CDK) to create resources. 2 | 3 | It differs from the [other CDK] variant in that it creates an application queue policy, 4 | using constants exported from the construct. 5 | 6 | Before you can run this you need to install Node and NPM. The instructions to do this for 7 | the most recent versions can be found [here](https://nodejs.org/en/download/). 8 | 9 | Then, install CDK (for more information, read the [Getting Started](https://docs.aws.amazon.com/cdk/latest/guide/getting_started.html) tutorial): 10 | 11 | ``` 12 | npm install -g aws-cdk 13 | ``` 14 | 15 | Then, from within this directory, run the following commands: 16 | 17 | ``` 18 | npm install 19 | 20 | cdk synth > template.yml 21 | ``` 22 | 23 | As with the CFNDSL example, this creates a CloudFormation template, and you can use the Console 24 | or CLI to manage the stack. Alternatively, you can let CDK deploy the stack for you (it will use 25 | the name `UsersAndGroupsStack`): 26 | 27 | ``` 28 | cdk deploy 29 | ``` 30 | 31 | And also destroy it: 32 | 33 | ``` 34 | cdk destroy 35 | ``` 36 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CDK-2/bin/app.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import 'source-map-support/register'; 3 | import cdk = require('@aws-cdk/core'); 4 | 5 | import { MultiQueueStack } from '../lib/stack'; 6 | 7 | const app = new cdk.App(); 8 | new MultiQueueStack(app, 'MultiQueueStack'); 9 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CDK-2/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node bin/app.ts" 3 | } 4 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CDK-2/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cdk", 3 | "version": "0.1.0", 4 | "bin": { 5 | "cdk": "bin/cdk.js" 6 | }, 7 | "scripts": { 8 | "build": "tsc", 9 | "watch": "tsc -w", 10 | "cdk": "cdk" 11 | }, 12 | "devDependencies": { 13 | "typescript": "^3.3.3333", 14 | "ts-node": "^8.1.0", 15 | "aws-cdk": "^1.1.0" 16 | }, 17 | "dependencies": { 18 | "@aws-cdk/aws-iam": "^1.3.0", 19 | "@aws-cdk/aws-sqs": "^1.19.0", 20 | "@aws-cdk/core": "^1.1.0", 21 | "source-map-support": "^0.5.9" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CDK-2/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target":"ES2018", 4 | "module": "commonjs", 5 | "lib": ["es2016", "es2017.object", "es2017.string"], 6 | "declaration": true, 7 | "strict": true, 8 | "noImplicitAny": true, 9 | "strictNullChecks": true, 10 | "noImplicitThis": true, 11 | "alwaysStrict": true, 12 | "noUnusedLocals": false, 13 | "noUnusedParameters": false, 14 | "noImplicitReturns": true, 15 | "noFallthroughCasesInSwitch": false, 16 | "inlineSourceMap": true, 17 | "inlineSources": true, 18 | "experimentalDecorators": true, 19 | "strictPropertyInitialization":false 20 | }, 21 | "exclude": ["cdk.out"] 22 | } 23 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CDK/.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | *.d.ts 3 | node_modules 4 | 5 | # CDK asset staging directory 6 | .cdk.staging 7 | cdk.out 8 | 9 | # for example code we'll expect this to be regenerated 10 | package-lock.json 11 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CDK/.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CDK/README.md: -------------------------------------------------------------------------------- 1 | This variant of the project uses the AWS Cloud Development Kit (CDK) to create resources. 2 | 3 | Before you can run this you need to install Node and NPM. The instructions to do this for 4 | the most recent versions can be found [here](https://nodejs.org/en/download/). 5 | 6 | Then, install CDK (for more information, read the [Getting Started](https://docs.aws.amazon.com/cdk/latest/guide/getting_started.html) tutorial): 7 | 8 | ``` 9 | npm install -g aws-cdk 10 | ``` 11 | 12 | Then, from within this directory, run the following commands: 13 | 14 | ``` 15 | npm install 16 | 17 | cdk synth > template.yml 18 | ``` 19 | 20 | As with the CFNDSL example, this creates a CloudFormation template, and you can use the Console 21 | or CLI to manage the stack. Alternatively, you can let CDK deploy the stack for you (it will use 22 | the name `UsersAndGroupsStack`): 23 | 24 | ``` 25 | cdk deploy 26 | ``` 27 | 28 | And also destroy it: 29 | 30 | ``` 31 | cdk destroy 32 | ``` 33 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CDK/bin/app.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import 'source-map-support/register'; 3 | import cdk = require('@aws-cdk/core'); 4 | 5 | import { MultiQueueStack } from '../lib/stack'; 6 | 7 | const app = new cdk.App(); 8 | new MultiQueueStack(app, 'MultiQueueStack'); 9 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CDK/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node bin/app.ts" 3 | } 4 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CDK/lib/stack.ts: -------------------------------------------------------------------------------- 1 | import cdk = require('@aws-cdk/core'); 2 | import iam = require('@aws-cdk/aws-iam'); 3 | import sqs = require('@aws-cdk/aws-sqs'); 4 | 5 | import { StandardQueue } from './standard_queue'; 6 | 7 | export class MultiQueueStack extends cdk.Stack { 8 | constructor(scope: cdk.Construct, id: string, props?: cdk.StackProps) { 9 | super(scope, id, props); 10 | 11 | // good habit: create "self" as reference to "this" so it can be used inside functions 12 | const self = this; 13 | 14 | const q1 = new StandardQueue(self, "Foo", { 15 | queueName: "Foo" 16 | }) 17 | 18 | const q2 = new StandardQueue(self, "Bar", { 19 | queueName: "Bar" 20 | }) 21 | 22 | const q3 = new StandardQueue(self, "Baz", { 23 | queueName: "Baz" 24 | }) 25 | 26 | const appRole = new iam.Role(self, "ApplicationRole", { 27 | roleName: self.stackName + "-ApplicationRole", 28 | assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'), 29 | managedPolicies: [ q1.producerPolicy, q2.producerPolicy, q3.producerPolicy ] 30 | }) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CDK/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cdk", 3 | "version": "0.1.0", 4 | "bin": { 5 | "cdk": "bin/cdk.js" 6 | }, 7 | "scripts": { 8 | "build": "tsc", 9 | "watch": "tsc -w", 10 | "cdk": "cdk" 11 | }, 12 | "devDependencies": { 13 | "typescript": "^3.3.3333", 14 | "ts-node": "^8.1.0", 15 | "aws-cdk": "^1.1.0" 16 | }, 17 | "dependencies": { 18 | "@aws-cdk/aws-iam": "^1.3.0", 19 | "@aws-cdk/aws-sqs": "^1.19.0", 20 | "@aws-cdk/core": "^1.1.0", 21 | "source-map-support": "^0.5.9" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CDK/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target":"ES2018", 4 | "module": "commonjs", 5 | "lib": ["es2016", "es2017.object", "es2017.string"], 6 | "declaration": true, 7 | "strict": true, 8 | "noImplicitAny": true, 9 | "strictNullChecks": true, 10 | "noImplicitThis": true, 11 | "alwaysStrict": true, 12 | "noUnusedLocals": false, 13 | "noUnusedParameters": false, 14 | "noImplicitReturns": true, 15 | "noFallthroughCasesInSwitch": false, 16 | "inlineSourceMap": true, 17 | "inlineSources": true, 18 | "experimentalDecorators": true, 19 | "strictPropertyInitialization":false 20 | }, 21 | "exclude": ["cdk.out"] 22 | } 23 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/CloudFormation/README.md: -------------------------------------------------------------------------------- 1 | This variant of the project uses CloudFormation by itself to create resources. It provides a 2 | baseline against which the other tools can be compared. 3 | 4 | I prefer to use the AWS Console for CloudFormation, because (1) it's easier to enter parameters, 5 | and (2) when you're updating a stack you can see the changes that will happen before clicking OK. 6 | However, if you want to use the CLI, here are the commands (replace `UserCreationExample` with 7 | whatever stack name you want to use): 8 | 9 | ``` 10 | aws cloudformation create-stack --stack-name UserCreationExample --capabilities CAPABILITY_NAMED_IAM --template-body file://template.yml 11 | ``` 12 | 13 | And to delete the stack: 14 | 15 | ``` 16 | aws cloudformation delete-stack --stack-name UserCreationExample 17 | ``` 18 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/README.md: -------------------------------------------------------------------------------- 1 | This project is another comparison of infrastructure management tools for AWS. It 2 | demonstrates the use of Terraform modules and CDK constructs to produce multiple 3 | SQS queues with a consistent configuration. 4 | 5 | There are four implementations: 6 | 7 | * [CloudFormation](CloudFormation/README.md), as a baseline. 8 | * [CDK](CDK/README.md), showing the use of user-defined constructs. 9 | * [CDK-2](CDK-2/README.md), an extension that creates a combined queue policy. 10 | * [Terraform](Terraform/README.md), which uses an in-project module to create the queue 11 | and related objects. 12 | 13 | 14 | ## General Notes 15 | 16 | To run, you must have the ability to create SQS queues and IAM policies.I strongly 17 | recommend running in a "sandbox" account so that you won't interfere with operations. 18 | 19 | Each variant lives in its own sub-directory, so that it can create artifacts without 20 | interfering with the others (this is particularly important for Terraform and CDK). 21 | The README for each variant describes how to run it. 22 | 23 | 24 | ## Resources Created 25 | 26 | Each variant of this project creates the following resources: 27 | 28 | * Threee "primary" queues, named `Foo`, `Bar`, and `Baz`. 29 | * Thread "dead letter" queues, named `Foo-DLQ`, `Bar-DLQ`, and `Baz-DLQ`. 30 | * A consumer and producer policy for each queue (named `SQS-Foo-Consumer`, 31 | `SQS-Foo-Producer`, and so on). 32 | * An "application role" that references the producer policies for all queues. 33 | * (CDK-2 only) A consumer and producer policy for the queues as a group, named 34 | `SQS-STACKNAME-Consumer` and `SQS-STACKNAME-Producer`. 35 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/Terraform/.gitignore: -------------------------------------------------------------------------------- 1 | # normally you would check this in; we don't because it's an example 2 | .terraform.lock.hcl 3 | 4 | # normally you would store state information externally; checking-in risks exposing secrets 5 | terraform.tfstate 6 | terraform.tfstate.backup 7 | 8 | # this is the local storage for Terraform modules and config; never check it in 9 | .terraform/ 10 | 11 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/Terraform/README.md: -------------------------------------------------------------------------------- 1 | This variant of the project uses Terraform to create resources. 2 | 3 | To start, you must [download Terraform for your system](https://www.terraform.io/downloads.html). 4 | If you run Linux, it makes more sense to unpack into `$HOME/bin` rather than install in a system 5 | directory: you'll find that you update the deployment frequently. 6 | 7 | 8 | Once you've done that, create the resources is a two-step process (both of which must be run from 9 | the child directory): 10 | 11 | ``` 12 | terraform init 13 | ``` 14 | 15 | Will download the AWS provider, and 16 | 17 | ``` 18 | terraform apply 19 | ``` 20 | 21 | Will performs the infrastructure changes. You have to manually approve this step. 22 | 23 | It may turn into a three-step process: sometimes the users or groups aren't ready to have roles applied, 24 | and you see an error about the resource not existing. Running `terraform apply` a second time resolves 25 | this. 26 | 27 | To destroy the resources: 28 | 29 | ``` 30 | terraform destroy 31 | ``` 32 | 33 | Note that my `.gitignore` file explicitly excludes the `tfstate` files. This is _not_ something that 34 | you'd do in a real deployment; those files should be checked-in alongside the template. 35 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/Terraform/main.tf: -------------------------------------------------------------------------------- 1 | ## 2 | ## Creates several SQS queues with related resources, using a module, 3 | ## then combines their policies into a role. 4 | ## 5 | 6 | terraform { 7 | required_providers { 8 | aws = { 9 | source = "hashicorp/aws" 10 | version = ">= 4.0.0" 11 | } 12 | } 13 | } 14 | 15 | data "aws_region" "current" {} 16 | 17 | 18 | module "foo_queue" { 19 | source = "./modules/create_sqs" 20 | queue_name = "Foo" 21 | } 22 | 23 | 24 | module "bar_queue" { 25 | source = "./modules/create_sqs" 26 | queue_name = "Bar" 27 | } 28 | 29 | 30 | module "baz_queue" { 31 | source = "./modules/create_sqs" 32 | queue_name = "Baz" 33 | retry_count = 5 34 | } 35 | 36 | 37 | resource "aws_iam_role" "application_role" { 38 | name = "Example-${data.aws_region.current.name}" 39 | description = "Created by Terraform SQS module example" 40 | 41 | assume_role_policy = jsonencode({ 42 | "Version" = "2012-10-17", 43 | "Statement" = [ 44 | { 45 | "Effect" = "Allow" 46 | "Action" = "sts:AssumeRole" 47 | "Principal" = { "Service" = "ec2.amazonaws.com" } 48 | } 49 | ] 50 | }) 51 | } 52 | 53 | resource "aws_iam_role_policy_attachment" "application_role_foo_producer" { 54 | role = aws_iam_role.application_role.name 55 | policy_arn = module.foo_queue.producer_policy_arn 56 | } 57 | 58 | resource "aws_iam_role_policy_attachment" "application_role_bar_producer" { 59 | role = aws_iam_role.application_role.name 60 | policy_arn = module.bar_queue.producer_policy_arn 61 | } 62 | 63 | resource "aws_iam_role_policy_attachment" "application_role_baz_producer" { 64 | role = aws_iam_role.application_role.name 65 | policy_arn = module.baz_queue.producer_policy_arn 66 | } 67 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/Terraform/modules/create_sqs/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chariotsolutions/aws-examples/cd636134fe14eb7314114d97290b5bd6124c25dc/infrastructure-tools-comparison-2/Terraform/modules/create_sqs/README.md -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/Terraform/modules/create_sqs/main.tf: -------------------------------------------------------------------------------- 1 | ## 2 | ## A module that creates an SQS queue, along with a related dead-letter queue, 3 | ## and policies to read/write both 4 | ## 5 | 6 | terraform { 7 | required_providers { 8 | aws = { 9 | source = "hashicorp/aws" 10 | version = ">= 4.0.0" 11 | } 12 | } 13 | } 14 | 15 | data "aws_region" "current" {} 16 | 17 | 18 | resource "aws_sqs_queue" "base_queue" { 19 | name = var.queue_name 20 | message_retention_seconds = var.retention_period 21 | visibility_timeout_seconds = var.visibility_timeout 22 | redrive_policy = jsonencode({ 23 | "deadLetterTargetArn" = aws_sqs_queue.deadletter_queue.arn, 24 | "maxReceiveCount" = var.retry_count 25 | }) 26 | } 27 | 28 | 29 | resource "aws_sqs_queue" "deadletter_queue" { 30 | name = "${var.queue_name}-DLQ" 31 | message_retention_seconds = var.retention_period 32 | visibility_timeout_seconds = var.visibility_timeout 33 | } 34 | 35 | 36 | resource "aws_iam_policy" "consumer_policy" { 37 | name = "SQS-${var.queue_name}-${data.aws_region.current.name}-Consumer" 38 | description = "Attach this policy to consumers of SQS queue ${var.queue_name}" 39 | policy = data.aws_iam_policy_document.consumer_policy.json 40 | } 41 | 42 | 43 | data "aws_iam_policy_document" "consumer_policy" { 44 | statement { 45 | actions = [ 46 | "sqs:ChangeMessageVisibility", 47 | "sqs:ChangeMessageVisibilityBatch", 48 | "sqs:DeleteMessage", 49 | "sqs:DeleteMessageBatch", 50 | "sqs:GetQueueAttributes", 51 | "sqs:GetQueueUrl", 52 | "sqs:ReceiveMessage" 53 | ] 54 | resources = [ 55 | aws_sqs_queue.base_queue.arn, 56 | aws_sqs_queue.deadletter_queue.arn 57 | ] 58 | } 59 | } 60 | 61 | 62 | resource "aws_iam_policy" "producer_policy" { 63 | name = "SQS-${var.queue_name}-${data.aws_region.current.name}-Producer" 64 | description = "Attach this policy to producers for SQS queue ${var.queue_name}" 65 | policy = data.aws_iam_policy_document.producer_policy.json 66 | } 67 | 68 | 69 | data "aws_iam_policy_document" "producer_policy" { 70 | statement { 71 | actions = [ 72 | "sqs:GetQueueAttributes", 73 | "sqs:GetQueueUrl", 74 | "sqs:SendMessage", 75 | "sqs:SendMessageBatch" 76 | ] 77 | resources = [ 78 | aws_sqs_queue.base_queue.arn 79 | ] 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/Terraform/modules/create_sqs/outputs.tf: -------------------------------------------------------------------------------- 1 | output "base_queue_url" { 2 | value = aws_sqs_queue.base_queue.id 3 | } 4 | 5 | 6 | output "deadletter_queue_url" { 7 | value = aws_sqs_queue.deadletter_queue.id 8 | } 9 | 10 | 11 | output "consumer_policy_arn" { 12 | value = aws_iam_policy.consumer_policy.arn 13 | } 14 | 15 | 16 | output "producer_policy_arn" { 17 | value = aws_iam_policy.producer_policy.arn 18 | } 19 | 20 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison-2/Terraform/modules/create_sqs/variables.tf: -------------------------------------------------------------------------------- 1 | variable "queue_name" { 2 | description = "The name of the queue. Used as a prefix for related resource names." 3 | type = string 4 | } 5 | 6 | 7 | variable "retention_period" { 8 | description = "Time (in seconds) that messages will remain in queue before being purged" 9 | type = number 10 | default = 86400 11 | } 12 | 13 | 14 | variable "visibility_timeout" { 15 | description = "Time (in seconds) that messages will be unavailable after being read" 16 | type = number 17 | default = 60 18 | } 19 | 20 | 21 | variable "retry_count" { 22 | description = "The number of times that a message will be delivered before being moved to dead-letter queue" 23 | type = number 24 | default = 3 25 | } 26 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison/CDK/.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | *.d.ts 3 | node_modules 4 | 5 | # CDK asset staging directory 6 | .cdk.staging 7 | cdk.out 8 | 9 | # for example code we'll expect this to be regenerated 10 | package-lock.json 11 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison/CDK/.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison/CDK/README.md: -------------------------------------------------------------------------------- 1 | This variant of the project uses the AWS Cloud Development Kit (CDK) to create resources. 2 | 3 | Before you can run this you need to install Node and NPM. The instructions to do this for 4 | the most recent versions can be found [here](https://nodejs.org/en/download/). 5 | 6 | Then, install CDK (for more information, read the [Getting Started](https://docs.aws.amazon.com/cdk/latest/guide/getting_started.html) tutorial): 7 | 8 | ``` 9 | npm install -g aws-cdk 10 | ``` 11 | 12 | Then, from within this directory, run the following commands: 13 | 14 | ``` 15 | npm install 16 | 17 | cdk synth > template.yml 18 | ``` 19 | 20 | As with the CFNDSL example, this creates a CloudFormation template, and you can use the Console 21 | or CLI to manage the stack. Alternatively, you can let CDK deploy the stack for you (it will use 22 | the name `UsersAndGroupsStack`): 23 | 24 | ``` 25 | cdk deploy 26 | ``` 27 | 28 | And also destroy it: 29 | 30 | ``` 31 | cdk destroy 32 | ``` 33 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison/CDK/bin/users_and_groups.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import 'source-map-support/register'; 3 | import cdk = require('@aws-cdk/core'); 4 | 5 | import { UsersAndGroupsStack } from '../lib/stack'; 6 | 7 | const app = new cdk.App(); 8 | new UsersAndGroupsStack(app, 'UsersAndGroupsStack'); 9 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison/CDK/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node bin/users_and_groups.ts" 3 | } 4 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison/CDK/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cdk", 3 | "version": "0.1.0", 4 | "bin": { 5 | "cdk": "bin/cdk.js" 6 | }, 7 | "scripts": { 8 | "build": "tsc", 9 | "watch": "tsc -w", 10 | "cdk": "cdk" 11 | }, 12 | "devDependencies": { 13 | "typescript": "^3.3.3333", 14 | "ts-node": "^8.1.0", 15 | "aws-cdk": "^1.1.0" 16 | }, 17 | "dependencies": { 18 | "@aws-cdk/aws-iam": "^1.3.0", 19 | "@aws-cdk/core": "^1.1.0", 20 | "source-map-support": "^0.5.9" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison/CDK/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target":"ES2018", 4 | "module": "commonjs", 5 | "lib": ["es2016", "es2017.object", "es2017.string"], 6 | "declaration": true, 7 | "strict": true, 8 | "noImplicitAny": true, 9 | "strictNullChecks": true, 10 | "noImplicitThis": true, 11 | "alwaysStrict": true, 12 | "noUnusedLocals": false, 13 | "noUnusedParameters": false, 14 | "noImplicitReturns": true, 15 | "noFallthroughCasesInSwitch": false, 16 | "inlineSourceMap": true, 17 | "inlineSources": true, 18 | "experimentalDecorators": true, 19 | "strictPropertyInitialization":false 20 | }, 21 | "exclude": ["cdk.out"] 22 | } 23 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison/CFNDSL/.gitignore: -------------------------------------------------------------------------------- 1 | *.json 2 | *.yml 3 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison/CFNDSL/README.md: -------------------------------------------------------------------------------- 1 | This variant of the project uses the CFNDSL Ruby gem to create resources. I include this as an 2 | example of pre-CDK programmatic template generation; at this time I would not recommend using 3 | it unless you are already a Ruby shop. 4 | 5 | You first need to install CFNDSL per the instructions [here](https://www.rubydoc.info/gems/cfndsl) 6 | (and you may also need to install a compatible Ruby version as well). 7 | 8 | Then, generate the template using this command: 9 | 10 | ``` 11 | cfndsl --disable-binding -f yaml script.rb > template.yml 12 | ``` 13 | 14 | * `--disable-binding` suppresses a warning about locally-defined configuration. 15 | In normal use, you would provide the lists of users, groups, &c via external 16 | configuration files. 17 | * `-f yaml` specifies that the output should be [YAML](https://yaml.org/). 18 | Omit to generate JSON (in which case you probably want to add `-p`, for 19 | pretty-printing). 20 | 21 | Once you have the template, you can use the AWS Console or CloudFormation CLI to create and 22 | destroy the stack (CFNDSL doesn't let you deploy/undeploy directly). 23 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison/CFNDSL/script.rb: -------------------------------------------------------------------------------- 1 | ## 2 | ## Generates a template that will create a set of users, assign them to groups, and grant 3 | ## those groups permission to assume roles in associated accounts. 4 | ## 5 | 6 | require 'json' 7 | 8 | users = [ "user1", "user2", "user3" ] 9 | 10 | groups = [ "group1", "group2" ] 11 | 12 | group_members = { 13 | "user1" => [ "group1", "group2" ], 14 | "user2" => [ "group1" ], 15 | "user3" => [ "group2" ] 16 | } 17 | 18 | account_lookup = { 19 | "dev" => "123456789012", 20 | "prod" => "234567890123" 21 | } 22 | 23 | group_permissions = { 24 | "group1" => [ 25 | [ "dev", "FooAppDeveloper" ], 26 | [ "prod", "FooAppReadOnly" ] 27 | ], 28 | "group2" => [ 29 | [ "dev", "BarAppDeveloper" ], 30 | [ "prod", "BarAppReadOnly" ] 31 | ] 32 | } 33 | 34 | 35 | CloudFormation do 36 | Description "Manages the account's users" 37 | 38 | users.each do |user| 39 | IAM_User("#{user}") do 40 | UserName user 41 | ManagedPolicyArns [ FnSub("arn:aws:iam::${AWS::AccountId}:policy/BasicUserPolicy") ] 42 | Groups group_members[user].map { |group| Ref("#{group}") } 43 | end 44 | end 45 | 46 | groups.each do |group| 47 | IAM_Group("#{group}") do 48 | GroupName group 49 | end 50 | 51 | policy_document = { 52 | "Version" => "2012-10-17", 53 | "Statement" => [{ 54 | "Effect" => "Allow", 55 | "Action" => [ "sts:AssumeRole" ], 56 | "Resource" => group_permissions[group].map { |acct_role| 57 | account_id = account_lookup[acct_role[0]] 58 | role_name = acct_role[1] 59 | "arn:aws:iam::#{account_id}:role/#{role_name}" 60 | } 61 | }] 62 | }.to_json 63 | 64 | IAM_Policy("#{group}Policy") do 65 | PolicyName "#{group}-AssumeRolePolicy" 66 | PolicyDocument policy_document 67 | Groups [ group ] 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison/CloudFormation/README.md: -------------------------------------------------------------------------------- 1 | This variant of the project uses CloudFormation by itself to create resources. It provides a 2 | baseline against which the other tools can be compared. 3 | 4 | I prefer to use the AWS Console for CloudFormation, because (1) it's easier to enter parameters, 5 | and (2) when you're updating a stack you can see the changes that will happen before clicking OK. 6 | However, if you want to use the CLI, here are the commands (replace `UserCreationExample` with 7 | whatever stack name you want to use): 8 | 9 | ``` 10 | aws cloudformation create-stack --stack-name UserCreationExample --capabilities CAPABILITY_NAMED_IAM --template-body file://template.yml 11 | ``` 12 | 13 | And to delete the stack: 14 | 15 | ``` 16 | aws cloudformation delete-stack --stack-name UserCreationExample 17 | ``` 18 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison/Terraform/.gitignore: -------------------------------------------------------------------------------- 1 | # normally you would check this in; we don't because it's an example 2 | .terraform.lock.hcl 3 | 4 | # normally you would store state information externally; checking-in risks exposing secrets 5 | terraform.tfstate 6 | terraform.tfstate.backup 7 | 8 | # this is the local storage for Terraform modules and config; never check it in 9 | .terraform/ 10 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison/Terraform/README.md: -------------------------------------------------------------------------------- 1 | This variant of the project uses Terraform to create resources. 2 | 3 | To start, you must [download Terraform for your system](https://www.terraform.io/downloads.html). 4 | If you run Linux, it makes more sense to unpack into `$HOME/bin` rather than install in a system 5 | directory: you'll find that you update the deployment frequently. 6 | 7 | 8 | Once you've done that, create the resources is a two-step process (both of which must be run from 9 | the child directory): 10 | 11 | ``` 12 | terraform init 13 | ``` 14 | 15 | Will download the AWS provider, and 16 | 17 | ``` 18 | terraform apply 19 | ``` 20 | 21 | Will performs the infrastructure changes. You have to manually approve this step. 22 | 23 | It may turn into a three-step process: sometimes the users or groups aren't ready to have roles applied, 24 | and you see an error about the resource not existing. Running `terraform apply` a second time resolves 25 | this. 26 | 27 | To destroy the resources: 28 | 29 | ``` 30 | terraform destroy 31 | ``` 32 | 33 | Note that my `.gitignore` file explicitly excludes the `tfstate` files. This is _not_ something that 34 | you'd do in a real deployment; those files should be checked-in alongside the template. 35 | -------------------------------------------------------------------------------- /infrastructure-tools-comparison/Terraform/main.tf: -------------------------------------------------------------------------------- 1 | ## 2 | ## Creates a set of users, assigns them to groups, and grants those groups 3 | ## permission to assume roles in associated accounts. 4 | ## 5 | 6 | 7 | terraform { 8 | required_providers { 9 | aws = { 10 | source = "hashicorp/aws" 11 | version = ">= 4.0.0" 12 | } 13 | } 14 | } 15 | 16 | 17 | ## 18 | ## For this example, all configuration is hardcoded; could be replaced 19 | ## with external variable definitions 20 | ## 21 | 22 | 23 | data "aws_caller_identity" "current" {} 24 | 25 | variable "users" { 26 | type = list 27 | default = [ "user1", "user2", "user3" ] 28 | } 29 | 30 | variable "groups" { 31 | type = list 32 | default = [ "group1", "group2" ] 33 | } 34 | 35 | variable "group_members" { 36 | type = map(list(string)) 37 | default = { 38 | "user1" = [ "group1", "group2" ], 39 | "user2" = [ "group1" ], 40 | "user3" = [ "group2" ] 41 | } 42 | } 43 | 44 | variable "account_id_lookup" { 45 | type = map 46 | default = { 47 | "dev" = "123456789012", 48 | "prod" = "234567890123" 49 | } 50 | } 51 | 52 | variable "group_permissions" { 53 | type = map(list(list(string))) 54 | default = { 55 | "group1" = [ 56 | [ "dev", "FooAppDeveloper" ], 57 | [ "prod", "FooAppReadOnly" ] 58 | ], 59 | "group2" = [ 60 | [ "dev", "BarAppDeveloper" ], 61 | [ "prod", "BarAppReadOnly" ] 62 | ] 63 | } 64 | } 65 | 66 | ## 67 | ## User creation 68 | ## 69 | 70 | resource "aws_iam_user" "users" { 71 | for_each = toset(var.users) 72 | name = each.key 73 | force_destroy = true 74 | } 75 | 76 | resource "aws_iam_user_policy_attachment" "base_user_policy_attachment" { 77 | for_each = toset(var.users) 78 | user = each.key 79 | policy_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/BasicUserPolicy" 80 | } 81 | 82 | ## 83 | ## Group creation 84 | ## 85 | 86 | resource "aws_iam_group" "groups" { 87 | for_each = toset(var.groups) 88 | name = each.key 89 | } 90 | 91 | ## 92 | ## Group membership 93 | ## 94 | 95 | resource "aws_iam_user_group_membership" "group-membership" { 96 | for_each = var.group_members 97 | user = each.key 98 | groups = each.value 99 | } 100 | 101 | ## 102 | ## Permission assignment 103 | ## 104 | 105 | data "aws_iam_policy_document" "group-policies" { 106 | for_each = var.group_permissions 107 | statement { 108 | sid = "AssumeRole" 109 | actions = [ "sts:AssumeRole" ] 110 | resources = [ 111 | for acct_role in each.value : 112 | "arn:aws:iam::${var.account_id_lookup[acct_role[0]]}:role/${acct_role[1]}" 113 | ] 114 | } 115 | } 116 | 117 | resource "aws_iam_group_policy" "group-policies" { 118 | for_each = toset(var.groups) 119 | name = "group-policies-${each.key}" 120 | group = each.key 121 | policy = data.aws_iam_policy_document.group-policies[each.key].json 122 | } 123 | -------------------------------------------------------------------------------- /lambda_build_in_docker/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | *.zip 3 | -------------------------------------------------------------------------------- /lambda_build_in_docker/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Chariot Solutions 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /lambda_build_in_docker/README.md: -------------------------------------------------------------------------------- 1 | An example of using a Docker container to build and deploy Lambdas. 2 | 3 | For more information, see [this blog post](). 4 | 5 | 6 | ## Building the Docker image 7 | 8 | The `docker` directory contains the configuration files -- Dockerfile and Makefile -- used for 9 | this example. 10 | 11 | ``` 12 | docker build -t build-environment:latest docker/ 13 | ``` 14 | 15 | 16 | ## Creating an empty Lambda 17 | 18 | The empty Lambda, along with its log stream and security group, are created using CloudFormation, from the 19 | template `cloudformation.yml`. Create using the AWS Console or 20 | [this utility script](https://github.com/kdgregory/aws-misc/blob/trunk/utils/cf-runner.py). 21 | 22 | The template uses the name "Example" for this Lambda. You can change the name via a template 23 | parameter, but beware that you'll have to use the changed name in the next step. 24 | 25 | 26 | ## Building and deploying the actual Lambda 27 | 28 | To just build the Lambda, storing its bundle in the current directory: 29 | 30 | ``` 31 | docker run --rm --user $(id -u):$(id -g) -v $(pwd):/build -e DEPLOY_DIR=/build build-environment:latest 32 | 33 | ``` 34 | 35 | If you want to both build and deploy, you must provide the container with AWS credentials. The easiest 36 | way to do that is to define and pass environment variables: 37 | 38 | ``` 39 | docker run --rm -v $(pwd):/build -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION build-environment:latest deploy 40 | ``` 41 | -------------------------------------------------------------------------------- /lambda_build_in_docker/cloudformation.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | Description: "Lambda that's deployed by a Makefile running within a Docker container" 3 | 4 | Parameters: 5 | 6 | LambdaName: 7 | Description: "Name for the Lambda function and associated resources" 8 | Type: "String" 9 | Default: "Example" 10 | 11 | Resources: 12 | 13 | LambdaLogGroup: 14 | Type: "AWS::Logs::LogGroup" 15 | DeletionPolicy: "Delete" 16 | Properties: 17 | LogGroupName: !Sub "/aws/lambda/${LambdaName}" 18 | RetentionInDays: 7 19 | 20 | 21 | LambdaRole: 22 | Type: "AWS::IAM::Role" 23 | Properties: 24 | Path: "/lambda/" 25 | RoleName: !Sub "${LambdaName}-ExecutionRole-${AWS::Region}" 26 | AssumeRolePolicyDocument: 27 | Version: "2012-10-17" 28 | Statement: 29 | Effect: "Allow" 30 | Principal: 31 | Service: "lambda.amazonaws.com" 32 | Action: "sts:AssumeRole" 33 | ManagedPolicyArns: 34 | - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" 35 | #Policies: 36 | 37 | 38 | LambdaFunction: 39 | Type: "AWS::Lambda::Function" 40 | Properties: 41 | FunctionName: !Ref LambdaName 42 | Description: "Example for deploying from within a Docker container" 43 | Role: !GetAtt LambdaRole.Arn 44 | Runtime: "python3.9" 45 | Handler: "index.lambda_handler" 46 | Code: 47 | ZipFile: | 48 | def lambda_handler(event, context): 49 | raise Exception("this is a dummy handler; please build and upload real handler") 50 | MemorySize: 512 51 | Timeout: 15 52 | -------------------------------------------------------------------------------- /lambda_build_in_docker/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazon/aws-lambda-python:3.9 2 | 3 | RUN yum install -y make zip 4 | 5 | RUN pip install awscli 6 | 7 | COPY Makefile / 8 | 9 | WORKDIR /build 10 | 11 | ENV DEPLOY_DIR=/tmp/deploy 12 | 13 | ENTRYPOINT ["/usr/bin/make", "--environment-overrides", "--directory=/build", "--makefile=/Makefile"] 14 | -------------------------------------------------------------------------------- /lambda_build_in_docker/docker/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: default deploy package test init clean 2 | 3 | LAMBDA_NAME ?= Example 4 | 5 | DEPLOY_DIR ?= /tmp/deploy 6 | ARTIFACT ?= example.zip 7 | 8 | SRC_DIR := /build/src 9 | LIB_DIR := /tmp/lib 10 | 11 | default: package 12 | 13 | deploy: package 14 | aws lambda update-function-code --function-name $(LAMBDA_NAME) --zip-file fileb://$(DEPLOY_DIR)/$(ARTIFACT) 15 | 16 | package: test 17 | mkdir -p ${DEPLOY_DIR} 18 | rm -f ${DEPLOY_DIR}/${ARTIFACT} 19 | cd $(SRC_DIR) ; zip -qr ${DEPLOY_DIR}/${ARTIFACT} * 20 | cd $(LIB_DIR) ; zip -qr ${DEPLOY_DIR}/${ARTIFACT} * 21 | 22 | test: init 23 | # run any unit tests here 24 | 25 | init: 26 | mkdir -p ${LIB_DIR} 27 | pip install -r /build/requirements.txt -t $(LIB_DIR) --upgrade 28 | 29 | clean: 30 | rm $(DEPLOY_DIR)/$(ARTIFACT) 31 | -------------------------------------------------------------------------------- /lambda_build_in_docker/requirements.txt: -------------------------------------------------------------------------------- 1 | psycopg2-binary 2 | -------------------------------------------------------------------------------- /lambda_build_in_docker/src/index.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | import logging 4 | import os 5 | 6 | import psycopg2 7 | 8 | def lambda_handler(event, context): 9 | print("I've been installed!") 10 | -------------------------------------------------------------------------------- /lambda_container_images/cloudformation/cloudformation-ecr.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | Description: "Creates an ECR repository that will hold a Lambda container" 3 | 4 | Parameters: 5 | 6 | ImageName: 7 | Description: "Name of the repository (this is the simple name for the image)" 8 | Type: "String" 9 | Default: "container-example" 10 | 11 | Resources: 12 | 13 | ECRRepository: 14 | Type: "AWS::ECR::Repository" 15 | Properties: 16 | RepositoryName: !Ref ImageName 17 | ImageTagMutability: MUTABLE 18 | RepositoryPolicyText: 19 | Version: "2008-10-17" 20 | Statement: 21 | - Sid: "AllowLambda" 22 | Effect: "Allow" 23 | Principal: 24 | Service: 25 | - "lambda.amazonaws.com" 26 | Action: 27 | - "ecr:BatchGetImage" 28 | - "ecr:DeleteRepositoryPolicy" 29 | - "ecr:GetDownloadUrlForLayer" 30 | - "ecr:GetRepositoryPolicy" 31 | - "ecr:SetRepositoryPolicy" 32 | -------------------------------------------------------------------------------- /lambda_container_images/cloudformation/cloudformation-lambda.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | Description: "Creates a Lambda function that's deployed as a container" 3 | 4 | Parameters: 5 | 6 | LambdaName: 7 | Description: "Name for the Lambda function and associated resources" 8 | Type: "String" 9 | Default: "ContainerExample" 10 | 11 | ImageName: 12 | Description: "The base name of an image stored in ECR" 13 | Type: "String" 14 | Default: "container-example" 15 | 16 | ImageTag: 17 | Description: "The tag of that image" 18 | Type: "String" 19 | Default: "latest" 20 | 21 | Resources: 22 | 23 | LambdaLogGroup: 24 | Type: "AWS::Logs::LogGroup" 25 | DeletionPolicy: "Delete" 26 | Properties: 27 | LogGroupName: !Sub "/aws/lambda/${LambdaName}" 28 | RetentionInDays: 7 29 | 30 | 31 | LambdaRole: 32 | Type: "AWS::IAM::Role" 33 | Properties: 34 | Path: "/lambda/" 35 | RoleName: !Sub "${LambdaName}-ExecutionRole-${AWS::Region}" 36 | AssumeRolePolicyDocument: 37 | Version: "2012-10-17" 38 | Statement: 39 | Effect: "Allow" 40 | Principal: 41 | Service: "lambda.amazonaws.com" 42 | Action: "sts:AssumeRole" 43 | ManagedPolicyArns: 44 | - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" 45 | #Policies: 46 | 47 | 48 | LambdaFunction: 49 | Type: "AWS::Lambda::Function" 50 | Properties: 51 | FunctionName: !Ref LambdaName 52 | Description: "Lambda function that uses a container image" 53 | Role: !GetAtt LambdaRole.Arn 54 | PackageType: Image 55 | Code: 56 | ImageUri: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${ImageName}:${ImageTag}" 57 | MemorySize: 256 58 | Timeout: 10 59 | Environment: 60 | Variables: 61 | EXAMPLE: "dummy envar" 62 | -------------------------------------------------------------------------------- /lambda_container_images/lambda/.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | lambda.zip 3 | -------------------------------------------------------------------------------- /lambda_container_images/lambda/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazon/aws-lambda-python:3.9 2 | 3 | COPY requirements.txt /tmp 4 | RUN pip install -r /tmp/requirements.txt 5 | 6 | COPY src/ /var/task 7 | 8 | CMD [ "handler.lambda_handler" ] 9 | -------------------------------------------------------------------------------- /lambda_container_images/lambda/requirements.txt: -------------------------------------------------------------------------------- 1 | psycopg2-binary 2 | -------------------------------------------------------------------------------- /lambda_container_images/lambda/src/handler.py: -------------------------------------------------------------------------------- 1 | import psycopg2 2 | 3 | def lambda_handler(event, context): 4 | print("lambda handler executing") 5 | -------------------------------------------------------------------------------- /lambda_container_images/terraform/.gitignore: -------------------------------------------------------------------------------- 1 | .terraform/ 2 | terraform.tfstate 3 | terraform.tfstate.backup 4 | -------------------------------------------------------------------------------- /lambda_container_images/terraform/ecr/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" {} 2 | 3 | 4 | resource "aws_ecr_repository" "lambda_container" { 5 | name = var.image_name 6 | image_tag_mutability = "MUTABLE" 7 | 8 | image_scanning_configuration { 9 | scan_on_push = true 10 | } 11 | } 12 | 13 | 14 | resource "aws_ecr_repository_policy" "lambda_container_policy" { 15 | repository = aws_ecr_repository.lambda_container.name 16 | policy = data.aws_iam_policy_document.lambda_container_policy.json 17 | } 18 | 19 | 20 | data "aws_iam_policy_document" "lambda_container_policy" { 21 | statement { 22 | sid = "AllowLambda" 23 | effect = "Allow" 24 | principals { 25 | type = "Service" 26 | identifiers = [ "lambda.amazonaws.com" ] 27 | } 28 | actions = [ 29 | "ecr:BatchGetImage", 30 | "ecr:DeleteRepositoryPolicy", 31 | "ecr:GetDownloadUrlForLayer", 32 | "ecr:GetRepositoryPolicy", 33 | "ecr:SetRepositoryPolicy" 34 | ] 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /lambda_container_images/terraform/ecr/outputs.tf: -------------------------------------------------------------------------------- 1 | output "repository_url" { 2 | description = "The URL (registry ID / image name) of this repository" 3 | value = aws_ecr_repository.lambda_container.repository_url 4 | } 5 | -------------------------------------------------------------------------------- /lambda_container_images/terraform/ecr/variables.tf: -------------------------------------------------------------------------------- 1 | variable "image_name" { 2 | description = "Name of the repository (this is the simple name for the image)" 3 | type = string 4 | default = "container-example" 5 | } 6 | -------------------------------------------------------------------------------- /lambda_container_images/terraform/lambda/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" {} 2 | 3 | data "aws_caller_identity" "current" {} 4 | data "aws_region" "current" {} 5 | 6 | locals { 7 | aws_account_id = data.aws_caller_identity.current.account_id 8 | aws_region = data.aws_region.current.name 9 | } 10 | 11 | # CloudWatch Log Group 12 | 13 | resource "aws_cloudwatch_log_group" "lambda_log_group" { 14 | name = "/aws/lambda/${var.lambda_name}" 15 | retention_in_days = 3 16 | } 17 | 18 | # Execution Role 19 | 20 | resource "aws_iam_role" "lambda_execution_role" { 21 | name = "${var.lambda_name}-${local.aws_region}-ExecutionRole" 22 | path = "/lambda/" 23 | assume_role_policy = data.aws_iam_policy_document.lambda_trust_policy.json 24 | } 25 | 26 | resource "aws_iam_role_policy_attachment" "lambda_execution_role_attach-1" { 27 | role = aws_iam_role.lambda_execution_role.name 28 | policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" 29 | } 30 | 31 | data "aws_iam_policy_document" "lambda_trust_policy" { 32 | statement { 33 | actions = ["sts:AssumeRole"] 34 | principals { 35 | type = "Service" 36 | identifiers = ["lambda.amazonaws.com"] 37 | } 38 | } 39 | } 40 | 41 | # The Lambda itself 42 | 43 | resource "aws_lambda_function" "lambda" { 44 | function_name = var.lambda_name 45 | description = "Lambda function that uses a container image" 46 | 47 | package_type = "Image" 48 | image_uri = "${local.aws_account_id}.dkr.ecr.${local.aws_region}.amazonaws.com/${var.image_name}:${var.image_tag}" 49 | 50 | role = aws_iam_role.lambda_execution_role.arn 51 | 52 | memory_size = 256 53 | timeout = 15 54 | } 55 | -------------------------------------------------------------------------------- /lambda_container_images/terraform/lambda/variables.tf: -------------------------------------------------------------------------------- 1 | variable "lambda_name" { 2 | description = "The name for the Lambda itself; also used for dependent resources" 3 | type = string 4 | default = "ContainerExample" 5 | } 6 | 7 | variable "image_name" { 8 | description = "The name of the Lambda's image in ECR" 9 | type = string 10 | default = "container-example" 11 | } 12 | 13 | variable "image_tag" { 14 | description = "The version of the image to load" 15 | type = string 16 | default = "latest" 17 | } 18 | 19 | -------------------------------------------------------------------------------- /lambda_four_ways/go/.gitignore: -------------------------------------------------------------------------------- 1 | bootstrap 2 | lambda.zip 3 | -------------------------------------------------------------------------------- /lambda_four_ways/go/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: default build upload deploy clean 2 | 3 | BUCKET ?= must_be_provided 4 | KEY ?= lambda_four_ways/go/lambda.zip 5 | LAMBDA_NAME ?= LambdaFourWays 6 | 7 | ARTIFACT := lambda.zip 8 | 9 | default: package 10 | 11 | deploy: lambda.zip 12 | aws lambda update-function-code --function-name $(LAMBDA_NAME) --zip-file fileb://$(ARTIFACT) 13 | 14 | upload: lambda.zip 15 | aws s3 cp ${ARTIFACT} s3://${BUCKET}/${KEY} 16 | 17 | lambda.zip: init lambda.go 18 | GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o bootstrap -tags lambda.norpc . 19 | zip $(ARTIFACT) bootstrap 20 | 21 | init: 22 | go get 23 | 24 | clean: 25 | ; 26 | -------------------------------------------------------------------------------- /lambda_four_ways/go/go.mod: -------------------------------------------------------------------------------- 1 | module example/dynamo_lambda 2 | 3 | go 1.22.1 4 | 5 | require ( 6 | github.com/aws/aws-lambda-go v1.47.0 7 | github.com/aws/aws-sdk-go-v2/config v1.27.11 8 | github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.13.13 9 | github.com/aws/aws-sdk-go-v2/service/dynamodb v1.31.1 10 | github.com/google/uuid v1.6.0 11 | ) 12 | 13 | require ( 14 | github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect 15 | github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect 16 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect 17 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect 18 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect 19 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect 20 | github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.20.4 // indirect 21 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect 22 | github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.6 // indirect 23 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect 24 | github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect 25 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect 26 | github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect 27 | github.com/aws/smithy-go v1.20.2 // indirect 28 | github.com/jmespath/go-jmespath v0.4.0 // indirect 29 | ) 30 | -------------------------------------------------------------------------------- /lambda_four_ways/go/types.go: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # 3 | # MIT No Attribution 4 | # 5 | # Copyright Chariot Solutions 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 8 | # software and associated documentation files (the "Software"), to deal in the Software 9 | # without restriction, including without limitation the rights to use, copy, modify, 10 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 11 | # permit persons to whom the Software is furnished to do so. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 14 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 15 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 16 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 17 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 18 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | # 20 | ################################################################################ 21 | 22 | package main 23 | 24 | import ( 25 | "fmt" 26 | ) 27 | 28 | 29 | // this is heavily abridged from the actual event, and contains only those fields that we use 30 | type Event struct { 31 | HttpMethod *string `json:"httpMethod"` 32 | RequestPath *string `json:"path"` 33 | PathParameters *map[string]string `json:"pathParameters"` 34 | Body *string `json:"body"` 35 | IsBase64Encoded *bool `json:"isBase64Encoded"` 36 | } 37 | 38 | 39 | type Response struct { 40 | StatusCode string `json:"statusCode"` 41 | Headers map[string]string `json:"headers"` 42 | IsBase64Encoded bool `json:"isBase64Encoded"` 43 | Body string `json:"body"` 44 | } 45 | 46 | 47 | func (e Event) String() string { 48 | return fmt.Sprintf("%v %v", *e.HttpMethod, *e.RequestPath) 49 | } 50 | -------------------------------------------------------------------------------- /lambda_four_ways/java/.gitignore: -------------------------------------------------------------------------------- 1 | # Maven 2 | target/ 3 | 4 | # IntelliJ 5 | .idea/ 6 | 7 | # Eclipse 8 | .classpath 9 | .project 10 | .settings/ 11 | -------------------------------------------------------------------------------- /lambda_four_ways/java/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: default build upload deploy clean 2 | 3 | BUCKET ?= must_be_provided 4 | KEY ?= lambda_four_ways/java/lambda.zip 5 | LAMBDA_NAME ?= LambdaFourWays 6 | 7 | BUILD_DIR := $(PWD)/target 8 | ARTIFACT := example-dynamdb-lambda-1.0-deployment.zip 9 | 10 | default: package 11 | 12 | deploy: build 13 | aws lambda update-function-code --function-name $(LAMBDA_NAME) --zip-file fileb://$(BUILD_DIR)/$(ARTIFACT) 14 | 15 | upload: build 16 | aws s3 cp ${BUILD_DIR}/${ARTIFACT} s3://${BUCKET}/${KEY} 17 | 18 | build: 19 | mvn package 20 | 21 | clean: 22 | rm -rf ${BUILD_DIR} 23 | -------------------------------------------------------------------------------- /lambda_four_ways/java/src/assembly/deployment.xml: -------------------------------------------------------------------------------- 1 | 3 | deployment 4 | false 5 | 6 | zip 7 | 8 | 9 | 10 | /lib 11 | true 12 | false 13 | runtime 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /lambda_four_ways/java/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{ISO8601} %-5level [%thread] %logger{24} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /lambda_four_ways/javascript/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | *.zip 3 | -------------------------------------------------------------------------------- /lambda_four_ways/javascript/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: default build upload deploy clean 2 | 3 | BUCKET ?= must_be_provided 4 | KEY ?= lambda_four_ways/javascript/lambda.zip 5 | LAMBDA_NAME ?= LambdaFourWays 6 | 7 | ARTIFACT := lambda.zip 8 | LIB_DIR := node_modules/ 9 | 10 | default: build 11 | 12 | deploy: build 13 | aws lambda update-function-code --function-name $(LAMBDA_NAME) --zip-file fileb://$(ARTIFACT) 14 | 15 | upload: build 16 | aws s3 cp ${ARTIFACT} s3://${BUCKET}/${KEY} 17 | 18 | build: init 19 | zip -q -r ${ARTIFACT} lambda.mjs node_modules 20 | 21 | init: 22 | npm install 23 | 24 | clean: 25 | rm -rf ${ARTIFACT} 26 | rm -rf ${LIB_DIR} 27 | -------------------------------------------------------------------------------- /lambda_four_ways/javascript/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dynamo-lambda", 3 | "version": "1.0.0", 4 | "description": "A simple Lambda to add, retrieve, and update DynamoDB items", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "@aws-sdk/util-dynamodb": "^3.624.0", 13 | "url-parse": "^1.5.10", 14 | "uuid": "^9.0.1" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /lambda_four_ways/python/.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | -------------------------------------------------------------------------------- /lambda_four_ways/python/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: default build upload deploy clean 2 | 3 | BUCKET ?= must_be_provided 4 | KEY ?= lambda_four_ways/python/lambda.zip 5 | LAMBDA_NAME ?= LambdaFourWays 6 | 7 | BUILD_DIR := $(PWD)/build 8 | ARTIFACT := lambda.zip 9 | 10 | default: package 11 | 12 | deploy: build 13 | aws lambda update-function-code --function-name $(LAMBDA_NAME) --zip-file fileb://$(BUILD_DIR)/$(ARTIFACT) 14 | 15 | upload: build 16 | aws s3 cp ${BUILD_DIR}/${ARTIFACT} s3://${BUCKET}/${KEY} 17 | 18 | build: init 19 | zip ${BUILD_DIR}/${ARTIFACT} lambda.py 20 | 21 | init: 22 | mkdir -p ${BUILD_DIR} 23 | 24 | clean: 25 | rm -rf ${BUILD_DIR} 26 | -------------------------------------------------------------------------------- /rds-flyway-migrations/README.md: -------------------------------------------------------------------------------- 1 | # RDS Flyway Migrations Sample 2 | 3 | This sample project creates an RDS instance within the private subnet of a VPC, and installs a CodeBuild project to run Flyway database migrations to provision the database. 4 | 5 | Setup steps: 6 | 7 | 1. Install AWS CLI 8 | 2. Access an AWS account with rights to create an RDS instance, VPC, and create and run CodeBuilds. 9 | 3. Run the deploy.sh script - two positional parameters are 10 | - the name of the stack to create 11 | - the region to create your stack within 12 | 13 | - Example: 14 | ```bash 15 | ./deploy.sh rds-flyway-migrations us-east-2 16 | ``` 17 | 4. Run the project created within CodeBuild to perform the migrations 18 | 5. Observe the logs to see that the environment variables for the username, database name and password are not exposed to the logs 19 | 6. You may also use the RDS control panel to perform queries against the database 20 | 21 | 22 | -------------------------------------------------------------------------------- /rds-flyway-migrations/deploy.sh: -------------------------------------------------------------------------------- 1 | # Use with: ./deploy.sh stackname region 2 | aws cloudformation deploy --stack-name $1 --region $2 --capabilities CAPABILITY_NAMED_IAM --template-file cloudformation.yml 3 | -------------------------------------------------------------------------------- /rds-flyway-migrations/flyway/migrations/V2023.02.08.001__config-db-options.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTENSION IF NOT EXISTS pgcrypto; -------------------------------------------------------------------------------- /rds-flyway-migrations/flyway/migrations/V2023.02.08.002__create-user-table.sql: -------------------------------------------------------------------------------- 1 | /* New table for application users */ 2 | create table user_accounts ( 3 | id uuid primary key default gen_random_uuid(), 4 | cognito_id varchar not null, 5 | email_address varchar not null, 6 | first_name varchar not null, 7 | last_name varchar not null, 8 | create_date date not null, 9 | unique(cognito_id) 10 | ); 11 | 12 | -------------------------------------------------------------------------------- /rds-flyway-migrations/flyway/migrations/V2023.02.08.003__create-role-tables.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS app_roles ( 2 | role_id uuid primary key default gen_random_uuid(), 3 | role_name varchar not null 4 | ); 5 | 6 | CREATE TABLE IF NOT EXISTS user_app_roles ( 7 | user_id uuid references user_accounts, 8 | role_id uuid references app_roles, 9 | active boolean not null default true, 10 | primary key (user_id, role_id) 11 | ); 12 | 13 | insert into app_roles(role_name) values ('admin'); 14 | insert into app_roles(role_name) values ('user'); 15 | 16 | select * from app_roles; 17 | 18 | -------------------------------------------------------------------------------- /sandbox-policies/BasicUserPolicy.iam: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Action": [ 7 | "iam:ChangePassword", 8 | "iam:CreateAccessKey", 9 | "iam:CreateVirtualMFADevice", 10 | "iam:DeactivateMFADevice", 11 | "iam:DeleteAccessKey", 12 | "iam:DeleteVirtualMFADevice", 13 | "iam:EnableMFADevice", 14 | "iam:GetAccessKeyLastUsed", 15 | "iam:GetLoginProfile", 16 | "iam:GetSSHPublicKey", 17 | "iam:GetUser", 18 | "iam:GetUserPolicy", 19 | "iam:ListAccessKeys", 20 | "iam:ListAttachedUserPolicies", 21 | "iam:ListGroupsForUser", 22 | "iam:ListMFADevices", 23 | "iam:ResyncMFADevice", 24 | "iam:UpdateAccessKey", 25 | "iam:UpdateLoginProfile", 26 | "iam:UpdateSSHPublicKey", 27 | "iam:UpdateUser", 28 | "iam:UploadSSHPublicKey" 29 | ], 30 | "Resource": [ 31 | "arn:aws:iam::366425516243:mfa/${aws:username}", 32 | "arn:aws:iam::366425516243:mfa/${aws:username}/*", 33 | "arn:aws:iam::366425516243:user/${aws:username}" 34 | ] 35 | } 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /sandbox-policies/PreventAdminRoleDeletion.scp: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "Statement1", 6 | "Effect": "Deny", 7 | "Action": [ 8 | "iam:DeleteRole", 9 | "iam:DeleteRolePolicy", 10 | "iam:DetachRolePolicy", 11 | "iam:UpdateRole" 12 | ], 13 | "Resource": [ 14 | "arn:aws:iam::*:role/OrganizationAccountAccessRole", 15 | "arn:aws:iam::*:role/OrganizationAccessRole" 16 | ] 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /sandbox-policies/README.md: -------------------------------------------------------------------------------- 1 | This directory contains polices -- both IAM and SCP -- that we use at Chariot to manage our 2 | developer sandboxes. They're saved here as general examples, referenced from 3 | [this blog post](https://chariotsolutions.com/blog/post/building-developer-sandboxes-on-aws/) 4 | but also form part of our infrastructure-as-code codebase. 5 | 6 | * `BasicUserPolicy` (IAM) 7 | 8 | This policy is granted to all users in the root account. It allows them to log in, change 9 | their password, and assign a virtual MFA device. We also have a very permissive read-only 10 | policy that grants access to other IAM information. 11 | 12 | * `PreventAdminRoleDeletion` (SCP) 13 | 14 | Each of our sandbox accounts has a role that grants administrator access. This role must 15 | be protected against changes, because the only alternative for managing these accounts is 16 | to log in as the account root user. 17 | 18 | One quirk of our particular implementation is that we protect two different role names. 19 | `OrganizationAccountAccessRole` is the default name for child accounts created via the 20 | AWS console. `OrganizationAccessRole` is a Chariot-specific name that was used by 21 | accident when creating a large number of workshop accounts. Rather than replace the 22 | latter with the former, it was easier to just protect both. 23 | 24 | * `SandboxGuardrail` (SCP) 25 | 26 | The basic execution limits for our sandbox accounts. This prevents the creation of 27 | expensive EC2 and RDS instance types, limits creation actions to the US regions, and 28 | prevents any use of a private certificate manager (which costs $400/month). 29 | 30 | In the blog post I specify the "non-blessed region" statement as a simple "deny all 31 | actions on all services" rule. For this policy I get a little more selective, denying 32 | actions that will cost us money, while still allowing reads and free-service creates. 33 | This bloats the policy, perhaps without any real benefit; it would be nice if AWS 34 | allowed a wildcard service name in the actions (so we could use "*:Create*"). 35 | -------------------------------------------------------------------------------- /springboot-iam-auth/.gitignore: -------------------------------------------------------------------------------- 1 | # Maven 2 | target/ 3 | 4 | # Eclipse 5 | .classpath 6 | .project 7 | .settings/ 8 | 9 | # Vim 10 | *.swp 11 | -------------------------------------------------------------------------------- /springboot-iam-auth/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | 7 | org.springframework.boot 8 | spring-boot-starter-parent 9 | 2.2.1.RELEASE 10 | 11 | 12 | com.chariotsolutions.example 13 | spring-boot-iam-auth 14 | 0.0.1-SNAPSHOT 15 | 16 | jar 17 | 18 | 19 | Uses a custom datasource to retrieve RDS credentials from IAM. 20 | 21 | 22 | 23 | 1.8 24 | 1.11.600 25 | 26 | 27 | 28 | 29 | com.amazonaws 30 | aws-java-sdk-rds 31 | ${aws-sdk.version} 32 | 33 | 34 | org.postgresql 35 | postgresql 36 | 37 | 38 | org.slf4j 39 | slf4j-api 40 | 41 | 42 | 43 | org.springframework.boot 44 | spring-boot-starter-jdbc 45 | 46 | 47 | org.springframework.boot 48 | spring-boot-starter-test 49 | test 50 | 51 | 52 | org.junit.vintage 53 | junit-vintage-engine 54 | 55 | 56 | 57 | 58 | org.springframework.boot 59 | spring-boot-starter-actuator 60 | 61 | 62 | 63 | 64 | 65 | 66 | org.springframework.boot 67 | spring-boot-maven-plugin 68 | 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /springboot-iam-auth/src/main/java/com/chariotsolutions/example/springboot/Application.java: -------------------------------------------------------------------------------- 1 | package com.chariotsolutions.example.springboot; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.boot.context.properties.ConfigurationProperties; 6 | import org.springframework.boot.jdbc.DataSourceBuilder; 7 | import org.springframework.context.annotation.Bean; 8 | 9 | import com.zaxxer.hikari.HikariDataSource; 10 | 11 | 12 | @SpringBootApplication 13 | public class Application 14 | { 15 | public static void main(String[] args) 16 | { 17 | SpringApplication.run(Application.class, args); 18 | } 19 | 20 | 21 | @Bean 22 | @ConfigurationProperties(prefix = "spring.datasource") 23 | public HikariDataSource dataSource() 24 | { 25 | HikariDataSource ds = DataSourceBuilder.create().type(HikariDataSource.class).build(); 26 | return ds; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /springboot-iam-auth/src/main/java/com/chariotsolutions/example/springboot/Runner.java: -------------------------------------------------------------------------------- 1 | package com.chariotsolutions.example.springboot; 2 | 3 | import java.sql.Timestamp; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.boot.CommandLineRunner; 10 | import org.springframework.jdbc.core.JdbcTemplate; 11 | import org.springframework.stereotype.Component; 12 | 13 | 14 | /** 15 | * This class performs the actual retrieval operation. It is invoked by the 16 | * application main. 17 | */ 18 | @Component 19 | public class Runner 20 | implements CommandLineRunner 21 | { 22 | private Logger logger = LoggerFactory.getLogger(getClass()); 23 | 24 | @Autowired 25 | private JdbcTemplate jdbcTemplate; 26 | 27 | @Override 28 | public void run(String... args) throws Exception 29 | { 30 | logger.info("application started"); 31 | 32 | Timestamp jdbcUser = jdbcTemplate.queryForObject("select current_timestamp", Timestamp.class); 33 | logger.info("database timestamp: {}", jdbcUser); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /springboot-iam-auth/src/main/java/com/chariotsolutions/example/springboot/datasource/IAMAuthDataSource.java: -------------------------------------------------------------------------------- 1 | package com.chariotsolutions.example.springboot.datasource; 2 | 3 | import java.sql.Connection; 4 | import java.sql.SQLException; 5 | 6 | import org.postgresql.ds.PGSimpleDataSource; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; 11 | import com.amazonaws.regions.DefaultAwsRegionProviderChain; 12 | import com.amazonaws.services.rds.auth.GetIamAuthTokenRequest; 13 | import com.amazonaws.services.rds.auth.RdsIamAuthTokenGenerator; 14 | 15 | 16 | /** 17 | * Retrieves IAM-generated credentials for the configured user. 18 | *

19 | * Before you can use this, you must grant rds_iam to the user. 20 | */ 21 | public class IAMAuthDataSource 22 | extends PGSimpleDataSource 23 | { 24 | private final static long serialVersionUID = 1L; 25 | 26 | private Logger logger = LoggerFactory.getLogger(getClass()); 27 | 28 | 29 | @Override 30 | public Connection getConnection(String user, String password) 31 | throws SQLException 32 | { 33 | // I'd like to do this in constructor, but it can throw SQLException 34 | setProperty("ssl", "true"); 35 | setProperty("sslmode", "require"); 36 | 37 | logger.debug("requesting IAM token for user {}", user); 38 | 39 | // adapted from https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.Connecting.Java.html 40 | RdsIamAuthTokenGenerator generator = RdsIamAuthTokenGenerator.builder() 41 | .credentials(new DefaultAWSCredentialsProviderChain()) 42 | .region((new DefaultAwsRegionProviderChain()).getRegion()) 43 | .build(); 44 | 45 | GetIamAuthTokenRequest request = GetIamAuthTokenRequest.builder() 46 | .hostname(getServerName()) 47 | .port(getPortNumber()) 48 | .userName(user) 49 | .build(); 50 | 51 | String authToken = generator.getAuthToken(request); 52 | 53 | return super.getConnection(user, authToken); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /springboot-iam-auth/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.main.web-application-type=NONE 2 | 3 | # this is the HikariCP connection pool -- settings appropriate for CLI app, not web-app 4 | spring.datasource.minimumIdle=0 5 | spring.datasource.maximumPoolSize=1 6 | 7 | # this is configuration for the underlying datasource 8 | spring.datasource.dataSourceClassName=com.chariotsolutions.example.springboot.datasource.IAMAuthDataSource 9 | spring.datasource.dataSourceProperties.serverName=${PGHOST} 10 | spring.datasource.dataSourceProperties.portNumber=${PGPORT} 11 | spring.datasource.dataSourceProperties.user=${PGUSER} 12 | spring.datasource.dataSourceProperties.databaseName=${PGDATABASE} 13 | -------------------------------------------------------------------------------- /springboot-iam-auth/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{ISO8601} %-5level [%thread] %logger{24} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /springboot-secman-auth/.gitignore: -------------------------------------------------------------------------------- 1 | # Maven 2 | target/ 3 | 4 | # Eclipse 5 | .classpath 6 | .project 7 | .settings/ 8 | 9 | # Vim 10 | *.swp 11 | -------------------------------------------------------------------------------- /springboot-secman-auth/README.md: -------------------------------------------------------------------------------- 1 | An example Sprint Boot program that uses the [AWS Labs Secrets Manager JDBC driver](https://github.com/aws/aws-secretsmanager-jdbc) 2 | to connect to an RDS database. 3 | 4 | 5 | ## Prerequisites 6 | 7 | * Install Maven and a Java JDK (not JRE) on your development computer. 8 | * Create an RDS Postgres database, along with a Secrets Manager secret that identifies a user 9 | in that database. [This CloudFormation template](src/cloudformation/rds_secretsmanager.yml) 10 | will do that, with the database master user stored in the secret. 11 | * Ensure that you can connect to this database server from your development computer. 12 | 13 | 14 | ## Running 15 | 16 | Set the environment variables `PGHOST`, `PGPORT`, and `PGDATABASE` to reference your database 17 | instance and the environment variable `SECRET_NAME` to the name or ARN of the secret holding 18 | the username and password. 19 | 20 | From the project root directory, execute `mvn spring-boot:run`. 21 | 22 | After the messages from Maven as it builds the project, you should see output like the following: 23 | 24 | ``` 25 | . ____ _ __ _ _ 26 | /\\ / ___'_ __ _ _(_)_ __ __ _ \ \ \ \ 27 | ( ( )\___ | '_ | '_| | '_ \/ _` | \ \ \ \ 28 | \\/ ___)| |_)| | | | | || (_| | ) ) ) ) 29 | ' |____| .__|_| |_|_| |_\__, | / / / / 30 | =========|_|==============|___/=/_/_/_/ 31 | :: Spring Boot :: (v2.2.1.RELEASE) 32 | 33 | 2020-08-07 16:22:27,877 INFO [main] c.c.e.s.Application - Starting Application on kgregory with PID 20364 (/home/kgregory/Workspace/aws-examples/springboot-secman-auth/target/classes started by kgregory in /home/kgregory/Workspace/aws-examples/springboot-secman-auth) 34 | 2020-08-07 16:22:27,878 DEBUG [main] c.c.e.s.Application - Running with Spring Boot v2.2.1.RELEASE, Spring v5.2.1.RELEASE 35 | 2020-08-07 16:22:27,878 INFO [main] c.c.e.s.Application - No active profile set, falling back to default profiles: default 36 | 2020-08-07 16:22:28,483 INFO [main] c.c.e.s.Application - Started Application in 0.752 seconds (JVM running for 0.996) 37 | 2020-08-07 16:22:28,483 INFO [main] c.c.e.springboot.Runner - application started 38 | 2020-08-07 16:22:29,397 INFO [main] c.c.e.springboot.Runner - database timestamp: 2020-08-07 16:22:29.380214 39 | ``` 40 | 41 | The last line shows that the program was able to connect to the database and execute a simple query. 42 | 43 | 44 | ## Implementation 45 | 46 | This is a simple Spring Boot application consisting of two classes: 47 | 48 | * `Application` provides the `main()` method, which initializes the application context. In a more complex example 49 | it would also provide bean factory methods. 50 | * `Runner` contains the actual application code. It is a Spring component, and is autowired with a `JdbcTemplate` 51 | instance (which is, in turn, autowired with the default datasource). It uses that object to execute a simple 52 | SQL command that retrieves the current time from the database server. 53 | 54 | The application configuration file configures the default datasource, and also explicitly tells Spring Boot that 55 | this is _not_ a web-application. 56 | -------------------------------------------------------------------------------- /springboot-secman-auth/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | 7 | org.springframework.boot 8 | spring-boot-starter-parent 9 | 2.2.1.RELEASE 10 | 11 | 12 | com.chariotsolutions.example 13 | spring-boot-secman-auth 14 | 0.0.1-SNAPSHOT 15 | 16 | jar 17 | 18 | 19 | Example using the AWSLabs Secrets Manager JDBC drivers with Spring Boot. 20 | See https://github.com/aws/aws-secretsmanager-jdbc 21 | 22 | 23 | 24 | 25 | 1.8 26 | 27 | 1.0.5 28 | 29 | 30 | 31 | 32 | com.amazonaws.secretsmanager 33 | aws-secretsmanager-jdbc 34 | ${aws-jdbc.version} 35 | 36 | 37 | org.postgresql 38 | postgresql 39 | 40 | 41 | org.slf4j 42 | slf4j-api 43 | 44 | 45 | org.springframework.boot 46 | spring-boot-starter-jdbc 47 | 48 | 49 | org.springframework.boot 50 | spring-boot-starter-test 51 | test 52 | 53 | 54 | org.junit.vintage 55 | junit-vintage-engine 56 | 57 | 58 | 59 | 60 | org.springframework.boot 61 | spring-boot-starter-actuator 62 | 63 | 64 | 65 | 66 | 67 | 68 | org.springframework.boot 69 | spring-boot-maven-plugin 70 | 71 | 72 | 73 | 74 | 75 | -------------------------------------------------------------------------------- /springboot-secman-auth/src/main/java/com/chariotsolutions/example/springboot/Application.java: -------------------------------------------------------------------------------- 1 | package com.chariotsolutions.example.springboot; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | 7 | @SpringBootApplication 8 | public class Application 9 | { 10 | public static void main(String[] args) 11 | { 12 | SpringApplication.run(Application.class, args); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /springboot-secman-auth/src/main/java/com/chariotsolutions/example/springboot/Runner.java: -------------------------------------------------------------------------------- 1 | package com.chariotsolutions.example.springboot; 2 | 3 | import java.sql.Timestamp; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.boot.CommandLineRunner; 10 | import org.springframework.jdbc.core.JdbcTemplate; 11 | import org.springframework.stereotype.Component; 12 | 13 | 14 | /** 15 | * This class performs the actual retrieval operation. It is invoked by the 16 | * application main. 17 | */ 18 | @Component 19 | public class Runner 20 | implements CommandLineRunner 21 | { 22 | private Logger logger = LoggerFactory.getLogger(getClass()); 23 | 24 | @Autowired 25 | private JdbcTemplate jdbcTemplate; 26 | 27 | @Override 28 | public void run(String... args) throws Exception 29 | { 30 | logger.info("application started"); 31 | 32 | Timestamp jdbcUser = jdbcTemplate.queryForObject("select current_timestamp", Timestamp.class); 33 | logger.info("database timestamp: {}", jdbcUser); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /springboot-secman-auth/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.main.web-application-type=NONE 2 | 3 | # this is the HikariCP connection pool -- settings appropriate for CLI app, not web-app 4 | spring.datasource.minimumIdle=0 5 | spring.datasource.maximumPoolSize=1 6 | 7 | # note that this configuration relies on environment variables; you can either set them or hardcode the values 8 | spring.datasource.url=jdbc-secretsmanager:postgresql://${PGHOST}:${PGPORT}/${PGDATABASE} 9 | spring.datasource.driver-class-name=com.amazonaws.secretsmanager.sql.AWSSecretsManagerPostgreSQLDriver 10 | spring.datasource.username=${SECRET_NAME} 11 | -------------------------------------------------------------------------------- /springboot-secman-auth/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{ISO8601} %-5level [%thread] %logger{24} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | tmp* 3 | -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/cloudformation/undeploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################################################################ 3 | ## 4 | ## Undeployes the example: removes all buckets (which may contain content) and then destroys 5 | ## the CloudFormation stack. Must be run using appropriate IAM permissions. 6 | ## 7 | ## Invocation: 8 | ## 9 | ## undeploy.sh STACK_NAME BASE_BUCKET_NAME 10 | ## 11 | ## Where: 12 | ## 13 | ## STACK_NAME is the name of the deployed stack, unique within the current region 14 | ## BASE_BUCKET_NAME is used to construct the names of the three staging buckets. This 15 | ## must be unique across AWS. 16 | ## 17 | ## Note: you must have the AWS CLI installed for this script to work. 18 | ## 19 | ################################################################################################ 20 | 21 | if [[ $# -ne 2 ]] ; then 22 | echo "invocation: undeploy.sh STACK_NAME BASE_BUCKET_NAME" 23 | exit 1 24 | fi 25 | 26 | STACK_NAME=$1 27 | BASE_BUCKET_NAME=$2 28 | 29 | ARCHIVE_BUCKET_NAME="${BASE_BUCKET_NAME}-archive" 30 | STATIC_BUCKET_NAME="${BASE_BUCKET_NAME}-static" 31 | UPLOADS_BUCKET_NAME="${BASE_BUCKET_NAME}-uploads" 32 | 33 | 34 | echo "" 35 | echo "deleting all buckets" 36 | echo "" 37 | 38 | aws s3 rb s3://${ARCHIVE_BUCKET_NAME} --force 39 | aws s3 rb s3://${STATIC_BUCKET_NAME} --force 40 | aws s3 rb s3://${UPLOADS_BUCKET_NAME} --force 41 | 42 | 43 | echo "" 44 | echo "deleting CloudFormation stack" 45 | echo "" 46 | 47 | aws cloudformation delete-stack --stack-name ${STACK_NAME} 48 | 49 | 50 | echo "" 51 | echo "waiting for stack deletion to finish (should be only a few minutes)" 52 | echo "" 53 | 54 | aws cloudformation wait stack-delete-complete --stack-name ${STACK_NAME} 55 | -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/docs/webapp-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chariotsolutions/aws-examples/cd636134fe14eb7314114d97290b5bd6124c25dc/two_buckets_and_a_lambda/docs/webapp-architecture.png -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/docs/webapp-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chariotsolutions/aws-examples/cd636134fe14eb7314114d97290b5bd6124c25dc/two_buckets_and_a_lambda/docs/webapp-ui.png -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/lambda/credentials.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # 3 | # MIT No Attribution 4 | # 5 | # Copyright Chariot Solutions 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 8 | # software and associated documentation files (the "Software"), to deal in the Software 9 | # without restriction, including without limitation the rights to use, copy, modify, 10 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 11 | # permit persons to whom the Software is furnished to do so. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 14 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 15 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 16 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 17 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 18 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | # 20 | ################################################################################ 21 | 22 | """ Credentials Lambda. 23 | 24 | This Lambda assumes a role that has permissions to upload a single file, and returns 25 | the credentials for that role session to the caller. 26 | """ 27 | 28 | import boto3 29 | import json 30 | import logging 31 | import os 32 | 33 | bucket = os.environ['UPLOAD_BUCKET'] 34 | role_arn = os.environ['ASSUMED_ROLE_ARN'] 35 | 36 | sts_client = boto3.client('sts') 37 | 38 | logger = logging.getLogger(__name__) 39 | logger.setLevel(logging.DEBUG) 40 | 41 | def lambda_handler(event, context): 42 | body = json.loads(event['body']) 43 | key = body['key'] 44 | 45 | session_name = f"{context.function_name}-{context.aws_request_id}" 46 | session_policy = { 47 | 'Version': '2012-10-17', 48 | 'Statement': [ 49 | { 50 | 'Effect': 'Allow', 51 | 'Action': 's3:PutObject', 52 | 'Resource': f"arn:aws:s3:::{bucket}/{key}" 53 | } 54 | ] 55 | } 56 | 57 | logger.info(f"generating restricted credentials for: s3://{bucket}/{key} for session {session_name}") 58 | 59 | response = sts_client.assume_role( 60 | RoleArn=role_arn, 61 | RoleSessionName=session_name, 62 | Policy=json.dumps(session_policy) 63 | ) 64 | creds = response['Credentials'] 65 | 66 | return { 67 | 'statusCode': 200, 68 | 'headers': { 69 | 'Content-Type': 'application/json' 70 | }, 71 | 'body': json.dumps({ 72 | 'access_key': creds['AccessKeyId'], 73 | 'secret_key': creds['SecretAccessKey'], 74 | 'session_token': creds['SessionToken'], 75 | 'region': os.environ['AWS_REGION'], 76 | 'bucket': bucket 77 | }) 78 | } 79 | -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/lambda/processor.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # 3 | # MIT No Attribution 4 | # 5 | # Copyright Chariot Solutions 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 8 | # software and associated documentation files (the "Software"), to deal in the Software 9 | # without restriction, including without limitation the rights to use, copy, modify, 10 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 11 | # permit persons to whom the Software is furnished to do so. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 14 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 15 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 16 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 17 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 18 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | # 20 | ################################################################################ 21 | 22 | """ Skeleton Processing Lambda 23 | 24 | This Lambda simply reports the content length of the new S3 file, then 25 | moves it to the archive bucket. 26 | """ 27 | 28 | import boto3 29 | import logging 30 | import os 31 | import urllib.parse 32 | 33 | archive_bucket = os.environ['ARCHIVE_BUCKET'] 34 | 35 | logger = logging.getLogger(__name__) 36 | logger.setLevel(logging.DEBUG) 37 | 38 | s3_client = boto3.client('s3') 39 | 40 | def lambda_handler(event, context): 41 | for record in event.get('Records', []): 42 | eventName = record['eventName'] 43 | bucket = record['s3']['bucket']['name'] 44 | raw_key = record['s3']['object']['key'] 45 | key = urllib.parse.unquote_plus(raw_key) 46 | try: 47 | logger.info(f"processing s3://{bucket}/{key}") 48 | process(bucket, key) 49 | logger.info(f"moving s3://{bucket}/{key} to s3://{archive_bucket}/{key}") 50 | archive(bucket, key) 51 | except Exception as ex: 52 | logger.exception(f"unhandled exception processing s3://{bucket}/{key}") 53 | 54 | 55 | def process(bucket, key): 56 | meta = s3_client.head_object(Bucket=bucket, Key=key) 57 | logger.info(f"processing s3://{bucket}/{key} filesize = {meta['ContentLength']}") 58 | 59 | 60 | def archive(bucket, key): 61 | s3_client.copy( 62 | CopySource={'Bucket': bucket, 'Key': key }, 63 | Bucket=archive_bucket, 64 | Key=key) 65 | s3_client.delete_object(Bucket=bucket, Key=key) 66 | -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/lambda/signed_url.py: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # 3 | # MIT No Attribution 4 | # 5 | # Copyright Chariot Solutions 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 8 | # software and associated documentation files (the "Software"), to deal in the Software 9 | # without restriction, including without limitation the rights to use, copy, modify, 10 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 11 | # permit persons to whom the Software is furnished to do so. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 14 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 15 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 16 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 17 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 18 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | # 20 | ################################################################################ 21 | 22 | """ Presigned URL Lambda 23 | 24 | This Lambda uses its own credentials to create a presigned URL that will allow 25 | the user to upload a file to S3. 26 | """ 27 | 28 | import boto3 29 | import json 30 | import logging 31 | import os 32 | 33 | bucket = os.environ['UPLOAD_BUCKET'] 34 | 35 | s3_client = boto3.client('s3') 36 | 37 | logger = logging.getLogger(__name__) 38 | logger.setLevel(logging.DEBUG) 39 | 40 | def lambda_handler(event, context): 41 | body = json.loads(event['body']) 42 | key = body['key'] 43 | content_type = body['type'] 44 | 45 | logger.info(f"generating presigned URL for: s3://{bucket}/{key} ({content_type})") 46 | 47 | params = { 48 | 'Bucket': bucket, 49 | 'Key': key, 50 | 'ContentType': content_type 51 | } 52 | url = s3_client.generate_presigned_url('put_object', params) 53 | 54 | return { 55 | 'statusCode': 200, 56 | 'headers': { 57 | 'Content-Type': 'application/json' 58 | }, 59 | 'body': json.dumps({ 60 | 'url': url 61 | }) 62 | } 63 | -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Two Buckets and a Lambda 4 | 5 | 6 | 7 |

Select your file: 8 | 9 |

10 |

11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/static/js/credentials.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This code retrieves credentials from the service and uses them with the 3 | * AWS SDK to perform a (potential) multi-part upload. 4 | */ 5 | 6 | const credentialsUpload = (function() { 7 | 8 | const rootUrl = window.location.href.replace(/[^/]*$/, ""); 9 | const queryUrl = rootUrl + "api/credentials"; 10 | 11 | async function requestCredentials(selectedFile) { 12 | console.log("requesting credentials for " + selectedFile.name); 13 | const request = { 14 | method: 'POST', 15 | cache: 'no-cache', 16 | headers: { 17 | 'Content-Type': 'application/json' 18 | }, 19 | body: JSON.stringify({ 20 | key: selectedFile.name, 21 | }) 22 | }; 23 | const response = await fetch(queryUrl, request); 24 | if (response.ok) { 25 | return response.json(); 26 | } 27 | else { 28 | console.log("failed to retrieve credentials: " + response.status); 29 | } 30 | } 31 | 32 | async function uploadFile(selectedFile, accessKeyId, secretAccessKey, sessionToken, region, bucket) { 33 | AWS.config.region = region; 34 | AWS.config.credentials = new AWS.Credentials(accessKeyId, secretAccessKey, sessionToken); 35 | const s3 = new AWS.S3(); 36 | 37 | console.log("uploading " + selectedFile.name); 38 | const params = { 39 | Bucket: bucket, 40 | Key: selectedFile.name, 41 | ContentType: selectedFile.type, 42 | Body: selectedFile 43 | }; 44 | const upload = new AWS.S3.ManagedUpload({ params: params }); 45 | upload.on('httpUploadProgress', function(evt) { 46 | console.log("uploaded " + evt.loaded + " of " + evt.total + " bytes for " + selectedFile.name); 47 | }); 48 | return upload.promise(); 49 | } 50 | 51 | return async function() { 52 | const selectedFile = document.getElementById('fileselector').files[0]; 53 | if (typeof selectedFile == 'undefined') { 54 | alert("Choose a file first"); 55 | return; 56 | } 57 | const creds = await requestCredentials(selectedFile); 58 | if (creds) { 59 | await uploadFile(selectedFile, creds.access_key, creds.secret_key, creds.session_token, creds.region, creds.bucket); 60 | alert("upload via restricted credentials complete"); 61 | } 62 | } 63 | 64 | })(); 65 | -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/static/js/signed-url.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This code retrieves a signed URL from the server and uses it to upload a file. 3 | */ 4 | 5 | const signedUrlUpload = (function() { 6 | 7 | const rootUrl = window.location.href.replace(/[^/]*$/, ""); 8 | const queryUrl = rootUrl + "api/signedurl"; 9 | 10 | async function requestSignedUrl(fileName, fileType) { 11 | console.log("requesting URL for " + fileName); 12 | const request = { 13 | method: 'POST', 14 | cache: 'no-cache', 15 | headers: { 16 | 'Content-Type': 'application/json' 17 | }, 18 | body: JSON.stringify({ 19 | key: fileName, 20 | type: fileType 21 | }) 22 | }; 23 | const response = await fetch(queryUrl, request); 24 | if (response.ok) { 25 | content = await response.json(); 26 | return content.url; 27 | } 28 | else { 29 | console.log("failed to retrieve signed URL: " + response.status) 30 | } 31 | } 32 | 33 | async function loadFileContent(selectedFile) { 34 | console.log("retrieving content for " + selectedFile.name); 35 | return new Promise(function(success, failure) { 36 | const reader = new FileReader(); 37 | reader.onload = (e) => success(e.target.result); 38 | reader.onabort = failure; 39 | reader.readAsArrayBuffer(selectedFile); 40 | }); 41 | } 42 | 43 | async function uploadFile(fileName, fileType, content, url) { 44 | console.log("uploading " + fileName); 45 | const request = { 46 | method: 'PUT', 47 | mode: 'cors', 48 | cache: 'no-cache', 49 | headers: { 50 | 'Content-Type': fileType 51 | }, 52 | body: content 53 | }; 54 | const response = await fetch(url, request); 55 | console.log("upload status: " + response.status); 56 | } 57 | 58 | return async function() { 59 | const selectedFile = document.getElementById('fileselector').files[0]; 60 | if (typeof selectedFile == 'undefined') { 61 | alert("Choose a file first"); 62 | return; 63 | } 64 | 65 | const fileName = selectedFile.name; 66 | const fileType = selectedFile.type || "application/x-octet-stream"; 67 | const url = await requestSignedUrl(fileName, fileType); 68 | const content = await loadFileContent(selectedFile); 69 | if (url && content) { 70 | await uploadFile(fileName, fileType, content, url); 71 | alert("upload via signed URL complete"); 72 | } 73 | } 74 | 75 | })(); 76 | 77 | -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/terraform/.gitignore: -------------------------------------------------------------------------------- 1 | *.zip 2 | 3 | # terraform cache; should never be checked in 4 | .terraform/ 5 | 6 | # this would normally be checked in, but isn't because this is an example 7 | .terraform.lock.hcl 8 | 9 | # for real-world usage, state should be stored externally 10 | terraform.tfstate 11 | terraform.tfstate.backup 12 | -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/terraform/buckets.tf: -------------------------------------------------------------------------------- 1 | ## 2 | ## Creates and configures all buckets 3 | ## 4 | ## The "static" and "archive" buckets are simple. The "upload" bucket, which involves 5 | ## user interaction, requires more configuration. 6 | ## 7 | ## I also populate the static content here. Definitely not recommended as a production 8 | ## practice, but we only have a few files. They're defined as an array of maps, and 9 | ## iterated using count; if you add files to an existing deployment, add them to the 10 | ## end of the list. 11 | ## 12 | 13 | locals { 14 | static_files = { 15 | "index.html" = "text/html; charset=utf-8" 16 | "js/credentials.js" = "text/javascript" 17 | "js/signed-url.js" = "text/javascript" 18 | } 19 | } 20 | 21 | 22 | resource "aws_s3_bucket" "static" { 23 | bucket = "${var.base_bucket_name}-static" 24 | force_destroy = true 25 | } 26 | 27 | resource "aws_s3_bucket_acl" "static" { 28 | bucket = aws_s3_bucket.static.id 29 | acl = "public-read" 30 | } 31 | 32 | resource "aws_s3_bucket_public_access_block" "static" { 33 | bucket = aws_s3_bucket.static.id 34 | block_public_acls = false 35 | block_public_policy = true 36 | ignore_public_acls = false 37 | restrict_public_buckets = true 38 | } 39 | 40 | resource "aws_s3_object" "static" { 41 | for_each = local.static_files 42 | bucket = aws_s3_bucket.static.id 43 | key = each.key 44 | source = "${path.root}/../static/${each.key}" 45 | content_type = each.value 46 | acl = "public-read" 47 | } 48 | 49 | 50 | resource "aws_s3_bucket" "upload" { 51 | bucket = "${var.base_bucket_name}-upload" 52 | force_destroy = true 53 | } 54 | 55 | resource "aws_s3_bucket_cors_configuration" "upload" { 56 | bucket = aws_s3_bucket.upload.id 57 | cors_rule { 58 | allowed_methods = ["PUT", "POST"] 59 | allowed_origins = ["*"] 60 | allowed_headers = ["*"] 61 | expose_headers = ["ETag"] 62 | max_age_seconds = 3000 63 | } 64 | } 65 | 66 | resource "aws_s3_bucket_lifecycle_configuration" "upload" { 67 | bucket = aws_s3_bucket.upload.id 68 | 69 | rule { 70 | id = "DeleteIncompleteMultipartUploads" 71 | status = "Enabled" 72 | abort_incomplete_multipart_upload { 73 | days_after_initiation = 1 74 | } 75 | } 76 | } 77 | 78 | resource "aws_s3_bucket_notification" "bucket_notification" { 79 | depends_on = [ aws_lambda_permission.processing_lambda ] 80 | bucket = aws_s3_bucket.upload.id 81 | lambda_function { 82 | lambda_function_arn = module.processing_lambda.lambda.arn 83 | events = [ "s3:ObjectCreated:*" ] 84 | } 85 | 86 | } 87 | 88 | 89 | resource "aws_s3_bucket" "archive" { 90 | bucket = "${var.base_bucket_name}-archive" 91 | force_destroy = true 92 | } 93 | -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/terraform/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | aws = { 4 | source = "hashicorp/aws" 5 | version = ">= 4.0.0" 6 | } 7 | } 8 | } 9 | 10 | provider "aws" { 11 | } 12 | 13 | 14 | data "aws_caller_identity" "current" {} 15 | data "aws_region" "current" {} 16 | 17 | 18 | locals { 19 | aws_account_id = data.aws_caller_identity.current.account_id 20 | aws_region = data.aws_region.current.name 21 | } 22 | -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/terraform/modules/lambda/output.tf: -------------------------------------------------------------------------------- 1 | output "lambda" { 2 | description = "The Lambda function" 3 | value = aws_lambda_function.lambda 4 | } 5 | 6 | 7 | output "execution_role" { 8 | description = "The Lambda's execution role" 9 | value = aws_iam_role.execution_role 10 | } 11 | -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/terraform/modules/lambda/variables.tf: -------------------------------------------------------------------------------- 1 | variable "name" { 2 | description = "The name of the Lambda; also used as the base name for its execution role" 3 | type = string 4 | } 5 | 6 | variable "description" { 7 | description = "Describes the Lambda's purpose" 8 | type = string 9 | } 10 | 11 | variable "source_file" { 12 | description = "Path to the single-module file containing the Lambda source code" 13 | type = string 14 | } 15 | 16 | variable "envars" { 17 | description = "A map of environment variables" 18 | type = map(string) 19 | default = {} 20 | } 21 | 22 | variable "policy_statements" { 23 | description = "Policy statements that are added to the Lambda's execution role" 24 | type = list(any) 25 | } 26 | 27 | ## the variables defined below this point are intended as constants 28 | 29 | variable "handler_function" { 30 | description = "The simple name of the Lambda's handler function (will be combined with standard module name)" 31 | type = string 32 | default = "lambda_handler" 33 | } 34 | 35 | variable "memory_size" { 36 | description = "Amount of memory given to Lambda execution environment" 37 | type = number 38 | default = 512 39 | } 40 | 41 | variable "timeout" { 42 | description = "number of seconds that the Lambda is allowed to run" 43 | type = number 44 | default = 10 45 | } 46 | -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/terraform/outputs.tf: -------------------------------------------------------------------------------- 1 | output "api_gateway_url" { 2 | description = "The root URL for the example" 3 | value = aws_apigatewayv2_api.main.api_endpoint 4 | } 5 | 6 | -------------------------------------------------------------------------------- /two_buckets_and_a_lambda/terraform/variables.tf: -------------------------------------------------------------------------------- 1 | variable "base_name" { 2 | description = "Name used for most of the resources in this stack" 3 | type = string 4 | default = "TwoBuckets" 5 | } 6 | 7 | 8 | variable "base_bucket_name" { 9 | description = "The name used as a prefix for all buckets created by this stack" 10 | type = string 11 | } 12 | -------------------------------------------------------------------------------- /untagged_ec2_cleanup/README.md: -------------------------------------------------------------------------------- 1 | # Untagged EC2 Cleanup 2 | 3 | This is an example Lambda function that will identify EC2 instances that don't meet all 4 | of the following requirements: 5 | 6 | * A non-empty `Name` tag 7 | * A non-empty `CreatedBy` tag 8 | * Either created within the past 7 days, or has a `DeleteAfter` tag that specifies a future 9 | date in the format `YYYY-MM-DD`, 10 | 11 | This Lambda is intended to be run against sandbox accounts, to ensure that developers don't 12 | start machines and forget about them. It is typically triggered via a scheduled CloudWatch Event. 13 | 14 | 15 | ## Deployment 16 | 17 | This function is deployed using a CloudFormation template that creates the following resources: 18 | 19 | * The Lambda function itself. 20 | * An execution role. 21 | * A CloudWatch Events rule that triggers the function. 22 | 23 | To create the stack, you must provide the following parameters: 24 | 25 | * `FunctionName` 26 | The name of the Lambda function. This is also used as the base name for the function's 27 | execution role and trigger. 28 | * `Accounts` 29 | A comma-separated list of the accounts to be examined. 30 | * `Regions` 31 | A comma-separated list of the regions to examine for those accounts. This defaults to the 32 | US regsions. 33 | * `RoleName` 34 | The name of a role that is present in all accounts and has permissions to examine and terminate 35 | EC2 instances. This defaults to `OrganizationAccountAccessRole`, which is the default admin role 36 | created when adding an account to an organization. 37 | * `Schedule` 38 | The CloudWatch Events schedule rule that will trigger the Lambda. This defaults to a CRON 39 | expression for 4 AM UTC. 40 | 41 | **As created, the trigger is disabled and the `terminate()` call is commented-out in the Lambda.** 42 | 43 | Before enabling, you should run the Lambda and verify from its output that it will not delete any 44 | unexpected instances. You can either run manually, using the "Test" feature from the AWS Console 45 | (any test event is fine; it's not used), or enable the trigger and examine the CloudWatch logs for 46 | the function after it runs. 47 | 48 | When you are ready to run for real, uncomment the `instance.terminate()` call at line 45. 49 | -------------------------------------------------------------------------------- /websocket_to_kinesis/.gitignore: -------------------------------------------------------------------------------- 1 | .venv/ 2 | -------------------------------------------------------------------------------- /websocket_to_kinesis/Dockerfile.client: -------------------------------------------------------------------------------- 1 | FROM python:3.12 2 | 3 | RUN pip install boto3 websockets 4 | 5 | COPY client.py /app/ 6 | 7 | CMD [ "python", "/app/client.py" ] 8 | -------------------------------------------------------------------------------- /websocket_to_kinesis/Dockerfile.server: -------------------------------------------------------------------------------- 1 | FROM python:3.12 2 | 3 | RUN pip install websockets 4 | 5 | COPY server.py /app/ 6 | 7 | CMD [ "python", "/app/server.py" ] 8 | -------------------------------------------------------------------------------- /websocket_to_kinesis/client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ################################################################################ 3 | # 4 | # MIT No Attribution 5 | # 6 | # Copyright Chariot Solutions 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 9 | # software and associated documentation files (the "Software"), to deal in the Software 10 | # without restriction, including without limitation the rights to use, copy, modify, 11 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 12 | # permit persons to whom the Software is furnished to do so. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 15 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 16 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 17 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 18 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 19 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | # 21 | ################################################################################ 22 | 23 | """ Connects to a web-socket specified via environment variable, reads all of its 24 | messages, and sends them to a Kinesis stream specified via another envar. 25 | """ 26 | 27 | import boto3 28 | import os 29 | import logging 30 | import random 31 | import time 32 | import websockets.sync.client as ws_client 33 | 34 | 35 | logging.basicConfig(level=logging.WARN, format="%(asctime)s - %(message)s") 36 | logger = logging.getLogger(__name__) 37 | logger.setLevel(logging.DEBUG) 38 | 39 | 40 | def main(kinesis_client, server_url, stream_name): 41 | while True: 42 | try: 43 | with ws_client.connect(server_url) as websocket: 44 | logger.info(f"opened connection {websocket.id} to {websocket.remote_address}") 45 | for msg in websocket: 46 | logger.debug(msg) 47 | kinesis_client.put_record(StreamName=stream_name, PartitionKey=str(random.random()), Data=msg.encode('utf-8')) 48 | logger.info("connection closed by server") 49 | except: 50 | logger.warning("exception while processing", exc_info=True) 51 | time.sleep(30) 52 | 53 | 54 | if __name__ == "__main__": 55 | kinesis_client = boto3.client('kinesis') 56 | server_url = os.environ['SERVER_URL'] 57 | stream_name = os.environ['STREAM_NAME'] 58 | main(kinesis_client, server_url, stream_name) 59 | -------------------------------------------------------------------------------- /websocket_to_kinesis/server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ################################################################################ 3 | # 4 | # MIT No Attribution 5 | # 6 | # Copyright Chariot Solutions 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 9 | # software and associated documentation files (the "Software"), to deal in the Software 10 | # without restriction, including without limitation the rights to use, copy, modify, 11 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 12 | # permit persons to whom the Software is furnished to do so. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 15 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 16 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 17 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 18 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 19 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | # 21 | ################################################################################ 22 | 23 | """ Sends a minute's worth of timestamp messages to a connected client, 24 | then terminates the connection. 25 | """ 26 | 27 | import asyncio 28 | from datetime import datetime, timezone 29 | import logging 30 | import os 31 | import time 32 | from websockets.server import serve 33 | 34 | 35 | logging.basicConfig(level=logging.WARN, format="%(asctime)s - %(message)s") 36 | logger = logging.getLogger(__name__) 37 | logger.setLevel(logging.DEBUG) 38 | 39 | listen_port = int(os.environ.get("LISTEN_PORT", "8000")) 40 | send_count = int(os.environ.get("SEND_COUNT", "60")) 41 | send_sleep = float(os.environ.get("SEND_SLEEP", "1.0")) 42 | 43 | 44 | async def main(): 45 | async with serve(send_messages, port=listen_port): 46 | logger.info(f"server started, listening on port {listen_port}") 47 | logger.info(f"send count: {send_count}, inter-message sleep: {send_sleep}") 48 | await asyncio.Future() # run forever 49 | 50 | 51 | async def send_messages(websocket): 52 | logger.info(f"received connection {websocket.id} from {websocket.remote_address}") 53 | for x in range(send_count): 54 | await websocket.send(datetime.now(tz=timezone.utc).isoformat()) 55 | time.sleep(send_sleep) 56 | await websocket.close() 57 | 58 | 59 | asyncio.run(main()) 60 | -------------------------------------------------------------------------------- /xray_glue/README.md: -------------------------------------------------------------------------------- 1 | This directory contains examples showing how to use AWS X-Ray with Glue jobs. 2 | 3 | * `example-1.py` 4 | 5 | A Python "shell job" that uses the AWS SDK to write segments directly to X-Ray. 6 | 7 | * `example-2.py` 8 | 9 | A Python "shell job" that uses the X-Ray SDK, along with a custom emitter that 10 | avoids the need for a daemon. 11 | 12 | * `example-3.py` 13 | 14 | A Python driver program that defines segments around PySpark operations. Uses 15 | the X-Ray SDK with a custom emitter. 16 | 17 | * `example-4.py` 18 | 19 | A Python driver program that writes explicit segments for a Pandas UDF (in 20 | addition to using the X-Ray SDK for overal operation tracing). 21 | 22 | You should be able to run each of these scripts from the Console, without setting 23 | any parameters. They should take a minute or two to run. 24 | 25 | When the script completes, look for its trace in the X-Ray Console (I recommend 26 | using the [new](https://console.aws.amazon.com/cloudwatch/home#xray:traces/query) 27 | Console page, under CloudWatch, rather than the legacy page). 28 | 29 | 30 | 31 | ## Deployment 32 | 33 | Because this example requires a pre-populated S3 bucket, it is deployed using 34 | a shell script: 35 | 36 | ``` 37 | scripts/deploy.sh BUCKET_NAME STACK_NAME 38 | ``` 39 | 40 | * `BUCKET_NAME` is the name of an S3 bucket that will be created to hold both 41 | scripts and data. 42 | 43 | * `STACK_NAME` is the name for the CloudFormation stack that defines the Glue 44 | jobs. Each job will have the stack name prepended to its name. 45 | 46 | To undeploy, use the counterpart shell script: 47 | 48 | ``` 49 | scripts/undeploy.sh BUCKET_NAME STACK_NAME 50 | ``` 51 | -------------------------------------------------------------------------------- /xray_glue/data/updateItemQuantity/2021/08/03/b8b8db59-f661-4326-9b90-a97e6d56aca9.json: -------------------------------------------------------------------------------- 1 | {"eventType": "updateItemQuantity", "eventId": "accdc474-404c-4a36-8d4d-dbbb1fa1cd71", "timestamp": "2021-08-02 21:31:07.044", "userId": "d7e0da45-02a1-4acf-a3a2-d299f1332bdc", "productId": "1036", "quantity": 6} 2 | {"eventType": "updateItemQuantity", "eventId": "1cde988b-ca7c-498a-aeb1-4b0b3b196fff", "timestamp": "2021-08-02 21:36:51.044", "userId": "c3d00a2b-c19a-413a-b041-72ce43e08df4", "productId": "1016", "quantity": 2} 3 | -------------------------------------------------------------------------------- /xray_glue/jobs/example-3.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python3 2 | 3 | """ Example X-Ray-enabled Glue job that measures the execution time of 4 | PySpark DataFrame operations. 5 | """ 6 | 7 | import binascii 8 | import boto3 9 | import json 10 | import pandas as pd 11 | import os 12 | import sys 13 | import time 14 | import uuid 15 | 16 | from awsglue.context import GlueContext 17 | from awsglue.utils import getResolvedOptions 18 | from pyspark.context import SparkContext 19 | from pyspark.sql.functions import udf, pandas_udf 20 | from pyspark.sql.types import StringType 21 | 22 | from aws_xray_sdk.core import xray_recorder 23 | 24 | 25 | ## 26 | ## X-Ray related stuff 27 | ## 28 | 29 | class CustomEmitter: 30 | """ This class replaces the default emitter in the X-Ray SDK, and writes 31 | directly to the X-Ray service rather than a daemon. 32 | """ 33 | 34 | def __init__(self): 35 | self.xray_client = None # lazily initialize 36 | 37 | def send_entity(self, entity): 38 | if not self.xray_client: 39 | self.xray_client = boto3.client('xray') 40 | segment_doc = json.dumps(entity.to_dict()) 41 | self.xray_client.put_trace_segments(TraceSegmentDocuments=[segment_doc]) 42 | 43 | def set_daemon_address(self, address): 44 | pass 45 | 46 | @property 47 | def ip(self): 48 | return None 49 | 50 | @property 51 | def port(self): 52 | return None 53 | 54 | 55 | xray_recorder.configure( 56 | emitter=CustomEmitter(), 57 | context_missing='LOG_ERROR', 58 | sampling=False) 59 | 60 | 61 | ## 62 | ## A UDF 63 | ## 64 | 65 | def unique_ids(series: pd.Series) -> pd.Series: 66 | time.sleep(0.25) 67 | if series is not None: 68 | arr = [str(uuid.uuid4()) for value in series] 69 | row_count = len(arr) 70 | return pd.Series(arr) 71 | 72 | unique_ids = pandas_udf(unique_ids, StringType()).asNondeterministic() 73 | 74 | 75 | ## 76 | ## The Glue job 77 | ## 78 | 79 | args = getResolvedOptions(sys.argv, ['JOB_NAME']) 80 | job_name = args['JOB_NAME'] 81 | 82 | xray_recorder.begin_segment(job_name) 83 | 84 | sc = SparkContext() 85 | glueContext = GlueContext(sc) 86 | spark = glueContext.spark_session 87 | 88 | with xray_recorder.in_subsegment("generate_dataframe"): 89 | df1 = spark.range(100000) 90 | 91 | with xray_recorder.in_subsegment("add_unique_ids"): 92 | df2 = df1.withColumn('unique_id', unique_ids(df1.id)) 93 | df2.foreach(lambda x : x) # forces evaluation of dataframe operation 94 | 95 | with xray_recorder.in_subsegment("sort_dataframe"): 96 | df3 = df2.orderBy('unique_id') 97 | df3.foreach(lambda x : x) 98 | 99 | df3.show() 100 | 101 | xray_recorder.end_segment() 102 | 103 | -------------------------------------------------------------------------------- /xray_glue/scripts/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################################################ 3 | ## 4 | ## This script creates a bucket, uploads data into it, and uses a CloudFormation 5 | ## stack to deploy example jobs along with an X-Ray daemon. 6 | ## 7 | ## Invocation: 8 | ## 9 | ## deploy.sh BUCKET_NAME STACK_NAME 10 | ## 11 | ## Where: 12 | ## 13 | ## BUCKET_NAME is used to hold Glue job scripts, as well as for the source 14 | ## and destination of those scripts. 15 | ## STACK_NAME is the name of the stack to create with CloudFormation. 16 | ## 17 | ################################################################################ 18 | 19 | 20 | if [[ $# -ne 2 ]] ; then 21 | echo "invocation: deploy.sh BUCKET_NAME STACK_NAME" 22 | exit 1 23 | fi 24 | 25 | BUCKET_NAME=$1 26 | STACK_NAME=$2 27 | 28 | 29 | ## 30 | ## Create bucket and upload content 31 | ## 32 | 33 | SCRIPT_PREFIX="jobs" 34 | DATA_PREFIX="json" 35 | DEST_PREFIX="avro" 36 | 37 | aws s3 mb s3://${BUCKET_NAME} 38 | if [[ $? -ne 0 ]] ; then 39 | echo "failed to create bucket" 40 | exit 2 41 | fi 42 | 43 | for f in jobs/*.py 44 | do 45 | aws s3 cp $f s3://${BUCKET_NAME}/${SCRIPT_PREFIX}/$(basename $f) 46 | done 47 | 48 | for f in $(find data -name *.json) 49 | do 50 | aws s3 cp $f s3://${BUCKET_NAME}/${f/data/$DATA_PREFIX} 51 | done 52 | 53 | ## 54 | ## Create CloudFormation stack 55 | ## 56 | 57 | cat > /tmp/$$.cfparams <