├── source ├── tools │ ├── requirements.txt │ ├── policyBulk.json.in │ ├── simpleTemplateBody2.json.in │ ├── simpleTemplateBody.json │ ├── bulk-result.py │ ├── enable-registry-events.sh │ ├── sample-pol1.json │ ├── sample-pol2.json │ ├── list-thing.py │ ├── create-device-type.sh │ ├── create-device-attrs.sh │ ├── create-device-attrs-type.sh │ ├── iot-dr-random-tests.sh │ ├── create-device.sh │ ├── create-device-pca.sh │ ├── iot-search-devices.py │ ├── list-all-things.py │ ├── delete-things.py │ ├── bulk-bench.sh │ ├── iot-dr-run-tests.sh │ ├── iot-devices-cmp.py │ └── iot-dr-shadow-cmp.py ├── lambda │ ├── iot-dr-layer │ │ └── requirements.txt │ ├── iot-dr-r53-health-check │ │ ├── requirements.txt │ │ └── iot-dr-r53-health-checker.py │ ├── iot-dr-region-syncer │ │ ├── requirements.txt │ │ ├── run-region-syncer.sh │ │ ├── Dockerfile │ │ ├── Dockerfile-r2d │ │ ├── Dockerfile-r2r │ │ ├── build-docker-image.sh │ │ ├── build-docker-image-r2d.sh │ │ ├── build-docker-image-r2r.sh │ │ ├── ECSTaskRole-Policy.json │ │ ├── iot-region-to-region-syncer.py │ │ └── iot-region-to-ddb-syncer.py │ ├── iot-mr-cross-region │ │ └── lambda_function.py │ ├── sfn-iot-mr-dynamo-trigger │ │ └── lambda_function.py │ ├── iot-dr-missing-device-replication │ │ └── lambda_function.py │ ├── sfn-iot-mr-shadow-syncer │ │ └── lambda_function.py │ ├── iot-dr-custom-launch-solution │ │ └── lambda_function.py │ ├── sfn-iot-mr-thing-type-crud │ │ └── lambda_function.py │ ├── sfn-iot-mr-thing-crud │ │ └── lambda_function.py │ ├── sfn-iot-mr-thing-group-crud │ │ └── lambda_function.py │ └── iot-mr-jitr │ │ └── lambda_function.py ├── images │ └── arch.png ├── launch-solution.yml ├── build.sh ├── jupyter │ ├── 01_IoTDR_Shared.ipynb │ ├── 04_IoTDR_Device_Certs.ipynb │ ├── 03_IoTDR_Reg_PCA.ipynb │ └── 05_IoTDR_JITR_Device.ipynb └── launch-solution-code-build.sh ├── CODE_OF_CONDUCT.md ├── CHANGELOG.md ├── NOTICE.txt ├── deployment ├── run-unit-tests.sh └── build-s3-dist.sh ├── CONTRIBUTING.md └── LICENSE.txt /source/tools/requirements.txt: -------------------------------------------------------------------------------- 1 | awsiotsdk 2 | dnspython 3 | -------------------------------------------------------------------------------- /source/lambda/iot-dr-layer/requirements.txt: -------------------------------------------------------------------------------- 1 | dynamodb-json==1.3 2 | -------------------------------------------------------------------------------- /source/lambda/iot-dr-r53-health-check/requirements.txt: -------------------------------------------------------------------------------- 1 | awsiotsdk 2 | -------------------------------------------------------------------------------- /source/lambda/iot-dr-region-syncer/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | dynamodb-json==1.3 3 | -------------------------------------------------------------------------------- /source/images/arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/disaster-recovery-for-aws-iot/HEAD/source/images/arch.png -------------------------------------------------------------------------------- /source/lambda/iot-dr-region-syncer/run-region-syncer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | echo "launching region syncer" 7 | python3 lambda_function.py 8 | echo "regions syncer finished" 9 | 10 | 11 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct (at) amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [1.0.0] - 2021-03-31 8 | ### Added 9 | - Initial release of code 10 | -------------------------------------------------------------------------------- /source/lambda/iot-dr-region-syncer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7.9 2 | 3 | RUN mkdir iot-dr-region-syncer 4 | WORKDIR iot-dr-region-syncer 5 | 6 | 7 | # Install Python dependencies 8 | COPY requirements.txt . 9 | RUN pip3 install -r requirements.txt 10 | 11 | # Copy Python files 12 | COPY iot-region-to-region-syncer.py . 13 | COPY device_replication.py . 14 | 15 | CMD ["python3", "iot-region-to-region-syncer.py"] 16 | -------------------------------------------------------------------------------- /source/lambda/iot-dr-region-syncer/Dockerfile-r2d: -------------------------------------------------------------------------------- 1 | FROM python:3.7.9 2 | 3 | RUN mkdir iot-dr-region-syncer 4 | WORKDIR iot-dr-region-syncer 5 | 6 | 7 | # Install Python dependencies 8 | COPY requirements.txt . 9 | RUN pip3 install -r requirements.txt 10 | 11 | # Copy Python files 12 | COPY iot-region-to-ddb-syncer.py . 13 | COPY device_replication.py . 14 | 15 | CMD ["python3", "iot-region-to-ddb-syncer.py"] 16 | -------------------------------------------------------------------------------- /source/lambda/iot-dr-region-syncer/Dockerfile-r2r: -------------------------------------------------------------------------------- 1 | FROM python:3.7.9 2 | 3 | RUN mkdir iot-dr-region-syncer 4 | WORKDIR iot-dr-region-syncer 5 | 6 | 7 | # Install Python dependencies 8 | COPY requirements.txt . 9 | RUN pip3 install -r requirements.txt 10 | 11 | # Copy Python files 12 | COPY iot-region-to-region-syncer.py . 13 | COPY device_replication.py . 14 | 15 | CMD ["python3", "iot-region-to-region-syncer.py"] 16 | -------------------------------------------------------------------------------- /source/lambda/iot-mr-cross-region/lambda_function.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import json 7 | import logging 8 | 9 | logger = logging.getLogger(__name__) 10 | logger.setLevel(logging.INFO) 11 | 12 | def lambda_handler(event, context): 13 | logger.info("context: {}".format(context)) 14 | logger.info("event: {}".format(event)) 15 | # TODO implement 16 | return { 17 | 'statusCode': 200, 18 | 'body': json.dumps({"return": "message"}) 19 | } 20 | -------------------------------------------------------------------------------- /source/tools/policyBulk.json.in: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Action": [ 6 | "iot:Connect" 7 | ], 8 | "Resource": "*", 9 | "Effect": "Allow" 10 | }, 11 | { 12 | "Action": [ 13 | "iot:Publish" 14 | ], 15 | "Resource": [ 16 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topic/*" 17 | ], 18 | "Effect": "Allow" 19 | }, 20 | { 21 | "Action": [ 22 | "iot:Receive" 23 | ], 24 | "Resource": [ 25 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topic/*" 26 | ], 27 | "Effect": "Allow" 28 | }, 29 | { 30 | "Action": [ 31 | "iot:Subscribe" 32 | ], 33 | "Resource": [ 34 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topicfilter/*" 35 | ], 36 | "Effect": "Allow" 37 | } 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /source/lambda/iot-dr-region-syncer/build-docker-image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | IMG="iot-dr-region-syncer" 7 | TAG="$(date '+%Y-%m-%d_%H-%M-%S')" 8 | ECR="AWS_ACCOUNT.dkr.ecr.AWS_REGION.amazonaws.com/iot-dr-region-syncer" 9 | 10 | echo "building docker image \"$TAG\"" 11 | 12 | cp ../iot-dr-layer/device_replication.py . 13 | 14 | docker build --no-cache --tag $IMG:$TAG . 15 | 16 | echo "tagging for ECR" 17 | docker tag $IMG:$TAG $ECR:$TAG 18 | 19 | echo "ecr login" 20 | aws ecr get-login-password \ 21 | --region eu-west-2 \ 22 | | docker login \ 23 | --username AWS \ 24 | --password-stdin AWS_ACCOUNT.dkr.ecr.AWS_REGION.amazonaws.com 25 | 26 | echo "push image" 27 | docker push $ECR:$TAG 28 | 29 | echo "For Fargate" 30 | echo "-----------" 31 | echo "${IMG}_${TAG}" 32 | echo "$ECR:$TAG" 33 | -------------------------------------------------------------------------------- /source/lambda/iot-dr-region-syncer/build-docker-image-r2d.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | IMG="iot-region-to-ddb-syncer" 7 | TAG="$(date '+%Y-%m-%d_%H-%M-%S')" 8 | ECR="AWS_ACCOUNT.dkr.ecr.AWS_REGION.amazonaws.com/$IMG" 9 | 10 | echo "building docker image \"$TAG\"" 11 | 12 | cp ../iot-dr-layer/device_replication.py . 13 | 14 | docker build --no-cache --tag $IMG:$TAG -f Dockerfile-r2d . 15 | 16 | echo "tagging for ECR" 17 | docker tag $IMG:$TAG $ECR:$TAG 18 | 19 | echo "ecr login" 20 | aws ecr get-login-password \ 21 | --region eu-west-2 \ 22 | | docker login \ 23 | --username AWS \ 24 | --password-stdin AWS_ACCOUNT.dkr.ecr.AWS_REGION.amazonaws.com 25 | 26 | echo "push image" 27 | docker push $ECR:$TAG 28 | 29 | echo "For Fargate" 30 | echo "-----------" 31 | echo "${IMG}_${TAG}" 32 | echo "$ECR:$TAG" 33 | -------------------------------------------------------------------------------- /source/lambda/iot-dr-region-syncer/build-docker-image-r2r.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | IMG="iot-region-to-region-syncer" 7 | TAG="$(date '+%Y-%m-%d_%H-%M-%S')" 8 | ECR="AWS_ACCOUNT.dkr.ecr.AWS_REGION.amazonaws.com/$IMG" 9 | 10 | echo "building docker image \"$TAG\"" 11 | 12 | cp ../iot-dr-layer/device_replication.py . 13 | 14 | docker build --no-cache --tag $IMG:$TAG -f Dockerfile-r2r . 15 | 16 | echo "tagging for ECR" 17 | docker tag $IMG:$TAG $ECR:$TAG 18 | 19 | echo "ecr login" 20 | aws ecr get-login-password \ 21 | --region eu-west-2 \ 22 | | docker login \ 23 | --username AWS \ 24 | --password-stdin AWS_ACCOUNT.dkr.ecr.AWS_REGION.amazonaws.com 25 | 26 | echo "push image" 27 | docker push $ECR:$TAG 28 | 29 | echo "For Fargate" 30 | echo "-----------" 31 | echo "${IMG}_${TAG}" 32 | echo "$ECR:$TAG" 33 | -------------------------------------------------------------------------------- /source/tools/simpleTemplateBody2.json.in: -------------------------------------------------------------------------------- 1 | { 2 | "Parameters" : { 3 | "ThingName" : { 4 | "Type" : "String" 5 | }, 6 | "SerialNumber" : { 7 | "Type" : "String" 8 | }, 9 | "Location" : { 10 | "Type" : "String", 11 | "Default" : "WA" 12 | }, 13 | "CSR" : { 14 | "Type" : "String" 15 | } 16 | }, 17 | "Resources" : { 18 | "thing" : { 19 | "Type" : "AWS::IoT::Thing", 20 | "Properties" : { 21 | "ThingName" : {"Ref" : "ThingName"}, 22 | "AttributePayload" : { "serialNumber" : {"Ref" : "SerialNumber"}} 23 | } 24 | }, 25 | "certificate" : { 26 | "Type" : "AWS::IoT::Certificate", 27 | "Properties" : { 28 | "CertificateSigningRequest": {"Ref" : "CSR"}, 29 | "Status" : "ACTIVE" 30 | } 31 | }, 32 | "policy" : { 33 | "Type" : "AWS::IoT::Policy", 34 | "Properties" : { 35 | "PolicyName": "__POLICY_NAME__" 36 | } 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Disaster Recovery for AWS IoT 2 | Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except 4 | in compliance with the License. A copy of the License is located at http://www.apache.org/licenses/ 5 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, 6 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for the 7 | specific language governing permissions and limitations under the License. 8 | 9 | ********************** 10 | THIRD PARTY COMPONENTS 11 | ********************** 12 | This software includes third party software subject to the following copyrights: 13 | 14 | AWS SDK - Apache License Version 2.0 15 | AWS IoT SDK - Apache License Version 2.0 16 | pyOpenSSL - Apache License Version 2.0 17 | dynamoDB-json - Apache License Version 2.0 18 | boto3 - Apache License Version 2.0 19 | simplejson - Massachusetts Institute of Technology (MIT) license 20 | dnspython - ISC license -------------------------------------------------------------------------------- /source/tools/simpleTemplateBody.json: -------------------------------------------------------------------------------- 1 | { 2 | "Parameters" : { 3 | "ThingName" : { 4 | "Type" : "String" 5 | }, 6 | "SerialNumber" : { 7 | "Type" : "String" 8 | }, 9 | "Location" : { 10 | "Type" : "String", 11 | "Default" : "WA" 12 | }, 13 | "CSR" : { 14 | "Type" : "String" 15 | } 16 | }, 17 | "Resources" : { 18 | "thing" : { 19 | "Type" : "AWS::IoT::Thing", 20 | "Properties" : { 21 | "ThingName" : {"Ref" : "ThingName"}, 22 | "AttributePayload" : { "serialNumber" : {"Ref" : "SerialNumber"}} 23 | } 24 | }, 25 | "certificate" : { 26 | "Type" : "AWS::IoT::Certificate", 27 | "Properties" : { 28 | "CertificateSigningRequest": {"Ref" : "CSR"}, 29 | "Status" : "ACTIVE" 30 | } 31 | }, 32 | "policy" : { 33 | "Type" : "AWS::IoT::Policy", 34 | "Properties" : { 35 | "PolicyDocument": "{\"Version\": \"2012-10-17\",\"Statement\": [{\"Effect\": \"Allow\",\"Action\": [\"iot:*\"],\"Resource\": [\"*\"]}]}" 36 | } 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /source/tools/bulk-result.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0. 5 | 6 | import json 7 | import os 8 | import sys 9 | 10 | 11 | def process_line(line): 12 | d = json.loads(line) 13 | crt = d["response"]["CertificatePem"] 14 | thing = d["response"]["ResourceArns"]["thing"].split('/')[1] 15 | print("creating file {}.crt for thing {}".format(thing, thing)) 16 | file = open(thing + ".crt", "w") 17 | file.write(crt) 18 | file.close() 19 | 20 | def process_results(file): 21 | try: 22 | with open(file) as f: 23 | for line in f: 24 | process_line(line) 25 | f.close() 26 | except Exception as e: 27 | print("error opening file {}: {}".format(file,e)) 28 | return None 29 | 30 | def main(argv): 31 | if len(argv) == 0: 32 | print("usage: {} ".format(os.path.basename(__file__))) 33 | sys.exit(1) 34 | 35 | process_results(argv[0]) 36 | 37 | if __name__ == "__main__": 38 | main(sys.argv[1:]) 39 | -------------------------------------------------------------------------------- /source/launch-solution.yml: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | version: 0.2 5 | 6 | phases: 7 | install: 8 | commands: 9 | - echo "$(date) - installing aws cli v2" 10 | - which aws 11 | - rm -f $(which aws) 12 | - hash -r 13 | - CWD=$(pwd) 14 | - echo "$CWD" 15 | - cd /tmp/ 16 | - wget --quiet https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip 17 | - unzip -q awscli-exe-linux-x86_64.zip 18 | - ./aws/install --update 19 | - rm -rf aws 20 | - rm -f awscli-exe-linux-x86_64.zip 21 | - aws --version 22 | - cd $CWD 23 | pre_build: 24 | commands: 25 | - echo "$(date) - starting pre_build in directory $(pwd)" 26 | - #env | sort 27 | - echo "BUCKET_RESOURCES $BUCKET_RESOURCES" 28 | - ls -la 29 | - chmod +x launch-solution-code-build.sh 30 | build: 31 | commands: 32 | - echo "$(date) - launching IoT DR solution from directory $(pwd)" 33 | - ./launch-solution-code-build.sh 34 | post_build: 35 | commands: 36 | - echo "$(date) - finished launching IoT DR solution from directory $(pwd)" 37 | -------------------------------------------------------------------------------- /source/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | set -e 7 | 8 | echo "building lambda installation packages" 9 | cd lambda 10 | 11 | cd iot-mr-jitr 12 | pip install pyOpenSSL -t . 13 | cd .. 14 | 15 | for lambda in iot-mr-jitr iot-mr-cross-region \ 16 | sfn-iot-mr-dynamo-trigger sfn-iot-mr-thing-crud \ 17 | sfn-iot-mr-thing-group-crud sfn-iot-mr-thing-type-crud \ 18 | sfn-iot-mr-shadow-syncer \ 19 | iot-dr-missing-device-replication 20 | do 21 | echo " creating zip for \"$lambda\"" 22 | rm -f ${lambda}.zip 23 | cd $lambda 24 | python -m py_compile *.py 25 | rm -rf __pycache__ 26 | zip ../${lambda}.zip -r . 27 | cd .. 28 | done 29 | 30 | # layer 31 | echo "creating lambda layer installation package" 32 | cd iot-dr-layer 33 | rm -rf python 34 | mkdir python 35 | pip install dynamodb-json==1.3 --no-deps -t python 36 | pip install simplejson==3.17.2 -t python 37 | python -m py_compile device_replication.py 38 | rm -rf __pycache__ 39 | cp device_replication.py python/ 40 | 41 | rm -f ../iot-dr-layer.zip 42 | zip ../iot-dr-layer.zip -r python 43 | cd .. 44 | 45 | echo "ZIP files:" 46 | pwd 47 | ls -l *.zip 48 | -------------------------------------------------------------------------------- /source/tools/enable-registry-events.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # Check to see if input has been provided: 7 | if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then 8 | echo "Please provide the root-region name, primary region & secondary region" 9 | echo "For example: ./enable-registry-events.sh us-east-1 eu-central-1 eu-west-1" 10 | exit 1 11 | fi 12 | REGIONS=($1 $2 $3) 13 | for region in "${!REGIONS[@]}"; 14 | do 15 | echo "enabling registry events & indexing in:" ${REGIONS[region]} 16 | IND=$(aws iot update-indexing-configuration --region ${REGIONS[region]} --thing-indexing-configuration 'thingIndexingMode=REGISTRY_AND_SHADOW,thingConnectivityIndexingMode=STATUS') 17 | REG=$(aws iot update-event-configurations --region ${REGIONS[region]} --cli-input-json '{"eventConfigurations": {"THING_TYPE": {"Enabled": true},"JOB_EXECUTION": {"Enabled": true},"THING_GROUP_HIERARCHY": {"Enabled": true},"CERTIFICATE": {"Enabled": true},"THING_TYPE_ASSOCIATION": {"Enabled": true},"THING_GROUP_MEMBERSHIP": {"Enabled": true},"CA_CERTIFICATE": {"Enabled": true},"THING":{"Enabled": true},"JOB": {"Enabled": true},"POLICY": {"Enabled": true},"THING_GROUP": {"Enabled": true}}}'); 18 | done -------------------------------------------------------------------------------- /deployment/run-unit-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # This assumes all of the OS-level configuration has been completed and git repo has already been cloned 7 | # 8 | # This script should be run from the repo's deployment directory 9 | # cd deployment 10 | # ./run-unit-tests.sh 11 | # 12 | 13 | set -e 14 | # Get reference for all important folders 15 | template_dir="$PWD" 16 | source_dir="$template_dir/../source" 17 | 18 | echo "------------------------------------------------------------------------------" 19 | echo "[Init] Unit tests" 20 | echo "------------------------------------------------------------------------------" 21 | 22 | cd $source_dir/lambda 23 | 24 | echo "python version: $(python3 --version)" 25 | 26 | for lambda in iot-mr-jitr iot-mr-cross-region \ 27 | sfn-iot-mr-dynamo-trigger sfn-iot-mr-thing-crud \ 28 | sfn-iot-mr-thing-group-crud sfn-iot-mr-thing-type-crud \ 29 | sfn-iot-mr-shadow-syncer \ 30 | iot-dr-missing-device-replication \ 31 | iot-dr-create-r53-checker \ 32 | iot-dr-launch-solution \ 33 | iot-dr-layer 34 | do 35 | echo "py_compile for \"$lambda\"" 36 | cd $lambda 37 | python3 -m py_compile *.py 38 | rm -rf __pycache__ 39 | cd .. 40 | done 41 | -------------------------------------------------------------------------------- /source/tools/sample-pol1.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Action": [ 6 | "iot:Connect" 7 | ], 8 | "Resource": "*", 9 | "Effect": "Allow" 10 | }, 11 | { 12 | "Action": [ 13 | "iot:Publish" 14 | ], 15 | "Resource": [ 16 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topic/dr/*", 17 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topic/$aws/things/${iot:Connection.Thing.ThingName}/shadow/*" 18 | ], 19 | "Effect": "Allow" 20 | }, 21 | { 22 | "Action": [ 23 | "iot:Receive" 24 | ], 25 | "Resource": [ 26 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topic/dr/*", 27 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topic/$aws/things/${iot:Connection.Thing.ThingName}/shadow/*", 28 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topic/$aws/events/*" 29 | ], 30 | "Effect": "Allow" 31 | }, 32 | { 33 | "Action": [ 34 | "iot:Subscribe" 35 | ], 36 | "Resource": [ 37 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topicfilter/dr/*", 38 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topicfilter/$aws/things/${iot:Connection.Thing.ThingName}/shadow/*", 39 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topicfilter/$aws/events/*" 40 | ], 41 | "Effect": "Allow" 42 | } 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /source/tools/sample-pol2.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Action": [ 6 | "iot:Connect" 7 | ], 8 | "Resource": [ 9 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:client/${iot:Connection.Thing.ThingName}", 10 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:client/${iot:Connection.Thing.ThingName}-*" 11 | ], 12 | "Effect": "Allow" 13 | }, 14 | { 15 | "Action": [ 16 | "iot:Publish" 17 | ], 18 | "Resource": [ 19 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topic/dr/${iot:Connection.Thing.ThingName}/*", 20 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topic/$aws/things/${iot:Connection.Thing.ThingName}/shadow/*" 21 | ], 22 | "Effect": "Allow" 23 | }, 24 | { 25 | "Action": [ 26 | "iot:Receive" 27 | ], 28 | "Resource": [ 29 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topic/dr/${iot:Connection.Thing.ThingName}/*", 30 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topic/$aws/things/${iot:Connection.Thing.ThingName}/shadow/*", 31 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topic/$aws/events/*" 32 | ], 33 | "Effect": "Allow" 34 | }, 35 | { 36 | "Action": [ 37 | "iot:Subscribe" 38 | ], 39 | "Resource": [ 40 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topicfilter/dr/${iot:Connection.Thing.ThingName}/*", 41 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topicfilter/$aws/things/${iot:Connection.Thing.ThingName}/shadow/*", 42 | "arn:aws:iot:AWS_REGION:AWS_ACCOUNT_ID:topicfilter/$aws/events/*" 43 | ], 44 | "Effect": "Allow" 45 | } 46 | ] 47 | } 48 | -------------------------------------------------------------------------------- /source/tools/list-thing.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0. 5 | 6 | import boto3 7 | import json 8 | import sys 9 | 10 | THING_NAME=None 11 | try: 12 | THING_NAME=sys.argv[1] 13 | except IndexError: 14 | print('usage: {} '.format(sys.argv[0])) 15 | sys.exit(1) 16 | 17 | 18 | c_iot = boto3.client('iot') 19 | 20 | def print_response(response): 21 | del response['ResponseMetadata'] 22 | print(json.dumps(response, indent=2, default=str)) 23 | 24 | 25 | try: 26 | response = c_iot.describe_thing(thingName=THING_NAME) 27 | print('THING') 28 | print_response(response) 29 | print('----------------------------------------') 30 | 31 | response = c_iot.list_thing_principals(thingName=THING_NAME) 32 | 33 | for principal in response['principals']: 34 | print('PRINCIPAL: {}'.format(principal)) 35 | response = c_iot.describe_certificate(certificateId=principal.split('/')[-1]) 36 | print('CERTIFICATE') 37 | print(' creationDate: {}'.format(response['certificateDescription']['creationDate'])) 38 | print(' validity: {}'.format(response['certificateDescription']['validity'])) 39 | print(' certificateMode: {}'.format(response['certificateDescription']['certificateMode'])) 40 | print('----------------------------------------') 41 | 42 | response = c_iot.list_principal_policies(principal=principal) 43 | print('POLICIES') 44 | 45 | for policy in response['policies']: 46 | response = c_iot.get_policy(policyName=policy['policyName']) 47 | print_response(response) 48 | except Exception as e: 49 | print('ERROR: {}'.format(e)) -------------------------------------------------------------------------------- /source/tools/create-device-type.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # 7 | # create-device.sh - provision a device with AWS IoT Core 8 | # 9 | 10 | if [ -z $1 ]; then 11 | echo "usage: $0 " 12 | exit 1 13 | fi 14 | 15 | THING_NAME=$1 16 | 17 | POLICY_NAME="" 18 | test ! -z $2 && POLICY_NAME=$2 19 | 20 | 21 | echo "Provisioning thing \"$THING_NAME\" in AWS IoT Core..." 22 | 23 | if aws iot describe-thing --thing-name $THING_NAME > /dev/null 2>&1; then 24 | echo "ERROR: device exists already. Exiting..."; 25 | aws iot describe-thing --thing-name $THING_NAME 26 | exit 1 27 | fi 28 | 29 | TMP_FILE=$(mktemp) 30 | 31 | echo " create thing" 32 | aws iot create-thing --thing-name $THING_NAME --thing-type-name dr-type03 33 | 34 | echo " create device key and certificate" 35 | aws iot create-keys-and-certificate --set-as-active \ 36 | --public-key-outfile $THING_NAME.public.key \ 37 | --private-key-outfile $THING_NAME.private.key \ 38 | --certificate-pem-outfile $THING_NAME.certificate.pem > $TMP_FILE 39 | 40 | CERTIFICATE_ARN=$(jq -r ".certificateArn" $TMP_FILE) 41 | CERTIFICATE_ID=$(jq -r ".certificateId" $TMP_FILE) 42 | echo " certificate arn: $CERTIFICATE_ARN" 43 | echo " echo certificate id: $CERTIFICATE_ID" 44 | 45 | if [ -z $POLICY_NAME ]; then 46 | POLICY_NAME=${THING_NAME}_Policy 47 | echo " create IoT policy \"$POLICY_NAME\"" 48 | aws iot create-policy --policy-name $POLICY_NAME \ 49 | --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action": "iot:*","Resource":"*"}]}' 50 | else 51 | echo "using provided policy \"$POLICY_NAME\"" 52 | fi 53 | 54 | sleep 10 55 | 56 | echo " attach policy to certificate" 57 | aws iot attach-policy --policy-name $POLICY_NAME \ 58 | --target $CERTIFICATE_ARN 59 | 60 | sleep 10 61 | 62 | echo " attach certificate to thing" 63 | aws iot attach-thing-principal --thing-name $THING_NAME \ 64 | --principal $CERTIFICATE_ARN 65 | 66 | rm $TMP_FILE 67 | -------------------------------------------------------------------------------- /source/tools/create-device-attrs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # 7 | # create-device.sh - provision a device with AWS IoT Core 8 | # 9 | 10 | if [ -z $1 ]; then 11 | echo "usage: $0 " 12 | exit 1 13 | fi 14 | 15 | THING_NAME=$1 16 | 17 | POLICY_NAME="" 18 | test ! -z $2 && POLICY_NAME=$2 19 | 20 | 21 | echo "Provisioning thing \"$THING_NAME\" in AWS IoT Core..." 22 | 23 | if aws iot describe-thing --thing-name $THING_NAME > /dev/null 2>&1; then 24 | echo "ERROR: device exists already. Exiting..."; 25 | aws iot describe-thing --thing-name $THING_NAME 26 | exit 1 27 | fi 28 | 29 | TMP_FILE=$(mktemp) 30 | 31 | echo " create thing" 32 | aws iot create-thing --thing-name $THING_NAME --attribute-payload {\"attributes\":{\"foo\":\"bar\"}} 33 | 34 | echo " create device key and certificate" 35 | aws iot create-keys-and-certificate --set-as-active \ 36 | --public-key-outfile $THING_NAME.public.key \ 37 | --private-key-outfile $THING_NAME.private.key \ 38 | --certificate-pem-outfile $THING_NAME.certificate.pem > $TMP_FILE 39 | 40 | CERTIFICATE_ARN=$(jq -r ".certificateArn" $TMP_FILE) 41 | CERTIFICATE_ID=$(jq -r ".certificateId" $TMP_FILE) 42 | echo " certificate arn: $CERTIFICATE_ARN" 43 | echo " echo certificate id: $CERTIFICATE_ID" 44 | 45 | if [ -z $POLICY_NAME ]; then 46 | POLICY_NAME=${THING_NAME}_Policy 47 | echo " create IoT policy \"$POLICY_NAME\"" 48 | aws iot create-policy --policy-name $POLICY_NAME \ 49 | --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action": "iot:*","Resource":"*"}]}' 50 | else 51 | echo "using provided policy \"$POLICY_NAME\"" 52 | fi 53 | 54 | sleep 10 55 | 56 | echo " attach policy to certificate" 57 | aws iot attach-policy --policy-name $POLICY_NAME \ 58 | --target $CERTIFICATE_ARN 59 | 60 | sleep 10 61 | 62 | echo " attach certificate to thing" 63 | aws iot attach-thing-principal --thing-name $THING_NAME \ 64 | --principal $CERTIFICATE_ARN 65 | 66 | rm $TMP_FILE 67 | -------------------------------------------------------------------------------- /source/tools/create-device-attrs-type.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # 7 | # create-device.sh - provision a device with AWS IoT Core 8 | # 9 | 10 | if [ -z $1 ]; then 11 | echo "usage: $0 " 12 | exit 1 13 | fi 14 | 15 | THING_NAME=$1 16 | 17 | POLICY_NAME="" 18 | test ! -z $2 && POLICY_NAME=$2 19 | 20 | 21 | echo "Provisioning thing \"$THING_NAME\" in AWS IoT Core..." 22 | 23 | if aws iot describe-thing --thing-name $THING_NAME > /dev/null 2>&1; then 24 | echo "ERROR: device exists already. Exiting..."; 25 | aws iot describe-thing --thing-name $THING_NAME 26 | exit 1 27 | fi 28 | 29 | TMP_FILE=$(mktemp) 30 | 31 | echo " create thing" 32 | aws iot create-thing --thing-name $THING_NAME --thing-type-name dr-type03 --attribute-payload {\"attributes\":{\"foo\":\"bar\"}} 33 | 34 | echo " create device key and certificate" 35 | aws iot create-keys-and-certificate --set-as-active \ 36 | --public-key-outfile $THING_NAME.public.key \ 37 | --private-key-outfile $THING_NAME.private.key \ 38 | --certificate-pem-outfile $THING_NAME.certificate.pem > $TMP_FILE 39 | 40 | CERTIFICATE_ARN=$(jq -r ".certificateArn" $TMP_FILE) 41 | CERTIFICATE_ID=$(jq -r ".certificateId" $TMP_FILE) 42 | echo " certificate arn: $CERTIFICATE_ARN" 43 | echo " echo certificate id: $CERTIFICATE_ID" 44 | 45 | if [ -z $POLICY_NAME ]; then 46 | POLICY_NAME=${THING_NAME}_Policy 47 | echo " create IoT policy \"$POLICY_NAME\"" 48 | aws iot create-policy --policy-name $POLICY_NAME \ 49 | --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action": "iot:*","Resource":"*"}]}' 50 | else 51 | echo "using provided policy \"$POLICY_NAME\"" 52 | fi 53 | 54 | sleep 10 55 | 56 | echo " attach policy to certificate" 57 | aws iot attach-policy --policy-name $POLICY_NAME \ 58 | --target $CERTIFICATE_ARN 59 | 60 | sleep 10 61 | 62 | echo " attach certificate to thing" 63 | aws iot attach-thing-principal --thing-name $THING_NAME \ 64 | --principal $CERTIFICATE_ARN 65 | 66 | rm $TMP_FILE 67 | -------------------------------------------------------------------------------- /source/tools/iot-dr-random-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # run-tests.sh 7 | 8 | 9 | usage() { 10 | echo "Usage: $0 -b thing_basename -d devices_dir" 1>&2 11 | exit 1 12 | } 13 | 14 | while getopts ":b:d:" options; do 15 | case "${options}" in 16 | b) 17 | THING_BASENAME=${OPTARG} 18 | ;; 19 | d) 20 | DEVICES_DIR=${OPTARG} 21 | ;; 22 | :) 23 | echo "Error: -${OPTARG} requires an argument." 24 | usage 25 | ;; 26 | *) 27 | usage 28 | ;; 29 | esac 30 | done 31 | 32 | [ -z "$DEVICES_DIR" ] && usage 33 | [ -z "$THING_BASENAME" ] && usage 34 | 35 | DIR=$(dirname $(realpath $0)) 36 | test ! -e $DIR/toolsrc && exit 1 37 | . $DIR/toolsrc 38 | 39 | echo "DIR: $DIR" 40 | echo "THING_BASENAME: $THING_BASENAME: DEVICES_DIR: $DEVICES_DIR" 41 | cd $DEVICES_DIR || exit 2 42 | 43 | NUM_THINGS=$(ls -1 *.key | wc -l) 44 | echo "NUM_THINGS: $NUM_THINGS" 45 | 46 | curl https://www.amazontrust.com/repository/AmazonRootCA1.pem -o root.ca.pem 47 | 48 | NUM_TESTS=2 49 | [ "$NUM_THINGS" -ge 100 ] && NUM_TESTS=5 50 | [ "$NUM_THINGS" -ge 1000 ] && NUM_TESTS=10 51 | 52 | echo "NUM_TESTS: $NUM_TESTS" 53 | cp /dev/null randoms 54 | for i in $(seq 1 $NUM_TESTS); do 55 | echo $((1 + $RANDOM % $NUM_THINGS)) >> randoms 56 | sleep 1 57 | done 58 | 59 | for n in $(cat randoms); do 60 | thing_name=${THING_BASENAME}${n} 61 | key=$thing_name.key 62 | cert=$thing_name.crt 63 | echo "RANDOM: thing_name: $thing_name IOT_ENDPOINT_PRIMARY: $IOT_ENDPOINT_PRIMARY IOT_ENDPOINT_SECONDARY: $IOT_ENDPOINT_SECONDARY" | tee -a ../randoms.log 64 | 65 | echo $DIR/iot-dr-pubsub.py --cert $cert --key $key --root-ca root.ca.pem --count 3 --interval 1 --endpoint $IOT_ENDPOINT_PRIMARY | tee -a ../randoms.log 66 | $DIR/iot-dr-pubsub.py --cert $cert --key $key --root-ca root.ca.pem --count 3 --interval 1 --endpoint $IOT_ENDPOINT_PRIMARY | tee ../${thing_name}-primary.log 67 | 68 | echo $DIR/iot-dr-pubsub.py --cert $cert --key $key --root-ca root.ca.pem --count 3 --interval 1 --endpoint $IOT_ENDPOINT_SECONDARY | tee -a ../randoms.log 69 | $DIR/iot-dr-pubsub.py --cert $cert --key $key --root-ca root.ca.pem --count 3 --interval 1 --endpoint $IOT_ENDPOINT_SECONDARY | tee ../${thing_name}-secondary.log 70 | done 71 | cd .. 72 | -------------------------------------------------------------------------------- /source/lambda/sfn-iot-mr-dynamo-trigger/lambda_function.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # 7 | # dynamodb trigger function 8 | # 9 | 10 | import boto3 11 | import json 12 | import logging 13 | import os 14 | import sys 15 | 16 | logger = logging.getLogger() 17 | for h in logger.handlers: 18 | logger.removeHandler(h) 19 | h = logging.StreamHandler(sys.stdout) 20 | FORMAT = '%(asctime)s [%(levelname)s] - %(filename)s:%(lineno)s - %(funcName)s - %(message)s' 21 | h.setFormatter(logging.Formatter(FORMAT)) 22 | logger.addHandler(h) 23 | logger.setLevel(logging.INFO) 24 | 25 | logger.debug('boto3 version: {}'.format(boto3.__version__)) 26 | 27 | STATEMACHINE_ARN = os.environ['STATEMACHINE_ARN'] 28 | 29 | c_sfn = boto3.client('stepfunctions') 30 | 31 | def lambda_handler(event, context): 32 | logger.info('event: {}'.format(event)) 33 | logger.debug(json.dumps(event, indent=4)) 34 | 35 | try: 36 | logger.info('length Records: {}'.format(len(event['Records']))) 37 | 38 | for record in event['Records']: 39 | logger.info('event type: {}'.format(record['dynamodb']['NewImage']['eventType']['S'])) 40 | item = record['dynamodb'] 41 | logger.info('item: {}'.format(item)) 42 | logger.info('event type: {}'.format(item['NewImage']['eventType']['S'])) 43 | logger.info('region: {} update region: {}'.format(os.environ['AWS_REGION'], item['NewImage']['aws:rep:updateregion']['S'])) 44 | 45 | if os.environ['AWS_REGION'] == item['NewImage']['aws:rep:updateregion']['S']: 46 | logger.info('item has been created in the same region and is not to be considered as replication - ignoring') 47 | return {'message': 'item has been created in the same region and is not to be considered as replication - ignoring'} 48 | 49 | input = json.dumps(item) 50 | logger.debug(input) 51 | 52 | logger.info('starting statemachine execution: STATEMACHINE_ARN: {}'.format(STATEMACHINE_ARN)) 53 | response = c_sfn.start_execution( 54 | stateMachineArn=STATEMACHINE_ARN, 55 | input=input 56 | ) 57 | logger.info('response: {}'.format(response)) 58 | 59 | return {'message': 'statemachine started'} 60 | except Exception as e: 61 | logger.error('{}'.format(e)) 62 | return {'message': 'error: {}'.format(e)} 63 | -------------------------------------------------------------------------------- /source/tools/create-device.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | 7 | # 8 | # create-device.sh - provision a device with AWS IoT Core 9 | # 10 | 11 | if [ -z $1 ]; then 12 | echo "usage: $0 " 13 | exit 1 14 | fi 15 | 16 | if [ -z $REGION ]; then 17 | echo "set the variable REGION to your aws region" 18 | exit 1 19 | fi 20 | 21 | THING_NAME=$1 22 | 23 | POLICY_NAME="" 24 | test ! -z $2 && POLICY_NAME=$2 25 | 26 | ACCOUNT_ID=$(aws sts get-caller-identity --output text |awk '{print $1}') 27 | 28 | echo "Provisioning thing \"$THING_NAME\" in AWS IoT Core..." 29 | 30 | if aws iot describe-thing --thing-name $THING_NAME > /dev/null 2>&1; then 31 | echo "ERROR: device exists already. Exiting..."; 32 | aws iot describe-thing --thing-name $THING_NAME 33 | exit 1 34 | fi 35 | 36 | TMP_FILE=$(mktemp) 37 | POL_FILE=$(mktemp) 38 | 39 | DIR=$(dirname $0) 40 | 41 | if [ -z "$AWS_DEFAULT_REGION" ]; then 42 | sed -e "s/AWS_REGION/$REGION/" -e "s/AWS_ACCOUNT_ID/$ACCOUNT_ID/" $DIR/sample-pol1.json > $POL_FILE 43 | else 44 | sed -e "s/AWS_REGION/$AWS_DEFAULT_REGION/" -e "s/AWS_ACCOUNT_ID/$ACCOUNT_ID/" $DIR/sample-pol1.json > $POL_FILE 45 | fi 46 | 47 | echo " create thing" 48 | aws iot create-thing --thing-name $THING_NAME 49 | 50 | echo " create device key and certificate" 51 | aws iot create-keys-and-certificate --set-as-active \ 52 | --public-key-outfile $THING_NAME.public.key \ 53 | --private-key-outfile $THING_NAME.private.key \ 54 | --certificate-pem-outfile $THING_NAME.certificate.pem > $TMP_FILE 55 | 56 | CERTIFICATE_ARN=$(jq -r ".certificateArn" $TMP_FILE) 57 | CERTIFICATE_ID=$(jq -r ".certificateId" $TMP_FILE) 58 | echo " certificate arn: $CERTIFICATE_ARN" 59 | echo " echo certificate id: $CERTIFICATE_ID" 60 | 61 | if [ -z $POLICY_NAME ]; then 62 | POLICY_NAME=${THING_NAME}_Policy 63 | echo " create IoT policy \"$POLICY_NAME\"" 64 | #aws iot create-policy --policy-name $POLICY_NAME \ 65 | # --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action": "iot:*","Resource":"*"}]}' 66 | aws iot create-policy --policy-name $POLICY_NAME --policy-document file://$POL_FILE 67 | 68 | else 69 | echo "using provided policy \"$POLICY_NAME\"" 70 | fi 71 | 72 | sleep 1 73 | 74 | echo " attach policy to certificate" 75 | aws iot attach-policy --policy-name $POLICY_NAME \ 76 | --target $CERTIFICATE_ARN 77 | 78 | sleep 1 79 | 80 | echo " attach certificate to thing" 81 | aws iot attach-thing-principal --thing-name $THING_NAME \ 82 | --principal $CERTIFICATE_ARN 83 | 84 | rm $TMP_FILE 85 | rm $POL_FILE 86 | -------------------------------------------------------------------------------- /source/lambda/iot-dr-region-syncer/ECSTaskRole-Policy.json: -------------------------------------------------------------------------------- 1 | Trust policy 2 | { 3 | "Version": "2012-10-17", 4 | "Statement": [ 5 | { 6 | "Sid": "", 7 | "Effect": "Allow", 8 | "Principal": { 9 | "Service": "ecs-tasks.amazonaws.com" 10 | }, 11 | "Action": "sts:AssumeRole" 12 | } 13 | ] 14 | } 15 | 16 | Inline Policy 17 | { 18 | "Version": "2012-10-17", 19 | "Statement": [ 20 | { 21 | "Action": [ 22 | "dynamodb:DeleteItem", 23 | "dynamodb:DescribeTable", 24 | "dynamodb:GetItem", 25 | "dynamodb:PutItem", 26 | "dynamodb:Query", 27 | "dynamodb:UpdateItem" 28 | ], 29 | "Resource": "*", 30 | "Effect": "Allow" 31 | }, 32 | { 33 | "Effect": "Allow", 34 | "Action": [ 35 | "logs:CreateLogGroup", 36 | "logs:CreateLogStream", 37 | "logs:PutLogEvents" 38 | ], 39 | "Resource": "arn:aws:logs:*:*:*" 40 | }, 41 | { 42 | "Effect": "Allow", 43 | "Action": [ 44 | "iot:AddThingToThingGroup", 45 | "iot:AttachPolicy", 46 | "iot:AttachThingPrincipal", 47 | "iot:CreateDynamicThingGroup", 48 | "iot:CreatePolicy", 49 | "iot:CreateThing", 50 | "iot:CreateThingGroup", 51 | "iot:CreateThingType", 52 | "iot:DeleteCertificate", 53 | "iot:DeleteDynamicThingGroup", 54 | "iot:DeletePolicy", 55 | "iot:DeleteThing", 56 | "iot:DeleteThingGroup", 57 | "iot:DeleteThingType", 58 | "iot:DeprecateThingType", 59 | "iot:DescribeCertificate", 60 | "iot:DescribeThing", 61 | "iot:DescribeThingGroup", 62 | "iot:DescribeThingType", 63 | "iot:DetachPolicy", 64 | "iot:DetachThingPrincipal", 65 | "iot:GetIndexingConfiguration", 66 | "iot:GetPolicy", 67 | "iot:ListAttachedPolicies", 68 | "iot:ListPrincipalPolicies", 69 | "iot:ListPrincipalThings", 70 | "iot:ListThingGroupsForThing", 71 | "iot:ListThingPrincipals", 72 | "iot:ListThings", 73 | "iot:ListThingTypes", 74 | "iot:ListThingsInThingGroup", 75 | "iot:RegisterCertificateWithoutCA", 76 | "iot:RemoveThingFromThingGroup", 77 | "iot:UpdateCertificate", 78 | "iot:UpdateThing", 79 | "iot:UpdateThingGroup", 80 | "iot:UpdateThingShadow" 81 | ], 82 | "Resource": "*" 83 | } 84 | ] 85 | } 86 | -------------------------------------------------------------------------------- /source/tools/create-device-pca.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | 7 | # 8 | # create-device.sh - provision a device with AWS IoT Core 9 | # 10 | 11 | if [ -z $1 ]; then 12 | echo "usage: $0 " 13 | exit 1 14 | fi 15 | 16 | THING_NAME=$1 17 | 18 | POLICY_NAME="" 19 | test ! -z $2 && POLICY_NAME=$2 20 | 21 | if [ -z $PCA_ARN ]; then 22 | echo "environment varriable PCA_ARN not set" 23 | echo "set it with export PCA_ARN=" 24 | exit 1 25 | fi 26 | 27 | PCA_REGION=$(echo $PCA_ARN |awk -F ':' '{print $4}') 28 | 29 | echo "Provisioning thing \"$THING_NAME\" in AWS IoT Core with certificate from PCA..." 30 | echo " PCA_ARN: $PCA_ARN" 31 | echo " PCA_REGION=$PCA_REGION" 32 | sleep 1 33 | 34 | if aws iot describe-thing --thing-name $THING_NAME > /dev/null 2>&1; then 35 | echo "ERROR: device exists already. Exiting..."; 36 | aws iot describe-thing --thing-name $THING_NAME 37 | exit 1 38 | fi 39 | 40 | TMP_FILE=$(mktemp) 41 | 42 | echo "requesting certificate from PCA: $PCA_ARN" 43 | openssl req -nodes -new -newkey rsa:2048 -keyout $THING_NAME.private.key -out $THING_NAME.csr -subj "/CN=$THING_NAME" 44 | CERTIFICATE_ARN_PCA=$(aws acm-pca issue-certificate --certificate-authority-arn $PCA_ARN --csr file://./$THING_NAME.csr --signing-algorithm "SHA256WITHRSA" --validity Value=365,Type="DAYS" --region $PCA_REGION | jq -r '.CertificateArn') 45 | echo "CERTIFICATE_ARN_PCA: $CERTIFICATE_ARN_PCA" 46 | aws acm-pca wait certificate-issued --certificate-authority-arn $PCA_ARN --certificate-arn $CERTIFICATE_ARN_PCA --region $PCA_REGION 47 | aws acm-pca get-certificate --certificate-authority-arn $PCA_ARN --certificate-arn $CERTIFICATE_ARN_PCA --region $PCA_REGION > $TMP_FILE 48 | jq -r '.Certificate' $TMP_FILE > $THING_NAME.certificate.pem 49 | 50 | echo "Provisioning thing \"$THING_NAME\" in AWS IoT Core..." 51 | 52 | echo " create thing" 53 | aws iot create-thing --thing-name $THING_NAME 54 | 55 | if [ -z $POLICY_NAME ]; then 56 | POLICY_NAME=${THING_NAME}_Policy 57 | echo " create IoT policy \"$POLICY_NAME\"" 58 | aws iot create-policy --policy-name $POLICY_NAME \ 59 | --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action": "iot:*","Resource":"*"}]}' 60 | else 61 | echo "using provided policy \"$POLICY_NAME\"" 62 | fi 63 | 64 | sleep 10 65 | 66 | echo " register certificate without CA" 67 | CERTIFICATE_ARN=$(aws iot register-certificate-without-ca --certificate-pem file://./$THING_NAME.certificate.pem --status ACTIVE | jq -r '.certificateArn') 68 | echo " CERTIFICATE_ARN: $CERTIFICATE_ARN" 69 | echo " attach policy to certificate" 70 | aws iot attach-policy --policy-name $POLICY_NAME --target $CERTIFICATE_ARN 71 | 72 | sleep 10 73 | 74 | echo " attach certificate to thing" 75 | aws iot attach-thing-principal --thing-name $THING_NAME --principal $CERTIFICATE_ARN 76 | 77 | rm $TMP_FILE 78 | -------------------------------------------------------------------------------- /source/lambda/iot-dr-missing-device-replication/lambda_function.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # 7 | # missing device replication 8 | # 9 | """IoT DR: replicate missing 10 | devices from one region to another.""" 11 | 12 | import logging 13 | import os 14 | import time 15 | 16 | import boto3 17 | 18 | from boto3.dynamodb.conditions import Key 19 | from device_replication import thing_exists, create_thing_with_cert_and_policy, delete_thing_create_error 20 | from dynamodb_json import json_util as ddb_json 21 | 22 | logger = logging.getLogger(__name__) 23 | logger.setLevel(logging.INFO) 24 | 25 | DYNAMODB_ERROR_TABLE = os.environ['DYNAMODB_ERROR_TABLE'] 26 | SECONDARY_REGION = os.environ['AWS_REGION'] 27 | 28 | logger.info('DYNAMODB_ERROR_TABLE: {} SECONDARY_REGION: {}'.format(DYNAMODB_ERROR_TABLE, SECONDARY_REGION)) 29 | 30 | def post_provision_thing(c_iot, c_dynamo, item): 31 | try: 32 | start_time = int(time.time()*1000) 33 | thing_name = item['thing_name'] 34 | primary_region = item['primary_region'] 35 | logger.info('thing_name: {} primary_region: {}'.format(thing_name, primary_region)) 36 | c_iot_p = boto3.client('iot', region_name = primary_region) 37 | 38 | # thing must exist in primary region 39 | if not thing_exists(c_iot_p, thing_name): 40 | logger.warn('thing_name "{}" does not exist in primary region: {}'.format(thing_name, primary_region)) 41 | return 'thing_name "{}" does not exist in primary region {}'.format(thing_name, primary_region) 42 | 43 | logger.info('trying to post provision thing_name: {}'.format(thing_name)) 44 | create_thing_with_cert_and_policy(c_iot, c_iot_p, thing_name, "", {}, primary_region, SECONDARY_REGION, 1, 0) 45 | 46 | delete_thing_create_error(c_dynamo, thing_name, DYNAMODB_ERROR_TABLE) 47 | 48 | end_time = int(time.time()*1000) 49 | duration = end_time - start_time 50 | logger.info('post_provision_thing duration: {}ms'.format(duration)) 51 | except Exception as e: 52 | logger.error('post_provision_thing: {}'.format(e)) 53 | 54 | 55 | def find_orphaned_things(c_dynamo, c_dynamo_resource, c_iot): 56 | table = c_dynamo_resource.Table(DYNAMODB_ERROR_TABLE) 57 | while True: 58 | if not table.global_secondary_indexes or table.global_secondary_indexes[0]['IndexStatus'] != 'ACTIVE': 59 | print('Waiting for index to backfill...') 60 | time.sleep(5) 61 | table.reload() 62 | else: 63 | break 64 | 65 | response = table.query( 66 | # Add the name of the index you want to use in your query. 67 | IndexName="action-index", 68 | KeyConditionExpression=Key('action').eq('create-thing'), 69 | ) 70 | logger.debug('response: {}'.format(response)) 71 | 72 | for item in response['Items']: 73 | item = ddb_json.loads(item) 74 | logger.info('item: {}'.format(item)) 75 | if 'primary_region' in item: 76 | post_provision_thing(c_iot, c_dynamo, item) 77 | else: 78 | logger.warn('cannot post provision device {} - primary region unknown'.format(item['thing_name'])) 79 | 80 | 81 | def lambda_handler(event, context): 82 | logger.info('event: {}'.format(event)) 83 | 84 | c_dynamo = boto3.client('dynamodb') 85 | c_dynamo_resource = boto3.resource('dynamodb') 86 | c_iot = boto3.client('iot') 87 | find_orphaned_things(c_dynamo, c_dynamo_resource, c_iot) 88 | 89 | return True 90 | -------------------------------------------------------------------------------- /source/tools/iot-search-devices.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0. 5 | 6 | import argparse 7 | import boto3 8 | import json 9 | import logging 10 | import sys 11 | 12 | logger = logging.getLogger() 13 | for h in logger.handlers: 14 | logger.removeHandler(h) 15 | h = logging.StreamHandler(sys.stdout) 16 | FORMAT = '%(asctime)s [%(levelname)s]: %(threadName)s-%(filename)s:%(lineno)s-%(funcName)s: %(message)s' 17 | h.setFormatter(logging.Formatter(FORMAT)) 18 | logger.addHandler(h) 19 | logger.setLevel(logging.INFO) 20 | #logger.setLevel(logging.DEBUG) 21 | 22 | parser = argparse.ArgumentParser(description="List all things for a given query string.") 23 | parser.add_argument('--query-string', required=True, help="Query string.") 24 | args = parser.parse_args() 25 | 26 | NUM_THINGS = 0 27 | 28 | 29 | def print_response(response): 30 | del response['ResponseMetadata'] 31 | logger.info(json.dumps(response, indent=2, default=str)) 32 | 33 | 34 | def get_next_token(response): 35 | next_token = None 36 | if 'nextToken' in response: 37 | next_token = response['nextToken'] 38 | 39 | #logger.info('next_token: {}'.format(next_token)) 40 | return next_token 41 | 42 | 43 | def search_things(max_results): 44 | global NUM_THINGS 45 | logger.info('args.query_string: {} max_results: {}'.format(args.query_string, max_results)) 46 | try: 47 | session = boto3.Session() 48 | region = session.region_name 49 | c_iot = session.client('iot') 50 | response = c_iot.search_index( 51 | indexName='AWS_Things', 52 | queryString=args.query_string, 53 | maxResults=max_results 54 | ) 55 | 56 | for thing in response['things']: 57 | logger.info('region: {} thing: {}'.format(region, thing)) 58 | NUM_THINGS += 1 59 | 60 | next_token = get_next_token(response) 61 | 62 | while next_token: 63 | session = boto3.Session() 64 | region = session.region_name 65 | c_iot = session.client('iot') 66 | response = c_iot.search_index( 67 | indexName='AWS_Things', 68 | nextToken=next_token, 69 | queryString=args.query_string, 70 | maxResults=max_results 71 | ) 72 | next_token = get_next_token(response) 73 | 74 | for thing in response['things']: 75 | logger.info('region: {} thing: {}'.format(region, thing)) 76 | NUM_THINGS += 1 77 | 78 | except Exception as e: 79 | logger.error('{}'.format(e)) 80 | 81 | 82 | def registry_indexing_enabled(): 83 | try: 84 | c_iot = boto3.Session().client('iot') 85 | response = c_iot.get_indexing_configuration() 86 | 87 | logger.info('thingIndexingMode: {}'.format(response['thingIndexingConfiguration']['thingIndexingMode'])) 88 | if response['thingIndexingConfiguration']['thingIndexingMode'] == 'OFF': 89 | return False 90 | 91 | return True 92 | except Exception as e: 93 | logger.error('{}'.format(e)) 94 | raise Exception(e) 95 | 96 | 97 | try: 98 | if not registry_indexing_enabled(): 99 | raise Exception('registry indexing must be enabled for this program to work') 100 | 101 | search_things(100) 102 | 103 | logger.info('region: {} query_string: {}: NUM_THINGS: {}'.format(boto3.Session().region_name, args.query_string, NUM_THINGS)) 104 | 105 | except Exception as e: 106 | logger.error('{}'.format(e)) 107 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check [existing open](https://github.com/awslabs/iot-dr/issues), or [recently closed](https://github.com/awslabs/iot-dr/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *master* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/awslabs/iot-dr/labels/help%20wanted) issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct (at) amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](https://github.com/awslabs/iot-dr/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | 61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 62 | -------------------------------------------------------------------------------- /source/lambda/sfn-iot-mr-shadow-syncer/lambda_function.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # 7 | # shadow syncer 8 | # 9 | """IoT DR: Lambda function 10 | for syncing classic device shadows.""" 11 | 12 | import json 13 | import logging 14 | import os 15 | import sys 16 | 17 | import boto3 18 | 19 | from botocore.config import Config 20 | from dynamodb_json import json_util as ddb_json 21 | 22 | logger = logging.getLogger() 23 | for h in logger.handlers: 24 | logger.removeHandler(h) 25 | h = logging.StreamHandler(sys.stdout) 26 | FORMAT = '%(asctime)s [%(levelname)s] - %(filename)s:%(lineno)s - %(funcName)s - %(message)s' 27 | h.setFormatter(logging.Formatter(FORMAT)) 28 | logger.addHandler(h) 29 | logger.setLevel(logging.INFO) 30 | 31 | ERRORS = [] 32 | IOT_ENDPOINT_PRIMARY = os.environ['IOT_ENDPOINT_PRIMARY'] 33 | IOT_ENDPOINT_SECONDARY = os.environ['IOT_ENDPOINT_SECONDARY'] 34 | 35 | class ShadowSyncerException(Exception): pass 36 | 37 | 38 | def get_iot_data_endpoint(region, iot_endpoints): 39 | try: 40 | logger.info('region: {} iot_endpoints: {}'.format(region, iot_endpoints)) 41 | iot_data_endpoint = None 42 | for endpoint in iot_endpoints: 43 | if region in endpoint: 44 | logger.info('region: {} in endpoint: {}'.format(region, endpoint)) 45 | iot_data_endpoint = endpoint 46 | break 47 | 48 | if iot_data_endpoint is None: 49 | logger.info('iot_data_endpoint not found calling describe_endpoint') 50 | iot_data_endpoint = boto3.client('iot').describe_endpoint(endpointType='iot:Data-ATS')['endpointAddress'] 51 | logger.info('iot_data_endpoint from describe_endpoint: {}'.format(iot_data_endpoint)) 52 | else: 53 | logger.info('iot_data_endpoint from iot_endpoints: {}'.format(iot_data_endpoint)) 54 | 55 | return iot_data_endpoint 56 | except Exception as e: 57 | logger.error('{}'.format(e)) 58 | raise ShadowSyncerException(e) 59 | 60 | 61 | def update_shadow(c_iot_data, thing_name, shadow): 62 | global ERRORS 63 | try: 64 | logger.info('update thing shadow: thing_name: {} payload: {}'.format(thing_name, shadow)) 65 | 66 | response = c_iot_data.update_thing_shadow( 67 | thingName=thing_name, 68 | payload=json.dumps(shadow).encode() 69 | ) 70 | 71 | logger.info('response: {}'.format(response)) 72 | except Exception as e: 73 | logger.error('update_shadow: {}'.format(e)) 74 | ERRORS.append('update_shadow: {}'.format(e)) 75 | 76 | 77 | def lambda_handler(event, context): 78 | global ERRORS 79 | logger.info('event: {}'.format(event)) 80 | logger.debug('context: {}'.format(context)) 81 | 82 | try: 83 | boto3_config = Config(retries = {'max_attempts': 12, 'mode': 'standard'}) 84 | 85 | iot_data_endpoint = get_iot_data_endpoint( 86 | os.environ['AWS_REGION'], 87 | [IOT_ENDPOINT_PRIMARY, IOT_ENDPOINT_SECONDARY] 88 | ) 89 | 90 | c_iot_data = boto3.client('iot-data', config=boto3_config, endpoint_url='https://{}'.format(iot_data_endpoint)) 91 | 92 | event = ddb_json.loads(event) 93 | logger.info('cleaned event: {}'.format(event)) 94 | if event['NewImage']['eventType'] == 'SHADOW_EVENT': 95 | thing_name = event['NewImage']['thing_name'] 96 | shadow = {'state': event['NewImage']['state']} 97 | logger.info('thing_name: {} shadow: {}'.format(thing_name, shadow)) 98 | 99 | update_shadow(c_iot_data, thing_name, shadow) 100 | else: 101 | logger.warn('eventType not a SHADOW_EVENT') 102 | 103 | except Exception as e: 104 | logger.error('{}'.format(e)) 105 | ERRORS.append('lambda_handler: {}'.format(e)) 106 | 107 | if ERRORS: 108 | error_message = ', '.join(ERRORS) 109 | logger.error('{}'.format(error_message)) 110 | raise ShadowSyncerException('{}'.format(error_message)) 111 | 112 | return {'message': 'shadow updated'} 113 | -------------------------------------------------------------------------------- /source/jupyter/01_IoTDR_Shared.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# AWS Disaster Recovery for IoT\n", 8 | "\n", 9 | "You can use this series of Jupyter notebooks to create optional resources or test features for an AWS IoT Disaster Recover (DR) setup. \n", 10 | "\n", 11 | "* `01_IoTDR_Shared` (this notebook): Set variables that are used in other notebooks of this series.\n", 12 | "* `02_IoTDR_ACM_PCA`: Setup your own private certificate authority with [AWS Certificate Manager Private Certificate Authority](https://docs.aws.amazon.com/acm-pca/latest/userguide/PcaWelcome.html). Your own CA can be registered with AWS IoT Core. It can be used to issue your device certificates. If you are using Just-in-Time Registration you must bring your own CA.\n", 13 | "* `03_IoTDR_Reg_PCA`: Register your private CA with AWS IoT Core.\n", 14 | "* `04_IoTDR_Device_Certs`: Issue certificates for devices with your private CA.\n", 15 | "* `05_IoTDR_JITR_Device`: Register a device with AWS IoT Core by using Just-in-Time Registration.\n", 16 | "\n", 17 | "### Permissions\n", 18 | "If you run the Jupyter notebooks on Amazon EC2 or an Amazon SageMaker notebook instance you need to add the following permissions to your instance profile.\n", 19 | "\n", 20 | "```\n", 21 | "{\n", 22 | " \"Effect\": \"Allow\",\n", 23 | " \"Action\": [\n", 24 | " \"acm-pca:*\",\n", 25 | " \"iot:*\"\n", 26 | " ],\n", 27 | " \"Resource\": \"*\"\n", 28 | "}\n", 29 | "```" 30 | ] 31 | }, 32 | { 33 | "cell_type": "markdown", 34 | "metadata": {}, 35 | "source": [ 36 | "## Library" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "from os.path import exists, join" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "## Shared Variables\n", 53 | "Variables which will be used in other notebooks of this series.\n", 54 | "\n", 55 | "Modify the variables to reflect your setup.\n", 56 | "\n", 57 | "* `aws_region_pca` AWS region where you are going to create the private CA. It can be in the primary or secondary or in another AWS region\n", 58 | "* `aws_region_primary` AWS IoT DR primary region\n", 59 | "* `aws_region_secondary` AWS IoT DR secondary region\n", 60 | "\n", 61 | "**Hint**: If you have already an ACM private CA and you want to use it in these examples set the variable `Sub_CN` to the common name of your private CA." 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "CA_subject = {\"C\": \"DE\", \"O\": \"AWS\", \"OU\": \"IoT\", \"ST\": \"Berlin\", \"L\": \"Berlin\", \"CN\": \"IoT DR CA\"}\n", 71 | "CA_directory = 'CA_{}'.format(CA_subject['CN'])\n", 72 | "CA_key = 'ca.key.pem'\n", 73 | "CA_cert = 'ca.crt.pem'\n", 74 | "\n", 75 | "PCA_directory = join(CA_directory, 'PCA')\n", 76 | "\n", 77 | "config = {}\n", 78 | "config['aws_region_pca'] = \"eu-west-1\"\n", 79 | "config['aws_region_primary'] = \"us-east-1\"\n", 80 | "config['aws_region_secondary'] = \"us-west-2\"\n", 81 | "config['CA_directory'] = CA_directory\n", 82 | "config['CA_key'] = CA_key\n", 83 | "config['CA_cert'] = CA_cert\n", 84 | "config['PCA_directory'] = PCA_directory\n", 85 | "config['CA_subject'] = CA_subject\n", 86 | "config['Sub_CN'] = 'Subordinated IoT Device CA'\n", 87 | "%store config" 88 | ] 89 | } 90 | ], 91 | "metadata": { 92 | "kernelspec": { 93 | "display_name": "conda_python3", 94 | "language": "python", 95 | "name": "conda_python3" 96 | }, 97 | "language_info": { 98 | "codemirror_mode": { 99 | "name": "ipython", 100 | "version": 3 101 | }, 102 | "file_extension": ".py", 103 | "mimetype": "text/x-python", 104 | "name": "python", 105 | "nbconvert_exporter": "python", 106 | "pygments_lexer": "ipython3", 107 | "version": "3.6.10" 108 | } 109 | }, 110 | "nbformat": 4, 111 | "nbformat_minor": 4 112 | } 113 | -------------------------------------------------------------------------------- /source/tools/list-all-things.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0. 5 | 6 | """IoT DR: list all things 7 | for a given query string.""" 8 | 9 | import logging 10 | import sys 11 | import time 12 | 13 | import boto3 14 | 15 | logger = logging.getLogger() 16 | for h in logger.handlers: 17 | logger.removeHandler(h) 18 | h = logging.StreamHandler(sys.stdout) 19 | FORMAT = '%(asctime)s [%(levelname)s] - %(filename)s:%(lineno)s - %(funcName)s - %(message)s' 20 | h.setFormatter(logging.Formatter(FORMAT)) 21 | logger.addHandler(h) 22 | logger.setLevel(logging.INFO) 23 | #logger.setLevel(logging.DEBUG) 24 | 25 | 26 | def get_next_token(response): 27 | next_token = None 28 | if 'nextToken' in response: 29 | next_token = response['nextToken'] 30 | 31 | #logger.info('next_token: {}'.format(next_token)) 32 | return next_token 33 | 34 | 35 | def get_search_things(c_iot, query_string, max_results): 36 | num_things = 0 37 | try: 38 | response = c_iot.search_index( 39 | indexName='AWS_Things', 40 | queryString=query_string, 41 | maxResults=max_results 42 | ) 43 | 44 | for thing in response['things']: 45 | num_things += 1 46 | logger.info('thing: {}'.format(thing)) 47 | 48 | next_token = get_next_token(response) 49 | 50 | while next_token: 51 | response = c_iot.search_index( 52 | indexName='AWS_Things', 53 | nextToken=next_token, 54 | queryString=query_string, 55 | maxResults=max_results 56 | ) 57 | next_token = get_next_token(response) 58 | 59 | for thing in response['things']: 60 | num_things += 1 61 | logger.info('thing: {}'.format(thing)) 62 | 63 | logger.info('num_things: {} query_string: {}'.format(num_things, query_string)) 64 | except Exception as e: 65 | logger.error('{}'.format(e)) 66 | 67 | 68 | def get_list_things(c_iot): 69 | num_things = 0 70 | try: 71 | paginator = c_iot.get_paginator("list_things") 72 | 73 | for page in paginator.paginate(): 74 | logger.debug('page: {}'.format(page)) 75 | logger.debug('things: {}'.format(page['things'])) 76 | for thing in page['things']: 77 | num_things += 1 78 | logger.info('thing: {}'.format(thing)) 79 | 80 | logger.info('num_things: {}'.format(num_things)) 81 | except Exception as e: 82 | logger.error('{}'.format(e)) 83 | 84 | 85 | def registry_indexing_enabled(c_iot): 86 | try: 87 | response = c_iot.get_indexing_configuration() 88 | logger.debug('response: {}'.format(response)) 89 | 90 | logger.info('thingIndexingMode: {}'.format(response['thingIndexingConfiguration']['thingIndexingMode'])) 91 | if response['thingIndexingConfiguration']['thingIndexingMode'] == 'OFF': 92 | return False 93 | 94 | return True 95 | except Exception as e: 96 | logger.error('{}'.format(e)) 97 | raise Exception(e) 98 | 99 | 100 | def list_all_things(query_string): 101 | c_iot = boto3.client('iot') 102 | region = c_iot.meta.region_name 103 | 104 | if registry_indexing_enabled(c_iot): 105 | logger.info('registry indexing enabled - using search_index to get things: query_string: {}'.format(query_string)) 106 | logger.info('region: {}'.format(region)) 107 | time.sleep(5) 108 | get_search_things(c_iot, query_string, 100) 109 | else: 110 | logger.info('registry indexing disabled - using list_things to get things') 111 | logger.info('region: {}'.format(region)) 112 | if sys.argv[1]: 113 | logger.warn('query string not supported when registry indexing is disabled') 114 | time.sleep(3) 115 | get_list_things(c_iot) 116 | 117 | return True 118 | 119 | 120 | # in case we run standalone, e.g. on Fargate 121 | if __name__ == '__main__': 122 | query_string = 'thingName:*' 123 | if len(sys.argv) > 1: 124 | query_string = 'thingName:{}'.format(sys.argv[1]) 125 | list_all_things(query_string) 126 | -------------------------------------------------------------------------------- /source/tools/delete-things.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0. 5 | 6 | # delete-things.py 7 | # 8 | # deletes things given by query string 9 | 10 | """Script to delete things from the device registry 11 | of IoT Core matched by a given query string.""" 12 | 13 | import argparse 14 | import logging 15 | import sys 16 | import time 17 | 18 | import boto3 19 | import boto3.session 20 | 21 | from device_replication import delete_thing 22 | 23 | logger = logging.getLogger() 24 | for h in logger.handlers: 25 | logger.removeHandler(h) 26 | h = logging.StreamHandler(sys.stdout) 27 | FORMAT = '%(asctime)s [%(levelname)s]: \ 28 | %(threadName)s-%(filename)s:%(lineno)s-%(funcName)s: %(message)s' 29 | h.setFormatter(logging.Formatter(FORMAT)) 30 | logger.addHandler(h) 31 | logger.setLevel(logging.INFO) 32 | #logger.setLevel(logging.DEBUG) 33 | 34 | parser = argparse.ArgumentParser(description="Delete things matching a given query string.") 35 | parser.add_argument('--region', required=True, help="AWS region.") 36 | parser.add_argument( 37 | '--query-string', required=True, 38 | help="Query string for example 'thingName:my-devices*'" 39 | ) 40 | parser.add_argument( 41 | '--retries', type=int, default=5, 42 | help="Number of retries in case of delete failure, default 5." 43 | ) 44 | parser.add_argument('--wait', type=int, default=1, help="Wait between retries, default 1.") 45 | parser.add_argument('-f', action='store_true', help="When true force delete without request.") 46 | args = parser.parse_args() 47 | 48 | NUM_THINGS_DELETED = 0 49 | POLICY_NAMES = {} 50 | THING_NAMES = [] 51 | NUM_ERRORS = 0 52 | 53 | session = boto3.session.Session(region_name=args.region) 54 | c_iot = session.client('iot', region_name=args.region) 55 | iot_data_endpoint = c_iot.describe_endpoint(endpointType='iot:Data-ATS')['endpointAddress'] 56 | 57 | logger.info("query_string: %s region: %s", args.query_string, args.region) 58 | logger.info("iot_data_endpoint: %s", iot_data_endpoint) 59 | 60 | response = c_iot.search_index(queryString=args.query_string) 61 | 62 | logger.info("response:\n%s", response) 63 | for thing in response["things"]: 64 | THING_NAMES.append(thing["thingName"]) 65 | 66 | while 'nextToken' in response: 67 | next_token = response['nextToken'] 68 | logger.info("next token: %s", next_token) 69 | response = c_iot.search_index( 70 | queryString=args.query_string, 71 | nextToken=next_token 72 | ) 73 | logger.info("response:\n%s", response) 74 | for thing in response["things"]: 75 | THING_NAMES.append(thing["thingName"]) 76 | 77 | if not THING_NAMES: 78 | logger.info("no things found matching query_string: %s", args.query_string) 79 | sys.exit(0) 80 | 81 | NUM_THINGS = len(THING_NAMES) 82 | 83 | if args.f is False: 84 | print("--------------------------------------\n") 85 | print("thing names to be DELETED:\n{}\n".format(THING_NAMES)) 86 | print("number of things to delete: {}\n".format(NUM_THINGS)) 87 | print("--------------------------------------\n") 88 | input("{} DEVICES FROM THE LIST ABOVE WILL BE DELETED \ 89 | INCLUDING CERTIFICATES, POLICIES AND SHADOWS \ 90 | \n== press to continue, to abort!\n".format(NUM_THINGS)) 91 | else: 92 | logger.info("-f is set - deleting without request: NUM_THINGS: %s", NUM_THINGS) 93 | time.sleep(1) 94 | 95 | 96 | for thing_name in THING_NAMES: 97 | THING_DELETED = False 98 | retries = args.retries 99 | wait = args.wait 100 | i = 1 101 | while THING_DELETED is False and i <= retries: 102 | try: 103 | logger.info("%s: THING NAME: %s", i, thing_name) 104 | i += 1 105 | delete_thing(c_iot, thing_name, iot_data_endpoint) 106 | THING_DELETED = True 107 | NUM_THINGS_DELETED += 1 108 | time.sleep(0.1) # avoid to run into api throttling 109 | break 110 | except Exception as delete_error: 111 | logger.error("delete thing thing_name: %s: %s", thing_name, delete_error) 112 | NUM_ERRORS += 1 113 | 114 | time.sleep(wait*i) 115 | 116 | 117 | logger.info("stats: NUM_THINGS: %s NUM_THINGS_DELETED: %s NUM_ERRORS: %s", 118 | NUM_THINGS, NUM_THINGS_DELETED, NUM_ERRORS 119 | ) 120 | -------------------------------------------------------------------------------- /source/lambda/iot-dr-custom-launch-solution/lambda_function.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import json 7 | import logging 8 | import sys 9 | import urllib.request 10 | 11 | from datetime import datetime 12 | 13 | import boto3 14 | 15 | logger = logging.getLogger() 16 | for h in logger.handlers: 17 | logger.removeHandler(h) 18 | h = logging.StreamHandler(sys.stdout) 19 | FORMAT = '%(asctime)s [%(levelname)s]: %(threadName)s-%(filename)s:%(lineno)s-%(funcName)s: %(message)s' 20 | h.setFormatter(logging.Formatter(FORMAT)) 21 | logger.addHandler(h) 22 | logger.setLevel(logging.INFO) 23 | 24 | 25 | SUCCESS = "SUCCESS" 26 | FAILED = "FAILED" 27 | 28 | 29 | def cfnresponse_send(event, context, responseStatus, responseData, physicalResourceId=None, noEcho=False): 30 | logger.info("event: {}".format(event)) 31 | logger.info("context: {}".format(context)) 32 | 33 | responseUrl = event['ResponseURL'] 34 | 35 | responseBody = {} 36 | responseBody['Status'] = responseStatus 37 | responseBody['Reason'] = 'See the details in CloudWatch Log Stream: {}'.format(context.log_stream_name) 38 | responseBody['PhysicalResourceId'] = physicalResourceId or context.log_stream_name 39 | responseBody['StackId'] = event['StackId'] 40 | responseBody['RequestId'] = event['RequestId'] 41 | responseBody['LogicalResourceId'] = event['LogicalResourceId'] 42 | responseBody['NoEcho'] = noEcho 43 | responseBody['Data'] = responseData 44 | 45 | json_responseBody = json.dumps(responseBody) 46 | 47 | logger.info("Response body: {}\n".format(json_responseBody)) 48 | 49 | headers = { 50 | 'content-type' : '', 51 | 'content-length' : str(len(json_responseBody)) 52 | } 53 | 54 | logger.info("responseUrl: {}".format(responseUrl)) 55 | try: 56 | req = urllib.request.Request(url=responseUrl, 57 | data=json_responseBody.encode(), 58 | headers=headers, 59 | method='PUT') 60 | with urllib.request.urlopen(req) as f: 61 | pass 62 | logger.info("urllib request: req: {} status: {} reason: {}".format(req, f.status, f.reason)) 63 | 64 | except Exception as e: 65 | logger.error("urllib request: {}".format(e)) 66 | 67 | 68 | def lambda_handler(event, context): 69 | logger.info('event: {}'.format(event)) 70 | 71 | responseData = {} 72 | 73 | if event['RequestType'] == 'Update': 74 | logger.info('update cycle') 75 | responseData = {'Success': 'Update pass'} 76 | cfnresponse_send(event, context, SUCCESS, responseData, 'CustomResourcePhysicalID') 77 | 78 | if event['RequestType'] == 'Delete': 79 | logger.info('delete cycle') 80 | client = boto3.client('lambda') 81 | response = client.list_tags( 82 | Resource=event['ServiceToken'] 83 | ) 84 | if 'Tags' in response and 'STACK_POSTFIX' in response['Tags']: 85 | iot_dr_primary_stack_name = 'IoTDRPrimary{}'.format(response['Tags']['STACK_POSTFIX']) 86 | logger.info('iot_dr_primary_stack_name: {}'.format(iot_dr_primary_stack_name)) 87 | else: 88 | logger.warn('no tag with name STACK_POSTFIX: delete stacks manually') 89 | 90 | responseData = {'Success': 'Delete pass'} 91 | cfnresponse_send(event, context, SUCCESS, responseData, 'CustomResourcePhysicalID') 92 | 93 | if event['RequestType'] == 'Create': 94 | cfn_result = FAILED 95 | responseData = {} 96 | try: 97 | primary_region = event['ResourceProperties']['PRIMARY_REGION'] 98 | secondary_region = event['ResourceProperties']['SECONDARY_REGION'] 99 | date_time = datetime.now().strftime('%Y%m%d%H%M%S') 100 | 101 | logger.info('primary_region: {} secondary_region: {} date_time: {}'. 102 | format(primary_region, secondary_region, date_time)) 103 | 104 | lambda_arn = event['ServiceToken'] 105 | logger.info('lambda_arn: {}'.format(lambda_arn)) 106 | 107 | client = boto3.client('lambda') 108 | response = client.tag_resource( 109 | Resource=lambda_arn, 110 | Tags={ 111 | 'STACK_POSTFIX': date_time 112 | } 113 | ) 114 | 115 | responseData['STACK_POSTFIX'] = date_time 116 | responseData['Success'] = 'Solution launch initiated' 117 | 118 | logger.info('responseData: {}'.format(responseData)) 119 | 120 | cfn_result = SUCCESS 121 | 122 | except Exception as e: 123 | logger.error('{}'.format(e)) 124 | raise Exception(e) 125 | 126 | cfnresponse_send(event, context, cfn_result, responseData, 'CustomResourcePhysicalID') 127 | -------------------------------------------------------------------------------- /source/tools/bulk-bench.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # bulk-bench.sh 7 | # shell script for doing a very basic benchmark 8 | # for bulk provisioing 9 | 10 | DIR=$(dirname $0) 11 | REAL_DIR=$(dirname $(realpath $0)) 12 | 13 | if [ -z $1 ] || [ -z $2 ]; then 14 | echo "usage: $0 []" 15 | exit 1 16 | fi 17 | 18 | THING_BASENAME=$1 19 | NUM_THINGS=$2 20 | TEMPLATE_BODY="${REAL_DIR}/simpleTemplateBody.json" 21 | if [ ! -z "$3" ]; then 22 | TEMPLATE_BODY=$3 23 | fi 24 | 25 | if [ -z "$S3_BUCKET" ]; then 26 | echo "you must set the shell variable S3_BUCKET to a bucket where you have write permissions to" 27 | echo "an S3 bucket is required for bulk provisioing" 28 | exit 1 29 | fi 30 | 31 | if [ -z "$ARN_IOT_PROVISIONING_ROLE" ]; then 32 | echo "you must set the shell variable ARN_IOT_PROVISIONING_ROLE to the arn of an IAM role" 33 | echo "which allows bulk provisioning devices in your account" 34 | exit 1 35 | fi 36 | 37 | if [ ! -e $TEMPLATE_BODY ]; then 38 | echo "cannot find provisioning template \"$TEMPLATE_BODY\"" 39 | exit 1 40 | fi 41 | 42 | 43 | DATE_TIME=$(date "+%Y-%m-%d_%H-%M-%S") 44 | BULK_JSON="bulk-${DATE_TIME}.json" 45 | 46 | OUT_DIR=${THING_BASENAME}-${DATE_TIME} 47 | mkdir $OUT_DIR || exit 1 48 | 49 | echo "starting bulk provisioning..." 50 | echo "THING_BASENAME: \"$THING_BASENAME\" NUM_THINGS: \"$NUM_THINGS\"" 51 | echo "TEMPLATE_BODY: \"$TEMPLATE_BODY\"" 52 | echo "OUT_DIR: \"$OUT_DIR\" BULK_JSON: \"$BULK_JSON\"" 53 | 54 | sleep 2 55 | 56 | TIME_START_GEN_KEYS=$(date +%s) 57 | cp /dev/null $OUT_DIR/$BULK_JSON 58 | for i in $(seq 1 $NUM_THINGS) ; do 59 | thing_name=${THING_BASENAME}${i} 60 | echo "${i}/${NUM_THINGS}: creating key/csr for \"$thing_name\"" 61 | openssl req -new -newkey rsa:2048 -nodes -keyout $OUT_DIR/${thing_name}.key -out $OUT_DIR/${thing_name}.csr -subj "/C=DE/ST=Berlin/L=Berlin/O=AWS/CN=${thing_name}" 62 | 63 | one_line_csr=$(awk 'NF {sub(/\r/, ""); printf "%s\\n",$0;}' $OUT_DIR/${thing_name}.csr) 64 | 65 | echo "{\"ThingName\": \"${thing_name}\", \"SerialNumber\": \"$i\", \"CSR\": \"$one_line_csr\"}" >> $OUT_DIR/$BULK_JSON 66 | done 67 | 68 | TIME_END_GEN_KEYS=$(date +%s) 69 | TIME_TOTAL_GEN_KEYS=$(expr $TIME_END_GEN_KEYS - $TIME_START_GEN_KEYS) 70 | 71 | echo "output written to $OUT_DIR/$BULK_JSON" 72 | 73 | echo "copying $OUT_DIR/$BULK_JSON to s3://$S3_BUCKET/" 74 | aws s3 cp $OUT_DIR/$BULK_JSON s3://$S3_BUCKET/ 75 | aws s3 ls s3://$S3_BUCKET/ 76 | 77 | TEMP_FILE=$(mktemp) 78 | echo "TEMP_FILE: $TEMP_FILE" 79 | 80 | TIME_START_BULK=$(date +%s) 81 | aws iot start-thing-registration-task \ 82 | --template-body file://$TEMPLATE_BODY \ 83 | --input-file-bucket $S3_BUCKET \ 84 | --input-file-key $BULK_JSON --role-arn $ARN_IOT_PROVISIONING_ROLE | tee $TEMP_FILE 85 | 86 | task_id=$(jq -r ".taskId" $TEMP_FILE) 87 | echo "task_id: $task_id" 88 | 89 | 90 | rc=1 91 | rc_err=1 92 | TEMP_FILE2=$(mktemp) 93 | echo "TEMP_FILE2: $TEMP_FILE2" 94 | while [[ $rc -ne 0 && $rc_err -ne 0 ]]; do 95 | echo "$(date '+%Y-%m-%d_%H-%M-%S'): task_id: $task_id" 96 | #echo "RESULTS" 97 | aws iot list-thing-registration-task-reports --report-type RESULTS --task-id $task_id | tee $TEMP_FILE2 98 | url=$(jq -r '.resourceLinks[]' $TEMP_FILE2) 99 | echo $url | grep ^https 100 | rc=$? 101 | echo "TYPE RESULTS list-thing-registration-task-reports: rc: $rc" 102 | if [ $rc -eq 0 ]; then 103 | echo " downloading results to $OUT_DIR/results.json" 104 | wget -O $OUT_DIR/results.json "$url" 105 | echo " results written to: $OUT_DIR/results.json" 106 | else 107 | echo " no results yet" 108 | fi 109 | 110 | #echo "ERRORS" 111 | err_url=$(aws iot list-thing-registration-task-reports --report-type ERRORS --task-id $task_id | jq -r '.resourceLinks[]') 112 | echo $err_url | grep -q '^https' 113 | rc_err=$? 114 | echo "TYPE ERRORS list-thing-registration-task-reports: rc_err: $rc_err" 115 | if [ $rc_err -eq 0 ]; then 116 | echo " errors detected, downloading to $OUT_DIR/errors.json" 117 | wget -O $OUT_DIR/errors.json "$err_url" 118 | echo " errors written to: $OUT_DIR/errors.json" 119 | echo "ERRORS detected!!! Consider stopping with Ctrl+C and analyse errors" 120 | sleep 5 121 | else 122 | echo " no errors detected" 123 | fi 124 | 125 | echo "----------------------------------------" 126 | 127 | if [[ $rc -ne 0 && $rc_err -ne 0 ]]; then 128 | sleep 5 129 | fi 130 | done 131 | 132 | if [ -e $OUT_DIR/results.json ] && [ -x $REAL_DIR/bulk-result.py ]; then 133 | cd $OUT_DIR 134 | $REAL_DIR/bulk-result.py results.json 135 | cd .. 136 | fi 137 | 138 | TIME_END_BULK=$(date +%s) 139 | TIME_TOTAL_BULK=$(expr $TIME_END_BULK - $TIME_START_BULK) 140 | 141 | echo "" 142 | 143 | echo "AWS IoT bulk provisioning results" 144 | echo "--------------------------------------------------------------" 145 | echo "THING_BASENAME: $THING_BASENAME" 146 | echo "NUM_THINGS: $NUM_THINGS" 147 | echo "START: $(date -d@$TIME_START_BULK +'%Y-%m-%d %H:%M:%S')" 148 | echo "END: $(date -d@$TIME_END_BULK +'%Y-%m-%d %H:%M:%S')" 149 | echo "time to generate keys and CSRs: $TIME_TOTAL_GEN_KEYS secs." 150 | echo "time for bulk provisioning: $TIME_TOTAL_BULK secs." 151 | -------------------------------------------------------------------------------- /source/lambda/sfn-iot-mr-thing-type-crud/lambda_function.py: -------------------------------------------------------------------------------- 1 | # 2 | # thing type crud 3 | # 4 | """IoT DR: Lambda function to handle 5 | thing type CreateUpdateDelete""" 6 | 7 | import logging 8 | import sys 9 | 10 | import boto3 11 | 12 | logger = logging.getLogger() 13 | for h in logger.handlers: 14 | logger.removeHandler(h) 15 | h = logging.StreamHandler(sys.stdout) 16 | FORMAT = '%(asctime)s [%(levelname)s] - %(filename)s:%(lineno)s - %(funcName)s - %(message)s' 17 | h.setFormatter(logging.Formatter(FORMAT)) 18 | logger.addHandler(h) 19 | logger.setLevel(logging.INFO) 20 | 21 | 22 | class ThingTypeCrudException(Exception): pass 23 | 24 | 25 | def deprecate_type(c_iot, thing_type_name, bool): 26 | logger.info('thing_type_name: {} undo deprecate bool: {}'.format(thing_type_name, bool)) 27 | try: 28 | response = c_iot.deprecate_thing_type(thingTypeName=thing_type_name, undoDeprecate=bool) 29 | logger.info('response: {}'.format(response)) 30 | except c_iot.exceptions.ResourceNotFoundException: 31 | logger.info('thing_type_name {} does not exist'.format(thing_type_name)) 32 | return False 33 | 34 | except Exception as e: 35 | logger.error('{}'.format(e)) 36 | raise(e) 37 | 38 | 39 | def thing_type_exists(c_iot, thing_type_name): 40 | logger.info("thing type exists: thing_type_name: {}".format(thing_type_name)) 41 | try: 42 | response = c_iot.describe_thing_type(thingTypeName=thing_type_name) 43 | logger.info('response: {}'.format(response)) 44 | return True 45 | 46 | except c_iot.exceptions.ResourceNotFoundException: 47 | logger.info('thing_type_name {} does not exist'.format(thing_type_name)) 48 | return False 49 | 50 | except Exception as e: 51 | logger.error('{}'.format(e)) 52 | raise(e) 53 | 54 | 55 | def delete_type(c_iot, thing_type_name): 56 | logger.info('thing_type_name: {}'.format(thing_type_name)) 57 | try: 58 | response = c_iot.delete_thing_type(thingTypeName=thing_type_name) 59 | logger.info('response: {}'.format(response)) 60 | except c_iot.exceptions.ResourceNotFoundException: 61 | logger.info('thing_type_name {} does not exist'.format(thing_type_name)) 62 | return False 63 | 64 | except Exception as e: 65 | logger.error('{}'.format(e)) 66 | raise(e) 67 | 68 | 69 | def update_thing_type(c_iot, thing_name, thing_type_name): 70 | try: 71 | if thing_type_name == None: 72 | response = c_iot.update_thing( 73 | thingName=thing_name, 74 | removeThingType=True 75 | ) 76 | else: 77 | response = c_iot.update_thing( 78 | thingName=thing_name, 79 | thingTypeName=thing_type_name, 80 | removeThingType=False 81 | ) 82 | logger.info("update thing type: {}".format(response)) 83 | except Exception as e: 84 | logger.error("update_thing_type: {}".format(e)) 85 | raise(e) 86 | 87 | 88 | def create_thing_type(c_iot, thing_type_name): 89 | logger.info("create thing type: thing_type_name: {}".format(thing_type_name)) 90 | try: 91 | if not thing_type_exists(c_iot, thing_type_name): 92 | response = c_iot.create_thing_type(thingTypeName=thing_type_name) 93 | logger.info("create_thing_type: response: {}".format(response)) 94 | else: 95 | logger.info("thing type exists already: {}".format(thing_type_name)) 96 | except c_iot.exceptions.ResourceAlreadyExistsException: 97 | logger.info('exists already thing_type_name: {}'.format(thing_type_name)) 98 | except Exception as e: 99 | logger.error("create_thing_type: {}".format(e)) 100 | raise(e) 101 | 102 | 103 | def lambda_handler(event, context): 104 | logger.info('event: {}'.format(event)) 105 | try: 106 | c_iot = boto3.client('iot') 107 | 108 | if event['NewImage']['eventType']['S'] == 'THING_TYPE_EVENT': 109 | if event['NewImage']['operation']['S'] == 'CREATED': 110 | thing_type_name = event['NewImage']['thingTypeName']['S'] 111 | logger.info("thing_type_name: {}".format(thing_type_name)) 112 | create_thing_type(c_iot, thing_type_name) 113 | 114 | elif event['NewImage']['operation']['S'] == 'UPDATED': 115 | if 'isDeprecated' in event['NewImage'] and event['NewImage']['isDeprecated']['BOOL'] is True: 116 | thing_type_name = event['NewImage']['thingTypeName']['S'] 117 | deprecate_type(c_iot, thing_type_name, False) 118 | 119 | elif 'isDeprecated' in event['NewImage'] and event['NewImage']['isDeprecated']['BOOL'] is False: 120 | thing_type_name = event['NewImage']['thingTypeName']['S'] 121 | deprecate_type(c_iot, thing_type_name, True) 122 | 123 | elif event['NewImage']['operation']['S'] == 'DELETED': 124 | thing_type_name = event['NewImage']['thingTypeName']['S'] 125 | logger.info("thing_type_name: {}".format(thing_type_name)) 126 | delete_type(c_iot, thing_type_name) 127 | 128 | elif event['NewImage']['eventType']['S'] == 'THING_TYPE_ASSOCIATION_EVENT': 129 | if event['NewImage']['operation']['S'] == 'ADDED': 130 | thing_name = event['NewImage']['thingName']['S'] 131 | thing_type_name = event['NewImage']['thingTypeName']['S'] 132 | logger.info("ADDED: thing_name: {} thing_type_name: {}".format(thing_name, thing_type_name)) 133 | update_thing_type(c_iot, thing_name, thing_type_name) 134 | 135 | elif event['NewImage']['operation']['S'] == 'REMOVED': 136 | thing_name = event['NewImage']['thingName']['S'] 137 | thing_type_name = event['NewImage']['thingTypeName']['S'] 138 | logger.info("REMOVED: thing_name: {} thing_type_name: {}".format(thing_name, thing_type_name)) 139 | update_thing_type(c_iot, thing_name, None) 140 | 141 | except Exception as e: 142 | logger.error(e) 143 | raise ThingTypeCrudException('{}'.format(e)) 144 | 145 | return {"message": "success"} 146 | -------------------------------------------------------------------------------- /source/lambda/sfn-iot-mr-thing-crud/lambda_function.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # 7 | # thing crud 8 | # 9 | """IoT DR: Lambda function to handle 10 | thing CreateUpdateDelete""" 11 | 12 | import logging 13 | import os 14 | import sys 15 | import time 16 | 17 | import boto3 18 | 19 | import device_replication 20 | 21 | from botocore.config import Config 22 | from device_replication import ( 23 | create_thing, create_thing_with_cert_and_policy, 24 | delete_thing_create_error, delete_thing, 25 | get_iot_data_endpoint 26 | ) 27 | from dynamodb_json import json_util as ddb_json 28 | 29 | logger = logging.getLogger() 30 | for h in logger.handlers: 31 | logger.removeHandler(h) 32 | h = logging.StreamHandler(sys.stdout) 33 | FORMAT = '%(asctime)s [%(levelname)s] - %(filename)s:%(lineno)s - %(funcName)s - %(message)s' 34 | h.setFormatter(logging.Formatter(FORMAT)) 35 | logger.addHandler(h) 36 | logger.setLevel(logging.INFO) 37 | 38 | ERRORS = [] 39 | DYNAMODB_ERROR_TABLE = os.environ['DYNAMODB_ERROR_TABLE'] 40 | CREATE_MODE = os.environ.get('CREATE_MODE', 'complete') 41 | IOT_ENDPOINT_PRIMARY = os.environ['IOT_ENDPOINT_PRIMARY'] 42 | IOT_ENDPOINT_SECONDARY = os.environ['IOT_ENDPOINT_SECONDARY'] 43 | 44 | 45 | class ThingCrudException(Exception): pass 46 | 47 | 48 | def update_table_create_thing_error(c_dynamo, thing_name, primary_region, error_message): 49 | logger.info('update_table_create_thing_error: thing_name: {}'.format(thing_name)) 50 | try: 51 | response = c_dynamo.update_item( 52 | TableName=DYNAMODB_ERROR_TABLE, 53 | Key={'thing_name': {'S': thing_name}, 'action': {'S': 'create-thing'}}, 54 | AttributeUpdates={ 55 | 'primary_region': {'Value': {'S': primary_region}}, 56 | 'error_message': {'Value': {'S': error_message}}, 57 | 'time_stamp': {'Value': {'N': str(int(time.time()*1000))}} 58 | } 59 | ) 60 | logger.info('update_table_create_thing_error: {}'.format(response)) 61 | except Exception as e: 62 | logger.error("update_table_create_thing_error: {}".format(e)) 63 | 64 | 65 | def lambda_handler(event, context): 66 | global ERRORS 67 | ERRORS = [] 68 | 69 | logger.info('event: {}'.format(event)) 70 | 71 | try: 72 | event = ddb_json.loads(event) 73 | logger.info('cleaned event: {}'.format(event)) 74 | 75 | boto3_config = Config(retries = {'max_attempts': 12, 'mode': 'standard'}) 76 | 77 | c_iot = boto3.client('iot', config=boto3_config) 78 | c_dynamo = boto3.client('dynamodb') 79 | 80 | secondary_region = os.environ['AWS_REGION'] 81 | logger.info('secondary_region: {}'.format(secondary_region)) 82 | 83 | if event['NewImage']['operation'] == 'CREATED': 84 | logger.info('operation: {}'.format(event['NewImage']['operation'])) 85 | thing_name = event['NewImage']['thingName'] 86 | logger.info('thing_name: {}'.format(thing_name)) 87 | attrs = {} 88 | if 'attributes' in event['NewImage'] and event['NewImage']['attributes']: 89 | attrs = {'attributes': {}} 90 | for key in event['NewImage']['attributes']: 91 | attrs['attributes'][key] = event['NewImage']['attributes'][key] 92 | 93 | if 'attributes' in attrs: 94 | attrs['merge'] = False 95 | logger.info('attrs: {}'.format(attrs)) 96 | 97 | thing_type_name = "" 98 | if 'thingTypeName' in event['NewImage']: 99 | thing_type_name = event['NewImage']['thingTypeName'] 100 | logger.info('thing_type_name: {}'.format(thing_type_name)) 101 | primary_region = event['NewImage']['aws:rep:updateregion'] 102 | logger.info('primary_region: {}'.format(primary_region)) 103 | logger.info('CREATE_MODE: {}'.format(CREATE_MODE)) 104 | 105 | c_iot_p = boto3.client('iot', config=boto3_config, region_name = primary_region) 106 | 107 | start_time = int(time.time()*1000) 108 | if CREATE_MODE == 'thing_only': 109 | create_thing(c_iot, c_iot_p, thing_name, thing_type_name, attrs) 110 | else: 111 | create_thing_with_cert_and_policy(c_iot, c_iot_p, thing_name, thing_type_name, attrs, 3, 2) 112 | end_time = int(time.time()*1000) 113 | duration = end_time - start_time 114 | logger.info('thing created: thing_name: {}: duration: {}ms'.format(thing_name, duration)) 115 | logger.info('thing created, deleting from dynamo if exists: thing_name: {}'.format(thing_name)) 116 | delete_thing_create_error(c_dynamo, thing_name, DYNAMODB_ERROR_TABLE) 117 | 118 | if event['NewImage']['operation'] == 'UPDATED': 119 | logger.info('operation: {}'.format(event['NewImage']['operation'])) 120 | thing_name = event['NewImage']['thingName'] 121 | logger.info("thing_name: {}".format(thing_name)) 122 | 123 | primary_region = event['NewImage']['aws:rep:updateregion'] 124 | logger.info('primary_region: {}'.format(primary_region)) 125 | 126 | c_iot_p = boto3.client('iot', config=boto3_config, region_name = primary_region) 127 | 128 | attrs = {} 129 | if 'attributes' in event['NewImage']: 130 | for key in event['NewImage']['attributes']: 131 | attrs[key] = event['NewImage']['attributes'][key] 132 | 133 | merge = True 134 | if attrs: 135 | merge = False 136 | 137 | thing_type_name = "" 138 | if 'S' in event['NewImage']['thingTypeName']: 139 | thing_type_name = event['NewImage']['thingTypeName']['S'] 140 | 141 | logger.info("thing_name: {} thing_type_name: {} attrs: {}". 142 | format(thing_name, thing_type_name, attrs)) 143 | update_thing(c_iot, c_iot_p, thing_name, thing_type_name, attrs, merge) 144 | 145 | if event['NewImage']['operation'] == 'DELETED': 146 | logger.info('operation: {}'.format(event['NewImage']['operation'])) 147 | thing_name = event['NewImage']['thingName'] 148 | logger.info("thing_name: {}".format(thing_name)) 149 | 150 | iot_data_endpoint = get_iot_data_endpoint( 151 | os.environ['AWS_REGION'], 152 | [IOT_ENDPOINT_PRIMARY, IOT_ENDPOINT_SECONDARY] 153 | ) 154 | 155 | delete_thing(c_iot, thing_name, iot_data_endpoint) 156 | delete_thing_create_error(c_dynamo, thing_name, DYNAMODB_ERROR_TABLE) 157 | 158 | except device_replication.DeviceReplicationCreateThingException as e: 159 | logger.error(e) 160 | ERRORS.append("lambda_handler: {}".format(e)) 161 | error_message = ', '.join(ERRORS) 162 | if event['NewImage']['operation'] == 'CREATED': 163 | update_table_create_thing_error(c_dynamo, thing_name, primary_region, error_message) 164 | 165 | except Exception as e: 166 | logger.error(e) 167 | ERRORS.append('lambda_handler: {}'.format(e)) 168 | 169 | if ERRORS: 170 | error_message = ', '.join(ERRORS) 171 | logger.error('{}'.format(error_message)) 172 | raise ThingCrudException('{}'.format(error_message)) 173 | 174 | return {'message': 'success'} 175 | -------------------------------------------------------------------------------- /source/lambda/iot-dr-region-syncer/iot-region-to-region-syncer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # 7 | # iot-region-to-region-syncer 8 | # 9 | 10 | import logging 11 | import os 12 | import sys 13 | import time 14 | import traceback 15 | 16 | from concurrent import futures 17 | 18 | import boto3 19 | 20 | from botocore.config import Config 21 | from device_replication import thing_exists, create_thing_with_cert_and_policy 22 | 23 | logger = logging.getLogger() 24 | for h in logger.handlers: 25 | logger.removeHandler(h) 26 | h = logging.StreamHandler(sys.stdout) 27 | FORMAT = '%(asctime)s [%(levelname)s]: %(threadName)s-%(filename)s:%(lineno)s-%(funcName)s: %(message)s' 28 | h.setFormatter(logging.Formatter(FORMAT)) 29 | logger.addHandler(h) 30 | logger.setLevel(logging.INFO) 31 | #logger.setLevel(logging.DEBUG) 32 | 33 | PRIMARY_REGION = os.environ['PRIMARY_REGION'] 34 | SECONDARY_REGION = os.environ['SECONDARY_REGION'] 35 | SYNC_MODE = os.environ.get('SYNC_MODE', 'smart') 36 | QUERY_STRING = os.environ.get('QUERY_STRING', 'thingName:*') 37 | MAX_WORKERS = int(os.environ.get('MAX_WORKERS', 10)) 38 | 39 | NUM_THINGS_SYNCED = 0 40 | NUM_THINGS_EXIST = 0 41 | NUM_ERRORS = 0 42 | 43 | logger.info('PRIMARY_REGION: {} SECONDARY_REGION: {} SYNC_MODE: {} QUERY_STRING: {} MAX_WORKERS: {}'. 44 | format(PRIMARY_REGION, SECONDARY_REGION, SYNC_MODE, QUERY_STRING, MAX_WORKERS)) 45 | logger.info('__name__: {}'.format(__name__)) 46 | 47 | 48 | def sync_thing(c_iot_p, c_iot_s, thing): 49 | global NUM_THINGS_SYNCED, NUM_THINGS_EXIST, NUM_ERRORS 50 | try: 51 | logger.info('thing: {}'.format(thing)) 52 | start_time = int(time.time()*1000) 53 | thing_name = thing['thingName'] 54 | 55 | if SYNC_MODE == "smart": 56 | if thing_exists(c_iot_s, thing_name): 57 | logger.info('thing_name {} exists already in secondary region {}'.format(thing_name, SECONDARY_REGION)) 58 | NUM_THINGS_EXIST += 1 59 | return 60 | 61 | thing_type_name = "" 62 | if 'thingTypeName' in thing: 63 | thing_type_name = thing['thingTypeName'] 64 | 65 | attrs = {} 66 | if 'attributes' in thing: 67 | attrs = {'attributes': {}} 68 | for key in thing['attributes']: 69 | attrs['attributes'][key] = thing['attributes'][key] 70 | 71 | if 'attributes' in attrs: 72 | attrs['merge'] = False 73 | 74 | logger.info('thing_name: {} thing_type_name: {} attrs: {}'.format(thing_name, thing_type_name, attrs)) 75 | 76 | create_thing_with_cert_and_policy(c_iot_s, c_iot_p, thing_name, thing_type_name, attrs, 2, 1) 77 | end_time = int(time.time()*1000) 78 | duration = end_time - start_time 79 | NUM_THINGS_SYNCED += 1 80 | logger.info('sync thing: thing_name: {} duration: {}ms'.format(thing_name, duration)) 81 | except Exception as e: 82 | logger.error('{}'.format(e)) 83 | NUM_ERRORS += 1 84 | traceback.print_stack() 85 | 86 | 87 | def get_next_token(response): 88 | next_token = None 89 | if 'nextToken' in response: 90 | next_token = response['nextToken'] 91 | 92 | #logger.info('next_token: {}'.format(next_token)) 93 | return next_token 94 | 95 | 96 | def get_search_things(c_iot_p, c_iot_s, query_string, max_results, executor): 97 | logger.info('query_string: {} max_results: {}'.format(query_string, max_results)) 98 | try: 99 | response = c_iot_p.search_index( 100 | indexName='AWS_Things', 101 | queryString=query_string, 102 | maxResults=max_results 103 | ) 104 | 105 | for thing in response['things']: 106 | executor.submit(sync_thing, c_iot_p, c_iot_s, thing) 107 | 108 | next_token = get_next_token(response) 109 | 110 | while next_token: 111 | response = c_iot_p.search_index( 112 | indexName='AWS_Things', 113 | nextToken=next_token, 114 | queryString=query_string, 115 | maxResults=max_results 116 | ) 117 | next_token = get_next_token(response) 118 | 119 | for thing in response['things']: 120 | executor.submit(sync_thing, c_iot_p, c_iot_s, thing) 121 | #sync_thing(c_iot_p, c_iot_s, thing) 122 | except Exception as e: 123 | logger.error('{}'.format(e)) 124 | 125 | 126 | def get_list_things(c_iot_p, c_iot_s): 127 | try: 128 | paginator = c_iot_p.get_paginator("list_things") 129 | 130 | for page in paginator.paginate(): 131 | logger.debug('page: {}'.format(page)) 132 | logger.debug('things: {}'.format(page['things'])) 133 | for thing in page['things']: 134 | sync_thing(c_iot_p, c_iot_s, thing) 135 | except Exception as e: 136 | logger.error('{}'.format(e)) 137 | 138 | 139 | def registry_indexing_enabled(c_iot_p): 140 | try: 141 | response = c_iot_p.get_indexing_configuration() 142 | logger.debug('response: {}'.format(response)) 143 | 144 | logger.info('thingIndexingMode: {}'.format(response['thingIndexingConfiguration']['thingIndexingMode'])) 145 | if response['thingIndexingConfiguration']['thingIndexingMode'] == 'OFF': 146 | return False 147 | 148 | return True 149 | except Exception as e: 150 | logger.error('{}'.format(e)) 151 | raise Exception(e) 152 | 153 | 154 | def lambda_handler(event, context): 155 | logger.info('syncer: start') 156 | global NUM_THINGS_SYNCED, NUM_THINGS_EXIST, NUM_ERRORS 157 | logger.info('event: {}'.format(event)) 158 | 159 | NUM_THINGS_SYNCED = 0 160 | NUM_THINGS_EXIST = 0 161 | NUM_ERRORS = 0 162 | 163 | if MAX_WORKERS > 50: 164 | logger.error('max allowed workers is 50 defined: {}'.format(MAX_WORKERS)) 165 | raise Exception('max allowed workers is 50 defined: {}'.format(MAX_WORKERS)) 166 | 167 | max_pool_connections = 10 168 | if MAX_WORKERS >= 10: 169 | max_pool_connections = round(MAX_WORKERS*1.2) 170 | 171 | logger.info('max_pool_connections: {}'.format(max_pool_connections)) 172 | 173 | boto3_config = Config( 174 | max_pool_connections = max_pool_connections, 175 | retries = {'max_attempts': 10, 'mode': 'standard'} 176 | ) 177 | 178 | c_iot_p = boto3.client('iot', config=boto3_config, region_name=PRIMARY_REGION) 179 | c_iot_s = boto3.client('iot', config=boto3_config, region_name=SECONDARY_REGION) 180 | 181 | executor = futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) 182 | logger.info('executor: started: {}'.format(executor)) 183 | 184 | if registry_indexing_enabled(c_iot_p): 185 | logger.info('registry indexing enabled - using search_index to get things') 186 | get_search_things(c_iot_p, c_iot_s, QUERY_STRING, 100, executor) 187 | else: 188 | logger.info('registry indexing disabled - using list_things to get things') 189 | get_list_things(c_iot_p, c_iot_s) 190 | 191 | logger.info('executor: waiting to finish') 192 | executor.shutdown(wait=True) 193 | logger.info('executor: shutted down') 194 | 195 | if SYNC_MODE == "smart": 196 | logger.info('syncer: stats: NUM_THINGS_SYNCED: {} NUM_THINGS_EXIST: {} NUM_ERRORS: {}'.format(NUM_THINGS_SYNCED, NUM_THINGS_EXIST, NUM_ERRORS)) 197 | else: 198 | logger.info('syncer: stats: NUM_THINGS_SYNCED: {} NUM_ERRORS: {}'.format(NUM_THINGS_SYNCED, NUM_ERRORS)) 199 | 200 | logger.info('syncer: stop') 201 | return True 202 | 203 | 204 | # in case we run standalone, e.g. on Fargate 205 | if __name__ == '__main__': 206 | logger.info('calling lambda_handler') 207 | lambda_handler({"no": "event"}, None) 208 | -------------------------------------------------------------------------------- /source/tools/iot-dr-run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # iot-dr-run-tests.sh 7 | 8 | 9 | usage() { 10 | echo "Usage: $0 -n number_of_things [ -b thing_basename ]" 1>&2 11 | exit 1 12 | } 13 | 14 | log() { 15 | echo "$(date +'%Y-%m-%d %H:%M:%S'): ${@}" | tee -a iot-dr-run-tests.log 16 | } 17 | 18 | while getopts ":n:b:" options; do 19 | case "${options}" in 20 | n) 21 | NUM_THINGS=${OPTARG} 22 | re_isanum='^[0-9]+$' 23 | if ! [[ $NUM_THINGS =~ $re_isanum ]]; then 24 | echo "Error: number_of_things must be a positive, whole number." 25 | usage 26 | elif [ $NUM_THINGS -eq "0" ]; then 27 | echo "Error: number_of_things must be greater than zero." 28 | usage 29 | fi 30 | ;; 31 | b) 32 | THING_BASENAME=${OPTARG} 33 | ;; 34 | :) 35 | echo "Error: -${OPTARG} requires an argument." 36 | usage 37 | ;; 38 | *) 39 | usage 40 | ;; 41 | esac 42 | done 43 | 44 | UUID=$(uuid) 45 | [ -z "$NUM_THINGS" ] && usage 46 | [ -z "$PRIMARY_REGION" ] && echo "shell variable PRIMARY_REGION not set" && exit 2 47 | [ -z "$SECONDARY_REGION" ] && echo "shell variable SECONDARY_REGION not set" && exit 2 48 | [ -z "$IOT_ENDPOINT_PRIMARY" ] && echo "shell variable IOT_ENDPOINT_PRIMARY not set" && exit 2 49 | [ -z "$IOT_ENDPOINT_SECONDARY" ] && echo "shell variable IOT_ENDPOINT_SECONDARY not set" && exit 2 50 | 51 | THING_INDEXING_PRIMARY=$(aws iot get-indexing-configuration --query 'thingIndexingConfiguration.thingIndexingMode' --region $PRIMARY_REGION --output text) 52 | [ "$THING_INDEXING_PRIMARY" == "OFF" ] && echo "thing indexing not enabled in region $PRIMARY_REGION" && exit 3 53 | 54 | THING_INDEXING_SECONDARY=$(aws iot get-indexing-configuration --query 'thingIndexingConfiguration.thingIndexingMode' --region $SECONDARY_REGION --output text) 55 | [ "$THING_INDEXING_SECONDARY" == "OFF" ] && echo "thing indexing not enabled in region $SECONDARY_REGION" && exit 3 56 | 57 | 58 | [ -z "$THING_BASENAME" ] && THING_BASENAME="dr-test-${UUID}-" 59 | QUERY_STRING="thingName:${THING_BASENAME}*" 60 | POLICY_NAME="dr-test-${UUID}_Policy" 61 | TEMPLATE_BODY="dr-test-${UUID}_templateBody.json" 62 | 63 | DIR=$(dirname $(realpath $0)) 64 | echo "DIR: $DIR" 65 | test ! -e $DIR/toolsrc && exit 1 66 | . $DIR/toolsrc 67 | 68 | for region in $PRIMARY_REGION $SECONDARY_REGION; do 69 | echo "CHECK if a thing exists in region $region for query string $QUERY_STRING" 70 | thing_exists=$(aws iot search-index --query-string $QUERY_STRING --query 'things' --output text --max-results 1 --region $region) 71 | if [ ! -z "$thing_exists" ]; then 72 | echo "ERROR: thing with basename ${THING_BASENAME}* already exist in region $region" 73 | echo " Thing: $thing_exists" 74 | echo " Please choose another basename" 75 | echo " QUERY_STRING \"$QUERY_STRING\" may not match any existing devices" 76 | exit 1 77 | else 78 | echo "no thing found" 79 | fi 80 | done 81 | 82 | START_TIME=$(date +%s) 83 | 84 | DATE_TIME=$(date "+%Y-%m-%d_%H-%M-%S") 85 | TEST_DIR=iot-dr-test-results-${DATE_TIME} 86 | if [ -d $TEST_DIR ]; then 87 | echo "TEST_DIR \"$TEST_DIR\" exists, exiting" 88 | exit 1 89 | else 90 | mkdir $TEST_DIR 91 | fi 92 | 93 | cd $TEST_DIR 94 | 95 | log "THING_BASENAME: $THING_BASENAME" 96 | log "NUM_THINGS: $NUM_THINGS" 97 | log "QUERY_STRING: \"$QUERY_STRING\"" 98 | log "POLICY_NAME: POLICY_NAME" 99 | log "TEMPLATE_BODY: $TEMPLATE_BODY" 100 | log "test results will be stored in directory $TEST_DIR" 101 | sleep 2 102 | 103 | cp /dev/null testrc 104 | echo "THING_BASENAME=$THING_BASENAME" >> testrc 105 | echo "NUM_THINGS=$NUM_THINGS" >> testrc 106 | echo "QUERY_STRING=$QUERY_STRING" >> testrc 107 | echo "POLICY_NAME=$POLICY_NAME" >> testrc 108 | 109 | log "CREATE POLICY and TEMPLATE BODY" 110 | ACCOUNT_ID=$(aws sts get-caller-identity --query 'Account' --output text) 111 | sed -e "s/AWS_REGION/$PRIMARY_REGION/g" -e "s/AWS_ACCOUNT_ID/$ACCOUNT_ID/g" $DIR/policyBulk.json.in > ${POLICY_NAME}.json 112 | sed -e "s/__POLICY_NAME__/$POLICY_NAME/g" $DIR/simpleTemplateBody2.json.in > $TEMPLATE_BODY 113 | AWS_PAGER="" aws iot create-policy --policy-name $POLICY_NAME --policy-document file://${POLICY_NAME}.json --region $PRIMARY_REGION 114 | CWD=$(pwd) 115 | echo "$CWD/$TEMPLATE_BODY" 116 | 117 | log "CREATE - THING_BASENAME: $THING_BASENAME NUM_THINGS: $NUM_THINGS" 118 | log AWS_DEFAULT_REGION=$PRIMARY_REGION $DIR/bulk-bench.sh $THING_BASENAME $NUM_THINGS "$CWD/$TEMPLATE_BODY" 119 | AWS_DEFAULT_REGION=$PRIMARY_REGION $DIR/bulk-bench.sh $THING_BASENAME $NUM_THINGS "$CWD/$TEMPLATE_BODY" | tee bulk-bench.log 120 | 121 | log "things created, waiting a minute for replication..." 122 | sleep 60 123 | 124 | log "SHADOW COMPARE" 125 | log $DIR/iot-dr-shadow-cmp.py --primary-region $PRIMARY_REGION --secondary-region $SECONDARY_REGION --num-tests $NUM_THINGS 126 | $DIR/iot-dr-shadow-cmp.py --primary-region $PRIMARY_REGION --secondary-region $SECONDARY_REGION --num-tests $NUM_THINGS | tee iot-dr-shadow-cmp.log 127 | 128 | log "PUB/SUB RANDOMS" 129 | DEVICES_DIR=$(find . -type d -name "$THING_BASENAME*") 130 | log $DIR/iot-dr-random-tests.sh -b $THING_BASENAME -d $DEVICES_DIR 131 | $DIR/iot-dr-random-tests.sh -b $THING_BASENAME -d $DEVICES_DIR | tee iot-dr-random-tests.log 132 | 133 | 134 | log "COMPARE - QUERY_STRING: $QUERY_STRING" 135 | log $DIR/iot-devices-cmp.py --primary-region $PRIMARY_REGION --secondary-region $SECONDARY_REGION --query-string "$QUERY_STRING" 136 | $DIR/iot-devices-cmp.py --primary-region $PRIMARY_REGION --secondary-region $SECONDARY_REGION --query-string "$QUERY_STRING" | tee iot-devices-cmp.log 137 | sleep 5 138 | 139 | log "DELETE - QUERY_STRING: $QUERY_STRING" 140 | log AWS_DEFAULT_REGION=$PRIMARY_REGION $DIR/delete-things.py --region $PRIMARY_REGION --query-string "$QUERY_STRING" -f 141 | AWS_DEFAULT_REGION=$PRIMARY_REGION $DIR/delete-things.py --region $PRIMARY_REGION --query-string "$QUERY_STRING" -f | tee delete-things.log 142 | 143 | log "Waiting a minute for index to be updated..." 144 | sleep 60 145 | 146 | log "ALL TESTS FINISHED" 147 | log "--------------------------------------------------------------" 148 | 149 | log "TEST IF ALL THINGS DELETED" 150 | AWS_PAGER="" AWS_DEFAULT_REGION=$PRIMARY_REGION aws iot search-index --query-string "$QUERY_STRING" --query 'things' --output text | tee things-in-$PRIMARY_REGION.log 151 | AWS_PAGER="" AWS_DEFAULT_REGION=$SECONDARY_REGION aws iot search-index --query-string "$QUERY_STRING" --query 'things' --output text | tee things-in-$SECONDARY_REGION.log 152 | 153 | if [ -s things-in-$PRIMARY_REGION.log ]; then 154 | log "NOT all things deleted in PRIMARY_REGION: $PRIMARY_REGION: QUERY_STRING: $QUERY_STRING" 155 | log "see: things-in-$PRIMARY_REGION.log" 156 | else 157 | log "ALL things deleted in PRIMARY_REGION: $PRIMARY_REGION: QUERY_STRING: $QUERY_STRING" 158 | fi 159 | 160 | if [ -s things-in-$SECONDARY_REGION.log ]; then 161 | log "NOT all things deleted in SECONDARY_REGION: $SECONDARY_REGION: QUERY_STRING: $QUERY_STRING" 162 | log "see: things-in-$SECONDARY_REGION.log" 163 | else 164 | log "ALL things deleted in SECONDARY_REGION: $SECONDARY_REGION: QUERY_STRING: $QUERY_STRING" 165 | fi 166 | log "--------------------------------------------------------------" 167 | 168 | log "ERRORS IN LOGS" 169 | grep -i error *.log|egrep -v 'no errors detected|TYPE ERRORS' 170 | log "--------------------------------------------------------------" 171 | 172 | 173 | END_TIME=$(date +%s) 174 | DURATION=$(expr $END_TIME - $START_TIME) 175 | log "AWS IoT DR test stats" 176 | log "--------------------------------------------------------------" 177 | log "THING_BASENAME: $THING_BASENAME" 178 | log "NUM_THINGS: $NUM_THINGS" 179 | log "QUERY_STRING: $QUERY_STRING" 180 | log "START: $(date -d@$START_TIME +'%Y-%m-%d %H:%M:%S')" 181 | log "END: $(date -d@$END_TIME +'%Y-%m-%d %H:%M:%S')" 182 | log "DURATION: $DURATION secs." 183 | log "RESULTS in directory \"$TEST_DIR\"" 184 | 185 | cd .. 186 | -------------------------------------------------------------------------------- /source/lambda/iot-dr-region-syncer/iot-region-to-ddb-syncer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # 7 | # iot-region-to-ddb-syncer 8 | # 9 | 10 | import hashlib 11 | import json 12 | import logging 13 | import os 14 | import sys 15 | import time 16 | import uuid 17 | 18 | import boto3 19 | 20 | from botocore.config import Config 21 | from device_replication import thing_exists 22 | from dynamodb_json import json_util as ddb_json 23 | 24 | logger = logging.getLogger() 25 | for h in logger.handlers: 26 | logger.removeHandler(h) 27 | h = logging.StreamHandler(sys.stdout) 28 | FORMAT = '%(asctime)s [%(levelname)s]: %(threadName)s-%(filename)s:%(lineno)s-%(funcName)s: %(message)s' 29 | h.setFormatter(logging.Formatter(FORMAT)) 30 | logger.addHandler(h) 31 | logger.setLevel(logging.INFO) 32 | #logger.setLevel(logging.DEBUG) 33 | 34 | PRIMARY_REGION = os.environ['PRIMARY_REGION'] 35 | SECONDARY_REGION = os.environ['SECONDARY_REGION'] 36 | SYNC_MODE = os.environ.get('SYNC_MODE', 'smart') 37 | QUERY_STRING = os.environ.get('QUERY_STRING', 'thingName:*') 38 | DYNAMODB_GLOBAL_TABLE = os.environ['DYNAMODB_GLOBAL_TABLE'] 39 | 40 | NUM_THINGS_TO_SYNC = 0 41 | NUM_THINGS_EXIST = 0 42 | NUM_ERRORS = 0 43 | 44 | logger.info('PRIMARY_REGION: {} SECONDARY_REGION: {} SYNC_MODE: {} QUERY_STRING: {}'. 45 | format(PRIMARY_REGION, SECONDARY_REGION, SYNC_MODE, QUERY_STRING)) 46 | logger.info('__name__: {}'.format(__name__)) 47 | 48 | 49 | def update_event(c_dynamodb, event): 50 | global NUM_THINGS_TO_SYNC, NUM_ERRORS 51 | try: 52 | response = c_dynamodb.put_item( 53 | TableName=DYNAMODB_GLOBAL_TABLE, 54 | Item=event 55 | ) 56 | logger.info('response: {}'.format(response)) 57 | NUM_THINGS_TO_SYNC += 1 58 | except Exception as e: 59 | logger.error("update_table_create_thing_error: {}".format(e)) 60 | NUM_ERRORS += 1 61 | 62 | 63 | def create_registry_event(c_iot_s, c_dynamodb, thing, account_id): 64 | global NUM_THINGS_EXIST, NUM_ERRORS 65 | logger.info('thing: {}'.format(thing)) 66 | try: 67 | thing_name = thing['thingName'] 68 | 69 | if SYNC_MODE == "smart": 70 | if thing_exists(c_iot_s, thing_name): 71 | logger.info('thing_name {} exists already in secondary region {}'.format(thing_name, c_iot_s.meta.region_name)) 72 | NUM_THINGS_EXIST += 1 73 | return 74 | 75 | # "uuid": "{}".format(uuid.uuid4()), 76 | event = { 77 | "uuid": "{}".format(hashlib.sha256(thing_name.encode()).hexdigest()), 78 | "accountId": str(account_id), 79 | "expires": int(time.time()+172800), 80 | "eventType" : "THING_EVENT", 81 | "eventId" : "{}".format(uuid.uuid4()), 82 | "timestamp" : int(time.time()*1000), 83 | "operation" : "CREATED" 84 | } 85 | 86 | event['thingName'] = thing_name 87 | 88 | thing_type_name = "" 89 | if 'thingTypeName' in thing: 90 | thing_type_name = thing['thingTypeName'] 91 | event['thingTypeName'] = thing_type_name 92 | 93 | attrs = {} 94 | if 'attributes' in thing: 95 | event['attributes'] = {} 96 | for key in thing['attributes']: 97 | event['attributes'][key] = thing['attributes'][key] 98 | 99 | logger.info('thing_name: {} thing_type_name: {} attrs: {}'.format(thing_name, thing_type_name, attrs)) 100 | 101 | update_event(c_dynamodb, json.loads(ddb_json.dumps(event))) 102 | except Exception as e: 103 | logger.error("update_table_create_thing_error: {}".format(e)) 104 | NUM_ERRORS += 1 105 | 106 | 107 | def get_next_token(response): 108 | next_token = None 109 | if 'nextToken' in response: 110 | next_token = response['nextToken'] 111 | 112 | #logger.info('next_token: {}'.format(next_token)) 113 | return next_token 114 | 115 | 116 | def get_search_things(c_iot_p, c_iot_s, c_dynamodb, account_id, query_string, max_results): 117 | logger.info('query_string: {} max_results: {}'.format(query_string, max_results)) 118 | try: 119 | response = c_iot_p.search_index( 120 | indexName='AWS_Things', 121 | queryString=query_string, 122 | maxResults=max_results 123 | ) 124 | 125 | for thing in response['things']: 126 | create_registry_event(c_iot_s, c_dynamodb, thing, account_id) 127 | 128 | next_token = get_next_token(response) 129 | 130 | while next_token: 131 | response = c_iot_p.search_index( 132 | indexName='AWS_Things', 133 | nextToken=next_token, 134 | queryString=query_string, 135 | maxResults=max_results 136 | ) 137 | next_token = get_next_token(response) 138 | 139 | for thing in response['things']: 140 | create_registry_event(c_iot_s, c_dynamodb, thing, account_id) 141 | 142 | except Exception as e: 143 | logger.error('{}'.format(e)) 144 | 145 | 146 | def get_list_things(c_iot_p, c_iot_s, c_dynamodb, account_id): 147 | try: 148 | paginator = c_iot_p.get_paginator("list_things") 149 | 150 | for page in paginator.paginate(): 151 | logger.debug('page: {}'.format(page)) 152 | logger.debug('things: {}'.format(page['things'])) 153 | for thing in page['things']: 154 | create_registry_event(c_iot_s, c_dynamodb, thing, account_id) 155 | except Exception as e: 156 | logger.error('{}'.format(e)) 157 | 158 | 159 | def registry_indexing_enabled(c_iot_p): 160 | try: 161 | response = c_iot_p.get_indexing_configuration() 162 | logger.debug('response: {}'.format(response)) 163 | 164 | logger.info('thingIndexingMode: {}'.format(response['thingIndexingConfiguration']['thingIndexingMode'])) 165 | if response['thingIndexingConfiguration']['thingIndexingMode'] == 'OFF': 166 | return False 167 | 168 | return True 169 | except Exception as e: 170 | logger.error('{}'.format(e)) 171 | raise Exception(e) 172 | 173 | 174 | def lambda_handler(event, context): 175 | logger.info('syncer: start') 176 | global NUM_THINGS_TO_SYNC, NUM_THINGS_EXIST, NUM_ERRORS 177 | logger.info('event: {}'.format(event)) 178 | 179 | NUM_THINGS_TO_SYNC = 0 180 | NUM_THINGS_EXIST = 0 181 | NUM_ERRORS = 0 182 | 183 | boto3_config = Config( 184 | max_pool_connections = 20, 185 | retries = {'max_attempts': 10, 'mode': 'standard'} 186 | ) 187 | 188 | c_iot_p = boto3.client('iot', config=boto3_config, region_name=PRIMARY_REGION) 189 | c_iot_s = boto3.client('iot', config=boto3_config, region_name=SECONDARY_REGION) 190 | c_dynamodb = boto3.client('dynamodb', region_name=PRIMARY_REGION) 191 | 192 | account_id = boto3.client('sts').get_caller_identity()['Account'] 193 | 194 | if registry_indexing_enabled(c_iot_p): 195 | logger.info('registry indexing enabled - using search_index to get things') 196 | get_search_things(c_iot_p, c_iot_s, c_dynamodb, account_id, QUERY_STRING, 100) 197 | else: 198 | logger.info('registry indexing disabled - using list_things to get things') 199 | get_list_things(c_iot_p, c_iot_s, c_dynamodb, account_id) 200 | 201 | if SYNC_MODE == "smart": 202 | logger.info('syncer: stats: NUM_THINGS_TO_SYNC: {} NUM_THINGS_EXIST: {} NUM_ERRORS: {}'.format(NUM_THINGS_TO_SYNC, NUM_THINGS_EXIST, NUM_ERRORS)) 203 | else: 204 | logger.info('syncer: stats: NUM_THINGS_TO_SYNC: {} NUM_ERRORS: {}'.format(NUM_THINGS_TO_SYNC, NUM_ERRORS)) 205 | 206 | logger.info('syncer: stop') 207 | return True 208 | 209 | 210 | # in case we run standalone, e.g. on Fargate 211 | if __name__ == '__main__': 212 | logger.info('calling lambda_handler') 213 | lambda_handler({"no": "event"}, None) 214 | -------------------------------------------------------------------------------- /source/lambda/sfn-iot-mr-thing-group-crud/lambda_function.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # 7 | # thing group crud 8 | # 9 | 10 | import logging 11 | 12 | import boto3 13 | 14 | logger = logging.getLogger(__name__) 15 | logger.setLevel(logging.INFO) 16 | 17 | 18 | ERRORS = [] 19 | 20 | class ThingGroupCrudException(Exception): pass 21 | 22 | 23 | def thing_group_exists(c_iot, thing_group_name): 24 | logger.info("thing group exists: thing_group_name: {}".format(thing_group_name)) 25 | try: 26 | response = c_iot.describe_thing_group(thingGroupName=thing_group_name) 27 | logger.info('response: {}'.format(response)) 28 | return True 29 | 30 | except c_iot.exceptions.ResourceNotFoundException: 31 | logger.info('thing_group_name {} does not exist'.format(thing_group_name)) 32 | return False 33 | 34 | except Exception as e: 35 | logger.error('{}'.format(e)) 36 | raise Exception(e) 37 | 38 | 39 | def create_thing_group(c_iot, thing_group_name, description, attrs, merge): 40 | logger.info("create thing group: thing_group_name: {}".format(thing_group_name)) 41 | global ERRORS 42 | try: 43 | if not thing_group_exists(c_iot, thing_group_name): 44 | response = c_iot.create_thing_group( 45 | thingGroupName=thing_group_name, 46 | thingGroupProperties={ 47 | 'thingGroupDescription': description, 48 | 'attributePayload': { 49 | 'attributes': attrs, 50 | 'merge': merge 51 | } 52 | } 53 | ) 54 | logger.info("create_thing_group: response: {}".format(response)) 55 | else: 56 | logger.info("thing group exists already: {}".format(thing_group_name)) 57 | except Exception as e: 58 | logger.error("create_thing_group: {}".format(e)) 59 | ERRORS.append("create_thing_group: {}".format(e)) 60 | 61 | 62 | def delete_thing_group(c_iot, thing_group_name): 63 | logger.info("delete thing group: thing_group_name: {}".format(thing_group_name)) 64 | global ERRORS 65 | try: 66 | response = c_iot.delete_thing_group(thingGroupName=thing_group_name) 67 | logger.info('delete_thing_group: {}'.format(response)) 68 | except Exception as e: 69 | logger.error("create_thing_group: {}".format(e)) 70 | ERRORS.append("create_thing_group: {}".format(e)) 71 | 72 | 73 | def update_thing_group(c_iot, thing_group_name, description, attrs, merge): 74 | logger.info("update thing group: thing_group_name: {}".format(thing_group_name)) 75 | global ERRORS 76 | try: 77 | create_thing_group(c_iot, thing_group_name, "", {}, True) 78 | response = c_iot.update_thing_group( 79 | thingGroupName=thing_group_name, 80 | thingGroupProperties={ 81 | 'thingGroupDescription': description, 82 | 'attributePayload': { 83 | 'attributes': attrs, 84 | 'merge': merge 85 | } 86 | } 87 | ) 88 | logger.info('update_thing_group: {}'.format(response)) 89 | except Exception as e: 90 | logger.error("create_thing_group: {}".format(e)) 91 | ERRORS.append("create_thing_group: {}".format(e)) 92 | 93 | 94 | 95 | def add_thing_to_group(c_iot, thing_group_name, thing_name): 96 | logger.info("add thing to group: thing_group_name: {} thing_name: {}".format(thing_group_name, thing_name)) 97 | global ERRORS 98 | try: 99 | create_thing_group(c_iot, thing_group_name, "", {}, True) 100 | response = c_iot.add_thing_to_thing_group( 101 | thingGroupName=thing_group_name, 102 | thingName=thing_name, 103 | overrideDynamicGroups=False) 104 | logger.info("add_thing_to_group: {}".format(response)) 105 | except Exception as e: 106 | logger.error("add_thing_to_group: {}".format(e)) 107 | ERRORS.append("add_thing_to_group: {}".format(e)) 108 | 109 | 110 | def remove_thing_from_group(c_iot, thing_group_name, thing_name): 111 | logger.info("remove thing from group: thing_group_name: {} thing_name: {}".format(thing_group_name, thing_name)) 112 | global ERRORS 113 | try: 114 | response = c_iot.remove_thing_from_thing_group( 115 | thingGroupName=thing_group_name, 116 | thingName=thing_name) 117 | logger.info("remove_thing_from_group: {}".format(response)) 118 | except Exception as e: 119 | logger.error("add_thing_to_group: {}".format(e)) 120 | ERRORS.append("add_thing_to_group: {}".format(e)) 121 | 122 | 123 | def lambda_handler(event, context): 124 | global ERRORS 125 | ERRORS = [] 126 | 127 | logger.info('event: {}'.format(event)) 128 | 129 | try: 130 | c_iot = boto3.client('iot') 131 | 132 | if event['NewImage']['eventType']['S'] == 'THING_GROUP_EVENT': 133 | thing_group_name = event['NewImage']['thingGroupName']['S'] 134 | logger.info("operation: {} thing_group_name: {}". 135 | format(event['NewImage']['operation']['S'], thing_group_name)) 136 | if event['NewImage']['operation']['S'] == 'CREATED': 137 | description = "" 138 | if 'S' in event['NewImage']['description']: 139 | description = event['NewImage']['description']['S'] 140 | 141 | attrs = {} 142 | if 'M' in event['NewImage']['attributes']: 143 | for key in event['NewImage']['attributes']['M']: 144 | attrs[key] = event['NewImage']['attributes']['M'][key]['S'] 145 | 146 | merge = True 147 | if attrs: 148 | merge = False 149 | logger.info('description: {} attrs: {}'.format(description, attrs)) 150 | create_thing_group(c_iot, thing_group_name, description, attrs, merge) 151 | elif event['NewImage']['operation']['S'] == 'DELETED': 152 | delete_thing_group(c_iot, thing_group_name) 153 | elif event['NewImage']['operation']['S'] == 'UPDATED': 154 | description = "" 155 | if 'S' in event['NewImage']['description']: 156 | description = event['NewImage']['description']['S'] 157 | 158 | attrs = {} 159 | if 'M' in event['NewImage']['attributes']: 160 | for key in event['NewImage']['attributes']['M']: 161 | attrs[key] = event['NewImage']['attributes']['M'][key]['S'] 162 | 163 | merge = True 164 | if attrs: 165 | merge = False 166 | logger.info('description: {} attrs: {}'.format(description, attrs)) 167 | update_thing_group(c_iot, thing_group_name, description, attrs, merge) 168 | elif event['NewImage']['eventType']['S'] == 'THING_GROUP_MEMBERSHIP_EVENT': 169 | group_arn = event['NewImage']['groupArn']['S'] 170 | thing_arn = event['NewImage']['thingArn']['S'] 171 | thing_group_name = group_arn.split('/')[-1] 172 | thing_name = thing_arn.split('/')[-1] 173 | logger.info("operation: {} group_arn: {} thing_arn: {} thing_group_name: {} thing_name: {}". 174 | format(event['NewImage']['operation']['S'], group_arn, thing_arn, thing_group_name, thing_name)) 175 | if event['NewImage']['operation']['S'] == 'ADDED': 176 | add_thing_to_group(c_iot, thing_group_name, thing_name) 177 | elif event['NewImage']['operation']['S'] == 'REMOVED': 178 | remove_thing_from_group(c_iot, thing_group_name, thing_name) 179 | 180 | except Exception as e: 181 | logger.error(e) 182 | ERRORS.append("lambda_handler: {}".format(e)) 183 | 184 | if ERRORS: 185 | error_message = ', '.join(ERRORS) 186 | logger.error('{}'.format(error_message)) 187 | raise ThingGroupCrudException('{}'.format(error_message)) 188 | 189 | return {"message": "success"} 190 | -------------------------------------------------------------------------------- /source/lambda/iot-mr-jitr/lambda_function.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | #required libraries 7 | import boto3 8 | import json 9 | import logging 10 | import os 11 | import sys 12 | 13 | from OpenSSL import crypto 14 | 15 | # configure logging 16 | logger = logging.getLogger() 17 | for h in logger.handlers: 18 | logger.removeHandler(h) 19 | h = logging.StreamHandler(sys.stdout) 20 | FORMAT = '%(asctime)s [%(levelname)s]: %(threadName)s-%(filename)s:%(lineno)s-%(funcName)s: %(message)s' 21 | h.setFormatter(logging.Formatter(FORMAT)) 22 | logger.addHandler(h) 23 | logger.setLevel(logging.INFO) 24 | 25 | IOT_POLICY_NAME = os.environ.get('IOT_POLICY_NAME', 'IoTDR-JITR_Policy') 26 | 27 | ERRORS = [] 28 | 29 | 30 | def get_thing_name(c_iot, certificate_id, response): 31 | try: 32 | cert_pem = response['certificateDescription']['certificatePem'] 33 | logger.info('cert_pem: {}'.format(cert_pem)) 34 | 35 | cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) 36 | 37 | subject = cert.get_subject() 38 | cn = subject.CN 39 | logger.info('subject: {} cn: {}'.format(subject, cn)) 40 | return cn 41 | except Exception as e: 42 | logger.warn('unable to get CN from certificate_id: {}: {}, using certificate_id as thing name'.format(certificate_id, e)) 43 | return certificate_id 44 | 45 | 46 | def thing_exists(c_iot, thing_name): 47 | try: 48 | logger.info('thing_name: {}'.format(thing_name)) 49 | response = c_iot.describe_thing(thingName=thing_name) 50 | print('response: {}'.format(response)) 51 | return True 52 | 53 | except c_iot.exceptions.ResourceNotFoundException: 54 | logger.info('thing_name "{}" does not exist'.format(thing_name)) 55 | return False 56 | 57 | except Exception as e: 58 | logger.error('{}'.format(e)) 59 | raise Exception(e) 60 | 61 | 62 | def create_thing(c_iot, thing_name): 63 | global ERRORS 64 | try: 65 | logger.info('thing_name: {}'.format(thing_name)) 66 | if not thing_exists(c_iot, thing_name): 67 | response = c_iot.create_thing(thingName=thing_name) 68 | logger.info("create_thing: response: {}".format(response)) 69 | else: 70 | logger.info("thing exists already: {}".format(thing_name)) 71 | except Exception as e: 72 | logger.error("create_thing: {}".format(e)) 73 | ERRORS.append("create_thing: {}".format(e)) 74 | 75 | 76 | def policy_exists(c_iot, policy_name): 77 | try: 78 | logger.info('policy_name: {}'.format(policy_name)) 79 | response = c_iot.get_policy(policyName=policy_name) 80 | print('response: {}'.format(response)) 81 | return True 82 | 83 | except c_iot.exceptions.ResourceNotFoundException: 84 | logger.info('policy_name: {}: does not exist'.format(policy_name)) 85 | return False 86 | 87 | except Exception as e: 88 | logger.error('{}'.format(e)) 89 | raise Exception(e) 90 | 91 | 92 | def create_iot_policy(c_iot, policy_name): 93 | global ERRORS 94 | policy_document = { 95 | "Version":"2012-10-17", 96 | "Statement":[ 97 | { 98 | "Effect": "Allow", 99 | "Action": [ 100 | "iot:Connect" 101 | ], 102 | "Resource": [ 103 | "arn:aws:iot:*:*:client/${iot:Connection.Thing.ThingName}" 104 | ] 105 | }, 106 | { 107 | "Effect": "Allow", 108 | "Action": [ 109 | "iot:Publish", 110 | "iot:Receive" 111 | ], 112 | "Resource": [ 113 | "arn:aws:iot:*:*:topic/dt/${iot:Connection.Thing.ThingName}/*", 114 | "arn:aws:iot:*:*:topic/cmd/${iot:Connection.Thing.ThingName}/*", 115 | "arn:aws:iot:*:*:topic/$aws/things/${iot:Connection.Thing.ThingName}/shadow/*" 116 | ] 117 | }, 118 | { 119 | "Effect": "Allow", 120 | "Action": [ 121 | "iot:Subscribe" 122 | ], 123 | "Resource": [ 124 | "arn:aws:iot:*:*:topicfilter/dt/${iot:Connection.Thing.ThingName}/*", 125 | "arn:aws:iot:*:*:topicfilter/cmd/${iot:Connection.Thing.ThingName}/*", 126 | "arn:aws:iot:*:*:topicfilter/$aws/things/${iot:Connection.Thing.ThingName}/shadow/*" 127 | ] 128 | } 129 | ] 130 | } 131 | 132 | try: 133 | logger.info('policy_name: {}'.format(policy_name)) 134 | if not policy_exists(c_iot, policy_name): 135 | response = c_iot.create_policy( 136 | policyName=policy_name, 137 | policyDocument=json.dumps(policy_document) 138 | ) 139 | logger.info("create_iot_policy: response: {}".format(response)) 140 | else: 141 | logger.info("policy exists already: {}".format(policy_name)) 142 | except c_iot.exceptions.ResourceAlreadyExistsException: 143 | logger.warn('policy_name {}: exists already - might have been created in a parallel thread'.format(policy_name)) 144 | except Exception as e: 145 | logger.error("create_iot_policy: {}".format(e)) 146 | ERRORS.append("create_iot_policy: {}".format(e)) 147 | 148 | 149 | def activate_certificate(c_iot, certificate_id): 150 | global ERRORS 151 | try: 152 | logger.info('certificate_id: {}'.format(certificate_id)) 153 | response = c_iot.update_certificate(certificateId=certificate_id, newStatus='ACTIVE') 154 | logger.info("activate_cert: response: {}".format(response)) 155 | except Exception as e: 156 | logger.error("activate_certificate: {}".format(e)) 157 | ERRORS.append("activate_certificate: {}".format(e)) 158 | 159 | 160 | def attach_policy(c_iot, thing_name, policy_name, response): 161 | global ERRORS 162 | try: 163 | logger.info('thing_name: {} policy_name: {}'.format(thing_name, policy_name)) 164 | certificate_arn = response['certificateDescription']['certificateArn'] 165 | logger.info("certificate_arn: {}".format(certificate_arn)) 166 | 167 | response = c_iot.attach_thing_principal(thingName=thing_name, principal=certificate_arn) 168 | logger.info("attach_thing_principal: response: {}".format(response)) 169 | 170 | response = c_iot.attach_policy(policyName=policy_name, target=certificate_arn) 171 | logger.info("attach_policy: response: {}".format(response)) 172 | except Exception as e: 173 | logger.error("attach_policy: {}".format(e)) 174 | ERRORS.append("attach_policy: {}".format(e)) 175 | 176 | 177 | def lambda_handler(event, context): 178 | logger.info("event: {}".format(event)) 179 | logger.info(json.dumps(event, indent=4)) 180 | 181 | region = os.environ["AWS_REGION"] 182 | logger.info("region: {}".format(region)) 183 | 184 | try: 185 | ca_certificate_id = event['caCertificateId'] 186 | certificate_id = event['certificateId'] 187 | certificate_status = event['certificateStatus'] 188 | 189 | logger.info("ca_certificate_id: " + ca_certificate_id) 190 | logger.info("certificate_id: " + certificate_id) 191 | logger.info("certificate_status: " + certificate_status) 192 | 193 | c_iot = boto3.client('iot') 194 | 195 | res_desc_cert = c_iot.describe_certificate(certificateId=certificate_id) 196 | logger.info('res_desc_cert: {}'.format(res_desc_cert)) 197 | 198 | thing_name = get_thing_name(c_iot, certificate_id, res_desc_cert) 199 | create_thing(c_iot, thing_name) 200 | create_iot_policy(c_iot, IOT_POLICY_NAME) 201 | activate_certificate(c_iot, certificate_id) 202 | attach_policy(c_iot, thing_name, IOT_POLICY_NAME, res_desc_cert) 203 | except Exception as e: 204 | logger.error('describe_certificate: {}'.format(e)) 205 | return {"status": "error", "message": '{}'.format(e)} 206 | 207 | if ERRORS: 208 | return {"status": "error", "message": '{}'.format(ERRORS)} 209 | 210 | return {"status": "success"} 211 | -------------------------------------------------------------------------------- /deployment/build-s3-dist.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # This assumes all of the OS-level configuration has been completed and git repo has already been cloned 7 | # 8 | # This script should be run from the repo's deployment directory 9 | # cd deployment 10 | # ./build-s3-dist.sh source-bucket-base-name solution-name version-code 11 | # 12 | # Paramenters: 13 | # - source-bucket-base-name: Name for the S3 bucket location where the template will source the Lambda 14 | # code from. The template will append '-[region_name]' to this bucket name. 15 | # For example: ./build-s3-dist.sh solutions v1.0.0 16 | # The template will then expect the source code to be located in the solutions-[region_name] bucket 17 | # 18 | # - solution-name: name of the solution for consistency 19 | # 20 | # - version-code: version of the package 21 | 22 | set -e 23 | 24 | # Check to see if input has been provided: 25 | if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then 26 | echo "Please provide the base source bucket name, trademark approved solution name and version where the lambda code will eventually reside." 27 | echo "For example: ./build-s3-dist.sh solutions trademarked-solution-name v1.0.0" 28 | exit 1 29 | fi 30 | 31 | DIST_OUTPUT_BUCKET=$1 32 | SOLUTION_NAME=$2 33 | VERSION=$3 34 | 35 | echo "DIST_OUTPUT_BUCKET: $DIST_OUTPUT_BUCKET SOLUTION_NAME: $SOLUTION_NAME VERSION: $VERSION" 36 | 37 | echo ./build-s3-dist.sh $1 $2 $3 38 | 39 | # Get reference for all important folders 40 | deployment_dir="$PWD" 41 | template_dist_dir="$deployment_dir/global-s3-assets" 42 | build_dist_dir="$deployment_dir/regional-s3-assets" 43 | source_dir="$deployment_dir/../source" 44 | template_dir="$deployment_dir/../source/cfn" 45 | opensource_dir="$deployment_dir/open-source" 46 | opensource_template_dir="$opensource_dir/deployment" 47 | 48 | echo "deployment_dir: $deployment_dir" 49 | 50 | echo "------------------------------------------------------------------------------" 51 | echo "[Init] Clean old dist, node_modules and bower_components folders" 52 | echo "------------------------------------------------------------------------------" 53 | echo "rm -rf $template_dist_dir" 54 | rm -rf $template_dist_dir 55 | echo "mkdir -p $template_dist_dir" 56 | mkdir -p $template_dist_dir 57 | echo "rm -rf $build_dist_dir" 58 | rm -rf $build_dist_dir 59 | echo "mkdir -p $build_dist_dir" 60 | mkdir -p $build_dist_dir 61 | 62 | echo "rm -rf $opensource_dir" 63 | rm -rf $opensource_dir 64 | echo "mkdir -p $opensource_dir" 65 | mkdir -p $opensource_dir 66 | echo "mkdir -p $opensource_template_dir" 67 | mkdir -p $opensource_template_dir 68 | 69 | echo "------------------------------------------------------------------------------" 70 | echo "[Packing] Templates" 71 | echo "------------------------------------------------------------------------------" 72 | 73 | #cd ../source/cfn 74 | for type in json template yaml; do 75 | count=$(ls -l *.$type 2>/dev/null|wc -l) 76 | echo "cp type: $type count: $count" 77 | if [ $count != 0 ]; then 78 | echo cp *.$type $template_dist_dir/ 79 | cp *.$type $template_dist_dir/ 80 | fi 81 | done 82 | 83 | cd $template_dist_dir 84 | 85 | # Rename all *.json and *.yaml to *.template 86 | for type in json yaml; do 87 | count=$(ls -l *.$type 2>/dev/null|wc -l) 88 | echo "rename type: $type count: $count" 89 | if [ $count != 0 ]; then 90 | for f in *.$type; do 91 | echo mv -- "$f" "${f%.$type}.template" 92 | mv -- "$f" "${f%.$type}.template" 93 | done 94 | fi 95 | done 96 | pwd 97 | ls -l 98 | 99 | echo "------------------------------------------------------------------------------" 100 | echo "[Build] preparing CloudFormation templates" 101 | echo "------------------------------------------------------------------------------" 102 | 103 | for template in $(find . -name "*.template"); do 104 | echo " template=$template" 105 | sed -i -e "s/__S3_BUCKET__/${DIST_OUTPUT_BUCKET}/g" \ 106 | -e "s/__SOLUTION_NAME__/${SOLUTION_NAME}/g" \ 107 | -e "s/__VERSION__/${VERSION}/g" $template 108 | done 109 | 110 | echo "------------------------------------------------------------------------------" 111 | echo "[Build] copying source dir to open-source dir" 112 | echo "------------------------------------------------------------------------------" 113 | echo "cp -r $source_dir $opensource_dir" 114 | cp -r $source_dir $opensource_dir 115 | echo "------------------------------------------------------------------------------" 116 | echo "opensource_dir: $opensource_dir/" 117 | echo "------------------------------------------------------------------------------" 118 | find $opensource_dir/ 119 | 120 | echo "------------------------------------------------------------------------------" 121 | echo "[Build] cleaning Jupyter notebooks" 122 | echo "------------------------------------------------------------------------------" 123 | 124 | cd $source_dir 125 | 126 | cd jupyter 127 | # 01_IoTDR_Shared.ipynb 128 | sed -r -i -e "s/config\['aws_region_pca'\] = .*$/config\['aws_region_pca'\] = \\\\\"REPLACE_WITH_AWS_REGION_FOR_PCA\\\\\"\\\n\",/g" \ 129 | -e "s/config\['aws_region_primary'\] = .*$/config\['aws_region_primary'\] = \\\\\"REPLACE_WITH_AWS_PRIMARY_REGION\\\\\"\\\n\",/g" \ 130 | -e "s/config\['aws_region_secondary'\] = .*$/config\['aws_region_secondary'\] = \\\\\"REPLACE_WITH_AWS_SECONDARY_REGION\\\\\"\\\n\",/g" \ 131 | -e "s/config\['Sub_CN'\] = .*$/config\['Sub_CN'\] = \\\\\"REPLACE_WITH_YOUR_PCA_CN\\\\\"\\\n\",/g" \ 132 | 01_IoTDR_Shared.ipynb 133 | 134 | # 04_IoTDR_Device_Certs.ipynb 135 | sed -r -i -e "s/thing_name = .*$/thing_name = 'REPLACE_WITH_THING_NAME_OF_YOUR_CHOICE'\\\n\",/g" \ 136 | 04_IoTDR_Device_Certs.ipynb 137 | 138 | # 05_IoTDR_JITR_Device.ipynb 139 | sed -r -i -e "s/thing_name = .*$/thing_name = 'REPLACE_WITH_THING_NAME_OF_YOUR_CHOICE'\\\n\",/g" \ 140 | 05_IoTDR_JITR_Device.ipynb 141 | cd .. 142 | 143 | echo "------------------------------------------------------------------------------" 144 | echo "[Build] zip packages" 145 | echo "------------------------------------------------------------------------------" 146 | 147 | echo "creating iot-dr-solution.zip" 148 | rm -f iot-dr-solution.zip 149 | ls -al 150 | 151 | zip -q iot-dr-solution.zip -r cfn jupyter tools lambda launch-solution-code-build.sh launch-solution.yml 152 | echo "cp iot-dr-solution.zip $build_dist_dir/" 153 | cp iot-dr-solution.zip $build_dist_dir/ 154 | 155 | echo "creating installation packages for lambda" 156 | pip3 --version 157 | cd lambda 158 | 159 | echo "installing pyOpenSSL for iot-mr-jitr" 160 | echo pip3 install pyOpenSSL -t iot-mr-jitr -q 161 | pip3 install pyOpenSSL -t iot-mr-jitr -q 162 | 163 | for lambda in iot-dr-launch-solution iot-mr-jitr iot-mr-cross-region \ 164 | sfn-iot-mr-dynamo-trigger sfn-iot-mr-thing-crud \ 165 | sfn-iot-mr-thing-group-crud sfn-iot-mr-thing-type-crud \ 166 | sfn-iot-mr-shadow-syncer \ 167 | iot-dr-missing-device-replication \ 168 | iot-dr-create-r53-checker 169 | do 170 | echo "creating lambda zip package for \"$lambda\"" 171 | rm -f ${lambda}.zip 172 | cd $lambda 173 | zip -q ../${lambda}.zip -r . 174 | cd .. 175 | done 176 | 177 | # layer 178 | cd iot-dr-layer 179 | rm -rf python 180 | mkdir python 181 | 182 | echo pip3 install dynamodb-json==1.3 --no-deps -t python -q 183 | pip3 install dynamodb-json==1.3 --no-deps -t python -q 184 | 185 | echo pip3 install simplejson==3.17.2 -t python -q 186 | pip3 install simplejson==3.17.2 -t python -q 187 | 188 | cp device_replication.py python/ 189 | 190 | rm -f ../iot-dr-layer.zip 191 | zip ../iot-dr-layer.zip -r python 192 | cd .. 193 | 194 | echo "ZIP files:" 195 | ls -l *.zip 196 | cp *.zip $build_dist_dir/ 197 | 198 | 199 | 200 | echo "------------------------------------------------------------------------------" 201 | echo "global-s3-assets: $template_dist_dir/" 202 | echo "------------------------------------------------------------------------------" 203 | ls -al $template_dist_dir/ 204 | #echo cat $template_dist_dir/disaster-recovery-for-aws-iot.template 205 | #cat $template_dist_dir/disaster-recovery-for-aws-iot.template 206 | 207 | echo "------------------------------------------------------------------------------" 208 | echo "regional-s3-assets: $build_dist_dir" 209 | echo "------------------------------------------------------------------------------" 210 | ls -al $build_dist_dir/ 211 | exit 0 212 | -------------------------------------------------------------------------------- /source/lambda/iot-dr-r53-health-check/iot-dr-r53-health-checker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import json 7 | import logging 8 | import os 9 | import sys 10 | import threading 11 | 12 | from datetime import datetime 13 | from uuid import uuid4 14 | 15 | from awscrt import io, mqtt 16 | from awsiot import mqtt_connection_builder 17 | 18 | logger = logging.getLogger() 19 | for h in logger.handlers: 20 | logger.removeHandler(h) 21 | h = logging.StreamHandler(sys.stdout) 22 | FORMAT = '%(asctime)s [%(levelname)s]: %(threadName)s:%(filename)s:%(funcName)s:%(lineno)s: %(message)s' 23 | h.setFormatter(logging.Formatter(FORMAT)) 24 | logger.addHandler(h) 25 | logger.setLevel(logging.INFO) 26 | #logger.setLevel(logging.DEBUG) 27 | 28 | CA = os.environ['CA'] 29 | CERT = os.environ['CERT'] 30 | CLIENT_ID = os.environ['CLIENT_ID'] 31 | COUNT = int(os.environ.get('COUNT', 2)) 32 | ENDPOINT = os.environ['ENDPOINT'] 33 | KEY = os.environ['KEY'] 34 | QUERY_STRING = os.environ['QUERY_STRING'] 35 | RECEIVE_TIMEOUT = float(os.environ.get('RECEIVE_TIMEOUT', 3)) 36 | 37 | #io.init_logging(getattr(io.LogLevel, 'Info'), 'stderr') 38 | io.init_logging(getattr(io.LogLevel, 'NoLogs'), 'stderr') 39 | RECEIVED_COUNT = 0 40 | RECEIVED_ALL_EVENT = threading.Event() 41 | 42 | # Callback when connection is accidentally lost. 43 | def on_connection_interrupted(connection, error, **kwargs): 44 | logger.info("connection interrupted: error: {}".format(error)) 45 | 46 | 47 | # Callback when an interrupted connection is re-established. 48 | def on_connection_resumed(connection, return_code, session_present, **kwargs): 49 | logger.info("connection resumed: return_code: {} session_present: {}".format(return_code, session_present)) 50 | 51 | if return_code == mqtt.ConnectReturnCode.ACCEPTED and not session_present: 52 | logger.info("Session did not persist. Resubscribing to existing topics...") 53 | resubscribe_future, _ = connection.resubscribe_existing_topics() 54 | 55 | # Cannot synchronously wait for resubscribe result because we're on the connection's event-loop thread, 56 | # evaluate result with a callback instead. 57 | resubscribe_future.add_done_callback(on_resubscribe_complete) 58 | 59 | 60 | def on_resubscribe_complete(resubscribe_future): 61 | resubscribe_results = resubscribe_future.result() 62 | logger.info("resubscribe results: {}".format(resubscribe_results)) 63 | 64 | for topic, qos in resubscribe_results['topics']: 65 | if qos is None: 66 | sys.exit("Server rejected resubscribe to topic: {}".format(topic)) 67 | 68 | 69 | # Callback when the subscribed topic receives a message 70 | def on_message_received(topic, payload, **kwargs): 71 | logger.info("message received: topic: {} payload: {}".format(topic, payload)) 72 | global RECEIVED_COUNT 73 | RECEIVED_COUNT += 1 74 | if RECEIVED_COUNT == COUNT: 75 | RECEIVED_ALL_EVENT.set() 76 | 77 | def lambda_handler(event, context): 78 | logger.info('r53-health-check: start') 79 | logger.info('event: {}'.format(event)) 80 | 81 | try: 82 | if COUNT < 1: 83 | raise Exception('COUNT must be greate or equal 1: defined: {}'.format(COUNT)) 84 | 85 | uuid = '{}'.format(uuid4()) 86 | client_id = '{}-{}'.format(CLIENT_ID, uuid) 87 | topic = 'dr/r53/check/{}/{}'.format(CLIENT_ID, uuid) 88 | logger.info('client_id: {} topic: {}'.format(client_id, topic)) 89 | 90 | if not 'queryStringParameters' in event: 91 | logger.error('queryStringParameters missing') 92 | return { 93 | 'statusCode': 503, 94 | 'body': json.dumps({ 'message': 'internal server error'}) 95 | } 96 | 97 | if not 'hashme' in event['queryStringParameters']: 98 | logger.error('hashme missing') 99 | return { 100 | 'statusCode': 503, 101 | 'body': json.dumps({ 'message': 'internal server error'}) 102 | } 103 | 104 | if event['queryStringParameters']['hashme'] != QUERY_STRING: 105 | logger.error('query string missmatch: rawQueryString: {}'.format(event['queryStringParameters']['hashme'])) 106 | return { 107 | 'statusCode': 503, 108 | 'body': json.dumps({ 'message': 'internal server error'}) 109 | } 110 | 111 | # Spin up resources 112 | event_loop_group = io.EventLoopGroup(1) 113 | host_resolver = io.DefaultHostResolver(event_loop_group) 114 | client_bootstrap = io.ClientBootstrap(event_loop_group, host_resolver) 115 | 116 | mqtt_connection = mqtt_connection_builder.mtls_from_path( 117 | endpoint=ENDPOINT, 118 | cert_filepath=CERT, 119 | pri_key_filepath=KEY, 120 | client_bootstrap=client_bootstrap, 121 | ca_filepath=CA, 122 | on_connection_interrupted=on_connection_interrupted, 123 | on_connection_resumed=on_connection_resumed, 124 | client_id=client_id, 125 | clean_session=False, 126 | keep_alive_secs=6) 127 | 128 | logger.info("connecting: endpoint: {} client_id: {}".format( 129 | ENDPOINT, client_id)) 130 | 131 | connect_future = mqtt_connection.connect() 132 | 133 | # Future.result() waits until a result is available 134 | connect_future.result() 135 | logger.info("connected to endpoint: {}".format(ENDPOINT)) 136 | 137 | # Subscribe 138 | logger.info("subscribing: topic: {}".format(topic)) 139 | subscribe_future, packet_id = mqtt_connection.subscribe( 140 | topic=topic, 141 | qos=mqtt.QoS.AT_LEAST_ONCE, 142 | callback=on_message_received) 143 | 144 | subscribe_result = subscribe_future.result() 145 | logger.info("subscribed: qos: {}".format(str(subscribe_result['qos']))) 146 | 147 | logger.info("sending {} message(s)".format(COUNT)) 148 | 149 | publish_count = 1 150 | while (publish_count <= COUNT): 151 | message = { 152 | "message": "R53 health check", 153 | "count": "{}".format(publish_count), 154 | "datetime": "{}".format(datetime.now().isoformat()) 155 | } 156 | 157 | if 'requestContext' in event and 'http' in event['requestContext'] and 'sourceIp' in event['requestContext']['http']: 158 | message['source_ip'] = {'source': 'http', 'ip': event['requestContext']['http']['sourceIp']} 159 | 160 | if 'requestContext' in event and 'identity' in event['requestContext'] and 'sourceIp' in event['requestContext']['identity']: 161 | message['source_ip'] = {'source': 'identity', 'ip': event['requestContext']['identity']['sourceIp']} 162 | 163 | logger.info("publishing: topic {}: message: {}".format(topic, message)) 164 | mqtt_connection.publish( 165 | topic=topic, 166 | payload=json.dumps(message), 167 | qos=mqtt.QoS.AT_LEAST_ONCE) 168 | #time.sleep(1) 169 | publish_count += 1 170 | 171 | # Wait for all messages to be received. 172 | # This waits forever if count was set to 0. 173 | if not RECEIVED_ALL_EVENT.is_set(): 174 | logger.info("waiting for all message(s) to be received: {}/{}".format(RECEIVED_COUNT, COUNT)) 175 | 176 | if not RECEIVED_ALL_EVENT.wait(RECEIVE_TIMEOUT): 177 | raise Exception('not all message received after timeout: received: {} expected: {} timeout: {}'.format( 178 | RECEIVED_COUNT, COUNT, RECEIVE_TIMEOUT)) 179 | 180 | logger.info("message(s) received: {}/{}".format(RECEIVED_COUNT, COUNT)) 181 | 182 | # Disconnect 183 | logger.info("initiating disconnect") 184 | disconnect_future = mqtt_connection.disconnect() 185 | disconnect_future.result() 186 | logger.info("disconnected") 187 | logger.info('r53-health-check: finished: messages: published/received: {}/{}'.format( 188 | publish_count-1, RECEIVED_COUNT)) 189 | 190 | return { 191 | 'statusCode': 200, 192 | 'body': json.dumps({ 'mqtt_status': 'healthy' }) 193 | } 194 | except Exception as e: 195 | logger.error('r53-health-check: finished: with errror: {}'.format(e)) 196 | return { 197 | 'statusCode': 503, 198 | 'body': json.dumps({ 'mqtt_status': 'unhealthy', 'error': '{}'.format(e)}) 199 | } 200 | 201 | if __name__ == '__main__': 202 | logger.info('calling lambda_handler') 203 | lambda_handler({"rawQueryString": QUERY_STRING}, None) 204 | -------------------------------------------------------------------------------- /source/tools/iot-devices-cmp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0. 5 | 6 | """IoT DR: compare device 7 | configuration in primary and 8 | secondary region.""" 9 | 10 | import argparse 11 | import json 12 | import logging 13 | import sys 14 | import time 15 | import traceback 16 | 17 | from concurrent import futures 18 | 19 | import boto3 20 | import boto3.session 21 | 22 | from botocore.config import Config 23 | 24 | 25 | logger = logging.getLogger() 26 | for h in logger.handlers: 27 | logger.removeHandler(h) 28 | h = logging.StreamHandler(sys.stdout) 29 | FORMAT = '%(asctime)s [%(levelname)s]: %(threadName)s-%(filename)s:%(lineno)s-%(funcName)s: %(message)s' 30 | h.setFormatter(logging.Formatter(FORMAT)) 31 | logger.addHandler(h) 32 | logger.setLevel(logging.INFO) 33 | #logger.setLevel(logging.DEBUG) 34 | 35 | parser = argparse.ArgumentParser(description="Compare device configuration in two regions") 36 | parser.add_argument('--primary-region', required=True, help="Primary aws region.") 37 | parser.add_argument('--secondary-region', required=True, help="Secondary aws region.") 38 | parser.add_argument('--max-workers', default=10, type=int, help="Maximum number of worker threads. Allowed maximum is 50.") 39 | parser.add_argument('--query-string', default='thingName:*', help="Query string.") 40 | args = parser.parse_args() 41 | 42 | NUM_THINGS_COMPARED = 0 43 | NUM_THINGS_NOTSYNCED = 0 44 | NUM_ERRORS = 0 45 | 46 | 47 | def print_response(response): 48 | del response['ResponseMetadata'] 49 | print(json.dumps(response, indent=2, default=str)) 50 | 51 | 52 | def get_device_status(c_iot, thing_name): 53 | logger.info('thing_name: {}'.format(thing_name)) 54 | 55 | try: 56 | device_status = {thing_name: {'policy': '', 'cert_id': ''}} 57 | response = c_iot.describe_thing(thingName=thing_name) 58 | logger.debug('response: {}'.format(response)) 59 | logger.info('exists: thing_name: {}'.format(thing_name)) 60 | 61 | response = c_iot.list_thing_principals(thingName=thing_name) 62 | 63 | for principal in response['principals']: 64 | #print('PRINCIPAL: {}'.format(principal)) 65 | cert_id = principal.split('/')[-1] 66 | device_status[thing_name]['cert_id'] = cert_id 67 | response = c_iot.describe_certificate(certificateId=principal.split('/')[-1]) 68 | 69 | response = c_iot.list_principal_policies(principal=principal) 70 | #print('POLICIES') 71 | 72 | for policy in response['policies']: 73 | response = c_iot.get_policy(policyName=policy['policyName']) 74 | device_status[thing_name]['policy'] = policy['policyName'] 75 | 76 | return device_status 77 | 78 | except c_iot.exceptions.ResourceNotFoundException: 79 | logger.info('replication error: thing does not exist: thing_name: {}'.format(thing_name)) 80 | return {} 81 | 82 | except Exception as e: 83 | logger.error('{}'.format(e)) 84 | raise Exception(e) 85 | 86 | 87 | 88 | def compare_device(thing_name): 89 | global NUM_THINGS_COMPARED, NUM_THINGS_NOTSYNCED, NUM_ERRORS 90 | try: 91 | logger.info('thing_name: {}'.format(thing_name)) 92 | start_time = int(time.time()*1000) 93 | device_status_primary = get_device_status(c_iot_p, thing_name) 94 | device_status_secondary = get_device_status(c_iot_s, thing_name) 95 | logger.info('thing_name: {} device_status_primary: {} device_status_secondary: {}'.format(thing_name, device_status_primary, device_status_secondary)) 96 | 97 | errors = [] 98 | if not thing_name in device_status_primary: 99 | errors.append('thing name does not exist in primary') 100 | elif not thing_name in device_status_secondary: 101 | errors.append('thing name does not exist in secondary') 102 | 103 | if thing_name in device_status_primary and thing_name in device_status_secondary: 104 | if device_status_primary[thing_name]['cert_id'] != device_status_secondary[thing_name]['cert_id']: 105 | errors.append('cert id missmatch') 106 | if device_status_primary[thing_name]['policy'] != device_status_secondary[thing_name]['policy']: 107 | errors.append('policy missmatch') 108 | 109 | if errors: 110 | logger.error('replication error: {}: primary: {} secondary: {}'.format(','.join(errors), device_status_primary, device_status_secondary)) 111 | NUM_ERRORS += 1 112 | 113 | end_time = int(time.time()*1000) 114 | duration = end_time - start_time 115 | NUM_THINGS_COMPARED += 1 116 | logger.info('compare device: thing_name: {} duration: {}ms'.format(thing_name, duration)) 117 | except Exception as e: 118 | logger.error('{}'.format(e)) 119 | NUM_ERRORS += 1 120 | traceback.print_stack() 121 | 122 | 123 | def get_next_token(response): 124 | next_token = None 125 | if 'nextToken' in response: 126 | next_token = response['nextToken'] 127 | 128 | return next_token 129 | 130 | 131 | def get_search_things(query_string, max_results): 132 | logger.info('query_string: {} max_results: {}'.format(query_string, max_results)) 133 | try: 134 | response = c_iot_p.search_index( 135 | indexName='AWS_Things', 136 | queryString=query_string, 137 | maxResults=max_results 138 | ) 139 | 140 | for thing in response['things']: 141 | executor.submit(compare_device, thing['thingName']) 142 | 143 | next_token = get_next_token(response) 144 | 145 | while next_token: 146 | response = c_iot_p.search_index( 147 | indexName='AWS_Things', 148 | nextToken=next_token, 149 | queryString=query_string, 150 | maxResults=max_results 151 | ) 152 | next_token = get_next_token(response) 153 | 154 | for thing in response['things']: 155 | executor.submit(compare_device, thing['thingName']) 156 | except Exception as e: 157 | logger.error('{}'.format(e)) 158 | 159 | 160 | def registry_indexing_enabled(): 161 | try: 162 | response = c_iot_p.get_indexing_configuration() 163 | logger.debug('response: {}'.format(response)) 164 | 165 | logger.info('thingIndexingMode: {}'.format(response['thingIndexingConfiguration']['thingIndexingMode'])) 166 | if response['thingIndexingConfiguration']['thingIndexingMode'] == 'OFF': 167 | return False 168 | 169 | return True 170 | except Exception as e: 171 | logger.error('{}'.format(e)) 172 | raise Exception(e) 173 | 174 | 175 | try: 176 | logger.info('cmp: start') 177 | logger.info('primary_region: {} secondary_region: {} query_string: {} max_workers: {}'. 178 | format(args.primary_region, args.secondary_region, args.query_string, args.max_workers)) 179 | time.sleep(2) 180 | 181 | NUM_THINGS_COMPARED = 0 182 | NUM_THINGS_NOTSYNCED = 0 183 | NUM_ERRORS = 0 184 | 185 | if args.max_workers > 50: 186 | logger.error('max allowed workers is 50 defined: {}'.format(args.max_workers)) 187 | raise Exception('max allowed workers is 50 defined: {}'.format(args.max_workers)) 188 | 189 | MAX_POOL_CONNECTIONS = 10 190 | if args.max_workers >= 10: 191 | MAX_POOL_CONNECTIONS = round(args.max_workers*1.2) 192 | 193 | logger.info('MAX_POOL_CONNECTIONS: {}'.format(MAX_POOL_CONNECTIONS)) 194 | 195 | boto3_config = Config( 196 | max_pool_connections = MAX_POOL_CONNECTIONS, 197 | retries = {'max_attempts': 10, 'mode': 'standard'} 198 | ) 199 | 200 | session_p = boto3.Session(region_name=args.primary_region) 201 | session_s = boto3.Session(region_name=args.secondary_region) 202 | c_iot_p = session_p.client('iot', config=boto3_config) 203 | c_iot_s = session_s.client('iot', config=boto3_config) 204 | 205 | executor = futures.ThreadPoolExecutor(max_workers=args.max_workers) 206 | logger.info('executor: started: {}'.format(executor)) 207 | 208 | if not registry_indexing_enabled(): 209 | logger.info('registry indexing enabled must be enabled in region: {}'.format(args.primary_region)) 210 | raise Exception('indexing not enabled in region: {}'.format(args.primary_region)) 211 | 212 | get_search_things(args.query_string, 100) 213 | 214 | 215 | logger.info('executor: waiting to finish') 216 | executor.shutdown(wait=True) 217 | logger.info('executor: shutted down') 218 | 219 | logger.info('cmp: stats: NUM_THINGS_COMPARED: {} NUM_THINGS_NOTSYNCED: {} NUM_ERRORS: {}'.format(NUM_THINGS_COMPARED, NUM_THINGS_NOTSYNCED, NUM_ERRORS)) 220 | 221 | logger.info('cmp: stop') 222 | except Exception as e: 223 | logger.error('{}'.format(e)) 224 | -------------------------------------------------------------------------------- /source/jupyter/04_IoTDR_Device_Certs.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## AWS IoT DR - Request device certificates from PCA\n", 8 | "\n", 9 | "First you create a key and a CSR. The CSR will be send to PCA which issues a certificate. These certificates can be used by your devices.\n", 10 | "\n", 11 | "Private, public key and the certificate will be stored on the local file system." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "from OpenSSL import crypto, SSL\n", 21 | "from os.path import exists, join\n", 22 | "from os import makedirs\n", 23 | "from shutil import copy\n", 24 | "from time import time, gmtime, localtime, strftime\n", 25 | "import boto3\n", 26 | "import json\n", 27 | "import time" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "metadata": {}, 33 | "source": [ 34 | "## Shared variables\n", 35 | "\n", 36 | "Import shared variables into this notebook." 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "%store -r config\n", 46 | "print(\"config: {}\".format(json.dumps(config, indent=4, default=str)))" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "## Some handy functions\n", 54 | "\n", 55 | "Create a key, a CSR and get the certificate from your private CA." 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "def create_csr(pkey, subject, digest=\"sha256\"):\n", 65 | " print(\"subject: {}\".format(subject))\n", 66 | " req = crypto.X509Req()\n", 67 | " subj = req.get_subject()\n", 68 | " \n", 69 | " for i in ['C', 'ST', 'L', 'O', 'OU', 'CN']:\n", 70 | " if i in subject:\n", 71 | " setattr(subj, i, subject[i])\n", 72 | "\n", 73 | " req.set_pubkey(pkey)\n", 74 | " req.sign(pkey, digest)\n", 75 | " return req\n", 76 | "\n", 77 | "\n", 78 | "def create_priv_key_and_csr(cert_dir, csr_file, key_file, subject):\n", 79 | " if not exists(cert_dir):\n", 80 | " print(\"creating directory: {}\".format(cert_dir))\n", 81 | " makedirs(cert_dir)\n", 82 | " \n", 83 | " priv_key = crypto.PKey()\n", 84 | " priv_key.generate_key(crypto.TYPE_RSA, 2048)\n", 85 | " #print(crypto.dump_privatekey(crypto.FILETYPE_PEM, priv_key).decode('utf-8'))\n", 86 | "\n", 87 | " key_file = join(cert_dir, key_file)\n", 88 | " f = open(key_file,\"w\")\n", 89 | " f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, priv_key).decode('utf-8'))\n", 90 | " f.close()\n", 91 | " \n", 92 | " csr = create_csr(priv_key, subject)\n", 93 | "\n", 94 | " csr_file = join(cert_dir, csr_file)\n", 95 | " f= open(csr_file,\"w\")\n", 96 | " f.write(crypto.dump_certificate_request(crypto.FILETYPE_PEM, csr).decode('utf-8'))\n", 97 | " f.close()\n", 98 | " \n", 99 | " return crypto.dump_certificate_request(crypto.FILETYPE_PEM, csr)\n", 100 | "\n", 101 | "\n", 102 | "def request_cert_from_pca(subject):\n", 103 | " device_key_file = '{}.device.key.pem'.format(subject['CN'])\n", 104 | " device_csr_file = '{}.device.csr.pem'.format(subject['CN'])\n", 105 | " device_cert_file = '{}.device.cert.pem'.format(subject['CN'])\n", 106 | " device_root_cert_file = '{}.device.root.cert.pem'.format(subject['CN'])\n", 107 | " \n", 108 | " device_csr = create_priv_key_and_csr(config['PCA_directory'], \n", 109 | " device_csr_file, \n", 110 | " device_key_file, \n", 111 | " subject)\n", 112 | " print(\"device_csr: {}\".format(device_csr))\n", 113 | "\n", 114 | " idempotency_token = '{}_id_token'.format(subject['CN'])\n", 115 | " response = c_acm_pca.issue_certificate(\n", 116 | " CertificateAuthorityArn = pca_arn,\n", 117 | " Csr = device_csr,\n", 118 | " SigningAlgorithm = 'SHA256WITHRSA',\n", 119 | " Validity= {\n", 120 | " 'Value': 365,\n", 121 | " 'Type': 'DAYS'\n", 122 | " },\n", 123 | " IdempotencyToken = idempotency_token\n", 124 | " )\n", 125 | "\n", 126 | " print(\"response: {}\".format(response))\n", 127 | "\n", 128 | " certificate_arn = response['CertificateArn']\n", 129 | " print(\"certificate_arn: {}\".format(certificate_arn))\n", 130 | " \n", 131 | " waiter = c_acm_pca.get_waiter('certificate_issued')\n", 132 | " try:\n", 133 | " waiter.wait(\n", 134 | " CertificateAuthorityArn=pca_arn,\n", 135 | " CertificateArn=certificate_arn\n", 136 | " )\n", 137 | " except botocore.exceptions.WaiterError as e:\n", 138 | " raise Exception(\"waiter: {}\".format(e))\n", 139 | " \n", 140 | " response = c_acm_pca.get_certificate(\n", 141 | " CertificateAuthorityArn = pca_arn,\n", 142 | " CertificateArn = certificate_arn\n", 143 | " )\n", 144 | " print(\"response: {}\".format(response))\n", 145 | " device_cert = response['Certificate']\n", 146 | " print(\"device_cert: {}\".format(device_cert))\n", 147 | "\n", 148 | " file_device_crt = join(config['PCA_directory'], device_cert_file)\n", 149 | " f = open(file_device_crt,\"w\")\n", 150 | " f.write(device_cert)\n", 151 | " f.close()\n", 152 | " \n", 153 | " file_root_device_crt = join(config['PCA_directory'], device_root_cert_file)\n", 154 | " f = open(file_root_device_crt,\"w\")\n", 155 | " f.write(device_cert)\n", 156 | " f.write(\"\\n\")\n", 157 | " f.write(pca_certificate)\n", 158 | " f.close()\n", 159 | " \n", 160 | " print(\"device_key_file: {}\".format(device_key_file))\n", 161 | " print(\"device_csr_file: {}\".format(device_csr_file))\n", 162 | " print(\"device_cert_file: {}\".format(device_cert_file))\n", 163 | " print(\"device_root_cert_file: {}\".format(device_root_cert_file))" 164 | ] 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "metadata": {}, 169 | "source": [ 170 | "## Boto3 client" 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": null, 176 | "metadata": {}, 177 | "outputs": [], 178 | "source": [ 179 | "c_acm_pca = boto3.client('acm-pca', region_name = config['aws_region_pca'])" 180 | ] 181 | }, 182 | { 183 | "cell_type": "markdown", 184 | "metadata": {}, 185 | "source": [ 186 | "## PCA certificate and ARN\n", 187 | "Get the CA certificate for the private CA as well as it's arn. The arn is required to issue certificates and for the private CA certificate will be stored together with the device certificate." 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": null, 193 | "metadata": {}, 194 | "outputs": [], 195 | "source": [ 196 | "response = c_acm_pca.list_certificate_authorities(MaxResults=50)\n", 197 | "\n", 198 | "for ca in response['CertificateAuthorities']:\n", 199 | " if ca['CertificateAuthorityConfiguration']['Subject']['CommonName'] == config['Sub_CN']:\n", 200 | " pca_arn = ca['Arn']\n", 201 | " break\n", 202 | "\n", 203 | "print(\"pca_arn: {}\".format(pca_arn))\n", 204 | "\n", 205 | "response = c_acm_pca.get_certificate_authority_certificate(\n", 206 | " CertificateAuthorityArn = pca_arn\n", 207 | ")\n", 208 | "#print(\"response: {}\".format(json.dumps(response, indent=4, default=str)))\n", 209 | "pca_certificate = response['Certificate']\n", 210 | "print(\"pca_certificate:\\n{}\".format(pca_certificate))" 211 | ] 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "metadata": {}, 216 | "source": [ 217 | "## Request device certificate\n", 218 | "Set the common name (CN) in your device certificate to be your thing or device name. Provide a thing name in the variable `thing_name`. \n", 219 | "\n", 220 | "The Just-in-Time Registration process will extract the CN from the certificate and use it as thing name." 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": null, 226 | "metadata": {}, 227 | "outputs": [], 228 | "source": [ 229 | "thing_name = \"dr-pca-04\"\n", 230 | "request_cert_from_pca({\"CN\": thing_name})" 231 | ] 232 | } 233 | ], 234 | "metadata": { 235 | "kernelspec": { 236 | "display_name": "conda_python3", 237 | "language": "python", 238 | "name": "conda_python3" 239 | }, 240 | "language_info": { 241 | "codemirror_mode": { 242 | "name": "ipython", 243 | "version": 3 244 | }, 245 | "file_extension": ".py", 246 | "mimetype": "text/x-python", 247 | "name": "python", 248 | "nbconvert_exporter": "python", 249 | "pygments_lexer": "ipython3", 250 | "version": "3.6.10" 251 | } 252 | }, 253 | "nbformat": 4, 254 | "nbformat_minor": 4 255 | } 256 | -------------------------------------------------------------------------------- /source/jupyter/03_IoTDR_Reg_PCA.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# AWS IoT DR - Register PCA with IoT Core\n", 8 | "\n", 9 | "Register the private CA in the primary and secondary region with AWS IoT Core." 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "## Libraries" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": null, 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "from OpenSSL import crypto, SSL\n", 26 | "from os.path import exists, join\n", 27 | "from os import makedirs\n", 28 | "from shutil import copy\n", 29 | "from time import time, gmtime, localtime, strftime\n", 30 | "import boto3\n", 31 | "import json\n", 32 | "import time" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": {}, 38 | "source": [ 39 | "## Shared variables\n", 40 | "\n", 41 | "Import shared variables into this notebook." 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": null, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "%store -r config\n", 51 | "print(\"config: {}\".format(json.dumps(config, indent=4, default=str)))" 52 | ] 53 | }, 54 | { 55 | "cell_type": "markdown", 56 | "metadata": {}, 57 | "source": [ 58 | "## Some handy functions\n", 59 | "\n", 60 | "Generate a key and create a certificate signing request (CSR)." 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": null, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [ 69 | "def create_csr(pkey, subject, digest=\"sha256\"):\n", 70 | " print(\"subject: {}\".format(subject))\n", 71 | " req = crypto.X509Req()\n", 72 | " subj = req.get_subject()\n", 73 | " \n", 74 | " for i in ['C', 'ST', 'L', 'O', 'OU', 'CN']:\n", 75 | " if i in subject:\n", 76 | " setattr(subj, i, subject[i])\n", 77 | "\n", 78 | " req.set_pubkey(pkey)\n", 79 | " req.sign(pkey, digest)\n", 80 | " return req\n", 81 | "\n", 82 | "\n", 83 | "def create_priv_key_and_csr(cert_dir, csr_file, key_file, subject):\n", 84 | " if not exists(cert_dir):\n", 85 | " print(\"creating directory: {}\".format(cert_dir))\n", 86 | " makedirs(cert_dir)\n", 87 | " \n", 88 | " priv_key = crypto.PKey()\n", 89 | " priv_key.generate_key(crypto.TYPE_RSA, 2048)\n", 90 | " #print(crypto.dump_privatekey(crypto.FILETYPE_PEM, priv_key).decode('utf-8'))\n", 91 | "\n", 92 | " key_file = join(cert_dir, key_file)\n", 93 | " f = open(key_file,\"w\")\n", 94 | " f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, priv_key).decode('utf-8'))\n", 95 | " f.close()\n", 96 | " \n", 97 | " csr = create_csr(priv_key, subject)\n", 98 | "\n", 99 | " csr_file = join(cert_dir, csr_file)\n", 100 | " f= open(csr_file,\"w\")\n", 101 | " f.write(crypto.dump_certificate_request(crypto.FILETYPE_PEM, csr).decode('utf-8'))\n", 102 | " f.close()\n", 103 | " \n", 104 | " return crypto.dump_certificate_request(crypto.FILETYPE_PEM, csr)" 105 | ] 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "metadata": {}, 110 | "source": [ 111 | "## Boto3 client\n", 112 | "Create a boto3 client for the acm-pca service endpoint." 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": null, 118 | "metadata": {}, 119 | "outputs": [], 120 | "source": [ 121 | "c_acm_pca = boto3.client('acm-pca', region_name = config['aws_region_pca'])" 122 | ] 123 | }, 124 | { 125 | "cell_type": "markdown", 126 | "metadata": {}, 127 | "source": [ 128 | "## PCA certificate and ARN\n", 129 | "Get the root certificate and the ARN from your private CA. They are required to register your private CA with AWS IoT Core." 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "response = c_acm_pca.list_certificate_authorities(MaxResults=50)\n", 139 | "\n", 140 | "for ca in response['CertificateAuthorities']:\n", 141 | " #print(ca['CertificateAuthorityConfiguration']['Subject']['CommonName'])\n", 142 | " if ca['CertificateAuthorityConfiguration']['Subject']['CommonName'] == config['Sub_CN']:\n", 143 | " pca_arn = ca['Arn']\n", 144 | " break\n", 145 | "\n", 146 | "print(\"pca_arn: {}\\n\".format(pca_arn))\n", 147 | "\n", 148 | "response = c_acm_pca.get_certificate_authority_certificate(\n", 149 | " CertificateAuthorityArn = pca_arn\n", 150 | ")\n", 151 | "print(\"response: {}\\n\".format(json.dumps(response, indent=4, default=str)))\n", 152 | "pca_certificate = response['Certificate']\n", 153 | "print(\"pca_certificate:\\n{}\".format(pca_certificate))" 154 | ] 155 | }, 156 | { 157 | "cell_type": "markdown", 158 | "metadata": {}, 159 | "source": [ 160 | "## Register private CA\n", 161 | "To register the private CA with AWS IoT Core you need to get a registration code. Then you create a certificate with the common name (CN) set to the registration code. This certificate will be used for the CA registration process.\n", 162 | "\n", 163 | "The private CA will be registered with AWS IoT Core in the primary and secondary region." 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": null, 169 | "metadata": {}, 170 | "outputs": [], 171 | "source": [ 172 | "for aws_region in [config['aws_region_primary'], config['aws_region_secondary']]: \n", 173 | " print(\"AWS REGION: {}\".format(aws_region))\n", 174 | " c_iot = boto3.client('iot', region_name = aws_region)\n", 175 | " time.sleep(2)\n", 176 | "\n", 177 | " response = c_iot.get_registration_code()\n", 178 | "\n", 179 | " print(\"response: {}\\n\".format(json.dumps(response, indent=4, default=str)))\n", 180 | " registration_code = response['registrationCode']\n", 181 | " print(\"registration_code: {}\\n\".format(registration_code))\n", 182 | "\n", 183 | " verification_csr = create_priv_key_and_csr(config['PCA_directory'], \n", 184 | " 'registration_csr_{}.pem'.format(aws_region), \n", 185 | " 'registration_key_{}.pem'.format(aws_region), \n", 186 | " {\"CN\": registration_code})\n", 187 | " print(\"verification_csr:\\n{}\\n\".format(verification_csr))\n", 188 | "\n", 189 | " idempotency_token = 'registration_cert'\n", 190 | " response = c_acm_pca.issue_certificate(\n", 191 | " CertificateAuthorityArn = pca_arn,\n", 192 | " Csr = verification_csr,\n", 193 | " SigningAlgorithm = 'SHA256WITHRSA',\n", 194 | " Validity= {\n", 195 | " 'Value': 365,\n", 196 | " 'Type': 'DAYS'\n", 197 | " },\n", 198 | " IdempotencyToken = idempotency_token\n", 199 | " )\n", 200 | "\n", 201 | " print(\"response: {}\\n\".format(json.dumps(response, indent=4, default=str)))\n", 202 | " certificate_arn = response['CertificateArn']\n", 203 | "\n", 204 | " print(\"certificate_arn: {}\\n\".format(certificate_arn))\n", 205 | "\n", 206 | " waiter = c_acm_pca.get_waiter('certificate_issued')\n", 207 | " try:\n", 208 | " waiter.wait(\n", 209 | " CertificateAuthorityArn=pca_arn,\n", 210 | " CertificateArn=certificate_arn\n", 211 | " )\n", 212 | " except botocore.exceptions.WaiterError as e:\n", 213 | " raise Exception(\"waiter: {}\".format(e))\n", 214 | " \n", 215 | " response = c_acm_pca.get_certificate(\n", 216 | " CertificateAuthorityArn = pca_arn,\n", 217 | " CertificateArn = certificate_arn\n", 218 | " )\n", 219 | " print(\"response: {}\".format(response))\n", 220 | " registration_certificate = response['Certificate']\n", 221 | "\n", 222 | " print(\"pca_certificate:\\n{}\\n\".format(pca_certificate))\n", 223 | " print(\"registration_certificate:\\n{}\\n\".format(registration_certificate))\n", 224 | " \n", 225 | " file_registration_crt = join(config['PCA_directory'], 'registration_cert_{}.pem'.format(aws_region))\n", 226 | " f = open(file_registration_crt,\"w\")\n", 227 | " f.write(registration_certificate)\n", 228 | " f.close()\n", 229 | "\n", 230 | " response = c_iot.register_ca_certificate(\n", 231 | " caCertificate = pca_certificate,\n", 232 | " verificationCertificate = registration_certificate,\n", 233 | " setAsActive = True,\n", 234 | " allowAutoRegistration = True\n", 235 | " )\n", 236 | "\n", 237 | " print(\"response: {}\\n\".format(json.dumps(response, indent=4, default=str)))\n", 238 | "\n", 239 | " certificate_id = response['certificateId']\n", 240 | " print(\"certificate_id: {}\\n\".format(certificate_id))\n", 241 | "\n", 242 | " response = c_iot.describe_ca_certificate(\n", 243 | " certificateId = certificate_id\n", 244 | " )\n", 245 | "\n", 246 | " print(\"response: {}\\n\".format(json.dumps(response, indent=4, default=str)))" 247 | ] 248 | } 249 | ], 250 | "metadata": { 251 | "kernelspec": { 252 | "display_name": "conda_python3", 253 | "language": "python", 254 | "name": "conda_python3" 255 | }, 256 | "language_info": { 257 | "codemirror_mode": { 258 | "name": "ipython", 259 | "version": 3 260 | }, 261 | "file_extension": ".py", 262 | "mimetype": "text/x-python", 263 | "name": "python", 264 | "nbconvert_exporter": "python", 265 | "pygments_lexer": "ipython3", 266 | "version": "3.6.10" 267 | } 268 | }, 269 | "nbformat": 4, 270 | "nbformat_minor": 4 271 | } 272 | -------------------------------------------------------------------------------- /source/tools/iot-dr-shadow-cmp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0. 5 | 6 | """IoT DR: test shadow synchronisation 7 | Shadows are created in the primary region 8 | and are updated. 9 | A get-shadow will be called in the secondary 10 | region and the result will be compared to 11 | the shadow content in the primary region. 12 | After running the test shadows will be deleted.""" 13 | 14 | import argparse 15 | import json 16 | import logging 17 | import random 18 | import sys 19 | import time 20 | import uuid 21 | 22 | from concurrent import futures 23 | 24 | import boto3 25 | import boto3.session 26 | 27 | from botocore.config import Config 28 | 29 | 30 | logger = logging.getLogger() 31 | for h in logger.handlers: 32 | logger.removeHandler(h) 33 | h = logging.StreamHandler(sys.stdout) 34 | FORMAT = '%(asctime)s [%(levelname)s]: %(threadName)s-%(filename)s:%(lineno)s-%(funcName)s: %(message)s' 35 | h.setFormatter(logging.Formatter(FORMAT)) 36 | logger.addHandler(h) 37 | logger.setLevel(logging.INFO) 38 | #logger.setLevel(logging.DEBUG) 39 | 40 | parser = argparse.ArgumentParser(description="Compare device configuration in two regions") 41 | parser.add_argument('--primary-region', required=True, help="Primary aws region.") 42 | parser.add_argument('--secondary-region', required=True, help="Secondary aws region.") 43 | parser.add_argument('--num-tests', default=10, type=int, help="Nunmber of tests to conduct.") 44 | parser.add_argument('--max-workers', default=10, type=int, help="Maximum number of worker threads. Allowed maximum is 50.") 45 | args = parser.parse_args() 46 | 47 | NUM_SHADOWS_COMPARED = 0 48 | NUM_SHADOWS_NOTSYNCED = 0 49 | NUM_ERRORS = 0 50 | 51 | THING_SHADOWS = {} 52 | 53 | 54 | def update_shadow(i, c_iot_p): 55 | global THING_SHADOWS 56 | try: 57 | thing_name = '{}'.format(uuid.uuid4()) 58 | i += 1 59 | shadow_payload = {'state':{'reported':{'temperature': '{}'.format(random.randrange(20, 40))}}} 60 | logger.info('i: {} thing_name: {} shadow_payload: {}'.format(i, thing_name, shadow_payload)) 61 | 62 | response = c_iot_p.update_thing_shadow( 63 | thingName=thing_name, 64 | payload=json.dumps(shadow_payload) 65 | ) 66 | logger.debug('response: {}'.format(response)) 67 | logger.info('i: {} thing_name: {} response HTTPStatusCode: {}'. 68 | format(i, thing_name, response['ResponseMetadata']['HTTPStatusCode'])) 69 | THING_SHADOWS[thing_name] = shadow_payload 70 | 71 | except Exception as e: 72 | logger.error('{}'.format(e)) 73 | 74 | 75 | def get_shadow(i, c_iot_s, thing_name): 76 | try: 77 | response = c_iot_s.get_thing_shadow( 78 | thingName=thing_name 79 | ) 80 | logger.debug('response: {}'.format(response)) 81 | payload = json.loads(response['payload'].read()) 82 | logger.info('i: {} thing_name: {}: payload: {}'.format(i, thing_name, payload)) 83 | return payload 84 | 85 | except c_iot_s.exceptions.ResourceNotFoundException: 86 | logger.warning('i: {} thing_name: {}: shadow does not exist'.format(i, thing_name)) 87 | return {} 88 | except Exception as e: 89 | logger.error('replication: {}'.format(e)) 90 | 91 | 92 | def compare_shadow(i, c_iot_s, thing_name, shadow_payload): 93 | global NUM_SHADOWS_COMPARED, NUM_SHADOWS_NOTSYNCED, NUM_ERRORS 94 | try: 95 | logger.info('i: {} thing_name: {} shadow_payload: {}'.format(i, thing_name, shadow_payload)) 96 | NUM_SHADOWS_COMPARED += 1 97 | shadow_payload_secondary = {} 98 | retries = 5 99 | wait = 2 100 | n = 1 101 | while not shadow_payload_secondary and n <= retries: 102 | logger.info('n: {}: get_shadow for thing_name: {}'.format(n, thing_name)) 103 | n += 1 104 | shadow_payload_secondary = get_shadow(i, c_iot_s, thing_name) 105 | if not shadow_payload_secondary: 106 | retry_in = wait*n 107 | logger.info('n: {} thing_name: {}: no shadow payload, retrying in {} secs.'.format(n, thing_name, retry_in)) 108 | time.sleep(retry_in) 109 | 110 | if not shadow_payload_secondary: 111 | logger.error('replication: thing_name: {}: shadow not replicated to secondary region'.format(thing_name)) 112 | NUM_SHADOWS_NOTSYNCED += 1 113 | return 114 | 115 | logger.info('i: {} thing_name: {} shadow_payload: {} shadow_payload_secondary: {}'.format(i, thing_name, shadow_payload, shadow_payload_secondary)) 116 | 117 | errors = [] 118 | temperature = "" 119 | temperature_secondary = "" 120 | if 'temperature' in shadow_payload['state']['reported']: 121 | temperature = shadow_payload['state']['reported']['temperature'] 122 | else: 123 | errors.append('thing_name: {} temperature not in shadow_payload'.format(thing_name)) 124 | 125 | if 'temperature' in shadow_payload_secondary['state']['reported']: 126 | temperature_secondary = shadow_payload_secondary['state']['reported']['temperature'] 127 | else: 128 | errors.append('thing_name: {}: temperature not in shadow_payload_secondary'.format(thing_name)) 129 | 130 | if errors: 131 | logger.error('replication: {}'.format(errors)) 132 | return 133 | 134 | logger.info('temperature: {} temperature_secondary: {}'.format(temperature, temperature_secondary)) 135 | if temperature != temperature_secondary: 136 | logger.error('replication: thing_name: {} shadows missmatch: temperature: {} temperature_secondary: {}'.format(thing_name, temperature, temperature_secondary)) 137 | return 138 | 139 | logger.info('i: {} thing_name: {} shadows match: temperature: {} temperature_secondary: {}'.format(i, thing_name, temperature, temperature_secondary)) 140 | 141 | except Exception as e: 142 | logger.error('{}'.format(e)) 143 | 144 | 145 | def delete_shadow(i, c_iot_data, thing_name): 146 | try: 147 | region = c_iot_data.meta.region_name 148 | logger.info('i: {} thing_name: {} region: {}'.format(i, thing_name, region)) 149 | response = c_iot_data.delete_thing_shadow( 150 | thingName=thing_name 151 | ) 152 | logger.debug('response: {}'.format(response)) 153 | logger.info('i: {} thing_name: {} region: {} response HTTPStatusCode: {}'. 154 | format(i, thing_name, region, response['ResponseMetadata']['HTTPStatusCode'])) 155 | 156 | except c_iot_s.exceptions.ResourceNotFoundException: 157 | logger.warning('thing_name: {}: shadow does not exist'.format(thing_name)) 158 | return {} 159 | except Exception as e: 160 | logger.error('replication: {}'.format(e)) 161 | 162 | 163 | try: 164 | logger.info('cmp: start') 165 | logger.info('primary_region: {} secondary_region: {} num_tests: {} max_workers: {}'. 166 | format(args.primary_region, args.secondary_region, args.num_tests, args.max_workers)) 167 | time.sleep(2) 168 | 169 | NUM_SHADOWS_COMPARED = 0 170 | NUM_SHADOWS_NOTSYNCED = 0 171 | NUM_ERRORS = 0 172 | 173 | if args.max_workers > 50: 174 | logger.error('max allowed workers is 50 defined: {}'.format(args.max_workers)) 175 | raise Exception('max allowed workers is 50 defined: {}'.format(args.max_workers)) 176 | 177 | max_pool_connections = 10 178 | if args.max_workers >= 10: 179 | max_pool_connections = round(args.max_workers*1.2) 180 | 181 | logger.info('max_pool_connections: {}'.format(max_pool_connections)) 182 | 183 | boto3_config = Config( 184 | max_pool_connections = max_pool_connections, 185 | retries = {'max_attempts': 10, 'mode': 'standard'} 186 | ) 187 | 188 | session_p = boto3.Session(region_name=args.primary_region) 189 | session_s = boto3.Session(region_name=args.secondary_region) 190 | 191 | endpoint_p = session_p.client('iot').describe_endpoint(endpointType='iot:Data-ATS')['endpointAddress'] 192 | endpoint_s = session_s.client('iot').describe_endpoint(endpointType='iot:Data-ATS')['endpointAddress'] 193 | 194 | c_iot_p = session_p.client('iot-data', config=boto3_config, endpoint_url='https://{}'.format(endpoint_p)) 195 | c_iot_s = session_s.client('iot-data', config=boto3_config, endpoint_url='https://{}'.format(endpoint_s)) 196 | 197 | executor = futures.ThreadPoolExecutor(max_workers=args.max_workers) 198 | logger.info('executor update_shadow: started: {}'.format(executor)) 199 | 200 | for x in range(args.num_tests): 201 | executor.submit(update_shadow, x, c_iot_p) 202 | 203 | logger.info('executor update_shadow: waiting to finish') 204 | executor.shutdown(wait=True) 205 | logger.info('executor update_shadow: shutted down') 206 | 207 | 208 | executor = futures.ThreadPoolExecutor(max_workers=args.max_workers) 209 | logger.info('executor compare_shadow: started: {}'.format(executor)) 210 | 211 | logger.info(THING_SHADOWS) 212 | y = 0 213 | for thing_name in THING_SHADOWS.keys(): 214 | y += 1 215 | executor.submit(compare_shadow, y, c_iot_s, thing_name, THING_SHADOWS[thing_name]) 216 | 217 | logger.info('executor compare_shadow: waiting to finish') 218 | executor.shutdown(wait=True) 219 | logger.info('executor compare_shadow: shutted down') 220 | 221 | executor = futures.ThreadPoolExecutor(max_workers=args.max_workers) 222 | logger.info('executor delete_shadow: started: {}'.format(executor)) 223 | z = 0 224 | for thing_name in THING_SHADOWS.keys(): 225 | z += 1 226 | executor.submit(delete_shadow, z, c_iot_p, thing_name) 227 | executor.submit(delete_shadow, z, c_iot_s, thing_name) 228 | 229 | logger.info('executor delete_shadow: waiting to finish') 230 | executor.shutdown(wait=True) 231 | logger.info('executor delete_shadow: shutted down') 232 | 233 | logger.info('cmp: stats: NUM_SHADOWS_COMPARED: {} NUM_SHADOWS_NOTSYNCED: {} NUM_ERRORS: {}'.format(NUM_SHADOWS_COMPARED, NUM_SHADOWS_NOTSYNCED, NUM_ERRORS)) 234 | 235 | logger.info('cmp: stop') 236 | except Exception as e: 237 | logger.error('{}'.format(e)) 238 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. -------------------------------------------------------------------------------- /source/launch-solution-code-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # 7 | # launch-solution-code-build.sh 8 | # 9 | 10 | set -e 11 | 12 | LAUNCH_VERSION="2021-02-10 01" 13 | echo "LAUNCH_VERSION: $LAUNCH_VERSION" 14 | 15 | DIR=$(pwd) 16 | 17 | if [ -z "$BUCKET_RESOURCES" ]; then 18 | echo "BUCKET_RESOURCES is not defined, exiting" 19 | exit 1 20 | fi 21 | 22 | if [ -z "$SOLUTION_NAME" ]; then 23 | echo "SOLUTION_NAME is not defined, exiting" 24 | exit 1 25 | fi 26 | 27 | if [ -z "$VERSION" ]; then 28 | echo "VERSION is not defined, exiting" 29 | exit 1 30 | fi 31 | 32 | if [ "$CREATE_HEALTH_CHECK" != "yes" ]; then 33 | CREATE_HEALTH_CHECK=no 34 | fi 35 | 36 | 37 | echo "BUCKET_RESOURCES: $BUCKET_RESOURCES SOLUTION_NAME: $SOLUTION_NAME VERSION: $VERSION CREATE_HEALTH_CHECK: $CREATE_HEALTH_CHECK" 38 | 39 | function dt () { date '+%Y-%m-%d %H:%M:%S'; } 40 | 41 | START_DATE=$(dt) 42 | 43 | if [ -z "$STACK_POSTFIX" ]; then 44 | echo "STACK_POSTFIX not defined creating STACK_POSTFIX" 45 | STACK_POSTFIX=$(date '+%Y%m%d%H%M%S') 46 | echo "new STACK_POSTFIX: $STACK_POSTFIX" 47 | else 48 | echo "STACK_POSTFIX set already: $STACK_POSTFIX" 49 | fi 50 | 51 | if [ -z "$UUID" ]; then 52 | echo "UUID not defined creating UUID" 53 | UUID=$(uuidgen | tr '[:upper:]' '[:lower:]') 54 | echo "new UUID: $UUID" 55 | else 56 | echo "UUID set already: $UUID" 57 | fi 58 | 59 | 60 | BUCKET_PRIMARY_REGION="iot-dr-primary-$UUID" 61 | BUCKET_SECONDARY_REGION="iot-dr-secondary-$UUID" 62 | 63 | echo "PRIMARY_REGION: $PRIMARY_REGION SECONDARY_REGION: $SECONDARY_REGION" 64 | echo "BUCKET_PRIMARY_REGION: $BUCKET_PRIMARY_REGION BUCKET_SECONDARY_REGION: $BUCKET_SECONDARY_REGION" 65 | ERROR="" 66 | 67 | echo "$(dt): creating buckets" 68 | aws s3 mb s3://$BUCKET_PRIMARY_REGION --region $PRIMARY_REGION 69 | 70 | echo "$(dt) enabling encryption for bucket \"$BUCKET_PRIMARY_REGION\"" 71 | aws s3api put-bucket-encryption --region $PRIMARY_REGION \ 72 | --bucket $BUCKET_PRIMARY_REGION \ 73 | --server-side-encryption-configuration '{"Rules": [{"ApplyServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}}]}' 74 | 75 | 76 | aws s3 mb s3://$BUCKET_SECONDARY_REGION --region $SECONDARY_REGION 77 | 78 | echo "$(dt) enabling encryption for bucket \"$BUCKET_SECONDARY_REGION\"" 79 | aws s3api put-bucket-encryption --region $SECONDARY_REGION \ 80 | --bucket $BUCKET_SECONDARY_REGION \ 81 | --server-side-encryption-configuration '{"Rules": [{"ApplyServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}}]}' 82 | 83 | echo "$(dt): syncing jupyter to S3: $BUCKET_PRIMARY_REGION" 84 | aws s3 sync jupyter s3://$BUCKET_PRIMARY_REGION/jupyter/ 85 | 86 | echo "$(dt): syncing tools to S3: $BUCKET_PRIMARY_REGION" 87 | cp lambda/iot-dr-layer/device_replication.py tools/ 88 | aws s3 sync tools s3://$BUCKET_PRIMARY_REGION/tools/ 89 | 90 | echo "$(dt): syncing region syncers to S3: $BUCKET_PRIMARY_REGION" 91 | aws s3 sync lambda s3://$BUCKET_PRIMARY_REGION/lambda/ 92 | 93 | echo "------------------------------" 94 | 95 | touch toolsrc 96 | 97 | DDB_TABLE_NAME="IoTDRGlobalTable${STACK_POSTFIX}" 98 | IOT_ENDPOINT_PRIMARY=$(aws iot describe-endpoint --endpoint-type iot:Data-ATS --query 'endpointAddress' --region $PRIMARY_REGION --output text) 99 | IOT_ENDPOINT_SECONDARY=$(aws iot describe-endpoint --endpoint-type iot:Data-ATS --query 'endpointAddress' --region $SECONDARY_REGION --output text) 100 | echo "$(dt): IOT_ENDPOINT_PRIMARY: $IOT_ENDPOINT_PRIMARY IOT_ENDPOINT_SECONDARY: $IOT_ENDPOINT_SECONDARY" 101 | 102 | # 103 | # primary region 104 | # 105 | STACK_NAME="IoTDRPrimary${STACK_POSTFIX}" 106 | CFN_TEMPLATE="${SOLUTION_NAME}/${VERSION}/disaster-recovery-for-aws-iot-primary-region.template" 107 | TEMPLATE_URL=https://$BUCKET_RESOURCES.s3.amazonaws.com/$CFN_TEMPLATE 108 | echo "$(dt): launching stack \"$STACK_NAME\" in region \"$PRIMARY_REGION\"" 109 | echo " TEMPLATE_URL: $TEMPLATE_URL" 110 | echo " DDB_TABLE_NAME: $DDB_TABLE_NAME" 111 | StackId=$(aws cloudformation create-stack \ 112 | --stack-name $STACK_NAME \ 113 | --template-url $TEMPLATE_URL \ 114 | --parameters ParameterKey=GlobalDynamoDBTableName,ParameterValue=$DDB_TABLE_NAME \ 115 | --capabilities CAPABILITY_IAM --region $PRIMARY_REGION --output text) 116 | 117 | echo "StackId: $StackId" 118 | echo "$(dt): waiting for stack \"$STACK_NAME\" to be created..." 119 | aws cloudformation wait stack-create-complete \ 120 | --stack-name $StackId \ 121 | --region $PRIMARY_REGION 122 | echo "$(dt): stack \"$STACK_NAME\" created" 123 | 124 | echo "DDB table" 125 | DDB_TABLE_NAME_PRIMARY=$(aws cloudformation describe-stack-resources \ 126 | --stack-name $STACK_NAME \ 127 | --region $PRIMARY_REGION \ 128 | --query 'StackResources[?LogicalResourceId == `ProvisioningDynamoDBTable`].PhysicalResourceId' \ 129 | --output text) 130 | echo "DDB_TABLE_NAME_PRIMARY: $DDB_TABLE_NAME_PRIMARY" 131 | echo "------------------------------" 132 | BULK_PROVISIONING_ROLE_NAME=$(aws cloudformation describe-stack-resources --stack-name $STACK_NAME --region $PRIMARY_REGION --query 'StackResources[?LogicalResourceId==`IoTBulkProvisioningRole`][PhysicalResourceId]' --output text) 133 | ARN_IOT_PROVISIONING_ROLE=$(aws iam get-role --role-name $BULK_PROVISIONING_ROLE_NAME --query 'Role.Arn' --output text) 134 | echo "export ARN_IOT_PROVISIONING_ROLE=$ARN_IOT_PROVISIONING_ROLE" >> toolsrc 135 | 136 | 137 | # 138 | # secondary region 139 | # 140 | STACK_NAME="IoTDRSecondary${STACK_POSTFIX}" 141 | CFN_TEMPLATE="${SOLUTION_NAME}/${VERSION}/disaster-recovery-for-aws-iot-secondary-region.template" 142 | TEMPLATE_URL=https://$BUCKET_RESOURCES.s3.amazonaws.com/$CFN_TEMPLATE 143 | echo "$(dt): launching stack \"$STACK_NAME\" in region \"$SECONDARY_REGION\"" 144 | echo " TEMPLATE_URL: $TEMPLATE_URL" 145 | echo " DDB_TABLE_NAME: $DDB_TABLE_NAME" 146 | StackId=$(aws cloudformation create-stack \ 147 | --stack-name $STACK_NAME \ 148 | --template-url $TEMPLATE_URL \ 149 | --parameters ParameterKey=GlobalDynamoDBTableName,ParameterValue=$DDB_TABLE_NAME ParameterKey=Postfix,ParameterValue=$STACK_POSTFIX \ 150 | ParameterKey=IoTEndpointPrimary,ParameterValue=$IOT_ENDPOINT_PRIMARY ParameterKey=IoTEndpointSecondary,ParameterValue=$IOT_ENDPOINT_SECONDARY \ 151 | --capabilities CAPABILITY_IAM --region $SECONDARY_REGION --output text) 152 | 153 | echo "StackId: $StackId" 154 | echo "$(dt): waiting for stack \"$STACK_NAME\" to be created..." 155 | aws cloudformation wait stack-create-complete \ 156 | --stack-name $StackId \ 157 | --region $SECONDARY_REGION 158 | echo "$(dt): stack \"$STACK_NAME\" created" 159 | 160 | echo "DDB table" 161 | DDB_TABLE_NAME_SECONDARY=$(aws cloudformation describe-stack-resources \ 162 | --stack-name $STACK_NAME \ 163 | --region $SECONDARY_REGION \ 164 | --query 'StackResources[?LogicalResourceId == `ProvisioningDynamoDBTable`].PhysicalResourceId' \ 165 | --output text) 166 | echo "DDB_TABLE_NAME_SECONDARY: $DDB_TABLE_NAME_SECONDARY" 167 | echo "------------------------------" 168 | 169 | # 170 | # create global table 171 | # 172 | if [ "$DDB_TABLE_NAME_PRIMARY" != "$DDB_TABLE_NAME_SECONDARY" ]; then 173 | ERROR="($dt): ERROR: DynamoDB table names in primary and secondary region differ. Global table cannot be created." 174 | else 175 | echo "$(dt): creating global DynamoDB table \"$DDB_TABLE_NAME_PRIMARY\"" 176 | aws dynamodb create-global-table \ 177 | --global-table-name $DDB_TABLE_NAME_PRIMARY \ 178 | --replication-group RegionName=$PRIMARY_REGION RegionName=$SECONDARY_REGION \ 179 | --region $PRIMARY_REGION 180 | fi 181 | sleep 2 182 | 183 | GLOBAL_TABLE_STATUS=$(aws dynamodb describe-global-table --global-table-name $DDB_TABLE_NAME_PRIMARY --region $PRIMARY_REGION --query 'GlobalTableDescription.GlobalTableStatus' --output text) 184 | echo "$(dt): GLOBAL_TABLE_STATUS: $GLOBAL_TABLE_STATUS" 185 | while [ "$GLOBAL_TABLE_STATUS" != "ACTIVE" ]; do 186 | sleep 5 187 | GLOBAL_TABLE_STATUS=$(aws dynamodb describe-global-table --global-table-name $DDB_TABLE_NAME_PRIMARY --region $PRIMARY_REGION --query 'GlobalTableDescription.GlobalTableStatus' --output text) 188 | echo "$(dt): GLOBAL_TABLE_STATUS: $GLOBAL_TABLE_STATUS" 189 | done 190 | 191 | 192 | # 193 | # R53 health checker primary region 194 | # 195 | STACK_NAME="R53HealthChecker${STACK_POSTFIX}" 196 | CFN_TEMPLATE="${SOLUTION_NAME}/${VERSION}/disaster-recovery-for-aws-iot-r53-health-checker.template" 197 | TEMPLATE_URL=https://$BUCKET_RESOURCES.s3.amazonaws.com/$CFN_TEMPLATE 198 | echo "$(dt): launching stack \"$STACK_NAME\" in region \"$PRIMARY_REGION\"" 199 | echo " TEMPLATE_URL: $TEMPLATE_URL" 200 | StackId=$(aws cloudformation create-stack \ 201 | --stack-name $STACK_NAME \ 202 | --template-url $TEMPLATE_URL \ 203 | --parameters ParameterKey=S3BucketForLambda,ParameterValue=$BUCKET_PRIMARY_REGION ParameterKey=CreateR53HealthCheck,ParameterValue=$CREATE_HEALTH_CHECK \ 204 | --capabilities CAPABILITY_IAM --region $PRIMARY_REGION --output text) 205 | 206 | echo "StackId: $StackId" 207 | echo "$(dt): waiting for stack \"$STACK_NAME\" to be created..." 208 | aws cloudformation wait stack-create-complete \ 209 | --stack-name $StackId \ 210 | --region $PRIMARY_REGION 211 | echo "$(dt): stack \"$STACK_NAME\" created" 212 | echo "------------------------------" 213 | 214 | # 215 | # R53 health checker secondary region 216 | # 217 | echo "Launching stack \"$STACK_NAME\" in region \"$SECONDARY_REGION\"" 218 | echo " TEMPLATE_URL: $TEMPLATE_URL" 219 | StackId=$(aws cloudformation create-stack \ 220 | --stack-name $STACK_NAME \ 221 | --template-url $TEMPLATE_URL \ 222 | --parameters ParameterKey=S3BucketForLambda,ParameterValue=$BUCKET_SECONDARY_REGION ParameterKey=CreateR53HealthCheck,ParameterValue=$CREATE_HEALTH_CHECK \ 223 | --capabilities CAPABILITY_IAM --region $SECONDARY_REGION --output text) 224 | 225 | echo "StackId: $StackId" 226 | echo "$(dt): waiting for stack \"$STACK_NAME\" to be created..." 227 | aws cloudformation wait stack-create-complete \ 228 | --stack-name $StackId \ 229 | --region $SECONDARY_REGION 230 | echo "$(dt): stack \"$STACK_NAME\" created" 231 | echo "------------------------------" 232 | 233 | echo "filling environment file" 234 | echo "export IOT_ENDPOINT_PRIMARY=$IOT_ENDPOINT_PRIMARY" >> toolsrc 235 | echo "export IOT_ENDPOINT_SECONDARY=$IOT_ENDPOINT_SECONDARY" >> toolsrc 236 | echo "export REGION=$PRIMARY_REGION" >> toolsrc 237 | echo "export S3_BUCKET=$BUCKET_PRIMARY_REGION" >> toolsrc 238 | echo "export DYNAMODB_GLOBAL_TABLE=$DDB_TABLE_NAME_PRIMARY" >> toolsrc 239 | echo "export PRIMARY_REGION=$PRIMARY_REGION" >> toolsrc 240 | echo "export SECONDARY_REGION=$SECONDARY_REGION" >> toolsrc 241 | 242 | aws s3 cp toolsrc s3://$BUCKET_PRIMARY_REGION/tools/toolsrc 243 | 244 | END_DATE=$(dt) 245 | echo "START_DATE: $START_DATE" 246 | echo "END_DATE: $END_DATE" 247 | if [ ! -z "$ERROR" ]; then 248 | echo "errors encountered: $ERROR" 249 | exit 1 250 | fi 251 | exit 0 252 | -------------------------------------------------------------------------------- /source/jupyter/05_IoTDR_JITR_Device.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# AWS IoT DR - register a device just-in-time\n", 8 | "\n", 9 | "In the previous notebooks of this series you have created the prerequisites for just-in-time registration with AWS IoT. A topic rule and a Lambda function which finally provisions your device has been created by the CloudFormation stack for the IoT DR solution.\n", 10 | "\n", 11 | "Your device will connect to AWS IoT Core with a certifcate issued by your private CA. Upon the first connect the device will get registered automatically." 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "## Libraries" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "from AWSIoTPythonSDK.MQTTLib import AWSIoTMQTTClient\n", 28 | "from os.path import join\n", 29 | "import boto3\n", 30 | "import json\n", 31 | "import logging\n", 32 | "import time\n", 33 | "import urllib.request" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "#### Note: If you get an error that the AWSIoTPythonSDK is not installed, install the SDK with the command below and import the libraries again!" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": null, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "!pip install AWSIoTPythonSDK -t ." 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": {}, 55 | "source": [ 56 | "## Amazon Root CA\n", 57 | "\n", 58 | "Get the Amazon Root CA which signed the certificate for IoT Core's MQTT message broker. The CA will be used when connecting a device to AWS IoT Core." 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "r = urllib.request.urlopen('https://www.amazontrust.com/repository/AmazonRootCA1.pem')\n", 68 | "cert_pem = r.read().decode()\n", 69 | "print(cert_pem)\n", 70 | "\n", 71 | "f = open('AmazonRootCA1.pem','w')\n", 72 | "f.write(cert_pem)\n", 73 | "f.close()" 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": {}, 79 | "source": [ 80 | "## Shared variables\n", 81 | "\n", 82 | "Import shared variables into this notebook." 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": null, 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "%store -r config\n", 92 | "print(\"config: {}\".format(json.dumps(config, indent=4, default=str)))" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "metadata": {}, 98 | "source": [ 99 | "## Boto3 clients" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": null, 105 | "metadata": {}, 106 | "outputs": [], 107 | "source": [ 108 | "c_iot_p = boto3.client('iot', region_name = config['aws_region_primary'])\n", 109 | "c_iot_s = boto3.client('iot', region_name = config['aws_region_secondary'])" 110 | ] 111 | }, 112 | { 113 | "cell_type": "markdown", 114 | "metadata": {}, 115 | "source": [ 116 | "## IoT endpoints\n", 117 | "\n", 118 | "Get the IoT endpoints for AWS IoT Core in the primary and secondary region. In the example below you will connect to the primary region. To try a connection to the secondary region replace the variable `iot_endpoint_primary` with `iot_endpoint_secondary` in the examples below." 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": null, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "response = c_iot_p.describe_endpoint(endpointType='iot:Data-ATS')\n", 128 | "iot_endpoint_primary = response['endpointAddress']\n", 129 | "print(\"iot_endpoint_primary: {}\".format(iot_endpoint_primary))\n", 130 | "\n", 131 | "response = c_iot_s.describe_endpoint(endpointType='iot:Data-ATS')\n", 132 | "iot_endpoint_secondary = response['endpointAddress']\n", 133 | "print(\"iot_endpoint_secondary: {}\".format(iot_endpoint_secondary))" 134 | ] 135 | }, 136 | { 137 | "cell_type": "markdown", 138 | "metadata": {}, 139 | "source": [ 140 | "## Connect the device\n", 141 | "\n", 142 | "When a device is reqistered automatically by JITR it will be disconnected automatically after the first connection attempt and the device is being registered. In the first connection attempt you need to provide the device certificate together with the root CA's certificate in one file. After the device has been register you only need to present the device certificate during connect.\n", 143 | "\n", 144 | "After the connection has timed out the code will wait some seconds. It will then configure the credentials to use the device certificate only and connect again to AWS IoT Core.\n", 145 | "\n", 146 | "\n", 147 | "**Before you connect your device go to the AWS IoT Console -> \"MQTT test client\" and subscribe to `$aws/events/#` and `cmd/+/pca`.**\n", 148 | "\n", 149 | "When a certificate is being registered automatically AWS IoT Core is publishing a message to the topic\n", 150 | "\n", 151 | "`$aws/events/certificates/registered/[certificateId]`\n", 152 | "\n", 153 | "As you have enable registry events for the solution you will also get messages when a thing is being created. These message are published to the topic\n", 154 | "\n", 155 | "`$aws/events/thing/[clientId]/created`\n", 156 | "\n", 157 | "Set the variable `thing_name` to the same value that you used in the notebook to issue a device certificate.\n", 158 | "\n", 159 | "Feel free to create more certificates and connect more things." 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "execution_count": null, 165 | "metadata": {}, 166 | "outputs": [], 167 | "source": [ 168 | "thing_name = 'dr-pca-04'\n", 169 | "\n", 170 | "root_ca = 'AmazonRootCA1.pem'\n", 171 | "\n", 172 | "device_key_file = '{}.device.key.pem'.format(thing_name)\n", 173 | "device_cert_file = '{}.device.cert.pem'.format(thing_name)\n", 174 | "device_root_cert_file = '{}.device.root.cert.pem'.format(thing_name)\n", 175 | "\n", 176 | "# AWS IoT Python SDK needs logging\n", 177 | "logger = logging.getLogger(\"AWSIoTPythonSDK.core\")\n", 178 | "#logger.setLevel(logging.DEBUG)\n", 179 | "logger.setLevel(logging.INFO)\n", 180 | "streamHandler = logging.StreamHandler()\n", 181 | "formatter = logging.Formatter(\"[%(asctime)s - %(levelname)s - %(filename)s:%(lineno)s - %(funcName)s - %(message)s\")\n", 182 | "streamHandler.setFormatter(formatter)\n", 183 | "logger.addHandler(streamHandler)\n", 184 | "\n", 185 | "myAWSIoTMQTTClient = None\n", 186 | "myAWSIoTMQTTClient = AWSIoTMQTTClient(thing_name)\n", 187 | "myAWSIoTMQTTClient.configureEndpoint(iot_endpoint_primary, 8883)\n", 188 | "myAWSIoTMQTTClient.configureCredentials(root_ca, \n", 189 | " join(config['PCA_directory'], device_key_file), \n", 190 | " join(config['PCA_directory'], device_root_cert_file))\n", 191 | "\n", 192 | "# AWSIoTMQTTClient connection configuration\n", 193 | "myAWSIoTMQTTClient.configureAutoReconnectBackoffTime(1, 32, 20)\n", 194 | "myAWSIoTMQTTClient.configureOfflinePublishQueueing(-1) # Infinite offline Publish queueing\n", 195 | "myAWSIoTMQTTClient.configureDrainingFrequency(2) # Draining: 2 Hz\n", 196 | "myAWSIoTMQTTClient.configureConnectDisconnectTimeout(10) # 10 sec\n", 197 | "myAWSIoTMQTTClient.configureMQTTOperationTimeout(5) # 5 sec\n", 198 | "\n", 199 | "# Connect and reconnect to AWS IoT\n", 200 | "try:\n", 201 | " myAWSIoTMQTTClient.connect()\n", 202 | "except Exception as e:\n", 203 | " logger.info('{}'.format(e))\n", 204 | " myAWSIoTMQTTClient.configureCredentials(root_ca, \n", 205 | " join(config['PCA_directory'], device_key_file), \n", 206 | " join(config['PCA_directory'], device_cert_file))\n", 207 | " time.sleep(5)\n", 208 | " myAWSIoTMQTTClient.connect()" 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "metadata": {}, 214 | "source": [ 215 | "## Verify\n", 216 | "\n", 217 | "Verify that the device has been created in the primary and the secondary region.\n", 218 | "\n", 219 | "### Primary region" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": null, 225 | "metadata": {}, 226 | "outputs": [], 227 | "source": [ 228 | "response = c_iot_p.describe_thing(thingName = thing_name)\n", 229 | "\n", 230 | "print(\"response: {}\".format(json.dumps(response, indent=4, default=str)))" 231 | ] 232 | }, 233 | { 234 | "cell_type": "markdown", 235 | "metadata": {}, 236 | "source": [ 237 | "### Secondary region" 238 | ] 239 | }, 240 | { 241 | "cell_type": "code", 242 | "execution_count": null, 243 | "metadata": {}, 244 | "outputs": [], 245 | "source": [ 246 | "response = c_iot_s.describe_thing(thingName = thing_name)\n", 247 | "\n", 248 | "print(\"response: {}\".format(json.dumps(response, indent=4, default=str)))" 249 | ] 250 | }, 251 | { 252 | "cell_type": "markdown", 253 | "metadata": {}, 254 | "source": [ 255 | "## Publish\n", 256 | "Publish a message in the primary region to verify that the device works as expected.\n", 257 | "\n", 258 | "**You have subsribed to \"cmd/+/pca# in the primary region?**" 259 | ] 260 | }, 261 | { 262 | "cell_type": "code", 263 | "execution_count": null, 264 | "metadata": {}, 265 | "outputs": [], 266 | "source": [ 267 | "topic = 'cmd/{}/pca'.format(thing_name)\n", 268 | "print(\"topic: {}\".format(topic))\n", 269 | "message = {\"provisioned\": \"through ACM PCA combined with JITR\", \"thing_name\": \"{}\".format(thing_name)}\n", 270 | "\n", 271 | "myAWSIoTMQTTClient.publish(topic, json.dumps(message), 0)" 272 | ] 273 | }, 274 | { 275 | "cell_type": "markdown", 276 | "metadata": {}, 277 | "source": [ 278 | "Disconnect the device from AWS IoT Core" 279 | ] 280 | }, 281 | { 282 | "cell_type": "code", 283 | "execution_count": null, 284 | "metadata": {}, 285 | "outputs": [], 286 | "source": [ 287 | "myAWSIoTMQTTClient.disconnect()" 288 | ] 289 | } 290 | ], 291 | "metadata": { 292 | "kernelspec": { 293 | "display_name": "conda_python3", 294 | "language": "python", 295 | "name": "conda_python3" 296 | }, 297 | "language_info": { 298 | "codemirror_mode": { 299 | "name": "ipython", 300 | "version": 3 301 | }, 302 | "file_extension": ".py", 303 | "mimetype": "text/x-python", 304 | "name": "python", 305 | "nbconvert_exporter": "python", 306 | "pygments_lexer": "ipython3", 307 | "version": "3.6.10" 308 | } 309 | }, 310 | "nbformat": 4, 311 | "nbformat_minor": 4 312 | } 313 | --------------------------------------------------------------------------------