├── .gitignore ├── CODE_OF_CONDUCT.md ├── LICENSE ├── NOTICE ├── README.md ├── THIRD-PARTY ├── diagrams └── StreamingSampleES_HLD.jpg ├── samconfig.toml ├── sample_scenarios ├── __init__.py ├── constants.py ├── delete_document.py ├── helpers.py ├── insert_documents.py ├── multiple_updates_to_a_document.py ├── requirements.txt ├── sample_data.py └── single_update_to_document.py ├── setup ├── __init__.py ├── provisioning_lambda.py └── requirements.txt ├── src ├── __init__.py ├── qldb_streaming_to_es_sample │ ├── __init__.py │ ├── app.py │ ├── clients │ │ ├── __init__.py │ │ └── elasticsearch.py │ ├── constants.py │ └── helpers │ │ ├── __init__.py │ │ └── filtered_records_generator.py └── requirements.txt ├── template.yaml └── tests ├── __init__.py └── unit ├── __init__.py ├── fixtures.py ├── test_constants.py ├── test_elasticsearch_client.py └── test_handler.py /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode 3 | 4 | ### Linux ### 5 | *~ 6 | 7 | # temporary files which can be created if a process still has a handle open of a deleted file 8 | .fuse_hidden* 9 | 10 | # KDE directory preferences 11 | .directory 12 | 13 | # Linux trash folder which might appear on any partition or disk 14 | .Trash-* 15 | 16 | # .nfs files are created when an open file is removed but is still being accessed 17 | .nfs* 18 | 19 | ### OSX ### 20 | *.DS_Store 21 | .AppleDouble 22 | .LSOverride 23 | 24 | # Icon must end with two \r 25 | Icon 26 | 27 | # Thumbnails 28 | ._* 29 | 30 | # Files that might appear in the root of a volume 31 | .DocumentRevisions-V100 32 | .fseventsd 33 | .Spotlight-V100 34 | .TemporaryItems 35 | .Trashes 36 | .VolumeIcon.icns 37 | .com.apple.timemachine.donotpresent 38 | 39 | # Directories potentially created on remote AFP share 40 | .AppleDB 41 | .AppleDesktop 42 | Network Trash Folder 43 | Temporary Items 44 | .apdisk 45 | 46 | ### PyCharm ### 47 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 48 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 49 | 50 | # User-specific stuff: 51 | .idea/**/workspace.xml 52 | .idea/**/tasks.xml 53 | .idea/dictionaries 54 | 55 | # Sensitive or high-churn files: 56 | .idea/**/dataSources/ 57 | .idea/**/dataSources.ids 58 | .idea/**/dataSources.xml 59 | .idea/**/dataSources.local.xml 60 | .idea/**/sqlDataSources.xml 61 | .idea/**/dynamic.xml 62 | .idea/**/uiDesigner.xml 63 | 64 | # Gradle: 65 | .idea/**/gradle.xml 66 | .idea/**/libraries 67 | 68 | # CMake 69 | cmake-build-debug/ 70 | 71 | # Mongo Explorer plugin: 72 | .idea/**/mongoSettings.xml 73 | 74 | ## File-based project format: 75 | *.iws 76 | 77 | ## Plugin-specific files: 78 | 79 | # IntelliJ 80 | /out/ 81 | 82 | # mpeltonen/sbt-idea plugin 83 | .idea_modules/ 84 | 85 | # JIRA plugin 86 | atlassian-ide-plugin.xml 87 | 88 | # Cursive Clojure plugin 89 | .idea/replstate.xml 90 | 91 | # Ruby plugin and RubyMine 92 | /.rakeTasks 93 | 94 | # Crashlytics plugin (for Android Studio and IntelliJ) 95 | com_crashlytics_export_strings.xml 96 | crashlytics.properties 97 | crashlytics-build.properties 98 | fabric.properties 99 | 100 | ### PyCharm Patch ### 101 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 102 | 103 | # *.iml 104 | # modules.xml 105 | # .idea/misc.xml 106 | # *.ipr 107 | 108 | # Sonarlint plugin 109 | .idea/sonarlint 110 | 111 | ### Python ### 112 | # Byte-compiled / optimized / DLL files 113 | __pycache__/ 114 | *.py[cod] 115 | *$py.class 116 | 117 | # C extensions 118 | *.so 119 | 120 | # Distribution / packaging 121 | .Python 122 | build/ 123 | develop-eggs/ 124 | dist/ 125 | downloads/ 126 | eggs/ 127 | .eggs/ 128 | lib/ 129 | lib64/ 130 | parts/ 131 | sdist/ 132 | var/ 133 | wheels/ 134 | *.egg-info/ 135 | .installed.cfg 136 | *.egg 137 | 138 | # PyInstaller 139 | # Usually these files are written by a python script from a template 140 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 141 | *.manifest 142 | *.spec 143 | 144 | # Installer logs 145 | pip-log.txt 146 | pip-delete-this-directory.txt 147 | 148 | # Unit test / coverage reports 149 | htmlcov/ 150 | .tox/ 151 | .coverage 152 | .coverage.* 153 | .cache 154 | .pytest_cache/ 155 | nosetests.xml 156 | coverage.xml 157 | *.cover 158 | .hypothesis/ 159 | 160 | # Translations 161 | *.mo 162 | *.pot 163 | 164 | # Flask stuff: 165 | instance/ 166 | .webassets-cache 167 | 168 | # Scrapy stuff: 169 | .scrapy 170 | 171 | # Sphinx documentation 172 | docs/_build/ 173 | 174 | # PyBuilder 175 | target/ 176 | 177 | # Jupyter Notebook 178 | .ipynb_checkpoints 179 | 180 | # pyenv 181 | .python-version 182 | 183 | # celery beat schedule file 184 | celerybeat-schedule.* 185 | 186 | # SageMath parsed files 187 | *.sage.py 188 | 189 | # Environments 190 | .env 191 | .venv 192 | env/ 193 | venv/ 194 | ENV/ 195 | env.bak/ 196 | venv.bak/ 197 | 198 | # Spyder project settings 199 | .spyderproject 200 | .spyproject 201 | 202 | # Rope project settings 203 | .ropeproject 204 | 205 | # mkdocs documentation 206 | /site 207 | 208 | # mypy 209 | .mypy_cache/ 210 | 211 | ### VisualStudioCode ### 212 | .vscode/* 213 | !.vscode/settings.json 214 | !.vscode/tasks.json 215 | !.vscode/launch.json 216 | !.vscode/extensions.json 217 | .history 218 | 219 | ### Windows ### 220 | # Windows thumbnail cache files 221 | Thumbs.db 222 | ehthumbs.db 223 | ehthumbs_vista.db 224 | 225 | # Folder config file 226 | Desktop.ini 227 | 228 | # Recycle Bin used on file shares 229 | $RECYCLE.BIN/ 230 | 231 | # Windows Installer files 232 | *.cab 233 | *.msi 234 | *.msm 235 | *.msp 236 | 237 | # Windows shortcuts 238 | *.lnk 239 | 240 | # Build folder 241 | 242 | */build/* 243 | 244 | # End of https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Amazon QLDB Streams Elasticsearch Integration Sample 2 | 3 | 4 | The sample in this project demonstrates how to integrate [Amazon Elasticsearch Service](https://aws.amazon.com/elasticsearch-service/) with [Amazon QLDB](https://aws.amazon.com/qldb/) using Streams. 5 | It consists of a AWS Lambda function written in Python which reads QLDB Streams and indexes documents to Amazon Elasticsearch. 6 | This sample is modelled around a department of motor vehicles (DMV) database that tracks the complete historical information about vehicle registrations. 7 | 8 | ![Amazon QLDB DMV Sample App and Streams](diagrams/StreamingSampleES_HLD.jpg) 9 | ## What does this sample application do ? 10 | 11 | The Sample demonstrates how you can replicate your documents in Amazon QLDB to Amazon Elasticsearch Service in near real time using [Amazon Kinesis](https://aws.amazon.com/kinesis/). 12 | 13 | ##### The following AWS technologies have been used: 14 | 15 | * [AWS Lambda](https://aws.amazon.com/lambda/) 16 | * [Amazon Kinesis Data Streams](https://aws.amazon.com/kinesis/data-streams/) 17 | * [Amazon Elasticsearch Service](https://aws.amazon.com/elasticsearch-service/) 18 | * [AWS Cognito](https://aws.amazon.com/cognito/) 19 | * [AWS SAM](https://aws.amazon.com/serverless/sam/) 20 | * [AWS CloudFormation](https://aws.amazon.com/cloudformation/) 21 | 22 | 23 | 24 | ##### What happens in the Sample: 25 | * QLDB captures every document revision that is committed to your journal and delivers this data to Amazon Kinesis Data Streams in near-real time. 26 | * Amazon Kinesis Data Streams triggers AWS Lambda for each batch of Stream Records. 27 | * The Lambda function indexes the documents to Elasticsearch. It indexes `Person` documents for only `insert` cases in QLDB and indexes `VehicleRegistration` for `insert and update` cases. 28 | * To view the documents you can login to the Kibana Dashboard. The endpoint is authenticated using AWS Cognito. You will be required to create a user and a temporary password to access Kibana Dashboard. This will be covered in the setup steps. 29 | * The sample includes Python scripts inside folder `sample_scenarios` to `insert, update and delete` data into the QLDB tables. 30 | 31 | 32 | ## Requirements 33 | 34 | 35 | ### SAM CLI 36 | 37 | [AWS SAM](https://aws.amazon.com/serverless/sam/) provides you with a command line tool, the AWS SAM CLI, that makes it easy for you to create and manage serverless applications. You need to install and configure a few things in order to use the AWS SAM CLI. See [AWS SAM CLI Installation](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html) for details. 38 | 39 | ### AWS CLI 40 | 41 | SAM requires an S3 bucket to host the source code for lambda function. We will be using the AWS CLI for creating the bucket. Please read [AWS CLI Configuration](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html#cli-quick-configuration) for help on how to configure the CLI. 42 | 43 | ### Python 3.4 or above 44 | 45 | The examples require Python 3.4 or above. Please see the link below for more detail to install Python: 46 | 47 | * [Python Installation](https://www.python.org/downloads/) 48 | 49 | ## Setting up the Sample 50 | 51 | It is required that you clone this repository. The project consists of two main directories: 52 | 53 | * `src` : This directory has the source for the Lambda function. 54 | * `sample_scenarios` : This consists of python scripts which can be used to insert, update and delete documents in `vehicle-registration`. These will be used after setup is complete to verify that the sample works as expected. 55 | 56 | ##### 1. Create a ledger named `vehicle-registration` 57 | 58 | Please Follow the steps listed [here](https://docs.aws.amazon.com/qldb/latest/developerguide/getting-started-step-1.html) to create a new Ledger. 59 | 60 | 61 | 62 | ##### 2. Create an S3 bucket 63 | 64 | We would need to create an S3 bucket. This S3 bucket would be used by SAM to host the source code of the lambda function. 65 | 66 | ``` 67 | export BUCKET_NAME=some_unique_valid_bucket_name 68 | aws s3 mb s3://$BUCKET_NAME 69 | ``` 70 | 71 | ##### 3. Run the following command in the root of the directory to build the source code and generate deployment artifacts. 72 | 73 | ``` 74 | sam build 75 | ``` 76 | 77 | ##### 4. Package the lambda function to S3 78 | 79 | ``` 80 | sam package \ 81 | --output-template-file packaged.yaml \ 82 | --s3-bucket $BUCKET_NAME 83 | ``` 84 | 85 | ##### 5. Deploy the lambda stack 86 | 87 | ``` 88 | sam deploy \ 89 | --template-file packaged.yaml \ 90 | --stack-name STACK_NAME \ 91 | --capabilities CAPABILITY_NAMED_IAM \ 92 | --parameter-overrides ParameterKey=ElasticsearchDomainName,ParameterValue=DOMAIN_NAME 93 | 94 | ``` 95 | *Replace `DOMAIN_NAME` with a domain name of your choice* 96 | 97 | *Replace `STACK_NAME` with a stack name of your choice* 98 | 99 | *Note: After the deployment completes, you should see `KibanaEndpoint` in the outputs. Please copy this, we will need it later.* 100 | 101 | The Deployment will create a Cloudformation Stack with name you specify in the deploy command. As part of the Stack, Cloudformation will create the following: 102 | * Create some IAM Roles. The Roles would be used by Lambda, Kinesis and Cognito. 103 | * Create an AWS Lambda function. The function would be responsible for parsing Stream Records, creating JSON documents and indexing/deleting them on Amazon Elasticsearch. 104 | * Create an Elasticsearch domain. 105 | * Create an AWS Cognito User Pool and Identity Pool 106 | * Create indexes `person_index` and `vehicle_index` on Elasticsearch using [Custom Resource](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-custom-resources.html). 107 | 108 | ##### 6. Create QLDB Stream 109 | 110 | - Sign in to the AWS Management Console, and open the Amazon QLDB console at https://console.aws.amazon.com/qldb. 111 | 112 | - In the navigation pane, choose Streams. 113 | 114 | - Choose Create QLDB stream. 115 | 116 | - On the Create QLDB stream page, enter the following settings: 117 | 118 | - Ledger – Select the ledger `vehicle-registration` from the drop down. 119 | 120 | - Start date and time – Leave this as the default. The default is current time. 121 | 122 | - End date and time – This can be left blank 123 | 124 | - Destination stream for journal data – Click browse and select 125 | `RegistrationStreamKinesis`. 126 | 127 | - Enable record aggregation in Kinesis Data Streams – Enables QLDB to publish multiple stream records in a single Kinesis Data Streams record. To learn more, see KPL Key Concepts. 128 | 129 | - IAM role – Select `RegistrationStreamsKinesisRole` from the dropdown 130 | 131 | - When the settings are as you want them, choose Create QLDB stream. 132 | 133 | - If your request submission is successful, the console returns to the main Streams page and lists your QLDB streams with their current status. 134 | 135 | 136 | ##### 7. Create a user and a temporary password for Kibana Dashboard. 137 | - Go to [AWS Cognito Console](https://console.aws.amazon.com/cognito/home) 138 | - Click on `Manage User Pools` 139 | - Click on `registrations_kibana_demo_userpool` 140 | - Click on `Users and groups` and click `Create user` 141 | - In the `Create user popup` : 142 | - Enter a username in `Username` field. (This will be needed later) 143 | - Enter a temporary password in `Password` field. (This will be needed later) 144 | - Uncheck all checkboxes 145 | - Click `Create user` button 146 | 147 | ##### 8. Verify if Elasticsearch setup is ready 148 | - Open the Kibana Endpoint you copied in step 5. 149 | - If you get a login dialog box, you are good to go. 150 | - If you get an unauthorized access error, then probably the Elasticsearch intialization has not finished. It usually takes 15 minutes. Check the status on [AWS Elasticsearch Console](https://console.aws.amazon.com/es/home). 151 | 152 | 153 | ##### 9. Login to Kibana. If you are doing it for the first time, it should ask you to reset the password. 154 | 155 | ##### 10. Follow these [steps](https://docs.aws.amazon.com/qldb/latest/developerguide/getting-started-step-2.html) to load Sample Data into the ledger. 156 | 157 | ##### 11. Create index pattern on kibana - `person_index`, `vehicle_index`. Check [here](https://www.elastic.co/guide/en/kibana/current/tutorial-define-index.html) on how to create index patterns on Kibana. 158 | 159 | You should see some documents for `person_index` and `vehicle_index`. 160 | 161 | 162 | 163 | ## Running Sample Scenarios 164 | 165 | Here we will insert and update some documents in QLDB and verify that those updates reflect on Elasticsearch. 166 | 167 | ##### Install dependencies required to run sample scenarios tasks. 168 | 169 | Run the following command in the root of the repository. 170 | 171 | ``` 172 | pip install -r sample_scenarios/requirements.txt 173 | ``` 174 | 175 | *Note: In case your Ledger name is not `vehicle-registration`, you will have to update the 176 | ledger name in `sample_scenarios/constants.py`* 177 | 178 | ##### Scenario 1: Insert some documents 179 | 180 | We will insert some more documents to the tables in the ledger and verify insertions in Elasticsearch. 181 | 182 | ``` 183 | python -m sample_scenarios.insert_documents 184 | ``` 185 | 186 | ##### Scenario 2: Update some documents 187 | 188 | We will update `PendingPenaltyTicketAmount` for some documents in the `VehicleRegistration` table. 189 | The updates should reflect on Elasticsearch. 190 | 191 | ``` 192 | python -m sample_scenarios.single_update_to_document 193 | ``` 194 | 195 | ##### Scenario 3: Update some documents 196 | 197 | We will update `PendingPenaltyTicketAmount` multiple times for a document in the `VehicleRegistration` table. 198 | The final update should reflect on Elasticsearch. 199 | 200 | ``` 201 | python -m sample_scenarios.multiple_updates_to_a_document 202 | ``` 203 | 204 | ## Note 205 | 206 | * This sample does not place the Elasticsearch domain in a VPC for the sake of simplicity. Refer [here](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-vpc.html) in case it is required. 207 | * You might want to change configurations of the Elasticsearch domain. Configurations such as availability zones, instance size, storage size highly depends on requirements. Refer [here](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-createupdatedomains.html#es-createdomains) for more details. 208 | 209 | 210 | 211 | ## Unit Tests 212 | 213 | Tests are defined in the `tests` folder in this project. Use PIP to install the [pytest](https://docs.pytest.org/en/latest/) and run unit tests. 214 | 215 | ```bash 216 | pip install pytest pytest-mock --user 217 | python -m pytest tests/ -v 218 | ``` 219 | 220 | ## Cleanup 221 | 222 | To delete the sample application that you created, use the AWS CLI. Assuming you used your project name for the stack name, you can run the following: 223 | 224 | ```bash 225 | aws cloudformation delete-stack --stack-name STACK_NAME 226 | ``` 227 | 228 | ## License 229 | 230 | This library is licensed under the MIT-0 License. 231 | -------------------------------------------------------------------------------- /THIRD-PARTY: -------------------------------------------------------------------------------- 1 | ** amazon.ion; version 0.5.0 -- https://pypi.org/project/amazon.ion/ 2 | Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | ** boto3; version 1.9.221 -- https://pypi.org/project/boto3/ 4 | boto3 5 | Copyright 2013-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. 6 | 7 | Apache License 8 | 9 | Version 2.0, January 2004 10 | 11 | http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND 12 | DISTRIBUTION 13 | 14 | 1. Definitions. 15 | 16 | "License" shall mean the terms and conditions for use, reproduction, and 17 | distribution as defined by Sections 1 through 9 of this document. 18 | 19 | "Licensor" shall mean the copyright owner or entity authorized by the 20 | copyright owner that is granting the License. 21 | 22 | "Legal Entity" shall mean the union of the acting entity and all other 23 | entities that control, are controlled by, or are under common control 24 | with that entity. For the purposes of this definition, "control" means 25 | (i) the power, direct or indirect, to cause the direction or management 26 | of such entity, whether by contract or otherwise, or (ii) ownership of 27 | fifty percent (50%) or more of the outstanding shares, or (iii) 28 | beneficial ownership of such entity. 29 | 30 | "You" (or "Your") shall mean an individual or Legal Entity exercising 31 | permissions granted by this License. 32 | 33 | "Source" form shall mean the preferred form for making modifications, 34 | including but not limited to software source code, documentation source, 35 | and configuration files. 36 | 37 | "Object" form shall mean any form resulting from mechanical 38 | transformation or translation of a Source form, including but not limited 39 | to compiled object code, generated documentation, and conversions to 40 | other media types. 41 | 42 | "Work" shall mean the work of authorship, whether in Source or Object 43 | form, made available under the License, as indicated by a copyright 44 | notice that is included in or attached to the work (an example is 45 | provided in the Appendix below). 46 | 47 | "Derivative Works" shall mean any work, whether in Source or Object form, 48 | that is based on (or derived from) the Work and for which the editorial 49 | revisions, annotations, elaborations, or other modifications represent, 50 | as a whole, an original work of authorship. For the purposes of this 51 | License, Derivative Works shall not include works that remain separable 52 | from, or merely link (or bind by name) to the interfaces of, the Work and 53 | Derivative Works thereof. 54 | 55 | "Contribution" shall mean any work of authorship, including the original 56 | version of the Work and any modifications or additions to that Work or 57 | Derivative Works thereof, that is intentionally submitted to Licensor for 58 | inclusion in the Work by the copyright owner or by an individual or Legal 59 | Entity authorized to submit on behalf of the copyright owner. For the 60 | purposes of this definition, "submitted" means any form of electronic, 61 | verbal, or written communication sent to the Licensor or its 62 | representatives, including but not limited to communication on electronic 63 | mailing lists, source code control systems, and issue tracking systems 64 | that are managed by, or on behalf of, the Licensor for the purpose of 65 | discussing and improving the Work, but excluding communication that is 66 | conspicuously marked or otherwise designated in writing by the copyright 67 | owner as "Not a Contribution." 68 | 69 | "Contributor" shall mean Licensor and any individual or Legal Entity on 70 | behalf of whom a Contribution has been received by Licensor and 71 | subsequently incorporated within the Work. 72 | 73 | 2. Grant of Copyright License. Subject to the terms and conditions of this 74 | License, each Contributor hereby grants to You a perpetual, worldwide, 75 | non-exclusive, no-charge, royalty-free, irrevocable copyright license to 76 | reproduce, prepare Derivative Works of, publicly display, publicly perform, 77 | sublicense, and distribute the Work and such Derivative Works in Source or 78 | Object form. 79 | 80 | 3. Grant of Patent License. Subject to the terms and conditions of this 81 | License, each Contributor hereby grants to You a perpetual, worldwide, 82 | non-exclusive, no-charge, royalty-free, irrevocable (except as stated in 83 | this section) patent license to make, have made, use, offer to sell, sell, 84 | import, and otherwise transfer the Work, where such license applies only to 85 | those patent claims licensable by such Contributor that are necessarily 86 | infringed by their Contribution(s) alone or by combination of their 87 | Contribution(s) with the Work to which such Contribution(s) was submitted. 88 | If You institute patent litigation against any entity (including a 89 | cross-claim or counterclaim in a lawsuit) alleging that the Work or a 90 | Contribution incorporated within the Work constitutes direct or contributory 91 | patent infringement, then any patent licenses granted to You under this 92 | License for that Work shall terminate as of the date such litigation is 93 | filed. 94 | 95 | 4. Redistribution. You may reproduce and distribute copies of the Work or 96 | Derivative Works thereof in any medium, with or without modifications, and 97 | in Source or Object form, provided that You meet the following conditions: 98 | 99 | (a) You must give any other recipients of the Work or Derivative Works a 100 | copy of this License; and 101 | 102 | (b) You must cause any modified files to carry prominent notices stating 103 | that You changed the files; and 104 | 105 | (c) You must retain, in the Source form of any Derivative Works that You 106 | distribute, all copyright, patent, trademark, and attribution notices 107 | from the Source form of the Work, excluding those notices that do not 108 | pertain to any part of the Derivative Works; and 109 | 110 | (d) If the Work includes a "NOTICE" text file as part of its 111 | distribution, then any Derivative Works that You distribute must include 112 | a readable copy of the attribution notices contained within such NOTICE 113 | file, excluding those notices that do not pertain to any part of the 114 | Derivative Works, in at least one of the following places: within a 115 | NOTICE text file distributed as part of the Derivative Works; within the 116 | Source form or documentation, if provided along with the Derivative 117 | Works; or, within a display generated by the Derivative Works, if and 118 | wherever such third-party notices normally appear. The contents of the 119 | NOTICE file are for informational purposes only and do not modify the 120 | License. You may add Your own attribution notices within Derivative Works 121 | that You distribute, alongside or as an addendum to the NOTICE text from 122 | the Work, provided that such additional attribution notices cannot be 123 | construed as modifying the License. 124 | 125 | You may add Your own copyright statement to Your modifications and may 126 | provide additional or different license terms and conditions for use, 127 | reproduction, or distribution of Your modifications, or for any such 128 | Derivative Works as a whole, provided Your use, reproduction, and 129 | distribution of the Work otherwise complies with the conditions stated in 130 | this License. 131 | 132 | 5. Submission of Contributions. Unless You explicitly state otherwise, any 133 | Contribution intentionally submitted for inclusion in the Work by You to the 134 | Licensor shall be under the terms and conditions of this License, without 135 | any additional terms or conditions. Notwithstanding the above, nothing 136 | herein shall supersede or modify the terms of any separate license agreement 137 | you may have executed with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, except 141 | as required for reasonable and customary use in describing the origin of the 142 | Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in 145 | writing, Licensor provides the Work (and each Contributor provides its 146 | Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 147 | KIND, either express or implied, including, without limitation, any 148 | warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or 149 | FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining 150 | the appropriateness of using or redistributing the Work and assume any risks 151 | associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, whether 154 | in tort (including negligence), contract, or otherwise, unless required by 155 | applicable law (such as deliberate and grossly negligent acts) or agreed to 156 | in writing, shall any Contributor be liable to You for damages, including 157 | any direct, indirect, special, incidental, or consequential damages of any 158 | character arising as a result of this License or out of the use or inability 159 | to use the Work (including but not limited to damages for loss of goodwill, 160 | work stoppage, computer failure or malfunction, or any and all other 161 | commercial damages or losses), even if such Contributor has been advised of 162 | the possibility of such damages. 163 | 164 | 9. Accepting Warranty or Additional Liability. While redistributing the Work 165 | or Derivative Works thereof, You may choose to offer, and charge a fee for, 166 | acceptance of support, warranty, indemnity, or other liability obligations 167 | and/or rights consistent with this License. However, in accepting such 168 | obligations, You may act only on Your own behalf and on Your sole 169 | responsibility, not on behalf of any other Contributor, and only if You 170 | agree to indemnify, defend, and hold each Contributor harmless for any 171 | liability incurred by, or claims asserted against, such Contributor by 172 | reason of your accepting any such warranty or additional liability. END OF 173 | TERMS AND CONDITIONS 174 | 175 | APPENDIX: How to apply the Apache License to your work. 176 | 177 | To apply the Apache License to your work, attach the following boilerplate 178 | notice, with the fields enclosed by brackets "[]" replaced with your own 179 | identifying information. (Don't include the brackets!) The text should be 180 | enclosed in the appropriate comment syntax for the file format. We also 181 | recommend that a file or class name and description of purpose be included on 182 | the same "printed page" as the copyright notice for easier identification 183 | within third-party archives. 184 | 185 | Copyright [yyyy] [name of copyright owner] 186 | 187 | Licensed under the Apache License, Version 2.0 (the "License"); 188 | 189 | you may not use this file except in compliance with the License. 190 | 191 | You may obtain a copy of the License at 192 | 193 | http://www.apache.org/licenses/LICENSE-2.0 194 | 195 | Unless required by applicable law or agreed to in writing, software 196 | 197 | distributed under the License is distributed on an "AS IS" BASIS, 198 | 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | 201 | See the License for the specific language governing permissions and 202 | 203 | limitations under the License. 204 | 205 | * For amazon.ion see also this required NOTICE: 206 | Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 207 | * For boto3 see also this required NOTICE: 208 | boto3 209 | Copyright 2013-2017 Amazon.com, Inc. or its affiliates. All Rights 210 | Reserved. 211 | 212 | ------ 213 | 214 | ** pytest-mock; version 1.13.0 -- https://pypi.org/project/pytest-mock/ 215 | Copyright (c) [2016] [Bruno Oliveira] 216 | 217 | MIT License 218 | 219 | Copyright (c) [2016] [Bruno Oliveira] 220 | 221 | Permission is hereby granted, free of charge, to any person obtaining a copy 222 | of this software and associated documentation files (the "Software"), to deal 223 | in the Software without restriction, including without limitation the rights 224 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 225 | copies of the Software, and to permit persons to whom the Software is 226 | furnished to do so, subject to the following conditions: 227 | 228 | The above copyright notice and this permission notice shall be included in all 229 | copies or substantial portions of the Software. 230 | 231 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 232 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 233 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 234 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 235 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 236 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 237 | SOFTWARE. 238 | 239 | ------ 240 | 241 | ** aws-kinesis-agg; version 1.1.2 -- https://pypi.org/project/aws-kinesis-agg 242 | Amazon Kinesis Producer Library Deaggregation Modules for AWS Lambda 243 | 244 | Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. 245 | 246 | Licensed under the Amazon Software License (the "License"). You may not use 247 | this file except in compliance with the License. A copy of the License is 248 | located at 249 | 250 | http://aws.amazon.com/asl/ 251 | 252 | or in the "license" file accompanying this file. This file is distributed on an 253 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or 254 | implied. See the License for the specific language governing permissions and 255 | limitations under the License. 256 | 257 | Amazon Software License 258 | 259 | 1. Definitions 260 | 261 | “Licensor” means any person or entity that distributes its Work. 262 | 263 | “Software” means the original work of authorship made available under this 264 | License. 265 | 266 | “Work” means the Software and any additions to or derivative works of the 267 | Software that are made available under this License. 268 | 269 | The terms “reproduce,” “reproduction,” “derivative works,” and “distribution” 270 | have the meaning as provided under U.S. copyright law; provided, however, that 271 | for the purposes of this License, derivative works shall not include works that 272 | remain separable from, or merely link (or bind by name) to the interfaces of, 273 | the Work. 274 | 275 | Works, including the Software, are “made available” under this License by 276 | including in or with the Work either (a) a copyright notice referencing the 277 | applicability of this License to the Work, or (b) a copy of this License. 278 | 2. License Grants 279 | 280 | 2.1 Copyright Grant. Subject to the terms and conditions of this License, each 281 | Licensor grants to you a perpetual, worldwide, non-exclusive, royalty-free, 282 | copyright license to reproduce, prepare derivative works of, publicly display, 283 | publicly perform, sublicense and distribute its Work and any resulting 284 | derivative works in any form. 285 | 286 | 2.2 Patent Grant. Subject to the terms and conditions of this License, each 287 | Licensor grants to you a perpetual, worldwide, non-exclusive, royalty-free 288 | patent license to make, have made, use, sell, offer for sale, import, and 289 | otherwise transfer its Work, in whole or in part. The foregoing license applies 290 | only to the patent claims licensable by Licensor that would be infringed by 291 | Licensor’s Work (or portion thereof) individually and excluding any 292 | combinations with any other materials or technology. 293 | 3. Limitations 294 | 295 | 3.1 Redistribution. You may reproduce or distribute the Work only if (a) you do 296 | so under this License, (b) you include a complete copy of this License with 297 | your distribution, and (c) you retain without modification any copyright, 298 | patent, trademark, or attribution notices that are present in the Work. 299 | 300 | 3.2 Derivative Works. You may specify that additional or different terms apply 301 | to the use, reproduction, and distribution of your derivative works of the Work 302 | (“Your Terms”) only if (a) Your Terms provide that the use limitation in 303 | Section 3.3 applies to your derivative works, and (b) you identify the specific 304 | derivative works that are subject to Your Terms. Notwithstanding Your Terms, 305 | this License (including the redistribution requirements in Section 3.1) will 306 | continue to apply to the Work itself. 307 | 308 | 3.3 Use Limitation. The Work and any derivative works thereof only may be used 309 | or intended for use with the web services, computing platforms or applications 310 | provided by Amazon.com, Inc. or its affiliates, including Amazon Web Services, 311 | Inc. 312 | 313 | 3.4 Patent Claims. If you bring or threaten to bring a patent claim against any 314 | Licensor (including any claim, cross-claim or counterclaim in a lawsuit) to 315 | enforce any patents that you allege are infringed by any Work, then your rights 316 | under this License from such Licensor (including the grants in Sections 2.1 and 317 | 2.2) will terminate immediately. 318 | 319 | 3.5 Trademarks. This License does not grant any rights to use any Licensor’s or 320 | its affiliates’ names, logos, or trademarks, except as necessary to reproduce 321 | the notices described in this License. 322 | 323 | 3.6 Termination. If you violate any term of this License, then your rights 324 | under this License (including the grants in Sections 2.1 and 2.2) will 325 | terminate immediately. 326 | 4. Disclaimer of Warranty. 327 | 328 | THE WORK IS PROVIDED “AS IS” WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 329 | EITHER EXPRESS OR IMPLIED, INCLUDING WARRANTIES OR CONDITIONS OF M 330 | ERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON-INFRINGEMENT. 331 | YOU BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER THIS LICENSE. SOME 332 | STATES’ CONSUMER LAWS DO NOT ALLOW EXCLUSION OF AN IMPLIED WARRANTY, SO THIS 333 | DISCLAIMER MAY NOT APPLY TO YOU. 334 | 5. Limitation of Liability. 335 | 336 | EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL THEORY, 337 | WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE SHALL ANY 338 | LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, 339 | INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF OR RELATED TO THIS LICENSE, 340 | THE USE OR INABILITY TO USE THE WORK (INCLUDING BUT NOT LIMITED TO LOSS OF 341 | GOODWILL, BUSINESS INTERRUPTION, LOST PROFITS OR DATA, COMPUTER FAILURE OR 342 | MALFUNCTION, OR ANY OTHER COMM ERCIAL DAMAGES OR LOSSES), EVEN IF THE LICENSOR 343 | HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 344 | 345 | Effective Date – April 18, 2008 © 2008 Amazon.com, Inc. or its affiliates. All 346 | rights reserved. -------------------------------------------------------------------------------- /diagrams/StreamingSampleES_HLD.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-qldb-streaming-amazon-opensearch-service-sample-python/da5e65a70a23f237437cf4e2683ec29eb0ed9fd0/diagrams/StreamingSampleES_HLD.jpg -------------------------------------------------------------------------------- /samconfig.toml: -------------------------------------------------------------------------------- 1 | version = 0.1 2 | [default] 3 | [default.deploy] 4 | [default.deploy.parameters] 5 | confirm_changeset = true 6 | capabilities = "CAPABILITY_NAMED_IAM" 7 | -------------------------------------------------------------------------------- /sample_scenarios/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | 17 | -------------------------------------------------------------------------------- /sample_scenarios/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | class Constants: 17 | """ 18 | Constant values used throughout this tutorial. 19 | """ 20 | LEDGER_NAME = "vehicle-registration" 21 | 22 | VEHICLE_REGISTRATION_TABLE_NAME = "VehicleRegistration" 23 | VEHICLE_TABLE_NAME = "Vehicle" 24 | PERSON_TABLE_NAME = "Person" 25 | DRIVERS_LICENSE_TABLE_NAME = "DriversLicense" 26 | 27 | LICENSE_NUMBER_INDEX_NAME = "LicenseNumber" 28 | LICENSE_PLATE_NUMBER_INDEX_NAME = "LicensePlateNumber" 29 | PERSON_ID_INDEX_NAME = "PersonId" 30 | 31 | 32 | RETRY_LIMIT = 4 -------------------------------------------------------------------------------- /sample_scenarios/delete_document.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | from logging import basicConfig, getLogger, INFO 17 | 18 | from sample_scenarios.constants import Constants 19 | from sample_scenarios.sample_data import convert_object_to_ion, SampleData, get_document_ids_from_dml_results 20 | from sample_scenarios.helpers import create_qldb_session 21 | from decimal import Decimal 22 | 23 | logger = getLogger(__name__) 24 | basicConfig(level=INFO) 25 | 26 | 27 | def delete_documents(transaction_executor): 28 | 29 | logger.info('Updating some documents in the {} table...'.format(Constants.VEHICLE_REGISTRATION_TABLE_NAME)) 30 | 31 | for vehicle_registration in SampleData.VEHICLE_REGISTRATION: 32 | statement = 'DELETE FROM {table_name} AS r \ 33 | WHERE r.LicensePlateNumber = ?' \ 34 | .format(table_name=Constants.VEHICLE_REGISTRATION_TABLE_NAME) 35 | 36 | logger.info('Deleting record from VehicleRegistration with License Number: {license_number}' 37 | .format(license_number=vehicle_registration["LicensePlateNumber"])) 38 | 39 | transaction_executor.execute_statement(statement, vehicle_registration["LicensePlateNumber"]) 40 | 41 | 42 | if __name__ == '__main__': 43 | """ 44 | Delete documents inserted by 'insert_documents.py'. 45 | """ 46 | try: 47 | with create_qldb_session() as session: 48 | session.execute_lambda(lambda executor: delete_documents(executor), 49 | lambda retry_attempt: logger.info('Retrying due to OCC conflict...')) 50 | logger.info('Documents deleted successfully!') 51 | except Exception: 52 | logger.exception('Error deleted documents.') 53 | -------------------------------------------------------------------------------- /sample_scenarios/helpers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | from pyqldb.driver.pooled_qldb_driver import PooledQldbDriver 17 | from .constants import Constants 18 | 19 | def create_qldb_session(): 20 | """ 21 | Retrieve a QLDB session object. 22 | 23 | :rtype: :py:class:`pyqldb.session.pooled_qldb_session.PooledQldbSession` 24 | :return: A pooled QLDB session object. 25 | """ 26 | pooled_qldb_driver = create_qldb_driver() 27 | qldb_session = pooled_qldb_driver.get_session() 28 | return qldb_session 29 | 30 | 31 | def create_qldb_driver(ledger_name=Constants.LEDGER_NAME, region_name=None, endpoint_url=None, boto3_session=None): 32 | """ 33 | Create a QLDB driver for creating sessions. 34 | 35 | :type ledger_name: str 36 | :param ledger_name: The QLDB ledger name. 37 | 38 | :type region_name: str 39 | :param region_name: See [1]. 40 | 41 | :type endpoint_url: str 42 | :param endpoint_url: See [1]. 43 | 44 | :type boto3_session: :py:class:`boto3.session.Session` 45 | :param boto3_session: The boto3 session to create the client with (see [1]). 46 | 47 | :rtype: :py:class:`pyqldb.driver.pooled_qldb_driver.PooledQldbDriver` 48 | :return: A pooled QLDB driver object. 49 | 50 | [1] https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html#boto3.session.Session.client 51 | """ 52 | qldb_driver = PooledQldbDriver(ledger_name=ledger_name, region_name=region_name, endpoint_url=endpoint_url, 53 | boto3_session=boto3_session) 54 | return qldb_driver -------------------------------------------------------------------------------- /sample_scenarios/insert_documents.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | from logging import basicConfig, getLogger, INFO 17 | 18 | from .constants import Constants 19 | from sample_scenarios.sample_data import convert_object_to_ion, SampleData, get_document_ids_from_dml_results 20 | from sample_scenarios.helpers import create_qldb_session 21 | 22 | logger = getLogger(__name__) 23 | basicConfig(level=INFO) 24 | 25 | 26 | def update_person_id(document_ids): 27 | """ 28 | Update the PersonId value for DriversLicense records and the PrimaryOwner value for VehicleRegistration records. 29 | """ 30 | new_drivers_licenses = SampleData.DRIVERS_LICENSE.copy() 31 | new_vehicle_registrations = SampleData.VEHICLE_REGISTRATION.copy() 32 | for i in range(len(SampleData.PERSON)): 33 | drivers_license = new_drivers_licenses[i] 34 | registration = new_vehicle_registrations[i] 35 | drivers_license.update({'PersonId': str(document_ids[i])}) 36 | registration['Owners']['PrimaryOwner'].update({'PersonId': str(document_ids[i])}) 37 | return new_drivers_licenses, new_vehicle_registrations 38 | 39 | 40 | def insert_documents(transaction_executor, table_name, documents): 41 | logger.info('Inserting some documents in the {} table...'.format(table_name)) 42 | statement = 'INSERT INTO {} ?'.format(table_name) 43 | cursor = transaction_executor.execute_statement(statement, convert_object_to_ion(documents)) 44 | list_of_document_ids = get_document_ids_from_dml_results(cursor) 45 | 46 | return list_of_document_ids 47 | 48 | 49 | def update_and_insert_documents(transaction_executor): 50 | """ 51 | Handle the insertion of documents and updating PersonIds all in a single transaction. 52 | """ 53 | 54 | list_ids = insert_documents(transaction_executor, Constants.PERSON_TABLE_NAME, SampleData.PERSON) 55 | 56 | logger.info("Updating PersonIds for 'DriversLicense' and PrimaryOwner for 'VehicleRegistration'...") 57 | new_licenses, new_registrations = update_person_id(list_ids) 58 | 59 | insert_documents(transaction_executor, Constants.VEHICLE_TABLE_NAME, SampleData.VEHICLE) 60 | insert_documents(transaction_executor, Constants.VEHICLE_REGISTRATION_TABLE_NAME, new_registrations) 61 | insert_documents(transaction_executor, Constants.DRIVERS_LICENSE_TABLE_NAME, new_licenses) 62 | 63 | 64 | if __name__ == '__main__': 65 | """ 66 | Insert documents into a table in a QLDB ledger. 67 | """ 68 | try: 69 | with create_qldb_session() as session: 70 | session.execute_lambda(lambda executor: update_and_insert_documents(executor), 71 | lambda retry_attempt: logger.info('Retrying due to OCC conflict...')) 72 | logger.info('Documents inserted successfully!') 73 | except Exception: 74 | logger.exception('Error inserting or updating documents.') 75 | -------------------------------------------------------------------------------- /sample_scenarios/multiple_updates_to_a_document.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | from logging import basicConfig, getLogger, INFO 17 | 18 | from sample_scenarios.sample_data import convert_object_to_ion, SampleData, get_document_ids_from_dml_results 19 | from sample_scenarios.helpers import create_qldb_session 20 | from decimal import Decimal 21 | from sample_scenarios.constants import Constants 22 | 23 | logger = getLogger(__name__) 24 | basicConfig(level=INFO) 25 | 26 | 27 | def update_documents(transaction_executor): 28 | logger.info('Updating some documents multiple times in the {} table...' 29 | .format(Constants.VEHICLE_REGISTRATION_TABLE_NAME)) 30 | 31 | 32 | for license_number, pending_amounts in SampleData.PENDING_AMOUNT_VALUES_FOR_MULTIPLE_UPDATES.items(): 33 | 34 | for pending_amount in pending_amounts: 35 | statement = 'UPDATE {table_name} SET PendingPenaltyTicketAmount = ? WHERE LicensePlateNumber = ?' \ 36 | .format(table_name=Constants.VEHICLE_REGISTRATION_TABLE_NAME) 37 | 38 | logger.info('Updating PendingPenaltyTicketAmount for License Number: {license_number}' 39 | ' to {amount}'.format(license_number=license_number, amount=pending_amount)) 40 | 41 | transaction_executor.execute_statement(statement, pending_amount, license_number) 42 | 43 | 44 | if __name__ == '__main__': 45 | """ 46 | Updating documents multiple times in VehicleRegistration table. 47 | """ 48 | try: 49 | with create_qldb_session() as session: 50 | session.execute_lambda(lambda executor: update_documents(executor), 51 | lambda retry_attempt: logger.info('Retrying due to OCC conflict...')) 52 | logger.info('Documents updated successfully!') 53 | except Exception: 54 | logger.exception('Error updating documents.') 55 | -------------------------------------------------------------------------------- /sample_scenarios/requirements.txt: -------------------------------------------------------------------------------- 1 | amazon.ion==0.5.0 2 | boto3==1.9.237 3 | botocore==1.12.237 4 | pyqldb~=2.0.0 -------------------------------------------------------------------------------- /sample_scenarios/sample_data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 17 | # SPDX-License-Identifier: MIT-0 18 | # 19 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 20 | # software and associated documentation files (the "Software"), to deal in the Software 21 | # without restriction, including without limitation the rights to use, copy, modify, 22 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 23 | # permit persons to whom the Software is furnished to do so. 24 | # 25 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 26 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 27 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 28 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 29 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 30 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 31 | from datetime import datetime 32 | from decimal import Decimal 33 | from logging import basicConfig, getLogger, INFO 34 | 35 | from amazon.ion.simple_types import IonPyBool, IonPyBytes, IonPyDecimal, IonPyDict, IonPyFloat, IonPyInt, IonPyList, \ 36 | IonPyNull, IonPySymbol, IonPyText, IonPyTimestamp 37 | from amazon.ion.simpleion import dumps, loads 38 | 39 | logger = getLogger(__name__) 40 | basicConfig(level=INFO) 41 | IonValue = (IonPyBool, IonPyBytes, IonPyDecimal, IonPyDict, IonPyFloat, IonPyInt, IonPyList, IonPyNull, IonPySymbol, 42 | IonPyText, IonPyTimestamp) 43 | 44 | 45 | class SampleData: 46 | """ 47 | Sample domain objects for use throughout this tutorial. 48 | """ 49 | DRIVERS_LICENSE = [ 50 | { 51 | 'PersonId': '', 52 | 'LicenseNumber': 'LEWISR261LL12', 53 | 'LicenseType': 'Learner', 54 | 'ValidFromDate': datetime(2016, 12, 20), 55 | 'ValidToDate': datetime(2020, 11, 15) 56 | }, 57 | { 58 | 'PersonId': '', 59 | 'LicenseNumber': 'LOGANB486CG12', 60 | 'LicenseType': 'Probationary', 61 | 'ValidFromDate': datetime(2016, 4, 6), 62 | 'ValidToDate': datetime(2020, 11, 15) 63 | }, 64 | { 65 | 'PersonId': '', 66 | 'LicenseNumber': '744 849 301 12', 67 | 'LicenseType': 'Full', 68 | 'ValidFromDate': datetime(2017, 12, 6), 69 | 'ValidToDate': datetime(2022, 10, 15) 70 | }, 71 | { 72 | 'PersonId': '', 73 | 'LicenseNumber': 'P626-168-229', 74 | 'LicenseType': 'Learner', 75 | 'ValidFromDate': datetime(2017, 8, 16), 76 | 'ValidToDate': datetime(2021, 11, 15) 77 | }, 78 | { 79 | 'PersonId': '', 80 | 'LicenseNumber': 'S152-780-97-415-01', 81 | 'LicenseType': 'Probationary', 82 | 'ValidFromDate': datetime(2015, 8, 15), 83 | 'ValidToDate': datetime(2021, 8, 21) 84 | } 85 | ] 86 | PERSON = [ 87 | { 88 | 'FirstName': 'Drew', 89 | 'LastName': 'Lawson', 90 | 'Address': '1719 University Street, Seattle, WA, 98109', 91 | 'DOB': datetime(1963, 8, 19), 92 | 'GovId': 'LEWISR261LL12', 93 | 'GovIdType': 'Driver License' 94 | }, 95 | { 96 | 'FirstName': 'Isaac', 97 | 'LastName': 'Howell', 98 | 'DOB': datetime(1967, 7, 3), 99 | 'Address': '43 Stockert Hollow Road, Everett, WA, 98203', 100 | 'GovId': 'LOGANB486CG12', 101 | 'GovIdType': 'Driver License' 102 | }, 103 | { 104 | 'FirstName': 'Ed', 105 | 'LastName': 'Morales', 106 | 'DOB': datetime(1974, 2, 10), 107 | 'Address': '4058 Melrose Street, Spokane Valley, WA, 99206', 108 | 'GovId': '744 849 301 12', 109 | 'GovIdType': 'SSN' 110 | }, 111 | { 112 | 'FirstName': 'Donna', 113 | 'LastName': 'Lucas', 114 | 'DOB': datetime(1976, 5, 22), 115 | 'Address': '4362 Ryder Avenue, Seattle, WA, 98101', 116 | 'GovId': 'P626-168-229', 117 | 'GovIdType': 'Passport' 118 | }, 119 | { 120 | 'FirstName': 'Kenneth', 121 | 'LastName': 'Porter', 122 | 'DOB': datetime(1997, 11, 15), 123 | 'Address': '4450 Honeysuckle Lane, Seattle, WA, 98101', 124 | 'GovId': 'S152-780-97-415-01', 125 | 'GovIdType': 'Passport' 126 | } 127 | ] 128 | VEHICLE = [ 129 | { 130 | 'VIN': '4M4PL11D75C109194', 131 | 'Type': 'Sedan', 132 | 'Year': 2011, 133 | 'Make': 'Audi', 134 | 'Model': 'A5', 135 | 'Color': 'Silver' 136 | }, 137 | { 138 | 'VIN': 'MK9RSDHF&FU074356', 139 | 'Type': 'Sedan', 140 | 'Year': 2015, 141 | 'Make': 'Tesla', 142 | 'Model': 'Model S', 143 | 'Color': 'Blue' 144 | }, 145 | { 146 | 'VIN': '4FFIT5G538M762986', 147 | 'Type': 'Motorcycle', 148 | 'Year': 2011, 149 | 'Make': 'Ducati', 150 | 'Model': 'Monster 1200', 151 | 'Color': 'Yellow' 152 | }, 153 | { 154 | 'VIN': '2FVCCDDSWXY59915', 155 | 'Type': 'Semi', 156 | 'Year': 2009, 157 | 'Make': 'Ford', 158 | 'Model': 'F 150', 159 | 'Color': 'Black' 160 | }, 161 | { 162 | 'VIN': '1C4RJFAG0FC625797', 163 | 'Type': 'Sedan', 164 | 'Year': 2019, 165 | 'Make': 'Mercedes', 166 | 'Model': 'CLK 350', 167 | 'Color': 'White' 168 | } 169 | ] 170 | VEHICLE_REGISTRATION = [ 171 | { 172 | 'VIN': '4M4PL11D75C109194', 173 | 'LicensePlateNumber': 'LEWISR261LL12', 174 | 'State': 'WA', 175 | 'City': 'Seattle', 176 | 'ValidFromDate': datetime(2017, 8, 21), 177 | 'ValidToDate': datetime(2020, 5, 11), 178 | 'PendingPenaltyTicketAmount': Decimal('90.25'), 179 | 'Owners': { 180 | 'PrimaryOwner': {'PersonId': ''}, 181 | 'SecondaryOwners': [] 182 | } 183 | }, 184 | { 185 | 'VIN': 'MK9RSDHF&FU074356', 186 | 'LicensePlateNumber': 'CA762X12', 187 | 'State': 'WA', 188 | 'City': 'Kent', 189 | 'PendingPenaltyTicketAmount': Decimal('130.75'), 190 | 'ValidFromDate': datetime(2017, 9, 14), 191 | 'ValidToDate': datetime(2020, 6, 25), 192 | 'Owners': { 193 | 'PrimaryOwner': {'PersonId': ''}, 194 | 'SecondaryOwners': [] 195 | } 196 | }, 197 | { 198 | 'VIN': '4FFIT5G538M762986', 199 | 'LicensePlateNumber': 'CD820Z12', 200 | 'State': 'WA', 201 | 'City': 'Everett', 202 | 'PendingPenaltyTicketAmount': Decimal('442.30'), 203 | 'ValidFromDate': datetime(2011, 3, 17), 204 | 'ValidToDate': datetime(2021, 3, 24), 205 | 'Owners': { 206 | 'PrimaryOwner': {'PersonId': ''}, 207 | 'SecondaryOwners': [] 208 | } 209 | }, 210 | { 211 | 'VIN': '2FVCCDDSWXY59915', 212 | 'LicensePlateNumber': 'LS477D12', 213 | 'State': 'WA', 214 | 'City': 'Tacoma', 215 | 'PendingPenaltyTicketAmount': Decimal('42.20'), 216 | 'ValidFromDate': datetime(2011, 10, 26), 217 | 'ValidToDate': datetime(2023, 9, 25), 218 | 'Owners': { 219 | 'PrimaryOwner': {'PersonId': ''}, 220 | 'SecondaryOwners': [] 221 | } 222 | }, 223 | { 224 | 'VIN': '1C4RJFAG0FC625797', 225 | 'LicensePlateNumber': 'TH393F12', 226 | 'State': 'WA', 227 | 'City': 'Olympia', 228 | 'PendingPenaltyTicketAmount': Decimal('30.45'), 229 | 'ValidFromDate': datetime(2013, 9, 2), 230 | 'ValidToDate': datetime(2024, 3, 19), 231 | 'Owners': { 232 | 'PrimaryOwner': {'PersonId': ''}, 233 | 'SecondaryOwners': [] 234 | } 235 | } 236 | ] 237 | 238 | PENDING_AMOUNT_VALUES_SINGLE_UPDATE = { 239 | VEHICLE_REGISTRATION[0]['LicensePlateNumber']: Decimal('142.20'), 240 | VEHICLE_REGISTRATION[1]['LicensePlateNumber']: Decimal('242.20'), 241 | VEHICLE_REGISTRATION[2]['LicensePlateNumber']: Decimal('452.20'), 242 | VEHICLE_REGISTRATION[3]['LicensePlateNumber']: Decimal('152.20'), 243 | VEHICLE_REGISTRATION[4]['LicensePlateNumber']: Decimal('352.20') 244 | } 245 | 246 | PENDING_AMOUNT_VALUES_FOR_MULTIPLE_UPDATES = { 247 | VEHICLE_REGISTRATION[0]['LicensePlateNumber']: [Decimal('195.20'), Decimal('190.20'), 248 | Decimal('300.20')]} 249 | 250 | def convert_object_to_ion(py_object): 251 | """ 252 | Convert a Python object into an Ion object. 253 | 254 | :type py_object: object 255 | :param py_object: The object to convert. 256 | 257 | :rtype: :py:class:`amazon.ion.simple_types.IonPyValue` 258 | :return: The converted Ion object. 259 | """ 260 | ion_object = loads(dumps(py_object)) 261 | return ion_object 262 | 263 | 264 | def to_ion_struct(key, value): 265 | """ 266 | Convert the given key and value into an Ion struct. 267 | 268 | :type key: str 269 | :param key: The key which serves as an unique identifier. 270 | 271 | :type value: str 272 | :param value: The value associated with a given key. 273 | 274 | :rtype: :py:class:`amazon.ion.simple_types.IonPyDict` 275 | :return: The Ion dictionary object. 276 | """ 277 | ion_struct = dict() 278 | ion_struct[key] = value 279 | return loads(str(ion_struct)) 280 | 281 | 282 | def get_document_ids(transaction_executor, table_name, field, value): 283 | """ 284 | Gets the document IDs from the given table. 285 | 286 | :type transaction_executor: :py:class:`pyqldb.session.executor.Executor` 287 | :param transaction_executor: An Executor object allowing for execution of statements within a transaction. 288 | 289 | :type table_name: str 290 | :param table_name: The table name to query. 291 | 292 | :type field: str 293 | :param field: A field to query. 294 | 295 | :type value: str 296 | :param value: The key of the given field. 297 | 298 | :rtype: list 299 | :return: A list of document IDs. 300 | """ 301 | query = "SELECT id FROM {} AS t BY id WHERE t.{} = '{}'".format(table_name, field, value) 302 | cursor = transaction_executor.execute_statement(query) 303 | list_of_ids = map(lambda table: table.get('id'), cursor) 304 | return list_of_ids 305 | 306 | 307 | def get_document_ids_from_dml_results(result): 308 | """ 309 | Return a list of modified document IDs as strings from DML results. 310 | 311 | :type result: :py:class:`pyqldb.cursor.stream_cursor.StreamCursor` 312 | :param: result: The result set from DML operation. 313 | 314 | :rtype: list 315 | :return: List of document IDs. 316 | """ 317 | ret_val = list(map(lambda x: x.get('documentId'), result)) 318 | return ret_val 319 | 320 | 321 | def print_result(cursor): 322 | """ 323 | Pretty print the result set. Returns the number of documents in the result set. 324 | 325 | :type cursor: :py:class:`pyqldb.cursor.stream_cursor.StreamCursor`/ 326 | :py:class:`pyqldb.cursor.buffered_cursor.BufferedCursor` 327 | :param cursor: An instance of the StreamCursor or BufferedCursor class. 328 | 329 | :rtype: int 330 | :return: Number of documents in the result set. 331 | """ 332 | result_counter = 0 333 | for row in cursor: 334 | # Each row would be in Ion format. 335 | logger.info(dumps(row, binary=False, indent=' ', omit_version_marker=True)) 336 | result_counter += 1 337 | return result_counter 338 | -------------------------------------------------------------------------------- /sample_scenarios/single_update_to_document.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | from logging import basicConfig, getLogger, INFO 17 | 18 | from sample_scenarios.sample_data import SampleData 19 | from sample_scenarios.helpers import create_qldb_session 20 | 21 | logger = getLogger(__name__) 22 | basicConfig(level=INFO) 23 | 24 | 25 | def update_documents(transaction_executor): 26 | logger.info('Updating some documents in the VehicleRegistration table...') 27 | 28 | for license_number, pending_amount in SampleData.PENDING_AMOUNT_VALUES_SINGLE_UPDATE.items(): 29 | statement = 'UPDATE VehicleRegistration SET PendingPenaltyTicketAmount = ? WHERE LicensePlateNumber = ?' 30 | 31 | logger.info('Updating PendingPenaltyTicketAmount for License Number: {license_number}' 32 | ' to {amount}'.format(license_number=license_number, amount=pending_amount)) 33 | 34 | transaction_executor.execute_statement(statement, pending_amount, license_number) 35 | 36 | 37 | if __name__ == '__main__': 38 | """ 39 | Updating documents in VehicleRegistration table in QLDB ledger. 40 | """ 41 | try: 42 | with create_qldb_session() as session: 43 | session.execute_lambda(lambda executor: update_documents(executor), 44 | lambda retry_attempt: logger.info('Retrying due to OCC conflict...')) 45 | logger.info('Documents updated successfully!') 46 | except Exception: 47 | logger.exception('Error updating documents.') 48 | -------------------------------------------------------------------------------- /setup/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | -------------------------------------------------------------------------------- /setup/provisioning_lambda.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | from __future__ import print_function 17 | from crhelper import CfnResource 18 | import logging 19 | import boto3 20 | import os 21 | from requests_aws4auth import AWS4Auth 22 | from elasticsearch import Elasticsearch, RequestsHttpConnection, RequestError 23 | 24 | logger = logging.getLogger(__name__) 25 | # Initialise the helper, all inputs are optional, this example shows the defaults 26 | helper = CfnResource(json_logging=False, log_level='DEBUG', boto_level='CRITICAL') 27 | 28 | service = 'es' 29 | INDEXES = ["person_index", "vehicle_registration_index"] 30 | es = None 31 | 32 | try: 33 | host = os.environ['ES_HOST'] 34 | session = boto3.Session() 35 | credentials = session.get_credentials() 36 | region = session.region_name 37 | awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service, session_token=credentials.token) 38 | es = Elasticsearch( 39 | hosts=[{'host': host, 'port': 443}], 40 | http_auth=awsauth, 41 | use_ssl=True, 42 | verify_certs=True, 43 | connection_class=RequestsHttpConnection, 44 | retry_on_timeout=True, 45 | max_retries=3 46 | ) 47 | 48 | except Exception as e: 49 | helper.init_failure(e) 50 | 51 | 52 | @helper.create 53 | def create(event, context): 54 | logger.info("Initiating index creation") 55 | helper.Data.update({"Status": "Initiated"}) 56 | 57 | for index in INDEXES: 58 | try: 59 | es.indices.create(index=index, body={'settings': {'index': {'gc_deletes': '1d'}}}) 60 | except RequestError as e: 61 | if e.error == "resource_already_exists_exception": 62 | es.indices.put_settings(index=index, body={'gc_deletes': '1d'}) 63 | else: 64 | raise e 65 | 66 | 67 | @helper.update 68 | def update(event, context): 69 | # no op 70 | pass 71 | 72 | 73 | @helper.delete 74 | def delete(event, context): 75 | # no op 76 | pass 77 | 78 | 79 | def lambda_handler(event, context): 80 | helper(event, context) 81 | -------------------------------------------------------------------------------- /setup/requirements.txt: -------------------------------------------------------------------------------- 1 | requests_aws4auth==0.9 2 | elasticsearch==7.5.1 3 | crhelper==2.0.5 -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | -------------------------------------------------------------------------------- /src/qldb_streaming_to_es_sample/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | -------------------------------------------------------------------------------- /src/qldb_streaming_to_es_sample/app.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | from aws_kinesis_agg.deaggregator import deaggregate_records 17 | import boto3 18 | import os 19 | from requests_aws4auth import AWS4Auth 20 | from .helpers.filtered_records_generator import filtered_records_generator 21 | from .clients.elasticsearch import ElasticsearchClient 22 | from .constants import Constants 23 | 24 | service = 'es' 25 | session = boto3.Session() 26 | credentials = session.get_credentials() 27 | region = session.region_name 28 | awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service, session_token=credentials.token) 29 | host = os.environ['ES_HOST'] 30 | 31 | elasticsearch_client = ElasticsearchClient(host=host, awsauth=awsauth) 32 | 33 | TABLE_TO_INDEX_MAP = {Constants.PERSON_TABLENAME : Constants.PERSON_INDEX, 34 | Constants.VEHICLE_REGISTRATION_TABLENAME : Constants.VEHICLE_REGISTRATION_INDEX} 35 | 36 | def lambda_handler(event, context): 37 | """ 38 | Triggered for a batch of kinesis records. 39 | Parses QLDB Journal streams and indexes documents to Elasticsearch for 40 | Person and Vehicle Registration Events. 41 | """ 42 | raw_kinesis_records = event['Records'] 43 | 44 | # Deaggregate all records in one call 45 | records = deaggregate_records(raw_kinesis_records) 46 | 47 | # Iterate through deaggregated records of Person and VehicleRegistration Table 48 | for record in filtered_records_generator(records, 49 | table_names=[Constants.PERSON_TABLENAME, 50 | Constants.VEHICLE_REGISTRATION_TABLENAME]): 51 | table_name = record["table_info"]["tableName"] 52 | revision_data = record["revision_data"] 53 | revision_metadata = record["revision_metadata"] 54 | version = revision_metadata["version"] 55 | document = None 56 | 57 | if revision_data: 58 | # if record is for Person table and is an insert event 59 | if (table_name == Constants.PERSON_TABLENAME) and (version == 0) and \ 60 | __fields_are_present(Constants.PERSON_TABLE_FIELDS, revision_data): 61 | 62 | document = __create_document(Constants.PERSON_TABLE_FIELDS, revision_data) 63 | elasticsearch_client.index(index=TABLE_TO_INDEX_MAP[table_name], 64 | id=revision_metadata["id"], body=document, version=version) 65 | 66 | # if record is for VehicleRegistration table and is an insert or update event 67 | elif table_name == Constants.VEHICLE_REGISTRATION_TABLENAME and \ 68 | __fields_are_present(Constants.VEHICLE_REGISTRATION_TABLE_FIELDS, revision_data): 69 | document = __create_document(Constants.VEHICLE_REGISTRATION_TABLE_FIELDS, revision_data) 70 | elasticsearch_client.index(index=TABLE_TO_INDEX_MAP[table_name], 71 | id=revision_metadata["id"], body=document, version=version) 72 | 73 | else: 74 | # delete record 75 | elasticsearch_client.delete(index=TABLE_TO_INDEX_MAP[table_name], 76 | id=revision_metadata["id"], version=version) 77 | 78 | 79 | return { 80 | 'statusCode': 200 81 | } 82 | 83 | 84 | def __create_document(fields, revision_data): 85 | document = {} 86 | 87 | for field in fields: 88 | document[field] = revision_data[field] 89 | 90 | return document 91 | 92 | 93 | def __fields_are_present(fields_list, revision_data): 94 | for field in fields_list: 95 | if not field in revision_data: 96 | return False 97 | 98 | return True 99 | -------------------------------------------------------------------------------- /src/qldb_streaming_to_es_sample/clients/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | -------------------------------------------------------------------------------- /src/qldb_streaming_to_es_sample/clients/elasticsearch.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | from elasticsearch import Elasticsearch, RequestsHttpConnection, NotFoundError 17 | from elasticsearch import SerializationError, ConflictError, RequestError 18 | 19 | 20 | class ElasticsearchClient: 21 | """ 22 | Elasticsearch wrapper 23 | """ 24 | es_client = None 25 | 26 | def __init__(self, host, awsauth): 27 | self.es_client = Elasticsearch( 28 | hosts=[{'host': host, 'port': 443}], 29 | http_auth=awsauth, 30 | use_ssl=True, 31 | verify_certs=True, 32 | connection_class=RequestsHttpConnection, 33 | retry_on_timeout=True, 34 | max_retries=3 35 | ) 36 | 37 | def index(self, index, id, body, version): 38 | """ 39 | Indexes documents to elasticsearch. 40 | Uses external version support to handle duplicates. 41 | https://www.elastic.co/blog/elasticsearch-versioning-support 42 | https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#index-version-types 43 | """ 44 | try: 45 | response = self.es_client.index(index=index, id=id, 46 | body=body, version=version, version_type="external") 47 | 48 | print("Indexed document with id: {id}, body: {body}" 49 | " and version: {version}".format(id=id, body=body, 50 | version=version)) 51 | 52 | return response 53 | 54 | except (SerializationError, ConflictError, 55 | RequestError) as e: # https://elasticsearch-py.readthedocs.io/en/master/exceptions.html#elasticsearch.ElasticsearchException 56 | print("Elasticsearch Exception occured while indexing id={id}, body={body} and" 57 | "version={version}. Error: {error}".format(id=id, body=body, version=version, 58 | error=str(e))) 59 | return None 60 | 61 | def delete(self, index, id, version): 62 | 63 | try: 64 | 65 | response = self.es_client.delete(index=index, id=id, version=version, version_type="external") 66 | print("Deleted document with id: {id}".format(id=id)) 67 | 68 | return response 69 | 70 | except (SerializationError, ConflictError, 71 | RequestError, NotFoundError) as e: # https://elasticsearch-py.readthedocs.io/en/master/exceptions.html#elasticsearch.ElasticsearchException 72 | print("Elasticsearch Exception occured while deleting id={id}. Error: {error}" 73 | .format(id=id, error=str(e))) 74 | return None 75 | -------------------------------------------------------------------------------- /src/qldb_streaming_to_es_sample/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | class Constants: 17 | 18 | PERSON_TABLENAME = "Person" 19 | VEHICLE_REGISTRATION_TABLENAME = "VehicleRegistration" 20 | 21 | PERSON_INDEX = "person_index" 22 | VEHICLE_REGISTRATION_INDEX = "vehicle_registration_index" 23 | 24 | PERSON_TABLE_FIELDS = ["FirstName","LastName", "GovId"] 25 | VEHICLE_REGISTRATION_TABLE_FIELDS = ["VIN", "LicensePlateNumber", "State", 26 | "PendingPenaltyTicketAmount"] 27 | 28 | 29 | -------------------------------------------------------------------------------- /src/qldb_streaming_to_es_sample/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | -------------------------------------------------------------------------------- /src/qldb_streaming_to_es_sample/helpers/filtered_records_generator.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | import amazon.ion.simpleion as ion 17 | import base64 18 | 19 | REVISION_DETAILS_RECORD_TYPE = "REVISION_DETAILS" 20 | 21 | 22 | def filtered_records_generator(kinesis_deaggregate_records, table_names=None): 23 | for record in kinesis_deaggregate_records: 24 | # Kinesis data in Python Lambdas is base64 encoded 25 | payload = base64.b64decode(record['kinesis']['data']) 26 | # payload is the actual ion binary record published by QLDB to the stream 27 | ion_record = ion.loads(payload) 28 | print("Ion record: ", (ion.dumps(ion_record, binary=False))) 29 | 30 | if ("recordType" in ion_record) and (ion_record["recordType"] == REVISION_DETAILS_RECORD_TYPE): 31 | table_info = get_table_info_from_revision_record(ion_record) 32 | 33 | if not table_names or (table_info and (table_info["tableName"] in table_names)): 34 | revision_data, revision_metadata = get_data_metdata_from_revision_record(ion_record) 35 | 36 | yield {"table_info": table_info, 37 | "revision_data": revision_data, 38 | "revision_metadata": revision_metadata} 39 | 40 | 41 | def get_data_metdata_from_revision_record(revision_record): 42 | """ 43 | Retrieves the data block from revision Revision Record 44 | 45 | Parameters: 46 | revision_record (string): The ion representation of Revision record from QLDB Streams 47 | """ 48 | 49 | revision_data = None 50 | revision_metadata = None 51 | 52 | if ("payload" in revision_record) and ("revision" in revision_record["payload"]): 53 | if "data" in revision_record["payload"]["revision"]: 54 | revision_data = revision_record["payload"]["revision"]["data"] 55 | else: 56 | revision_data = None 57 | if "metadata" in revision_record["payload"]["revision"]: 58 | revision_metadata = revision_record["payload"]["revision"]["metadata"] 59 | 60 | return [revision_data, revision_metadata] 61 | 62 | 63 | def get_table_info_from_revision_record(revision_record): 64 | """ 65 | Retrieves the table information block from revision Revision Record 66 | Table information contains the table name and table id 67 | 68 | Parameters: 69 | revision_record (string): The ion representation of Revision record from QLDB Streams 70 | """ 71 | 72 | if ("payload" in revision_record) and "tableInfo" in revision_record["payload"]: 73 | return revision_record["payload"]["tableInfo"] 74 | -------------------------------------------------------------------------------- /src/requirements.txt: -------------------------------------------------------------------------------- 1 | amazon.ion==0.5.0 2 | aws-kinesis-agg==1.1.2 3 | requests_aws4auth==0.9 4 | elasticsearch==7.5.1 -------------------------------------------------------------------------------- /template.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Transform: AWS::Serverless-2016-10-31 3 | Description: > 4 | SAM Template for QLDB Streams Elasticsearch Integration Sample Application. 5 | 6 | This template: 7 | 8 | 1) Creates a Kinesis Stream 9 | 2) Creates a Lambda 10 | 3) Maps lambda to the Kinesis Stream 11 | 4) Creates RegistrationStreamsKinesisRole which will be used by QLDB to write to Kinesis 12 | 5) Creates an Elasticsearch Domain 13 | 14 | # More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst 15 | Globals: 16 | Function: 17 | Timeout: 3 18 | 19 | Parameters: 20 | ElasticsearchDomainName: 21 | Description: Domain name for the elasticsearch endpoint 22 | Type: String 23 | 24 | Metadata: 25 | AWS::ServerlessRepo::Application: 26 | Name: amazon-qldb-streaming-elasticsearch-lambda-python 27 | Description: This sample demonstrates how to stream insertions into Amazon QLDB to Elasticsearch. 28 | SpdxLicenseId: Apache-2.0 29 | Labels: ['aws_qldb_sample', 'qldb_streams', 'elasticsearch'] 30 | HomePageUrl: https://github.com/aws-samples/amazon-qldb-streaming-elasticsearch-lambda-python 31 | SemanticVersion: 0.0.1 32 | SourceCodeUrl: https://github.com/aws-samples/amazon-qldb-streaming-elasticsearch-lambda-python 33 | 34 | Resources: 35 | 36 | KibanaCognitoUserpool: 37 | Type: AWS::Cognito::UserPool 38 | Properties: 39 | AdminCreateUserConfig: 40 | AllowAdminCreateUserOnly: true 41 | Policies: 42 | PasswordPolicy: 43 | MinimumLength: 8 44 | UserPoolName: registrations_kibana_demo_userpool 45 | 46 | KibanaCognitoIdentitypool: 47 | Type: AWS::Cognito::IdentityPool 48 | Properties: 49 | IdentityPoolName: registrations_kibana_demo_identitypool 50 | AllowUnauthenticatedIdentities: true 51 | 52 | KibanaCognitoIdentitypooldomain: 53 | Type: AWS::Cognito::UserPoolDomain 54 | Properties: 55 | Domain: 56 | Fn::Sub: ${ElasticsearchDomainName} 57 | UserPoolId: 58 | Ref: KibanaCognitoUserpool 59 | 60 | CognitoAuthUserIAMPolicy: 61 | Type: AWS::IAM::Policy 62 | Properties: 63 | PolicyDocument: 64 | Version: '2012-10-17' 65 | Statement: 66 | - Effect: Allow 67 | Action: 68 | - cognito-identity:* 69 | Resource: 70 | - "*" 71 | PolicyName: kibanacognitoauthuserpolicy 72 | Roles: 73 | - Ref: CognitoAuthUserIAMRole 74 | 75 | # Create a role for ES access Cognito 76 | SampleCognitoAccessForAmazonES: 77 | Type: "AWS::IAM::Role" 78 | Properties: 79 | RoleName: "SampleCognitoAccessForAmazonES" 80 | AssumeRolePolicyDocument: 81 | Version: "2012-10-17" 82 | Statement: 83 | - Effect: "Allow" 84 | Principal: 85 | Service: 86 | - "es.amazonaws.com" 87 | Action: 88 | - "sts:AssumeRole" 89 | ManagedPolicyArns: 90 | - "arn:aws:iam::aws:policy/AmazonESCognitoAccess" 91 | 92 | CognitoAuthUserIAMRole: 93 | Type: AWS::IAM::Role 94 | Properties: 95 | AssumeRolePolicyDocument: 96 | Version: '2012-10-17' 97 | Statement: 98 | - Effect: Allow 99 | Principal: 100 | Federated: cognito-identity.amazonaws.com 101 | Action: sts:AssumeRoleWithWebIdentity 102 | Condition: 103 | StringEquals: 104 | cognito-identity.amazonaws.com:aud: 105 | Ref: KibanaCognitoIdentitypool 106 | ForAnyValue:StringLike: 107 | cognito-identity.amazonaws.com:amr: authenticated 108 | RoleName: kibanacognitoauthuserrole 109 | CognitoIdentityPoolRoleAttachment: 110 | Type: AWS::Cognito::IdentityPoolRoleAttachment 111 | Properties: 112 | IdentityPoolId: 113 | Ref: KibanaCognitoIdentitypool 114 | Roles: 115 | authenticated: 116 | Fn::GetAtt: 117 | - CognitoAuthUserIAMRole 118 | - Arn 119 | unauthenticated: 120 | Fn::GetAtt: 121 | - CognitoAuthUserIAMRole 122 | - Arn 123 | 124 | 125 | RegistrationIndexerLambdaRole: # Used by lambda to read Kinesis Streams. 126 | Type: AWS::IAM::Role 127 | Properties: 128 | AssumeRolePolicyDocument: 129 | Version: 2012-10-17 130 | Statement: 131 | - Effect: Allow 132 | Principal: 133 | Service: 134 | - lambda.amazonaws.com 135 | - qldb.amazonaws.com 136 | Action: 137 | - sts:AssumeRole 138 | Path: / 139 | ManagedPolicyArns: 140 | - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole 141 | Policies: 142 | - PolicyName: root 143 | PolicyDocument: 144 | Version: 2012-10-17 145 | Statement: 146 | - Effect: Allow 147 | Action: 148 | - kinesis:ListStreams 149 | - kinesis:DescribeStream 150 | - kinesis:GetRecords 151 | - kinesis:GetShardIterator 152 | Resource: !GetAtt RegistrationStreamKinesis.Arn 153 | - Effect: Allow 154 | Action: 155 | - sqs:SendMessage 156 | Resource: !GetAtt RegistrationIndexerFailureQueue.Arn 157 | 158 | ProvisioningLambdaRole: # Used by Custom Resource Lambda to create settings for Elasticsearch Index 159 | Type: AWS::IAM::Role 160 | Properties: 161 | AssumeRolePolicyDocument: 162 | Version: 2012-10-17 163 | Statement: 164 | - Effect: Allow 165 | Principal: 166 | Service: 167 | - lambda.amazonaws.com 168 | Action: 169 | - sts:AssumeRole 170 | Path: / 171 | ManagedPolicyArns: 172 | - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole 173 | Policies: 174 | - PolicyName: root 175 | PolicyDocument: 176 | Version: 2012-10-17 177 | Statement: 178 | - Effect: Allow 179 | Action: es:UpdateElasticsearchDomainConfig 180 | Resource: 181 | Fn::Sub: arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${ElasticsearchDomainName} 182 | - Effect: Allow 183 | Action: iam:PassRole 184 | Resource: !GetAtt SampleCognitoAccessForAmazonES.Arn 185 | 186 | 187 | RegistrationStreamsKinesisRole: # Used by QLDB to write to Kinesis Streams 188 | Type: AWS::IAM::Role 189 | Properties: 190 | RoleName: RegistrationStreamsKinesisRole 191 | AssumeRolePolicyDocument: 192 | Version: '2012-10-17' 193 | Statement: 194 | - Effect: Allow 195 | Principal: 196 | Service: qldb.amazonaws.com 197 | Action: sts:AssumeRole 198 | Policies: 199 | - PolicyName: QLDBStreamKinesisPermissions 200 | PolicyDocument: 201 | Version: 2012-10-17 202 | Statement: 203 | - Effect: Allow 204 | Action: 205 | - kinesis:ListShards 206 | - kinesis:DescribeStream 207 | - kinesis:PutRecord* 208 | Resource: !GetAtt RegistrationStreamKinesis.Arn 209 | 210 | RegistrationIndexerLambda: 211 | Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction 212 | Properties: 213 | FunctionName: RegistrationIndexerLambda 214 | CodeUri: src 215 | Handler: qldb_streaming_to_es_sample.app.lambda_handler 216 | Runtime: python3.7 217 | Timeout: 900 218 | Role: !GetAtt RegistrationIndexerLambdaRole.Arn 219 | Events: 220 | Stream: 221 | Type: Kinesis 222 | Properties: 223 | Stream: !GetAtt RegistrationStreamKinesis.Arn 224 | StartingPosition: TRIM_HORIZON 225 | MaximumRetryAttempts: 0 226 | Environment: 227 | Variables: 228 | ES_HOST: !GetAtt ElasticsearchDomain.DomainEndpoint 229 | DeadLetterQueue: 230 | Type: SQS 231 | TargetArn: !GetAtt RegistrationIndexerFailureQueue.Arn 232 | 233 | 234 | ProvisioningLambda: 235 | Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction 236 | Properties: 237 | FunctionName: ElasticsearchSampleProvisioningLambda 238 | CodeUri: setup 239 | Handler: provisioning_lambda.lambda_handler 240 | Runtime: python3.7 241 | Timeout: 180 242 | Role: !GetAtt ProvisioningLambdaRole.Arn 243 | Environment: 244 | Variables: 245 | ES_HOST: !GetAtt ElasticsearchDomain.DomainEndpoint 246 | 247 | RegistrationStreamKinesis: 248 | Type: AWS::Kinesis::Stream 249 | Properties: 250 | Name: RegistrationStreamKinesis 251 | RetentionPeriodHours: 168 252 | ShardCount: 1 253 | StreamEncryption: 254 | EncryptionType: KMS 255 | KeyId: alias/aws/kinesis 256 | 257 | RegistrationIndexerFailureQueue: 258 | Type: AWS::SQS::Queue 259 | Properties: 260 | KmsMasterKeyId: alias/aws/sqs 261 | 262 | ElasticsearchDomain: 263 | Type: AWS::Elasticsearch::Domain 264 | Properties: 265 | DomainName: !Ref ElasticsearchDomainName 266 | ElasticsearchVersion: '7.1' 267 | CognitoOptions: 268 | Enabled: True 269 | IdentityPoolId: !Ref KibanaCognitoIdentitypool 270 | RoleArn: !GetAtt SampleCognitoAccessForAmazonES.Arn 271 | UserPoolId: !Ref KibanaCognitoUserpool 272 | ElasticsearchClusterConfig: 273 | InstanceCount: '1' 274 | InstanceType: 't2.small.elasticsearch' 275 | EBSOptions: 276 | EBSEnabled: 'true' 277 | Iops: 0 278 | VolumeSize: 10 279 | VolumeType: standard 280 | SnapshotOptions: 281 | AutomatedSnapshotStartHour: '0' 282 | AccessPolicies: 283 | Version: 2012-10-17 284 | Statement: 285 | - Effect: Allow 286 | Principal: 287 | AWS: 288 | - !GetAtt RegistrationIndexerLambdaRole.Arn 289 | - !GetAtt ProvisioningLambdaRole.Arn # The Provisioning Lambda needs permissions to create indexes on Elasticsearch 290 | - Fn::Sub: arn:aws:iam::${AWS::AccountId}:root 291 | - Fn::GetAtt: 292 | - CognitoAuthUserIAMRole 293 | - Arn 294 | Action: es:ESHttp* 295 | Resource: 296 | Fn::Sub: arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${ElasticsearchDomainName}/* 297 | AdvancedOptions: 298 | rest.action.multi.allow_explicit_index: 'true' 299 | 300 | ElasticsearchSetupResource: 301 | Type: "Custom::Setup" 302 | Properties: 303 | ServiceToken: !GetAtt ProvisioningLambda.Arn 304 | DependsOn: ElasticsearchDomain 305 | 306 | Outputs: 307 | RegistrationStreamsKinesisRole: 308 | Description: "IAM Role for QLDB. Will enable QLDB to write to Kinesis Streams" 309 | Value: !GetAtt RegistrationStreamsKinesisRole.Arn 310 | DomainEndpoint: 311 | Value: !GetAtt ElasticsearchDomain.DomainEndpoint 312 | KibanaEndpoint: 313 | Value: 314 | Fn::Sub: https://${ElasticsearchDomain.DomainEndpoint}/_plugin/kibana 315 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 17 | # SPDX-License-Identifier: MIT-0 18 | # 19 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 20 | # software and associated documentation files (the "Software"), to deal in the Software 21 | # without restriction, including without limitation the rights to use, copy, modify, 22 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 23 | # permit persons to whom the Software is furnished to do so. 24 | # 25 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 26 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 27 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 28 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 29 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 30 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | -------------------------------------------------------------------------------- /tests/unit/fixtures.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | import amazon.ion.simpleion as ion 17 | import base64 18 | import pytest 19 | 20 | def person_revision_details_ion_record(revision_version): 21 | person_revision_details_ion = """{ 22 | recordType: "REVISION_DETAILS", 23 | payload: { 24 | tableInfo: { 25 | tableName: "Person", 26 | }, 27 | revision: { 28 | data: { 29 | FirstName: "Nova", 30 | LastName: "Lewis", 31 | DOB: 1963-08-19T, 32 | GovId: "LEWISR261LL", 33 | GovIdType: "Driver License" 34 | }, 35 | metadata: { 36 | version: """ + str(revision_version) + """, 37 | id: "a8698243bnnmjy" 38 | } 39 | } 40 | } 41 | }""" 42 | 43 | return person_revision_details_ion 44 | 45 | def person_revision_details_ion_record_for_delete_scenario(): 46 | person_revision_details_ion = """{ 47 | recordType: "REVISION_DETAILS", 48 | payload: { 49 | tableInfo: { 50 | tableName: "Person", 51 | }, 52 | revision: { 53 | metadata: { 54 | version: 2, 55 | id: "a8698243bnnmjy" 56 | } 57 | } 58 | } 59 | }""" 60 | 61 | return person_revision_details_ion 62 | 63 | def vehicle_registration_revision_details_ion_record(revision_version): 64 | vehicle_registration_revision_details_ion = """{ 65 | recordType: "REVISION_DETAILS", 66 | payload: { 67 | tableInfo: { 68 | tableName: "VehicleRegistration", 69 | }, 70 | revision: { 71 | data: { 72 | VIN: "L12345", 73 | LicensePlateNumber: "1234567", 74 | State: "WA", 75 | PendingPenaltyTicketAmount: 127.5, 76 | }, 77 | metadata: { 78 | version: """ + str(revision_version) + """, 79 | id: "2136bjkdc8" 80 | } 81 | } 82 | } 83 | }""" 84 | 85 | return vehicle_registration_revision_details_ion 86 | 87 | def person_block_summary_ion_record(): 88 | PERSON_BLOCK_SUMMARY_ION = """{ 89 | recordType: "BLOCK_SUMMARY", 90 | payload: { 91 | transactionId: "0007KbqoyqAIch6XRbQ4iA", 92 | blockTimestamp: 2019-12-11T07:20:51.261Z, 93 | blockHash: {{lu425dAWsmvzxuNHTbn4ID4mLo0bWKkjLP2Uel4wrPQ=}}, 94 | entriesHash: {{RNGQGcOKCGLCo5S+hs1eboanNrocIzRiqzzq1s99G/Q=}}, 95 | previousBlockHash: {{pV28aszpqJH9LOO9oMsACDmXfdzdEW7HYxzuQVIjSDU=}}, 96 | entriesHashList: [ 97 | {{LmcGQjLlfScQQxbzaoglEpXpeN9bp7I/QUk690ncEpk=}}, 98 | {{vJFOcsNRM14gsIBSEnwPhMVgRAWf/4EUW5gPYbtmDv0=}}, 99 | {{KXJrG8t/KePERHasyztlv4kZPol4Q2buhWmy7iJrsiY=}}, 100 | {{Lz3XWBwtWyBA/Lhj+UoLhbajPQ8Mk9N4j0HJlrm2OTg=}} 101 | ], 102 | transactionInfo: { 103 | statements: [ 104 | { 105 | statement: "INSERT INTO Person <<\\n{\\n 'FirstName' : 'Testing new 600',\\n 'LastName' : 'Lewis',\\n 'DOB' : `1963-08-19T`,\\n 'GovId' : 'LEWISR261LL',\\n 'GovIdType' : 'Driver License',\\n 'Address' : '1719 University Street, Seattle, WA, 98109'\\n}\\n>>", 106 | startTime: 2019-12-11T07:20:51.223Z, 107 | statementDigest: {{t5wbRW+wIi/X0n3iPhFJtbt2qpzzgWkOXIFC4xJHp4o=}} 108 | } 109 | ], 110 | documents: { 111 | D35qd3e2prnJYmtKW6kok1: { 112 | tableName: "Person", 113 | tableId: "1SUXCa3wwV0GD7kV78RbSg", 114 | statements: [ 115 | 0 116 | ] 117 | } 118 | } 119 | }, 120 | revisionSummaries: [ 121 | { 122 | hash: {{vJFOcsNRM14gsIBSEnwPhMVgRAWf/4EUW5gPYbtmDv0=}}, 123 | documentId: "D35qd3e2prnJYmtKW6kok1" 124 | } 125 | ] 126 | } 127 | }""" 128 | 129 | return PERSON_BLOCK_SUMMARY_ION 130 | 131 | 132 | @pytest.fixture 133 | def deaggregated_stream_records(): 134 | def deaggregated_records(revision_version=0): 135 | return [{ 136 | 'kinesis': { 137 | 'kinesisSchemaVersion': '1.0', 138 | 'aggregated': True, 139 | 'data': base64.b64encode(ion.dumps(ion.loads(person_block_summary_ion_record()))).decode("utf-8") 140 | } 141 | }, { 142 | 'kinesis': { 143 | 'kinesisSchemaVersion': '1.0', 144 | 'aggregated': True, 145 | 'data': base64.b64encode( 146 | ion.dumps(ion.loads(person_revision_details_ion_record(revision_version)))).decode("utf-8") 147 | } 148 | }, { 149 | 'kinesis': { 150 | 'kinesisSchemaVersion': '1.0', 151 | 'aggregated': True, 152 | 'data': base64.b64encode( 153 | ion.dumps(ion.loads(vehicle_registration_revision_details_ion_record(revision_version)))).decode("utf-8") 154 | } 155 | }] 156 | 157 | return deaggregated_records 158 | 159 | 160 | @pytest.fixture 161 | def deaggregated_stream_records_for_delete_scenario(): 162 | def deaggregated_records(revision_version=0): 163 | return [{ 164 | 'kinesis': { 165 | 'kinesisSchemaVersion': '1.0', 166 | 'aggregated': True, 167 | 'data': base64.b64encode(ion.dumps( 168 | ion.loads(person_revision_details_ion_record_for_delete_scenario()))).decode("utf-8") 169 | } 170 | }, { 171 | 'kinesis': { 172 | 'kinesisSchemaVersion': '1.0', 173 | 'aggregated': True, 174 | 'data': base64.b64encode( 175 | ion.dumps(ion.loads(person_revision_details_ion_record(revision_version)))).decode("utf-8") 176 | } 177 | }, { 178 | 'kinesis': { 179 | 'kinesisSchemaVersion': '1.0', 180 | 'aggregated': True, 181 | 'data': base64.b64encode( 182 | ion.dumps(ion.loads(vehicle_registration_revision_details_ion_record(revision_version)))).decode("utf-8") 183 | } 184 | }] 185 | 186 | return deaggregated_records 187 | 188 | @pytest.fixture 189 | def elasticsearch_error(): 190 | def client_error_helper(error_class): 191 | client_error = error_class(400, "error", "info"); 192 | 193 | return client_error 194 | 195 | return client_error_helper 196 | -------------------------------------------------------------------------------- /tests/unit/test_constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | from decimal import Decimal 17 | from elasticsearch import ConnectionError,ImproperlyConfigured,SSLError 18 | from elasticsearch import SerializationError, ConflictError, RequestError 19 | 20 | class TestConstants: 21 | PERSON_DATA = {'FirstName': 'Nova', 'GovId': 'LEWISR261LL', 'LastName': 'Lewis'} 22 | VEHICLE_REGISTRATION_DATA = {'VIN': 'L12345', 'LicensePlateNumber': '1234567', 23 | 'State': 'WA', 'PendingPenaltyTicketAmount': Decimal('127.5')} 24 | PERSON_METADATA_ID = "a8698243bnnmjy" 25 | VEHICLE_REGISTRATION_METADATA_ID = "2136bjkdc8" 26 | 27 | EXCEPTIONS_THAT_SHOULD_BE_BUBBLED = [ConnectionError, ImproperlyConfigured, SSLError] 28 | EXCEPTIONS_THAT_SHOULD_BE_HANDLED = [SerializationError, ConflictError, RequestError] -------------------------------------------------------------------------------- /tests/unit/test_elasticsearch_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | from src.qldb_streaming_to_es_sample.clients.elasticsearch import ElasticsearchClient 17 | from requests_aws4auth import AWS4Auth 18 | from .test_constants import TestConstants 19 | from src.qldb_streaming_to_es_sample.constants import Constants 20 | from unittest.mock import MagicMock 21 | from .fixtures import elasticsearch_error 22 | import unittest 23 | 24 | region = 'us-east-1' 25 | service = 'es' 26 | awsauth = AWS4Auth("access_key", "secret_key", region, service, "session_token") 27 | host = "elasticsearch_host" 28 | 29 | elasticsearch_client = ElasticsearchClient(host=host, awsauth=awsauth) 30 | 31 | def test_indexing(): 32 | 33 | # Mock 34 | elasticsearch_client.es_client.index = MagicMock(return_value={"status":"success"}) 35 | 36 | # Trigger 37 | response= elasticsearch_client.index(body = TestConstants.PERSON_DATA, version=1, 38 | index = Constants.PERSON_INDEX, id = TestConstants.PERSON_DATA["GovId"]) 39 | 40 | # Verify 41 | elasticsearch_client.es_client.index.assert_called_once_with(body=TestConstants.PERSON_DATA, 42 | id=TestConstants.PERSON_DATA["GovId"], 43 | index=Constants.PERSON_INDEX, 44 | version=1,version_type='external') 45 | 46 | 47 | def test_bad_input_exceptions_are_handled_for_indexing(elasticsearch_error): 48 | 49 | for error_class in TestConstants.EXCEPTIONS_THAT_SHOULD_BE_HANDLED: 50 | error = elasticsearch_error(error_class) 51 | 52 | # Mock 53 | elasticsearch_client.es_client.index = MagicMock(side_effect=[error, None]) 54 | 55 | # Trigger 56 | response = elasticsearch_client.index(body=TestConstants.PERSON_DATA, version=1, 57 | index=Constants.PERSON_INDEX, id=TestConstants.PERSON_DATA["GovId"]) 58 | 59 | 60 | # Verify 61 | assert response == None 62 | 63 | 64 | def test_deletion(): 65 | 66 | # Mock 67 | elasticsearch_client.es_client.delete = MagicMock(return_value={"status":"success"}) 68 | 69 | # Trigger 70 | response= elasticsearch_client.delete(version=1, 71 | index = Constants.PERSON_INDEX, id = TestConstants.PERSON_DATA["GovId"]) 72 | 73 | # Verify 74 | elasticsearch_client.es_client.delete.assert_called_once_with(id=TestConstants.PERSON_DATA["GovId"], 75 | index=Constants.PERSON_INDEX, 76 | version=1,version_type='external') 77 | 78 | 79 | def test_bad_input_exceptions_are_handled_for_deletion(elasticsearch_error): 80 | 81 | for error_class in TestConstants.EXCEPTIONS_THAT_SHOULD_BE_HANDLED: 82 | error = elasticsearch_error(error_class) 83 | 84 | # Mock 85 | elasticsearch_client.es_client.delete = MagicMock(side_effect=[error, None]) 86 | 87 | # Trigger 88 | response = elasticsearch_client.delete(version=1, index=Constants.PERSON_INDEX, 89 | id=TestConstants.PERSON_DATA["GovId"]) 90 | 91 | 92 | # Verify 93 | assert response == None -------------------------------------------------------------------------------- /tests/unit/test_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | import sys, os 17 | 18 | sys.path.append(os.path.abspath('../../')) 19 | import aws_kinesis_agg 20 | from aws_kinesis_agg.deaggregator import deaggregate_records 21 | from .fixtures import deaggregated_stream_records 22 | from .fixtures import deaggregated_stream_records_for_delete_scenario 23 | from .fixtures import elasticsearch_error 24 | from src.qldb_streaming_to_es_sample.constants import Constants 25 | from unittest.mock import call 26 | from .test_constants import TestConstants 27 | import unittest 28 | 29 | sys.path.append(os.path.abspath('../../')) 30 | os.environ["ES_HOST"] = "htttp://es" 31 | 32 | from src.qldb_streaming_to_es_sample import app 33 | 34 | PERSON_INSERT_CALL = call(body=TestConstants.PERSON_DATA, 35 | id=TestConstants.PERSON_METADATA_ID, 36 | index=Constants.PERSON_INDEX, 37 | version=0) 38 | 39 | PERSON_DELETE_CALL = call(id=TestConstants.PERSON_METADATA_ID, 40 | index=Constants.PERSON_INDEX, 41 | version=2) 42 | 43 | VEHICLE_REGISTRATION_INSERT_CALL = call(body=TestConstants.VEHICLE_REGISTRATION_DATA, 44 | id=TestConstants.VEHICLE_REGISTRATION_METADATA_ID, 45 | index=Constants.VEHICLE_REGISTRATION_INDEX, 46 | version=0) 47 | 48 | test_case_instance = unittest.TestCase('__init__') 49 | 50 | def test_indexing_records_for_inserts(mocker, deaggregated_stream_records): 51 | deaggregated_records = deaggregated_stream_records(revision_version=0) 52 | 53 | # Mock 54 | mocker.patch('src.qldb_streaming_to_es_sample.app.deaggregate_records', return_value=deaggregated_records) 55 | mocker.patch('src.qldb_streaming_to_es_sample.app.elasticsearch_client.index', return_value={"status": "success"}) 56 | 57 | # Trigger 58 | response = app.lambda_handler({"Records": ["a dummy record"]}, "") 59 | 60 | # Verify 61 | calls = [PERSON_INSERT_CALL, VEHICLE_REGISTRATION_INSERT_CALL] 62 | 63 | app.elasticsearch_client.index.assert_has_calls(calls) 64 | assert response["statusCode"] == 200 65 | 66 | def test_deletion_records_for_delete_events(mocker, deaggregated_stream_records_for_delete_scenario): 67 | deaggregated_records = deaggregated_stream_records_for_delete_scenario(revision_version=0) 68 | 69 | # Mock 70 | mocker.patch('src.qldb_streaming_to_es_sample.app.deaggregate_records', return_value=deaggregated_records) 71 | mocker.patch('src.qldb_streaming_to_es_sample.app.elasticsearch_client.delete', return_value={"status": "success"}) 72 | mocker.patch('src.qldb_streaming_to_es_sample.app.elasticsearch_client.index', return_value={"status": "success"}) 73 | 74 | # Trigger 75 | response = app.lambda_handler({"Records": ["a dummy record"]}, "") 76 | 77 | # Verify 78 | calls = [PERSON_INSERT_CALL, VEHICLE_REGISTRATION_INSERT_CALL] 79 | 80 | app.elasticsearch_client.index.assert_has_calls(calls) 81 | app.elasticsearch_client.delete.assert_called_once_with(id=TestConstants.PERSON_METADATA_ID, 82 | index=Constants.PERSON_INDEX, version=2) 83 | assert response["statusCode"] == 200 84 | 85 | def test_no_indexing_person_record_for_updates(mocker, deaggregated_stream_records): 86 | deaggregated_records = deaggregated_stream_records(revision_version=1) 87 | 88 | # Mock 89 | mocker.patch('src.qldb_streaming_to_es_sample.app.deaggregate_records', return_value=deaggregated_records) 90 | mocker.patch('src.qldb_streaming_to_es_sample.app.elasticsearch_client.index', return_value={"status": "success"}) 91 | 92 | # Trigger 93 | reponse = app.lambda_handler({"Records": ["a dummy record"]}, "") 94 | 95 | # Verify 96 | app.elasticsearch_client.index.assert_called_once_with(body=TestConstants.VEHICLE_REGISTRATION_DATA, 97 | id=TestConstants.VEHICLE_REGISTRATION_METADATA_ID, 98 | index=Constants.VEHICLE_REGISTRATION_INDEX, 99 | version=1) 100 | 101 | assert reponse["statusCode"] == 200 102 | 103 | 104 | def test_config_exceptions_are_bubbled_for_index(mocker, deaggregated_stream_records, elasticsearch_error): 105 | deaggregated_records = deaggregated_stream_records(revision_version=1) 106 | 107 | # Mock 108 | mocker.patch('src.qldb_streaming_to_es_sample.app.deaggregate_records', return_value=deaggregated_records) 109 | 110 | 111 | for error_class in TestConstants.EXCEPTIONS_THAT_SHOULD_BE_BUBBLED: 112 | error = elasticsearch_error(error_class) 113 | mocker.patch('src.qldb_streaming_to_es_sample.app.elasticsearch_client.index', side_effect=[error, None]) 114 | 115 | # Verify 116 | test_case_instance.assertRaises(error_class, app.lambda_handler,{"Records": ["a dummy record"]}, "") 117 | 118 | 119 | def test_config_exceptions_are_bubbled_for_deletion(mocker, deaggregated_stream_records_for_delete_scenario, 120 | elasticsearch_error): 121 | deaggregated_records = deaggregated_stream_records_for_delete_scenario() 122 | 123 | # Mock 124 | mocker.patch('src.qldb_streaming_to_es_sample.app.deaggregate_records', return_value=deaggregated_records) 125 | 126 | 127 | for error_class in TestConstants.EXCEPTIONS_THAT_SHOULD_BE_BUBBLED: 128 | error = elasticsearch_error(error_class) 129 | mocker.patch('src.qldb_streaming_to_es_sample.app.elasticsearch_client.delete', side_effect=[error, None]) 130 | 131 | # Verify 132 | test_case_instance.assertRaises(error_class, app.lambda_handler,{"Records": ["a dummy record"]}, "") 133 | 134 | --------------------------------------------------------------------------------