├── .gitignore ├── 01_ProductRecommendations ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── bedrock-similarity-search.ipynb ├── bedrock-text-search.ipynb ├── bedrock_requirements.txt ├── data │ ├── FEIDEGGER_release_1.2.json │ └── amazon.csv ├── genai-pgvector-similarity-search.ipynb ├── opensource-similarity-search.ipynb ├── opensource_requirements.txt ├── requirements.txt └── static │ └── architecture.png ├── 02_RetrievalAugmentedGeneration ├── 02_QuestionAnswering_Bedrock_LLMs │ ├── .gitignore │ ├── CODE_OF_CONDUCT.md │ ├── CONTRIBUTING.md │ ├── LICENSE │ ├── README.md │ ├── app.py │ ├── data │ │ ├── Amazon Aurora FAQs.pdf │ │ └── Amazon Bedrock FAQs.pdf │ ├── env.example │ ├── htmlTemplates.py │ ├── rag_app.py │ ├── requirements.txt │ └── static │ │ ├── Powered-By_logo-stack_RGB_REV.png │ │ └── RAG_APG.png ├── 02_QuestionAnswering_Open_Source_LLMs │ ├── .gitignore │ ├── CITATION.cff │ ├── CODE_OF_CONDUCT.md │ ├── CONTRIBUTING.md │ ├── LICENSE │ ├── README.md │ ├── app.py │ ├── data │ │ ├── amazon-aurora-design-considerations-for-high-throughput-cloud-native-relational-databases.pdf │ │ └── constitution.pdf │ ├── env.example │ ├── htmlTemplates.py │ ├── psql.sh │ ├── requirements.txt │ └── static │ │ └── APG-pgvector-streamlit.png └── 02_ResponseStreaming │ ├── CODE_OF_CONDUCT.md │ ├── CONTRIBUTING.md │ ├── LICENSE │ ├── README.md │ ├── app.py │ ├── data │ ├── AMZN-2020-Shareholder-Letter.pdf │ ├── AMZN-2021-Shareholder-Letter.pdf │ ├── AMZN-2022-Shareholder-Letter.pdf │ └── AMZN-2023-Shareholder-Letter.pdf │ ├── env.example │ ├── htmlTemplates.py │ ├── requirements.txt │ ├── static │ ├── Powered-By_logo-stack_RGB_REV.png │ └── Streaming_Responses_RAG.png │ └── streaming_app.py ├── 03_SimilaritySearchSentimentAnalysis ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── data │ └── fictitious_hotel_reviews_trimmed_500.csv ├── env.example ├── pgvector_with_langchain_auroraml.ipynb ├── requirements1.txt ├── requirements2.txt └── static │ └── APG-pgvector-sagemaker.png ├── 04_AuroraML_Bedrock_MovieRecommendations ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── app.py ├── data │ ├── functions.sql │ └── movies.sql.gz ├── env.example ├── requirements.txt └── static │ ├── ARCH.png │ ├── Powered-By_logo-stack_RGB_REV.png │ └── Preview_App.png ├── 05_AuroraML_Bedrock_Chatbot ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── chatbot-app.py ├── chatbot.py ├── env.example ├── requirements.txt └── static │ ├── architecture.png │ └── postgres_cli.png ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DAT303 ├── 01_ProductRecommendations │ ├── data │ │ └── amazon.csv │ ├── images │ │ └── arch_product_recommendation.png │ └── product-recommendations.ipynb ├── 02_QuestionAndAnswering │ ├── .gitignore │ ├── CODE_OF_CONDUCT.md │ ├── CONTRIBUTING.md │ ├── LICENSE │ ├── README.md │ ├── app.py │ ├── data │ │ ├── Amazon Aurora FAQs.pdf │ │ └── Amazon Bedrock FAQs.pdf │ ├── env.example │ ├── htmlTemplates.py │ ├── rag_app.py │ ├── requirements.txt │ └── static │ │ ├── Powered-By_logo-stack_RGB_REV.png │ │ └── RAG_APG.png ├── 03_ResponseStreaming │ ├── CODE_OF_CONDUCT.md │ ├── CONTRIBUTING.md │ ├── LICENSE │ ├── README.md │ ├── app.py │ ├── data │ │ ├── AMZN-2019-Shareholder-Letter.pdf │ │ ├── AMZN-2020-Shareholder-Letter.pdf │ │ ├── AMZN-2021-Shareholder-Letter.pdf │ │ └── AMZN-2022-Shareholder-Letter.pdf │ ├── htmlTemplates.py │ ├── requirements.txt │ ├── static │ │ ├── Powered-By_logo-stack_RGB_REV.png │ │ └── Streaming_RAG_APG.png │ └── streaming_app.py ├── README.md └── scripts │ └── prereq.sh ├── DAT307 ├── data │ └── dump.sql.gz ├── knowledge-base │ ├── documents │ │ ├── Amazon-Aurora-DB-engines-for-Blue_Green-Deployments.pdf │ │ ├── Amazon-RDS-Blue_Green-Deployments.pdf │ │ ├── aurora-postgresql-relnotes.pdf │ │ └── rds-postgresql-relnotes.pdf │ └── runbooks │ │ ├── exercise │ │ └── rds_acu_capacity_remediation_steps.md │ │ ├── rds_cpu_remediation_steps.md │ │ └── rds_iops_remediation_steps.md ├── lambda │ ├── api-action-runbook-kb.py │ ├── api-get-incidents.py │ ├── api-list-runbook-kb.py │ ├── cw-ingest-to-dynamodb.py │ ├── idr-bedrock-agent-action-group-good.py │ ├── idr-bedrock-agent-action-group.py │ └── qa-bedrock-agent-action-group.py ├── loadtest │ └── stress_test.py ├── notebooks │ ├── .DS_Store │ ├── images │ │ └── gen_embeddings.png │ ├── module-1-part-1-tsne.ipynb │ └── module-1-part-2-embeddings-and-pgvector.ipynb ├── script │ ├── agent_action_group_for_acu.json │ ├── agent_action_group_for_acu.sh │ ├── cloudwatch_alarm_for_acu.sh │ ├── fix_cloud9.sh │ ├── lambda_update_for_acu.sh │ ├── misc │ │ ├── lambda │ │ │ ├── auto-remediate │ │ │ │ ├── alertToaction.py │ │ │ │ ├── idr-agent-action-group-fn.py │ │ │ │ └── idr-fn.py │ │ │ ├── idr_get_active_alerts │ │ │ │ └── idr_get_active_alerts.py │ │ │ ├── ingest-dynamo │ │ │ │ └── ingestdynamodb.py │ │ │ ├── list-runbook-steps-kb │ │ │ │ └── list-runbook-steps-kb.py │ │ │ ├── psycopg2_layer │ │ │ │ ├── Makefile │ │ │ │ ├── psycopg2.zip │ │ │ │ └── publish_lambda_layer.sh │ │ │ ├── publish_lambda.sh │ │ │ ├── runbook-steps-action │ │ │ │ └── runbook-steps-action.py │ │ │ ├── s3upload │ │ │ │ ├── Dockerfile │ │ │ │ ├── app.py │ │ │ │ └── requirements.txt │ │ │ └── stress.test │ │ │ │ └── stress.test.lambda.py │ │ └── validate_api.py │ └── prereq.sh └── ui │ ├── .streamlit │ └── config.toml.old │ ├── image │ ├── aws_logo.png │ ├── demo.png │ ├── idr_logo.png │ ├── incident_management.png │ └── powered_by_aws.png │ ├── index.py │ ├── page │ ├── all_incidents.py │ ├── app.orig.py │ ├── login_page.py │ ├── login_page.py.orig │ ├── pending_incidents.py │ └── signup_page.py │ ├── requirements.txt │ └── utils │ ├── apigw_handler.py │ ├── cognito_handler.py │ └── init_session.py ├── DAT326 ├── env_sample ├── requirements.txt ├── static │ └── AZFlights.jpg ├── travel_knowledge_base.csv └── valkey-chatbot.py ├── LICENSE ├── README.md ├── data ├── Amazon Aurora FAQs.pdf ├── Amazon Bedrock FAQs.pdf └── postgresql-16-US.pdf ├── scripts └── prereq.sh └── setup_cloud9.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints 2 | -------------------------------------------------------------------------------- /01_ProductRecommendations/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /01_ProductRecommendations/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /01_ProductRecommendations/LICENSE: -------------------------------------------------------------------------------- 1 | MIT No Attribution 2 | 3 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 13 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 15 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 16 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | 18 | -------------------------------------------------------------------------------- /01_ProductRecommendations/bedrock_requirements.txt: -------------------------------------------------------------------------------- 1 | pandarallel==1.6.5 2 | pgvector==0.3.6 3 | boto3==1.36.0 4 | psycopg==3.2.4 5 | numexpr==2.10.2 6 | awscli==1.37.0 7 | -------------------------------------------------------------------------------- /01_ProductRecommendations/opensource_requirements.txt: -------------------------------------------------------------------------------- 1 | psycopg2-binary==2.9.10 2 | pgvector==0.3.6 3 | tqdm==4.67.1 4 | boto3==1.36.0 5 | requests==2.32.3 6 | scikit-image==0.25.0 7 | -------------------------------------------------------------------------------- /01_ProductRecommendations/requirements.txt: -------------------------------------------------------------------------------- 1 | packaging==24.2 2 | setuptools==65.5.0 3 | -------------------------------------------------------------------------------- /01_ProductRecommendations/static/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aurora-postgresql-pgvector/81e197513381392765f190dbf0b25b90b2137e09/01_ProductRecommendations/static/architecture.png -------------------------------------------------------------------------------- /02_RetrievalAugmentedGeneration/02_QuestionAnswering_Bedrock_LLMs/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | -------------------------------------------------------------------------------- /02_RetrievalAugmentedGeneration/02_QuestionAnswering_Bedrock_LLMs/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. -------------------------------------------------------------------------------- /02_RetrievalAugmentedGeneration/02_QuestionAnswering_Bedrock_LLMs/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. -------------------------------------------------------------------------------- /02_RetrievalAugmentedGeneration/02_QuestionAnswering_Bedrock_LLMs/LICENSE: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy of this 7 | software and associated documentation files (the "Software"), to deal in the Software 8 | without restriction, including without limitation the rights to use, copy, modify, 9 | merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so. 11 | 12 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 13 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 14 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 15 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 16 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 17 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /02_RetrievalAugmentedGeneration/02_QuestionAnswering_Bedrock_LLMs/README.md: -------------------------------------------------------------------------------- 1 | # 🤖 Enterprise RAG Question-Answering System 2 | 3 | Build an enterprise-ready Retrieval Augmented Generation (RAG) application leveraging Amazon Web Services and open-source technologies. This implementation creates an intelligent question-answering system that combines the power of vector databases with state-of-the-art language models. 4 | 5 | ## 🎯 Overview 6 | 7 | This solution demonstrates the implementation of a production-ready RAG system using: 8 | 9 | - 🚀 Amazon Bedrock for accessing foundation models 10 | - 📊 pgvector extension on Amazon Aurora PostgreSQL for vector operations 11 | - 🧠 Anthropic's Claude for advanced language understanding 12 | - 💫 Titan Text for generating high-quality text embeddings 13 | - 🔗 LangChain for orchestrating AI components 14 | - 🖥️ Streamlit for creating an intuitive user interface 15 | 16 | ## 🏗️ Architecture 17 | 18 | ![Architecture](static/RAG_APG.png) 19 | 20 | ## ⚙️ System Workflow 21 | 22 | Our RAG implementation follows a sophisticated pipeline to deliver accurate answers: 23 | 24 | 1. **Document Ingestion** 📄 25 | - Processes PDF documents through advanced text extraction 26 | - Maintains document structure and metadata 27 | 28 | 2. **Semantic Processing** 🔄 29 | - Implements intelligent text chunking algorithms 30 | - Preserves context across document segments 31 | 32 | 3. **Vector Embedding** 🎯 33 | - Utilizes Amazon Bedrock's Titan Text for generating embeddings 34 | - Creates high-dimensional vector representations of content 35 | 36 | 4. **Query Processing** 💭 37 | - Accepts natural language questions 38 | - Converts queries into compatible vector representations 39 | 40 | 5. **Context Retrieval** 🔍 41 | - Performs semantic similarity matching 42 | - Identifies relevant document segments 43 | 44 | 6. **Answer Generation** ✨ 45 | - Leverages Anthropic's Claude for response synthesis 46 | - Ensures responses are grounded in source documents 47 | 48 | ## 🚀 Getting Started 49 | 50 | ### Prerequisites 51 | 52 | - Python 3.9 or higher 53 | - AWS account with Bedrock access 54 | - Amazon Aurora PostgreSQL cluster 55 | - Git 56 | 57 | ### Installation 58 | 59 | 1. Clone the repository: 60 | ```bash 61 | git clone [repository-url] 62 | cd [repository-name] 63 | ``` 64 | 65 | 2. Create and activate a virtual environment: 66 | ```bash 67 | python3.9 -m venv env 68 | source env/bin/activate 69 | ``` 70 | 71 | 3. Configure environment variables: 72 | ```bash 73 | # Create .env file with the following structure 74 | PGVECTOR_DRIVER='psycopg2' 75 | PGVECTOR_USER='' 76 | PGVECTOR_PASSWORD='' 77 | PGVECTOR_HOST='' 78 | PGVECTOR_PORT=5432 79 | PGVECTOR_DATABASE='' 80 | ``` 81 | 82 | 4. Install dependencies: 83 | ```bash 84 | pip install -r requirements.txt 85 | ``` 86 | 87 | ### Database Setup 88 | 89 | 1. Connect to your Aurora PostgreSQL cluster 90 | 2. Enable the pgvector extension: 91 | ```sql 92 | CREATE EXTENSION vector; 93 | ``` 94 | 95 | ## 💻 Usage 96 | 97 | 1. Launch the application: 98 | ```bash 99 | streamlit run app.py 100 | ``` 101 | 102 | 2. Navigate to the web interface in your browser 103 | 3. Upload PDF documents through the provided interface 104 | 4. Start asking questions about your documents 105 | 106 | ## 🔒 Security Considerations 107 | 108 | - Implement appropriate IAM roles and permissions 109 | - Secure database connections using SSL/TLS 110 | - Follow AWS security best practices for Bedrock access 111 | - Properly handle sensitive information in environment variables 112 | 113 | ## 📈 Performance Optimization 114 | 115 | - Utilize connection pooling for database operations 116 | - Implement caching strategies where appropriate 117 | - Configure proper chunk sizes for optimal retrieval 118 | - Monitor and adjust embedding dimensions based on requirements 119 | 120 | ## 🤝 Contributing 121 | 122 | This project is maintained for educational purposes and demonstrates AWS best practices. While we don't accept direct contributions, we encourage: 123 | 124 | - Creating issues for bugs or suggested improvements 125 | - Forking the repository for personal customization 126 | - Sharing your experiences and optimizations 127 | 128 | ## 📝 License 129 | 130 | This project is licensed under the [MIT-0 License](https://spdx.org/licenses/MIT-0.html) - see the LICENSE file for details. 131 | -------------------------------------------------------------------------------- /02_RetrievalAugmentedGeneration/02_QuestionAnswering_Bedrock_LLMs/data/Amazon Aurora FAQs.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aurora-postgresql-pgvector/81e197513381392765f190dbf0b25b90b2137e09/02_RetrievalAugmentedGeneration/02_QuestionAnswering_Bedrock_LLMs/data/Amazon Aurora FAQs.pdf -------------------------------------------------------------------------------- /02_RetrievalAugmentedGeneration/02_QuestionAnswering_Bedrock_LLMs/data/Amazon Bedrock FAQs.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aurora-postgresql-pgvector/81e197513381392765f190dbf0b25b90b2137e09/02_RetrievalAugmentedGeneration/02_QuestionAnswering_Bedrock_LLMs/data/Amazon Bedrock FAQs.pdf -------------------------------------------------------------------------------- /02_RetrievalAugmentedGeneration/02_QuestionAnswering_Bedrock_LLMs/env.example: -------------------------------------------------------------------------------- 1 | PGVECTOR_USER='' 2 | PGVECTOR_PASSWORD='' 3 | PGVECTOR_HOST='' 4 | PGVECTOR_PORT=5432 5 | PGVECTOR_DATABASE='' 6 | -------------------------------------------------------------------------------- /02_RetrievalAugmentedGeneration/02_QuestionAnswering_Bedrock_LLMs/htmlTemplates.py: -------------------------------------------------------------------------------- 1 | css = ''' 2 |
{metricname}

""" 29 | return lnk + htmlstr 30 | 31 | def all_incident_page(): 32 | incidents = get_incidents("all") 33 | if len(incidents) == 0 : 34 | dfall = pd.DataFrame(columns=["incidentActionTrace","incidentData","incidentStatus","incidentIdentifier","incidentRunbook","incidentTime","sk","incidentType","lastUpdateBy","pk","lastUpdate"]) 35 | else: 36 | dfall = pd.DataFrame(incidents) 37 | 38 | dfall = dfall.drop("incidentData" , axis=1) 39 | 40 | eventCount = len(incidents) 41 | instanceCount = str(dfall['incidentIdentifier'].nunique()) 42 | alertTypeCount = str(dfall['incidentType'].nunique()) 43 | 44 | st.set_page_config(page_title="DAT307-IDR: Amazon RDS Incidents", layout="wide") 45 | st.markdown(""" 46 | 54 | """, unsafe_allow_html=True) 55 | with st.sidebar: 56 | st.sidebar.image("image/idr_logo.png") 57 | st.subheader("DAT307 - Build a Generative AI incident detection and response system powered by Amazon Aurora") 58 | st.divider() 59 | 60 | if st.button("Pending incidents"): 61 | st.session_state['page'] = 'pending_incidents' 62 | st.rerun() 63 | if st.button("All incidents"): 64 | pass 65 | if st.button("Logout"): 66 | reset_session() 67 | st.rerun() 68 | 69 | st.sidebar.image("image/powered_by_aws.png",width=120) 70 | 71 | 72 | st.title("All incidents") 73 | st.subheader("Metric Summary", divider=True) 74 | col1, col2, col3 = st.columns(3) 75 | col1.markdown(get_kpi("fa-solid fa-circle-exclamation","Total incidents",eventCount), unsafe_allow_html=True) 76 | col2.markdown(get_kpi("fa-solid fa-server","Total Unique Instance",instanceCount), unsafe_allow_html=True) 77 | col3.markdown(get_kpi("fa-solid fa-bell","Total Unique Alert Type",alertTypeCount), unsafe_allow_html=True) 78 | 79 | col4, col5 = st.columns([10,1]) 80 | col4.markdown("#### Incident Summary") 81 | col4.write("Here are the list of active incidents") 82 | col4.write("Please select an incident to process by clicking the first column of the row") 83 | 84 | print("Display table output") 85 | print(dfall) 86 | event = col4.dataframe(dfall, 87 | on_select="rerun", 88 | selection_mode="single-row", 89 | hide_index=True, 90 | column_config={ 91 | "incidentType": "Incident Type", 92 | "pk": "Session ID", 93 | "incidentIdentifier": "Database Instance", 94 | "incidentStatus": "Incident Status", 95 | "incidentTime": "Incident Time" 96 | }, 97 | column_order=("pk","incidentIdentifier","incidentType","incidentStatus","incidentTime") 98 | ) 99 | col4.markdown("#### Event Details") 100 | col4.divider() 101 | rows = event['selection']['rows'] 102 | if len(rows) != 0: 103 | print(dfall) 104 | col4.write("Runbook information") 105 | print(dfall.iloc[rows[0]]['incidentRunbook']) 106 | if dfall.iloc[rows[0]]['incidentRunbook'] != "None": 107 | col4.json(dfall.iloc[rows[0]]['incidentRunbook']) 108 | col4.write("Action trace") 109 | if dfall.iloc[rows[0]]['incidentActionTrace'] != "None": 110 | col4.json(dfall.iloc[rows[0]]['incidentActionTrace']) 111 | -------------------------------------------------------------------------------- /DAT307/ui/page/app.orig.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | from utils.apigw_handler import get_incidents, get_runbook, incident_remediate 3 | from utils.init_session import reset_session 4 | import pandas as pd 5 | import json 6 | 7 | def get_kpi(iconname, metricname, metricvalue): 8 | wch_colour_box = (0,204,102) 9 | wch_colour_font = (0,0,0) 10 | fontsize = 32 11 | valign = "left" 12 | lnk = '' 13 | 14 | htmlstr = f"""

26 | {metricvalue} 27 |
{metricname}

""" 29 | return lnk + htmlstr 30 | 31 | def app_page(): 32 | incidents = get_incidents("pending") 33 | if len(incidents) == 0 : 34 | dfall = pd.DataFrame(columns=["incidentActionTrace","incidentData","incidentStatus","incidentIdentifier","incidentRunbook","incidentTime","sk","incidentType","lastUpdateBy","pk","lastUpdate"]) 35 | else: 36 | dfall = pd.DataFrame(incidents) 37 | 38 | eventCount = len(incidents) 39 | instanceCount = str(dfall['incidentIdentifier'].nunique()) 40 | alertTypeCount = str(dfall['incidentType'].nunique()) 41 | 42 | st.set_page_config(page_title="DAT307-IDR: Amazon RDS Incidents", layout="wide") 43 | 44 | with st.sidebar: 45 | st.sidebar.image("image/idr_logo.png") 46 | st.subheader("DAT307 - Build a Generative AI incident detection and response system powered by Amazon Aurora") 47 | st.divider() 48 | 49 | if st.button("Logout"): 50 | reset_session() 51 | st.rerun() 52 | 53 | st.sidebar.image("image/powered_by_aws.png",width=120) 54 | 55 | 56 | st.title("Incidents") 57 | st.subheader("Metric Summary", divider=True) 58 | col1, col2, col3 = st.columns(3) 59 | #col1.metric(label="Total Pending Events", value=eventCount, delta_color="inverse") 60 | #col2.metric(label="Total Unique Instance", value=instanceCount) 61 | #col3.metric(label="Total Unique Alert Type", value=alertTypeCount) 62 | col1.markdown(get_kpi("fa-solid fa-circle-exclamation","Total Pending Events",eventCount), unsafe_allow_html=True) 63 | col2.markdown(get_kpi("fa-solid fa-server","Total Unique Instance",instanceCount), unsafe_allow_html=True) 64 | col3.markdown(get_kpi("fa-solid fa-bell","Total Unique Alert Type",alertTypeCount), unsafe_allow_html=True) 65 | 66 | col4, col5 = st.columns([3,1]) 67 | col4.markdown("#### Event Summary") 68 | col4.write("Here are the list of active incidents") 69 | col4.write("Please select an incident to process by clicking the first column of the row") 70 | 71 | print("Display table output") 72 | print(dfall) 73 | event = col4.dataframe(dfall, 74 | on_select="rerun", 75 | selection_mode="single-row", 76 | hide_index=True, 77 | column_config={ 78 | "incidentType": "Incident Type", 79 | "pk": "Session ID", 80 | "incidentIdentifier": "Database Instance", 81 | "incidentStatus": "Incident Status", 82 | "incidentTime": "Incident Time" 83 | }, 84 | column_order=("pk","incidentIdentifier","incidentType","incidentStatus","incidentTime") 85 | ) 86 | #col4.markdown("#### Event Details") 87 | #col4.divider() 88 | col5.markdown("#### User Action") 89 | col5.write("Here are the actions that requires manual user intervention") 90 | runbook_action = col5.button("Get Runbook") 91 | remediate_action = col5.button("Remediate Incident") 92 | col5.divider() 93 | rows = event['selection']['rows'] 94 | pk = None 95 | description = None 96 | if len(rows) != 0: 97 | print(dfall) 98 | pk = dfall.iloc[rows[0]]['pk'] 99 | description = json.loads(dfall.iloc[rows[0]]['incidentData'])['configuration']['description'] 100 | print(pk) 101 | #col4.json(dfall.iloc[rows[0]].to_json(orient='records')) 102 | 103 | if runbook_action: 104 | if pk is None: 105 | col4.error("Please select an incident to get the runbook for the incident") 106 | return 107 | with col4.status("Retrieving incident runbook..."): 108 | runbook = get_runbook(pk,description) 109 | col4.markdown("***Runbook Instructions for " + pk + "***") 110 | col4.text_area("Runbook Instructions", runbook['runbook'],height=200, label_visibility="hidden") 111 | 112 | if remediate_action: 113 | if pk is None: 114 | col4.error("Please select an incident to auto-remediate the incident") 115 | return 116 | with col4.status("Remediating incident..."): 117 | incident = incident_remediate(pk,description) 118 | col4.markdown("***Status of auto remediation for " + pk + "***") 119 | col4.json(incident['result']) 120 | 121 | -------------------------------------------------------------------------------- /DAT307/ui/page/login_page.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | from utils.cognito_handler import authenticate_user 3 | 4 | # Pages 5 | def login_page(): 6 | #st.set_page_config(page_title="DAT307-IDR: User Login", layout="wide") 7 | st.set_page_config(page_title="DAT307-IDR: User Login") 8 | st.markdown(""" 9 | 17 | """, unsafe_allow_html=True) 18 | st.image("image/aws_logo.png",width=120) 19 | st.header("DAT307 - Build a Generative AI incident detection and response system powered by Amazon Aurora") 20 | with st.empty().container(border=True): 21 | col1, _, col2 = st.columns([10,1,10]) 22 | 23 | with col1: 24 | st.write("") 25 | st.write("") 26 | st.image("image/incident_management.png") 27 | 28 | with col2: 29 | #st.title("Login Page") 30 | #email = st.text_input("E-mail",value="test1@test.com") 31 | #password = st.text_input("Password", type="password",value="IDR@dat307") 32 | 33 | st.write("") 34 | st.write("") 35 | st.write("") 36 | st.write("") 37 | st.write("") 38 | st.write("") 39 | if st.button("**Login as Demo user**"): 40 | auth, token, message = authenticate_user() 41 | if auth: 42 | st.session_state['authenticated'] = True 43 | st.session_state['token'] = token 44 | st.session_state['page'] = 'pending_incidents' 45 | st.rerun() 46 | else: 47 | st.error(message) 48 | -------------------------------------------------------------------------------- /DAT307/ui/page/login_page.py.orig: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | from utils.cognito_handler import authenticate_user 3 | 4 | # Pages 5 | def login_page(): 6 | #st.set_page_config(page_title="DAT307-IDR: User Login", layout="wide") 7 | st.set_page_config(page_title="DAT307-IDR: User Login") 8 | st.markdown(""" 9 | 17 | """, unsafe_allow_html=True) 18 | st.image("image/aws_logo.png",width=120) 19 | st.header("DAT307 - Build a Generative AI incident detection and response system powered by Amazon Aurora") 20 | with st.empty().container(border=True): 21 | col1, _, col2 = st.columns([10,1,10]) 22 | 23 | with col1: 24 | st.write("") 25 | st.write("") 26 | st.image("image/incident_management.png") 27 | 28 | with col2: 29 | st.title("Login Page") 30 | email = st.text_input("E-mail",value="test1@test.com") 31 | password = st.text_input("Password", type="password",value="IDR@dat307") 32 | 33 | if st.button("Login"): 34 | if not (email and password): 35 | st.error("Please provide email and password") 36 | else: 37 | auth, token, message = authenticate_user(email, password) 38 | if auth: 39 | st.session_state['authenticated'] = True 40 | st.session_state['token'] = token 41 | st.session_state['page'] = 'pending_incidents' 42 | st.rerun() 43 | else: 44 | st.error(message) 45 | if st.button("Sign Up"): 46 | st.session_state['page'] = 'signup' 47 | st.rerun() 48 | -------------------------------------------------------------------------------- /DAT307/ui/page/signup_page.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | import re 3 | from utils.cognito_handler import sign_up_user 4 | from utils.init_session import reset_session 5 | 6 | def is_valid_email(email): 7 | """Check if the provided email is valid using regex.""" 8 | email_regex = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' 9 | return re.match(email_regex, email) is not None 10 | 11 | def input_field(input_param, type): 12 | """Render an input field based on the type and store the value in session state.""" 13 | if type == 'text': 14 | st.session_state[input_param] = st.text_input(input_param) 15 | elif type == 'number': 16 | st.session_state[input_param] = st.number_input(input_param, step=1) 17 | 18 | def signup_page(): 19 | st.set_page_config(page_title="DAT307-IDR: User Registration", layout="wide") 20 | st.markdown(""" 21 | 29 | """, unsafe_allow_html=True) 30 | 31 | """Render the signup page with optional extra input parameters and password confirmation.""" 32 | if st.session_state['verifying']: 33 | auth,message = sign_up_user(st.session_state['email'],st.session_state['password']) 34 | 35 | if auth : 36 | #with st.empty().container(border=True): 37 | #st.title(f"User {st.session_state['email']} created successfully. Please login") 38 | st.success(f"User {st.session_state['email']} created successfully. Please login", icon="✅") 39 | st.session_state['verifying'] = False 40 | else: 41 | print("I am here - showing exception: " + str(st.session_state['verifying'])) 42 | st.session_state["signup_error"] = message 43 | st.session_state['verifying'] = False 44 | st.error(st.session_state['signup_error']) 45 | st.rerun() 46 | 47 | else: 48 | with st.empty().container(border=True): 49 | st.title("Sign Up Page") 50 | 51 | # Email input with validation 52 | st.session_state['email'] = st.text_input("Email") 53 | if st.session_state['email'] and not is_valid_email(st.session_state['email']): 54 | st.error("Please enter a valid email address") 55 | 56 | # Password input 57 | st.session_state['password'] = st.text_input("Password", type='password') 58 | 59 | # Confirm password if required 60 | confirm_password = st.text_input("Confirm Password", type='password') 61 | print("Error state") 62 | print(st.session_state['signup_error']) 63 | if st.session_state['signup_error']: 64 | st.error(st.session_state['signup_error']) 65 | 66 | # Validate all required fields before proceeding 67 | if st.session_state['email'] and st.session_state['password'] and confirm_password \ 68 | and (st.session_state['password'] == confirm_password): 69 | 70 | if st.button("Register"): 71 | st.session_state['verifying'] = True 72 | st.rerun() 73 | else: 74 | if st.session_state['password'] != confirm_password: 75 | st.error("Passwords do not match") 76 | elif st.button("Register"): 77 | st.error("Please fill in all required fields") 78 | print("I am here - empty container") 79 | 80 | print("show another login button, state page is " + st.session_state['page']) 81 | with st.sidebar: 82 | st.sidebar.image("image/idr_logo.png") 83 | st.subheader("DAT307 - Build a Generative AI incident detection and response system powered by Amazon Aurora") 84 | st.divider() 85 | 86 | if st.button("Back to Login"): 87 | print("I am here - session state verifying") 88 | print(st.session_state['page']) 89 | st.session_state['verifying'] = False 90 | st.session_state['page'] = 'login' 91 | st.rerun() 92 | 93 | st.sidebar.image("image/powered_by_aws.png",width=120) -------------------------------------------------------------------------------- /DAT307/ui/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3==1.34.107 2 | botocore==1.34.107 3 | python-dotenv==1.0.1 4 | Requests==2.32.3 5 | streamlit==1.37.1 6 | -------------------------------------------------------------------------------- /DAT307/ui/utils/apigw_handler.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | from dotenv import load_dotenv 3 | import os 4 | import requests 5 | import sys 6 | 7 | load_dotenv() 8 | APIGWURL=os.getenv('APIGWURL') 9 | APIGWSTAGE = os.getenv('APIGWSTAGE') 10 | AWS_REGION=os.getenv('AWS_REGION') 11 | 12 | 13 | def get_incidents(incidentStatus): 14 | headers = {'Authorization': f"{st.session_state['token']}", 'Content-Type': 'application/json'} 15 | url = f'{APIGWURL}{APIGWSTAGE}/get-incidents' 16 | try: 17 | response = requests.get(url,params={"incidentStatus": incidentStatus}, headers = headers) 18 | response.raise_for_status() 19 | items = response.json()['Items'] 20 | if len(items) == 0 : 21 | print("No incidents available") 22 | 23 | return response.json()['Items'] 24 | except requests.exceptions.RequestException as e: 25 | print (f"Error in calling /get-incidents API: {e}") 26 | return None 27 | 28 | 29 | def get_runbook(id, description): 30 | headers = {'Authorization': f"{st.session_state['token']}", 'Content-Type': 'application/json'} 31 | url = f'{APIGWURL}{APIGWSTAGE}/get-incident-runbook' 32 | try: 33 | response = requests.get(url, params={"query": description, "id": id}, headers = headers) 34 | response.raise_for_status() 35 | return response.json() 36 | except requests.exceptions.RequestException as e: 37 | print (f"Error in calling /get-incident-runbook API: {e}") 38 | print(sys.exc_info()) 39 | return { "runbook" : 40 | {"ERROR" : "Error getting the runbook. Please make sure the required FM's are enable"} 41 | } 42 | 43 | def incident_remediate(id, description): 44 | print("Calling the agent to take action") 45 | headers = {'Authorization': f"{st.session_state['token']}", 'Content-Type': 'application/json'} 46 | url = f'{APIGWURL}{APIGWSTAGE}/post-incident-action' 47 | try: 48 | data = {'action':description, 'id': id} 49 | response = requests.post(url, headers = headers, json=data) 50 | response.raise_for_status() 51 | return response.json() 52 | except requests.exceptions.RequestException as e: 53 | print (f"Error in calling /post-incident-action API: {e}") 54 | print(sys.exc_info()) 55 | return {"result": 56 | {"ERROR" : "Error calling the remediate action. Please try preparing the agent"} 57 | } 58 | -------------------------------------------------------------------------------- /DAT307/ui/utils/cognito_handler.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | from dotenv import load_dotenv 3 | import os 4 | import boto3 5 | from botocore.exceptions import ClientError 6 | 7 | load_dotenv() 8 | APP_CLIENT_ID=os.getenv('APP_CLIENT_ID') 9 | USER_POOL_ID = os.getenv('USER_POOL_ID') 10 | AWS_REGION=os.getenv('AWS_REGION') 11 | 12 | cognito_idp_client = boto3.client('cognito-idp', region_name=AWS_REGION) 13 | 14 | def authenticate_user(username="demo@dat307.com", password="Welcome@reInvent2024"): 15 | try: 16 | response = cognito_idp_client.initiate_auth( 17 | ClientId=APP_CLIENT_ID, 18 | AuthFlow='USER_PASSWORD_AUTH', 19 | AuthParameters={ 'USERNAME': username,'PASSWORD': password } 20 | ) 21 | print (response) 22 | return True, response['AuthenticationResult']['IdToken'], None 23 | except ClientError as err: 24 | print(f"Couldn't login {username} due to {err.response['Error']['Message']}") 25 | return False, None, err.response["Error"]["Message"] 26 | 27 | def sign_up_user(username, password): 28 | try: 29 | kwargs = { 30 | "ClientId": APP_CLIENT_ID, 31 | "Username": username, 32 | "Password": password, 33 | "UserAttributes": [{"Name": "email", "Value": username}], 34 | } 35 | response = cognito_idp_client.sign_up(**kwargs) 36 | print(response) 37 | confirmed = response["UserConfirmed"] 38 | print(f"Created the user {username} successfully") 39 | return True, None 40 | 41 | except ClientError as err: 42 | if err.response["Error"]["Code"] == "UsernameExistsException": 43 | print(f"Couldn't sign up {username}. {err.response['Error']['Message']}") 44 | return False, err.response["Error"]["Message"] 45 | 46 | print(f"Couldn't sign up {username}. {err.response['Error']['Message']}") 47 | return False, err.response["Error"]["Message"] 48 | -------------------------------------------------------------------------------- /DAT307/ui/utils/init_session.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | 3 | def init_session(): 4 | if 'authenticated' not in st.session_state: 5 | st.session_state['authenticated'] = False 6 | if 'token' not in st.session_state: 7 | st.session_state['token'] = "" 8 | if 'page' not in st.session_state: 9 | st.session_state['page'] = 'login' 10 | if 'guest_mode' not in st.session_state: 11 | st.session_state['guest_mode'] = False 12 | if 'verifying' not in st.session_state: 13 | st.session_state['verifying'] = False 14 | if 'email' not in st.session_state: 15 | st.session_state['email'] = "" 16 | if 'password' not in st.session_state: 17 | st.session_state['password'] = "" 18 | if 'extra_input_params' not in st.session_state: 19 | st.session_state['extra_input_params'] = {} 20 | 21 | def reset_session(): 22 | st.session_state['authenticated'] = False 23 | st.session_state['page'] = 'login' 24 | st.session_state['guest_mode'] = False 25 | st.session_state['verifying'] = False 26 | st.session_state['otp'] = "" 27 | st.session_state['email'] = "" 28 | st.session_state['password'] = "" 29 | st.session_state['signup_error'] = "" 30 | 31 | -------------------------------------------------------------------------------- /DAT326/env_sample: -------------------------------------------------------------------------------- 1 | ELASTICACHE_HOST='<>' 2 | ELASTICACHE_PORT=6379 3 | DB_HOST='<>' 4 | DB_NAME='<>' 5 | DB_USER='<>' 6 | DB_PASSWORD='<>' 7 | AWS_REGION='us-east-1' -------------------------------------------------------------------------------- /DAT326/requirements.txt: -------------------------------------------------------------------------------- 1 | streamlit 2 | boto3 3 | redis 4 | psycopg 5 | psycopg-binary 6 | uuid 7 | python-dotenv -------------------------------------------------------------------------------- /DAT326/static/AZFlights.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aurora-postgresql-pgvector/81e197513381392765f190dbf0b25b90b2137e09/DAT326/static/AZFlights.jpg -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy of this 7 | software and associated documentation files (the "Software"), to deal in the Software 8 | without restriction, including without limitation the rights to use, copy, modify, 9 | merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so. 11 | 12 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 13 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 14 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 15 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 16 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 17 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Generative AI Use Cases with pgvector, Aurora PostgreSQL and Amazon Bedrock 2 | 3 | [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/) 4 | [![GitHub stars](https://img.shields.io/github/stars/aws-samples/aurora-postgresql-pgvector.svg)](https://github.com/aws-samples/aurora-postgresql-pgvector/stargazers) 5 | [![GitHub forks](https://img.shields.io/github/forks/aws-samples/aurora-postgresql-pgvector.svg)](https://github.com/aws-samples/aurora-postgresql-pgvector/network) 6 | [![GitHub issues](https://img.shields.io/github/issues/aws-samples/aurora-postgresql-pgvector.svg)](https://github.com/aws-samples/aurora-postgresql-pgvector/issues) 7 | [![GitHub pull requests](https://img.shields.io/github/issues-pr/aws-samples/aurora-postgresql-pgvector.svg)](https://github.com/aws-samples/aurora-postgresql-pgvector/pulls) 8 | [![License: MIT-0](https://img.shields.io/badge/License-MIT--0-yellow.svg)](https://spdx.org/licenses/MIT-0.html) 9 | 10 | > Explore powerful Generative AI applications using pgvector on Amazon Aurora PostgreSQL with Amazon Bedrock 11 | 12 | ## 🌟 Overview 13 | 14 | This repository demonstrates production-ready implementations using [**pgvector**](https://github.com/pgvector/pgvector), a powerful open-source PostgreSQL extension for vector similarity search. pgvector seamlessly integrates with PostgreSQL's native features, enabling sophisticated vector operations, indexing, and querying capabilities. 15 | 16 | ## 📚 Resources 17 | 18 | - 📖 [AWS Blog Post: Leverage pgvector and Amazon Aurora PostgreSQL for NLP, Chatbots and Sentiment Analysis](https://aws.amazon.com/blogs/database/leverage-pgvector-and-amazon-aurora-postgresql-for-natural-language-processing-chatbots-and-sentiment-analysis/) 19 | - 🎓 [AWS Workshop: Generative AI Use Cases with Aurora PostgreSQL and pgvector](https://catalog.workshops.aws/pgvector/en-US) 20 | 21 | ## 🚀 Use Cases 22 | 23 | This repository showcases the following production-ready implementations: 24 | 25 | 1. **Product Recommendations** 🛒 26 | - Implement intelligent product recommendation systems 27 | - Leverage vector similarity for personalized suggestions 28 | 29 | 2. **Retrieval Augmented Generation (RAG)** 🔄 30 | - Enhance LLM responses with relevant context 31 | - Implement efficient vector-based information retrieval 32 | 33 | 3. **Semantic Search and Sentiment Analysis** 🧠 34 | - Deploy sophisticated natural language search capabilities 35 | - Perform nuanced sentiment analysis on text data 36 | 37 | 4. **Knowledge Bases for Amazon Bedrock** 📚 38 | - Build scalable knowledge management systems 39 | - Integrate with Amazon Bedrock for enhanced AI capabilities 40 | 41 | 5. **Movie Recommendations** 🎬 42 | - Implement ML-based movie recommendation systems 43 | - Combine Aurora ML with Amazon Bedrock for sophisticated predictions 44 | 45 | 6. **Democratizing Data Insights with Amazon Q Business** 💼 46 | - Connect Amazon Q Business with Aurora PostgreSQL for enterprise-wide data access 47 | - Implement secure data exploration through user management and access control lists (ACLs) 48 | 49 | ## 🛠️ Getting Started 50 | 51 | 1. Clone the repository: 52 | ```bash 53 | git clone https://github.com/aws-samples/aurora-postgresql-pgvector.git 54 | cd aurora-postgresql-pgvector 55 | ``` 56 | 57 | 2. Follow the setup instructions in each use case directory for specific implementation details. 58 | 59 | ## 🤝 Contributing 60 | 61 | This repository is maintained for educational purposes and does not accept external contributions. However, you are encouraged to: 62 | - Fork the repository 63 | - Adapt the code for your specific needs 64 | - Share your learnings with the community 65 | 66 | ## 📄 License 67 | 68 | This project is licensed under the [MIT-0 License](https://spdx.org/licenses/MIT-0.html) - see the [LICENSE](LICENSE) file for details. 69 | 70 | ## 🔗 Related Projects 71 | 72 | - [pgvector](https://github.com/pgvector/pgvector) 73 | - [Amazon Aurora](https://aws.amazon.com/rds/aurora/) 74 | - [Amazon Bedrock](https://aws.amazon.com/bedrock/) 75 | 76 | --- 77 | 78 | **Note**: This repository is provided as-is and is intended for educational and demonstration purposes. 79 | -------------------------------------------------------------------------------- /data/Amazon Aurora FAQs.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aurora-postgresql-pgvector/81e197513381392765f190dbf0b25b90b2137e09/data/Amazon Aurora FAQs.pdf -------------------------------------------------------------------------------- /data/Amazon Bedrock FAQs.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aurora-postgresql-pgvector/81e197513381392765f190dbf0b25b90b2137e09/data/Amazon Bedrock FAQs.pdf -------------------------------------------------------------------------------- /data/postgresql-16-US.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aurora-postgresql-pgvector/81e197513381392765f190dbf0b25b90b2137e09/data/postgresql-16-US.pdf -------------------------------------------------------------------------------- /setup_cloud9.sh: -------------------------------------------------------------------------------- 1 | !/bin/bash 2 | # Install necessary packages 3 | sudo yum update -y 4 | sudo yum install -y git python3 python3-pip postgresql jq 5 | # Set up environment variables 6 | echo "export DB_CLUSTER_IDENTIFIER=${DB_CLUSTER_IDENTIFIER}" >> ~/.bashrc 7 | echo "export DEFAULT_CODE_REPO=${DEFAULT_CODE_REPO}" >> ~/.bashrc 8 | echo "export SECRETARN=${SECRETARN}" >> ~/.bashrc 9 | echo "export AWSREGION=${AWSREGION}" >> ~/.bashrc 10 | # Get DB Endpoint 11 | DBENDP=$(aws rds describe-db-clusters --db-cluster-identifier $DB_CLUSTER_IDENTIFIER --region $AWSREGION --query 'DBClusters[*].Endpoint' | jq -r '.[0]') 12 | echo "export DBENDP=$DBENDP" >> ~/ 13 | # Get database credentials 14 | CREDS=$(aws secretsmanager get-secret-value --secret-id $SECRETARN --region $AWSREGION | jq -r '.SecretString') 15 | DBUSER=$(echo $CREDS | jq -r '.username') 16 | DBPASS=$(echo $CREDS | jq -r '.password') 17 | echo "export PGHOST=$DBENDP" >> ~/.bashrc 18 | echo "export PGUSER=$DBUSER" >> ~/.bashrc 19 | echo "export PGPASSWORD=$DBPASS" >> ~/.bashrc 20 | echo "export PGDATABASE=postgres" >> ~/.bashrc 21 | # Create a .pgpass file for passwordless login 22 | echo "$DBENDP:5432:postgres:$DBUSER:$DBPASS" > ~/.pgpass 23 | chmod 600 ~/.pgpass 24 | # Source the updated .bashrc 25 | source ~/.bashrc 26 | # Print completion message 27 | echo "Cloud9 environment setup completed successfully." 28 | --------------------------------------------------------------------------------