├── .eslintrc.cjs ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── Pipfile ├── README.md ├── THIRD-PARTY-LICENSES ├── amplify.yml ├── amplify ├── .config │ └── project-config.json ├── README.md ├── backend │ ├── api │ │ ├── APIGatewayAuthStack.json │ │ └── clipcrunchapi │ │ │ └── cli-inputs.json │ ├── auth │ │ └── frontend1790dc25 │ │ │ └── cli-inputs.json │ ├── backend-config.json │ ├── custom │ │ ├── mediaConvertRolePolicy │ │ │ └── mediaConvertRolePolicy-cloudformation-template.json │ │ └── vectordbaccess │ │ │ ├── CreateOpensearchIndexLambdaFunction.py │ │ │ ├── convertPythonFileToCloudformationJSONZipfile.py │ │ │ └── vectordbaccess-cloudformation-template.json │ ├── function │ │ ├── CalculateVectorEmbeddingForImagesInS3IndexInOpensearch │ │ │ ├── CalculateVectorEmbeddingForImagesInS3IndexInOpensearch-cloudformation-template.json │ │ │ ├── Pipfile │ │ │ ├── Pipfile.lock │ │ │ ├── amplify.state │ │ │ ├── custom-policies.json │ │ │ ├── function-parameters.json │ │ │ ├── parameters.json │ │ │ └── src │ │ │ │ ├── index.py │ │ │ │ ├── setup.py │ │ │ │ └── src.egg-info │ │ │ │ ├── PKG-INFO │ │ │ │ ├── SOURCES.txt │ │ │ │ ├── dependency_links.txt │ │ │ │ └── top_level.txt │ │ ├── frontendClipCrunchersShared │ │ │ ├── frontendClipCrunchersShared-awscloudformation-template.json │ │ │ ├── layer-configuration.json │ │ │ ├── lib │ │ │ │ └── python │ │ │ │ │ ├── Pipfile │ │ │ │ │ └── Pipfile.lock │ │ │ └── parameters.json │ │ └── frontendf6b68d3c │ │ │ ├── Pipfile │ │ │ ├── Pipfile.lock │ │ │ ├── amplify.state │ │ │ ├── custom-policies.json │ │ │ ├── frontendf6b68d3c-cloudformation-template.json │ │ │ ├── function-parameters.json │ │ │ ├── parameters.json │ │ │ └── src │ │ │ ├── event.json │ │ │ ├── index.py │ │ │ ├── setup.py │ │ │ └── src.egg-info │ │ │ ├── PKG-INFO │ │ │ ├── SOURCES.txt │ │ │ ├── dependency_links.txt │ │ │ └── top_level.txt │ ├── storage │ │ └── s3storage │ │ │ └── cli-inputs.json │ ├── tags.json │ ├── types │ │ └── amplify-dependent-resources-ref.d.ts │ └── vectordb │ │ └── opensearch │ │ ├── parameters.json │ │ └── template.json ├── cli.json └── hooks │ └── README.md ├── clip-crunchers-blog-code.iml ├── diagrams └── blogpost-mm.drawio ├── img ├── solution-architecture.png └── solution.png ├── index.html ├── package-lock.json ├── package.json ├── public └── favicon.ico ├── src ├── App.jsx ├── Routes.jsx ├── components │ ├── NavBar.jsx │ ├── SideBar.jsx │ └── ToolsBar.jsx ├── main.jsx └── pages │ ├── FileUploadPage.jsx │ ├── LivePage.jsx │ ├── SearchPage.css │ ├── SearchPage.jsx │ ├── SummarizePage.jsx │ └── WebcamUploadPage.jsx └── vite.config.js /.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | env: { browser: true, es2020: true }, 4 | extends: [ 5 | 'eslint:recommended', 6 | 'plugin:react/recommended', 7 | 'plugin:react/jsx-runtime', 8 | 'plugin:react-hooks/recommended', 9 | ], 10 | ignorePatterns: ['dist', '.eslintrc.cjs'], 11 | parserOptions: { ecmaVersion: 'latest', sourceType: 'module' }, 12 | settings: { react: { version: '18.2' } }, 13 | plugins: ['react-refresh'], 14 | rules: { 15 | 'react-refresh/only-export-components': [ 16 | 'warn', 17 | { allowConstantExport: true }, 18 | ], 19 | }, 20 | } 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | 26 | #amplify-do-not-edit-begin 27 | amplify/\#current-cloud-backend 28 | amplify/.config/local-* 29 | amplify/logs 30 | amplify/mock-data 31 | amplify/mock-api-resources 32 | amplify/backend/amplify-meta.json 33 | amplify/backend/.temp 34 | build/ 35 | dist/ 36 | node_modules/ 37 | aws-exports.js 38 | awsconfiguration.json 39 | amplifyconfiguration.json 40 | amplifyconfiguration.dart 41 | amplify-build-config.json 42 | amplify-gradle-config.json 43 | amplifytools.xcconfig 44 | .secret-* 45 | **.sample 46 | #amplify-do-not-edit-end 47 | amplify/backend/function/frontendClipCrunchersShared/lib/python/lib/ 48 | amplify/team-provider-info.json 49 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT No Attribution 2 | 3 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 13 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 15 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 16 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | 18 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | 8 | [dev-packages] 9 | 10 | [requires] 11 | python_version = "3.11" 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Serverless Semantic Video Search Using a Vector Database and a Multi-Modal Generative AI Embeddings Model 2 | 3 | You can find the related blogpost to this repository here: 4 | [Implement serverless semantic search of image and live video with Amazon Titan Multimodal Embeddings!](https://aws.amazon.com/blogs/machine-learning/implement-serverless-semantic-search-of-image-and-live-video-with-amazon-titan-multimodal-embeddings/) 5 | 6 | Deploying the infrastructure requires you to have sufficient AWS privileges to do so. 7 | 8 | > [!WARNING] 9 | > **_This example is for experimental purposes only and is not production ready. The deployment of this sample can incur costs. Please ensure to remove infrastructure via the provided scripts when not needed anymore._** 10 | 11 | --- 12 | 13 | ![Screenshot of the solution](/img/solution.png) 14 | 15 | --- 16 | 1. [AWS Account Prerequisites](#aws-account-prerequisites) 17 | 1. [Deploy to Amplify](#deploy-to-amplify) 18 | 2. [Local Development Prerequisites](#local-development-prerequisites) 19 | 3. [Local Build](#local-build) 20 | 4. [Clean Up](#clean-up-resources) 21 | 5. [Usage Instructions](#usage-instructions) 22 | 6. [Solution Walkthrough](#solution-walkthrough) 23 | 24 | ## AWS Account Prerequisites 25 | 26 | * Enabled Model Access for Amazon Bedrock `Titan Multimodal Embeddings G1` using [instructions](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html) 27 | 28 | ## Deploy to Amplify 29 | 30 | [![amplifybutton](https://oneclick.amplifyapp.com/button.svg)](https://us-east-1.console.aws.amazon.com/amplify/home#/deploy?repo=https://github.com/aws-samples/Serverless-Semantic-Video-Search-Vector-Database-and-a-Multi-Modal-Generative-Al-Embeddings-Model) 31 | 32 | * Click the button above to deploy this solution with default parameters directly in your AWS account or [use the Amplify Console to setup Github Access](https://docs.aws.amazon.com/amplify/latest/userguide/setting-up-GitHub-access.html#setting-up-github-app). 33 | * In the select service role section, create a new service role and see [Amplify Service Role](#amplify-service-role) for required permissions used for the deployment role 34 | 35 | > [!CAUTION] 36 | > We advise you to restrict access to branches using a username and password to limit resource consumption by unintended users by following this [guide](https://docs.aws.amazon.com/amplify/latest/userguide/access-control.html). 37 | 38 | * Add a [SPA Redirect](#spa-redirect) 39 | 40 | ### Amplify Service Role 41 | 42 | * Attach **AdministratorAccess** rather than **AdministratorAccess-Amplify** 43 | * ***Optional:*** You can use **AdministratorAccess-Amplify** but add a new IAM policy with additional required permissions which may include: 44 | * "aoss:BatchGetCollection" 45 | * "aoss:CreateAccessPolicy" 46 | * "aoss:CreateCollection" 47 | * "aoss:GetSecurityPolicy" 48 | * "aoss:CreateSecurityPolicy" 49 | * "aoss:DeleteSecurityPolicy" 50 | * "aoss:DeleteCollection" 51 | * "aoss:DeleteAccessPolicy" 52 | * "aoss:TagResource" 53 | * "aoss:UntagResource" 54 | * "kms:Decrypt" 55 | * "kms:Encrypt" 56 | * "kms:DescribeKey" 57 | * "kms:CreateGrant" 58 | 59 | ## Local Development Prerequisites 60 | 61 | * [AWS CLI]((https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html)) 62 | * [python 3.11](https://www.python.org/downloads/release/python-3119/) 63 | * pip 24.0 or higher 64 | * [virtualenv 20.25.0 or higher](https://virtualenv.pypa.io/en/stable/installation.html) 65 | * [node v20.10.0 or higher](https://nodejs.org/en/download) 66 | * npm 10.5.0 or higher 67 | * [amplify CLI 12.10.1 or higher](https://docs.amplify.aws/javascript/tools/cli/start/set-up-cli/) 68 | * Use **us-east-1** for deployment region 69 | * See [Amplify Service Role](#amplify-service-role) for required permissions used for the deployment role 70 | 71 | --- 72 | ## Local Build 73 | 74 | * `amplify init` 75 | * `npm ci` 76 | * `amplify push` 77 | * `npm run dev` 78 | 79 | > [!IMPORTANT] 80 | > We advise you to run the application in a sandbox account and deploy the frontend locally. 81 | 82 | ### [Optional] Manually Deployed Cloud Hosted Frontend 83 | 84 | > [!CAUTION] 85 | > Using the Cloud hosted frontend with the default cognito settings of allowing any user to create and confirm an account will allow any user with knowledge of the deployed URL to upload images/video which has the potential to incur unexpected charges in your AWS account. You can implement a human review of new sign-up requests in cognito by following instructions in the Cognito Developer Guide for [Allowing users to sign up in your app but confirming them as a user pool administrator](https://docs.aws.amazon.com/cognito/latest/developerguide/signing-up-users-in-your-app.html#signing-up-users-in-your-app-and-confirming-them-as-admin) 86 | 87 | * [Deploy and host app](https://docs.amplify.aws/gen1/javascript/start/getting-started/hosting/) 88 | * Add a [SPA Redirect](#spa-redirect) 89 | 90 | ### SPA Redirect 91 | Follow the [instructions](https://docs.aws.amazon.com/amplify/latest/userguide/redirects.html#parts-of-a-redirect) to create a redirect for [single page web apps (SPA)](https://docs.aws.amazon.com/amplify/latest/userguide/redirects.html#redirects-for-single-page-web-apps-spa) 92 | 93 | ## Clean up resources 94 | 95 | * [Full Cleanup Instructions](https://repost.aws/knowledge-center/amplify-delete-application) 96 | * `amplify delete` for local build 97 | 98 | --- 99 | 100 | ### Usage Instructions 101 | 102 | 1. Use the `Sign In` button to log in. Use the `Create Account` tab located at the top of the website to sign up for a new user account with your Amazon Cognito integration. 103 | 104 | 2. After successfully signing in, choose from the left sidebar to upload an image or video: 105 | 106 | #### File Upload 107 | - Click on `Choose files` Button 108 | - Select the images or videos from your local drive 109 | - Click on `Upload Files` 110 | 111 | ### Webcam Upload 112 | - Click `Allow` when your browser asks for permissions to access your webcam 113 | - Click `Capture Image` and `Upload Image` when you want to upload a single image from your webcam 114 | - Click `Start Video Capture`, `Stop Video Capture` and finally `Upload Video` to upload a video from your webcam 115 | 116 | ### Search 117 | 118 | - Type your prompt in the `Search Videos` text field. Depending on your input in previous steps you can prompt i.e. `“Show me a person with glasses”` 119 | - Lower the `confidence parameter` closer to 0, if you see fewer results than you were originally expecting 120 | 121 | 122 | >[!TIP] 123 | >The confidence is not a linear scale from 0 to 100. This confidence represents the vector distance between the user's query and the image in the database where 0 represents completely opposite vectors and 100 represents the same vector datapoint. 124 | 125 | 126 | ## Solution Walkthrough 127 | 128 | ![Screenshot of the solution](/img/solution-architecture.png) 129 | 130 | [Raw Solution Architecture Diagram](/diagrams/blogpost-mm.drawio) 131 | 132 | ### AWS Services Used 133 | * Amazon Opensearch Serverless 134 | * Amazon Bedrock 135 | * AWS Lambda 136 | * AWS S3 137 | * Amazon Cognito 138 | * AWS Elemental MediaConvert 139 | * AWS Amplify [Deploying and hosting frontend and backend] 140 | * Amazon Cloudfront [optional when using cloud hosted front-end] 141 | 142 | ### Manual clip upload process 143 | 1. User manually uploads video clips to S3 bucket (console, CLI or SDK). 144 | 2. S3 Bucket that holds video clips trigger an (s3:ObjectCreated) event for each clip (mp4 or webm) stored in S3. 145 | 3. Lambda function is subscribed to S3 Bucket (s3:ObjectCreated) event and queues up a MediaConvert job to convert the video clip into JPEG images. 146 | 4. Converted images are saved by MediaConvert into an S3 bucket. 147 | 5. S3 Bucket triggers an (s3:ObjectCreated) event for each image (JPEG) stored in S3. 148 | 6. Lambda function is subscribed to the (s3:ObjectCreated) event and generates an embedding using Amazon Titan Multimodal Embeddings, for every new image (JPEG) stored in the S3 Bucket. 149 | 7. Lambda function stores the embeddings in an OpenSearch Serverless index. 150 | 151 | ### Automated video ingestion using Kinesis Video Stream 152 | 1. Alternatively, video clips can be ingested from a video source into a Kinesis Video Data Stream. 153 | 2. Kinesis Video Stream saves the video stream into video clips on the S3 Bucket. This triggers the same above path for steps 2-7. 154 | 155 | ### Website Image Search 156 | 1. Use browses the website. 157 | 2. CloudFront CDN fetches the static web files in S3. 158 | 3. User authenticates and get token from Cognito User Pool. 159 | 4. User makes a search requests to the website, passing the request to the API Gateway. 160 | 5. API Gateway forwards the request to a Lambda Function. 161 | 6. Lambda function passes the search query to Amazon Titan Multimodal Embeddings and converts the request into an embedding. 162 | 7. Lambda function passes the embedding as part of the search, OpenSearch returns matching embeddings and Lambda function returns the matching images to the user. 163 | 164 | ### Website Kinesis Integration 165 | While this solution doesn't create or manage a kinesis video stream, the website does include functionality for displaying a live kinesis video stream and replaying video clips from a kinesis video stream when an image is selected for self-managed kinesis video streams. 166 | 167 | You can turn on this functionality by setting the ***kinesisVideoStreamIntegration*** parameter in the [frontend cloudformation template](amplify/backend/function/frontendf6b68d3c/frontendf6b68d3c-cloudformation-template.json) to **True** and setting ***__KINESIS_VIDEO_STREAM_INTEGRATION__*** to **true** in [vite.config.js](vite.config.js) 168 | 169 | 170 | ## Suggested minimum changes if used in production environments 171 | 172 | > [!WARNING] 173 | > **Making all the changes below does not guarantee a production ready environment. Before using this solution in production, you should carefully review all the resources deployed and their associated configuration to ensure it meets all of your organization's AWS Well Architected Framework requirements.** 174 | 175 | * Opensearch configuration 176 | * [Audit logs should be enabled for Amazon Opensearch.](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/audit-logs.html#audit-log-enabling) 177 | * [Network access for Amazon OpenSearch Serverless](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-network.html) 178 | * AWS S3 configuration 179 | * [Server access logging](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerLogs.html) 180 | * [Lifecycle policies](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lifecycle-mgmt.html) 181 | * [Amazon Macie](https://docs.aws.amazon.com/macie/latest/user/what-is-macie.html) 182 | * Lambda configuration 183 | * [Enabling X-Ray](https://docs.aws.amazon.com/xray/latest/devguide/xray-services-lambda.html) 184 | * IAM configuration 185 | * [Permissions boundaries](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_boundaries.html) 186 | * Cognito configuration 187 | * [MFA configuration](https://docs.aws.amazon.com/cognito/latest/developerguide/user-pool-settings-mfa.html) 188 | 189 | ## Security 190 | 191 | See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. 192 | 193 | ## License 194 | 195 | This library is licensed under the MIT-0 License. See the LICENSE file. 196 | -------------------------------------------------------------------------------- /THIRD-PARTY-LICENSES: -------------------------------------------------------------------------------- 1 | The Serverless Semantic Video Search Using a Vector Database and a Multi-Modal Generative AI Embeddings Model Product includes the following third-party software/licensing: 2 | 3 | ** buffer - https://github.com/feross/buffer 4 | Copyright (c) Feross Aboukhadijeh, and other contributors. 5 | ** react - github.com/facebook/react 6 | Copyright (c) Meta Platforms, Inc. and affiliates. 7 | ** react-dom - https://github.com/facebook/react 8 | Copyright (c) Meta Platforms, Inc. and affiliates. 9 | ** react-hls-player - https://github.com/devcshort/react-hls 10 | Copyright (c) 2019 Christopher Short 11 | ** react-router-dom - https://github.com/remix-run/react-router 12 | Copyright (c) React Training LLC 2015-2019 Copyright (c) Remix Software Inc. 2020-2021 Copyright (c) Shopify Inc. 2022-2023 13 | ** react-webcam - https://github.com/mozmorris/react-webcam 14 | Copyright (c) 2018 Moz Morris 15 | 16 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 17 | 18 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 19 | 20 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | 22 | ** boto3 - https://github.com/boto/boto3 23 | ** aws-sdk-pandas - https://github.com/aws/aws-sdk-pandas 24 | ** amplify-ui - https://github.com/aws-amplify/amplify-ui 25 | ** cloudscape-design - https://github.com/cloudscape-design/components 26 | ** aws-amplify - https://github.com/aws-amplify/amplify-js 27 | 28 | Apache License 29 | Version 2.0, January 2004 30 | http://www.apache.org/licenses/ 31 | 32 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 33 | 34 | 1. Definitions. 35 | 36 | "License" shall mean the terms and conditions for use, reproduction, 37 | and distribution as defined by Sections 1 through 9 of this document. 38 | 39 | "Licensor" shall mean the copyright owner or entity authorized by 40 | the copyright owner that is granting the License. 41 | 42 | "Legal Entity" shall mean the union of the acting entity and all 43 | other entities that control, are controlled by, or are under common 44 | control with that entity. For the purposes of this definition, 45 | "control" means (i) the power, direct or indirect, to cause the 46 | direction or management of such entity, whether by contract or 47 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 48 | outstanding shares, or (iii) beneficial ownership of such entity. 49 | 50 | "You" (or "Your") shall mean an individual or Legal Entity 51 | exercising permissions granted by this License. 52 | 53 | "Source" form shall mean the preferred form for making modifications, 54 | including but not limited to software source code, documentation 55 | source, and configuration files. 56 | 57 | "Object" form shall mean any form resulting from mechanical 58 | transformation or translation of a Source form, including but 59 | not limited to compiled object code, generated documentation, 60 | and conversions to other media types. 61 | 62 | "Work" shall mean the work of authorship, whether in Source or 63 | Object form, made available under the License, as indicated by a 64 | copyright notice that is included in or attached to the work 65 | (an example is provided in the Appendix below). 66 | 67 | "Derivative Works" shall mean any work, whether in Source or Object 68 | form, that is based on (or derived from) the Work and for which the 69 | editorial revisions, annotations, elaborations, or other modifications 70 | represent, as a whole, an original work of authorship. For the purposes 71 | of this License, Derivative Works shall not include works that remain 72 | separable from, or merely link (or bind by name) to the interfaces of, 73 | the Work and Derivative Works thereof. 74 | 75 | "Contribution" shall mean any work of authorship, including 76 | the original version of the Work and any modifications or additions 77 | to that Work or Derivative Works thereof, that is intentionally 78 | submitted to Licensor for inclusion in the Work by the copyright owner 79 | or by an individual or Legal Entity authorized to submit on behalf of 80 | the copyright owner. For the purposes of this definition, "submitted" 81 | means any form of electronic, verbal, or written communication sent 82 | to the Licensor or its representatives, including but not limited to 83 | communication on electronic mailing lists, source code control systems, 84 | and issue tracking systems that are managed by, or on behalf of, the 85 | Licensor for the purpose of discussing and improving the Work, but 86 | excluding communication that is conspicuously marked or otherwise 87 | designated in writing by the copyright owner as "Not a Contribution." 88 | 89 | "Contributor" shall mean Licensor and any individual or Legal Entity 90 | on behalf of whom a Contribution has been received by Licensor and 91 | subsequently incorporated within the Work. 92 | 93 | 2. Grant of Copyright License. Subject to the terms and conditions of 94 | this License, each Contributor hereby grants to You a perpetual, 95 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 96 | copyright license to reproduce, prepare Derivative Works of, 97 | publicly display, publicly perform, sublicense, and distribute the 98 | Work and such Derivative Works in Source or Object form. 99 | 100 | 3. Grant of Patent License. Subject to the terms and conditions of 101 | this License, each Contributor hereby grants to You a perpetual, 102 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 103 | (except as stated in this section) patent license to make, have made, 104 | use, offer to sell, sell, import, and otherwise transfer the Work, 105 | where such license applies only to those patent claims licensable 106 | by such Contributor that are necessarily infringed by their 107 | Contribution(s) alone or by combination of their Contribution(s) 108 | with the Work to which such Contribution(s) was submitted. If You 109 | institute patent litigation against any entity (including a 110 | cross-claim or counterclaim in a lawsuit) alleging that the Work 111 | or a Contribution incorporated within the Work constitutes direct 112 | or contributory patent infringement, then any patent licenses 113 | granted to You under this License for that Work shall terminate 114 | as of the date such litigation is filed. 115 | 116 | 4. Redistribution. You may reproduce and distribute copies of the 117 | Work or Derivative Works thereof in any medium, with or without 118 | modifications, and in Source or Object form, provided that You 119 | meet the following conditions: 120 | 121 | (a) You must give any other recipients of the Work or 122 | Derivative Works a copy of this License; and 123 | 124 | (b) You must cause any modified files to carry prominent notices 125 | stating that You changed the files; and 126 | 127 | (c) You must retain, in the Source form of any Derivative Works 128 | that You distribute, all copyright, patent, trademark, and 129 | attribution notices from the Source form of the Work, 130 | excluding those notices that do not pertain to any part of 131 | the Derivative Works; and 132 | 133 | (d) If the Work includes a "NOTICE" text file as part of its 134 | distribution, then any Derivative Works that You distribute must 135 | include a readable copy of the attribution notices contained 136 | within such NOTICE file, excluding those notices that do not 137 | pertain to any part of the Derivative Works, in at least one 138 | of the following places: within a NOTICE text file distributed 139 | as part of the Derivative Works; within the Source form or 140 | documentation, if provided along with the Derivative Works; or, 141 | within a display generated by the Derivative Works, if and 142 | wherever such third-party notices normally appear. The contents 143 | of the NOTICE file are for informational purposes only and 144 | do not modify the License. You may add Your own attribution 145 | notices within Derivative Works that You distribute, alongside 146 | or as an addendum to the NOTICE text from the Work, provided 147 | that such additional attribution notices cannot be construed 148 | as modifying the License. 149 | 150 | You may add Your own copyright statement to Your modifications and 151 | may provide additional or different license terms and conditions 152 | for use, reproduction, or distribution of Your modifications, or 153 | for any such Derivative Works as a whole, provided Your use, 154 | reproduction, and distribution of the Work otherwise complies with 155 | the conditions stated in this License. 156 | 157 | 5. Submission of Contributions. Unless You explicitly state otherwise, 158 | any Contribution intentionally submitted for inclusion in the Work 159 | by You to the Licensor shall be under the terms and conditions of 160 | this License, without any additional terms or conditions. 161 | Notwithstanding the above, nothing herein shall supersede or modify 162 | the terms of any separate license agreement you may have executed 163 | with Licensor regarding such Contributions. 164 | 165 | 6. Trademarks. This License does not grant permission to use the trade 166 | names, trademarks, service marks, or product names of the Licensor, 167 | except as required for reasonable and customary use in describing the 168 | origin of the Work and reproducing the content of the NOTICE file. 169 | 170 | 7. Disclaimer of Warranty. Unless required by applicable law or 171 | agreed to in writing, Licensor provides the Work (and each 172 | Contributor provides its Contributions) on an "AS IS" BASIS, 173 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 174 | implied, including, without limitation, any warranties or conditions 175 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 176 | PARTICULAR PURPOSE. You are solely responsible for determining the 177 | appropriateness of using or redistributing the Work and assume any 178 | risks associated with Your exercise of permissions under this License. 179 | 180 | 8. Limitation of Liability. In no event and under no legal theory, 181 | whether in tort (including negligence), contract, or otherwise, 182 | unless required by applicable law (such as deliberate and grossly 183 | negligent acts) or agreed to in writing, shall any Contributor be 184 | liable to You for damages, including any direct, indirect, special, 185 | incidental, or consequential damages of any character arising as a 186 | result of this License or out of the use or inability to use the 187 | Work (including but not limited to damages for loss of goodwill, 188 | work stoppage, computer failure or malfunction, or any and all 189 | other commercial damages or losses), even if such Contributor 190 | has been advised of the possibility of such damages. 191 | 192 | 9. Accepting Warranty or Additional Liability. While redistributing 193 | the Work or Derivative Works thereof, You may choose to offer, 194 | and charge a fee for, acceptance of support, warranty, indemnity, 195 | or other liability obligations and/or rights consistent with this 196 | License. However, in accepting such obligations, You may act only 197 | on Your own behalf and on Your sole responsibility, not on behalf 198 | of any other Contributor, and only if You agree to indemnify, 199 | defend, and hold each Contributor harmless for any liability 200 | incurred by, or claims asserted against, such Contributor by reason 201 | of your accepting any such warranty or additional liability. -------------------------------------------------------------------------------- /amplify.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | backend: 3 | phases: 4 | preBuild: 5 | commands: 6 | - '# Install pre-requisites' 7 | - sudo dnf install python3.11 -y 8 | - sudo dnf install python3.11-pip -y 9 | - pip3.11 install --user pipenv 10 | - PATH="/root/.local/bin:${PATH}" 11 | - export PATH 12 | build: 13 | commands: 14 | - '# Execute Amplify CLI with the helper script' 15 | - amplifyPush --simple 16 | frontend: 17 | phases: 18 | preBuild: 19 | commands: 20 | - npm ci --cache .npm --prefer-offline 21 | build: 22 | commands: 23 | - npm run build 24 | artifacts: 25 | baseDirectory: dist 26 | files: 27 | - '**/*' 28 | cache: 29 | paths: 30 | - .npm/**/* -------------------------------------------------------------------------------- /amplify/.config/project-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "projectName": "ClipCrunchers", 3 | "version": "3.1", 4 | "frontend": "javascript", 5 | "javascript": { 6 | "framework": "react", 7 | "config": { 8 | "SourceDir": "src", 9 | "DistributionDir": "dist", 10 | "BuildCommand": "npm run-script build", 11 | "StartCommand": "npm run-script start" 12 | } 13 | }, 14 | "providers": [ 15 | "awscloudformation" 16 | ] 17 | } -------------------------------------------------------------------------------- /amplify/README.md: -------------------------------------------------------------------------------- 1 | # Getting Started with Amplify CLI 2 | This directory was generated by [Amplify CLI](https://docs.amplify.aws/cli). 3 | 4 | Helpful resources: 5 | - Amplify documentation: https://docs.amplify.aws. 6 | - Amplify CLI documentation: https://docs.amplify.aws/cli. 7 | - More details on this folder & generated files: https://docs.amplify.aws/cli/reference/files. 8 | - Join Amplify's community: https://amplify.aws/community/. 9 | -------------------------------------------------------------------------------- /amplify/backend/api/APIGatewayAuthStack.json: -------------------------------------------------------------------------------- 1 | { 2 | "Description": "API Gateway policy stack created using Amplify CLI", 3 | "AWSTemplateFormatVersion": "2010-09-09", 4 | "Parameters": { 5 | "authRoleName": { 6 | "Type": "String" 7 | }, 8 | "unauthRoleName": { 9 | "Type": "String" 10 | }, 11 | "env": { 12 | "Type": "String" 13 | }, 14 | "clipcrunchapi": { 15 | "Type": "String" 16 | } 17 | }, 18 | "Conditions": { 19 | "ShouldNotCreateEnvResources": { 20 | "Fn::Equals": [ 21 | { 22 | "Ref": "env" 23 | }, 24 | "NONE" 25 | ] 26 | } 27 | }, 28 | "Resources": { 29 | "PolicyAPIGWAuth1": { 30 | "Type": "AWS::IAM::ManagedPolicy", 31 | "Properties": { 32 | "PolicyDocument": { 33 | "Version": "2012-10-17", 34 | "Statement": [ 35 | { 36 | "Effect": "Allow", 37 | "Action": [ 38 | "execute-api:Invoke" 39 | ], 40 | "Resource": [ 41 | { 42 | "Fn::Join": [ 43 | "", 44 | [ 45 | "arn:aws:execute-api:", 46 | { 47 | "Ref": "AWS::Region" 48 | }, 49 | ":", 50 | { 51 | "Ref": "AWS::AccountId" 52 | }, 53 | ":", 54 | { 55 | "Ref": "clipcrunchapi" 56 | }, 57 | "/", 58 | { 59 | "Fn::If": [ 60 | "ShouldNotCreateEnvResources", 61 | "Prod", 62 | { 63 | "Ref": "env" 64 | } 65 | ] 66 | }, 67 | "/*/images/*" 68 | ] 69 | ] 70 | }, 71 | { 72 | "Fn::Join": [ 73 | "", 74 | [ 75 | "arn:aws:execute-api:", 76 | { 77 | "Ref": "AWS::Region" 78 | }, 79 | ":", 80 | { 81 | "Ref": "AWS::AccountId" 82 | }, 83 | ":", 84 | { 85 | "Ref": "clipcrunchapi" 86 | }, 87 | "/", 88 | { 89 | "Fn::If": [ 90 | "ShouldNotCreateEnvResources", 91 | "Prod", 92 | { 93 | "Ref": "env" 94 | } 95 | ] 96 | }, 97 | "/*/images" 98 | ] 99 | ] 100 | } 101 | ] 102 | } 103 | ] 104 | }, 105 | "Roles": [ 106 | { 107 | "Ref": "authRoleName" 108 | } 109 | ] 110 | } 111 | } 112 | } 113 | } -------------------------------------------------------------------------------- /amplify/backend/api/clipcrunchapi/cli-inputs.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "paths": { 4 | "/images": { 5 | "name": "/images", 6 | "lambdaFunction": "frontendf6b68d3c", 7 | "permissions": { 8 | "setting": "private", 9 | "auth": [ 10 | "create", 11 | "read", 12 | "update", 13 | "delete" 14 | ] 15 | } 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /amplify/backend/auth/frontend1790dc25/cli-inputs.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "cognitoConfig": { 4 | "identityPoolName": "frontend1790dc25_identitypool_1790dc25", 5 | "allowUnauthenticatedIdentities": false, 6 | "resourceNameTruncated": "fronte1790dc25", 7 | "userPoolName": "frontend1790dc25_userpool_1790dc25", 8 | "autoVerifiedAttributes": [ 9 | "email" 10 | ], 11 | "mfaConfiguration": "OFF", 12 | "mfaTypes": [ 13 | "SMS Text Message" 14 | ], 15 | "smsAuthenticationMessage": "Your authentication code is {####}", 16 | "smsVerificationMessage": "Your verification code is {####}", 17 | "emailVerificationSubject": "Your verification code", 18 | "emailVerificationMessage": "Your verification code is {####}", 19 | "defaultPasswordPolicy": false, 20 | "passwordPolicyMinLength": 8, 21 | "passwordPolicyCharacters": [], 22 | "requiredAttributes": [], 23 | "aliasAttributes": [], 24 | "userpoolClientGenerateSecret": false, 25 | "userpoolClientRefreshTokenValidity": 30, 26 | "userpoolClientWriteAttributes": [ 27 | "email" 28 | ], 29 | "userpoolClientReadAttributes": [ 30 | "email" 31 | ], 32 | "userpoolClientLambdaRole": "fronte1790dc25_userpoolclient_lambda_role", 33 | "userpoolClientSetAttributes": false, 34 | "sharedId": "1790dc25", 35 | "resourceName": "frontend1790dc25", 36 | "authSelections": "identityPoolAndUserPool", 37 | "useDefault": "default", 38 | "hostedUI": false, 39 | "triggers": {}, 40 | "userPoolGroupList": [], 41 | "serviceName": "Cognito", 42 | "usernameCaseSensitive": false, 43 | "useEnabledMfas": false, 44 | "authRoleArn": { 45 | "Fn::GetAtt": [ 46 | "AuthRole", 47 | "Arn" 48 | ] 49 | }, 50 | "unauthRoleArn": { 51 | "Fn::GetAtt": [ 52 | "UnauthRole", 53 | "Arn" 54 | ] 55 | }, 56 | "breakCircularDependency": false, 57 | "dependsOn": [], 58 | "parentStack": { 59 | "Ref": "AWS::StackId" 60 | }, 61 | "permissions": [], 62 | "userPoolGroups": false, 63 | "adminQueries": false, 64 | "authProviders": [] 65 | } 66 | } -------------------------------------------------------------------------------- /amplify/backend/backend-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "api": { 3 | "clipcrunchapi": { 4 | "dependsOn": [ 5 | { 6 | "attributes": [ 7 | "Name", 8 | "Arn" 9 | ], 10 | "category": "function", 11 | "resourceName": "frontendf6b68d3c" 12 | } 13 | ], 14 | "providerPlugin": "awscloudformation", 15 | "service": "API Gateway" 16 | } 17 | }, 18 | "auth": { 19 | "frontend1790dc25": { 20 | "customAuth": false, 21 | "dependsOn": [], 22 | "frontendAuthConfig": { 23 | "mfaConfiguration": "OFF", 24 | "mfaTypes": [ 25 | "SMS" 26 | ], 27 | "passwordProtectionSettings": { 28 | "passwordPolicyCharacters": [], 29 | "passwordPolicyMinLength": 8 30 | }, 31 | "signupAttributes": [], 32 | "socialProviders": [], 33 | "usernameAttributes": [], 34 | "verificationMechanisms": [ 35 | "EMAIL" 36 | ] 37 | }, 38 | "providerPlugin": "awscloudformation", 39 | "service": "Cognito" 40 | } 41 | }, 42 | "custom": { 43 | "mediaConvertRolePolicy": { 44 | "dependsOn": [ 45 | { 46 | "attributes": [ 47 | "Name", 48 | "Arn", 49 | "Region", 50 | "LambdaExecutionRole", 51 | "LambdaExecutionRoleArn", 52 | "MediaConvertExecutionRole" 53 | ], 54 | "category": "function", 55 | "resourceName": "CalculateVectorEmbeddingForImagesInS3IndexInOpensearch" 56 | }, 57 | { 58 | "attributes": [ 59 | "BucketName", 60 | "Region" 61 | ], 62 | "category": "storage", 63 | "resourceName": "s3storage" 64 | } 65 | ], 66 | "providerPlugin": "awscloudformation", 67 | "service": "customCloudformation" 68 | }, 69 | "vectordbaccess": { 70 | "dependsOn": [ 71 | { 72 | "attributes": [ 73 | "Name", 74 | "Arn", 75 | "Region", 76 | "LambdaExecutionRole", 77 | "LambdaExecutionRoleArn" 78 | ], 79 | "category": "function", 80 | "resourceName": "CalculateVectorEmbeddingForImagesInS3IndexInOpensearch" 81 | }, 82 | { 83 | "attributes": [ 84 | "Name", 85 | "Arn", 86 | "Region", 87 | "LambdaExecutionRole", 88 | "LambdaExecutionRoleArn" 89 | ], 90 | "category": "function", 91 | "resourceName": "frontendf6b68d3c" 92 | }, 93 | { 94 | "attributes": [ 95 | "BucketName", 96 | "Region" 97 | ], 98 | "category": "storage", 99 | "resourceName": "s3storage" 100 | }, 101 | { 102 | "attributes": [ 103 | "CollectionEndpoint", 104 | "opensearchServerlessCollectionARN", 105 | "opensearchServerlessCollectionName", 106 | "opensearchServerlessIndexName" 107 | ], 108 | "category": "vectordb", 109 | "resourceName": "opensearch" 110 | } 111 | ], 112 | "providerPlugin": "awscloudformation", 113 | "service": "customCloudformation" 114 | } 115 | }, 116 | "function": { 117 | "CalculateVectorEmbeddingForImagesInS3IndexInOpensearch": { 118 | "build": true, 119 | "dependsOn": [ 120 | { 121 | "attributes": [ 122 | "Arn" 123 | ], 124 | "category": "function", 125 | "resourceName": "frontendClipCrunchersShared" 126 | }, 127 | { 128 | "attributes": [ 129 | "CollectionEndpoint", 130 | "opensearchServerlessCollectionARN", 131 | "opensearchServerlessCollectionName", 132 | "opensearchServerlessIndexName" 133 | ], 134 | "category": "vectordb", 135 | "resourceName": "opensearch" 136 | } 137 | ], 138 | "providerPlugin": "awscloudformation", 139 | "service": "Lambda" 140 | }, 141 | "frontendClipCrunchersShared": { 142 | "build": true, 143 | "providerPlugin": "awscloudformation", 144 | "service": "LambdaLayer" 145 | }, 146 | "frontendf6b68d3c": { 147 | "build": true, 148 | "dependsOn": [ 149 | { 150 | "attributes": [ 151 | "BucketName" 152 | ], 153 | "category": "storage", 154 | "resourceName": "s3storage" 155 | }, 156 | { 157 | "attributes": [ 158 | "Arn" 159 | ], 160 | "category": "function", 161 | "resourceName": "frontendClipCrunchersShared" 162 | }, 163 | { 164 | "attributes": [ 165 | "CollectionEndpoint", 166 | "opensearchServerlessCollectionARN", 167 | "opensearchServerlessCollectionName", 168 | "opensearchServerlessIndexName" 169 | ], 170 | "category": "vectordb", 171 | "resourceName": "opensearch" 172 | } 173 | ], 174 | "providerPlugin": "awscloudformation", 175 | "service": "Lambda" 176 | } 177 | }, 178 | "hosting": {}, 179 | "parameters": { 180 | "AMPLIFY_function_CalculateVectorEmbeddingForImagesInS3IndexInOpensearch_deploymentBucketName": { 181 | "usedBy": [ 182 | { 183 | "category": "function", 184 | "resourceName": "CalculateVectorEmbeddingForImagesInS3IndexInOpensearch" 185 | } 186 | ] 187 | }, 188 | "AMPLIFY_function_CalculateVectorEmbeddingForImagesInS3IndexInOpensearch_s3Key": { 189 | "usedBy": [ 190 | { 191 | "category": "function", 192 | "resourceName": "CalculateVectorEmbeddingForImagesInS3IndexInOpensearch" 193 | } 194 | ] 195 | }, 196 | "AMPLIFY_function_frontendClipCrunchersShared_deploymentBucketName": { 197 | "usedBy": [ 198 | { 199 | "category": "function", 200 | "resourceName": "frontendClipCrunchersShared" 201 | } 202 | ] 203 | }, 204 | "AMPLIFY_function_frontendClipCrunchersShared_s3Key": { 205 | "usedBy": [ 206 | { 207 | "category": "function", 208 | "resourceName": "frontendClipCrunchersShared" 209 | } 210 | ] 211 | }, 212 | "AMPLIFY_function_frontendf6b68d3c_deploymentBucketName": { 213 | "usedBy": [ 214 | { 215 | "category": "function", 216 | "resourceName": "frontendf6b68d3c" 217 | } 218 | ] 219 | }, 220 | "AMPLIFY_function_frontendf6b68d3c_s3Key": { 221 | "usedBy": [ 222 | { 223 | "category": "function", 224 | "resourceName": "frontendf6b68d3c" 225 | } 226 | ] 227 | } 228 | }, 229 | "storage": { 230 | "s3storage": { 231 | "dependsOn": [ 232 | { 233 | "attributes": [ 234 | "Name", 235 | "Arn", 236 | "LambdaExecutionRole" 237 | ], 238 | "category": "function", 239 | "resourceName": "CalculateVectorEmbeddingForImagesInS3IndexInOpensearch" 240 | } 241 | ], 242 | "providerPlugin": "awscloudformation", 243 | "service": "S3" 244 | } 245 | }, 246 | "vectordb": { 247 | "opensearch": { 248 | "providerPlugin": "awscloudformation" 249 | } 250 | } 251 | } -------------------------------------------------------------------------------- /amplify/backend/custom/mediaConvertRolePolicy/mediaConvertRolePolicy-cloudformation-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Parameters": { 4 | "env": { 5 | "Type": "String" 6 | }, 7 | "functionCalculateVectorEmbeddingForImagesInS3IndexInOpensearchName": { 8 | "Type": "String", 9 | "Description": "Input parameter describing Name attribute for function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch resource" 10 | }, 11 | "functionCalculateVectorEmbeddingForImagesInS3IndexInOpensearchArn": { 12 | "Type": "String", 13 | "Description": "Input parameter describing Arn attribute for function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch resource" 14 | }, 15 | "functionCalculateVectorEmbeddingForImagesInS3IndexInOpensearchRegion": { 16 | "Type": "String", 17 | "Description": "Input parameter describing Region attribute for function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch resource" 18 | }, 19 | "functionCalculateVectorEmbeddingForImagesInS3IndexInOpensearchLambdaExecutionRole": { 20 | "Type": "String", 21 | "Description": "Input parameter describing LambdaExecutionRole attribute for function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch resource" 22 | }, 23 | "functionCalculateVectorEmbeddingForImagesInS3IndexInOpensearchLambdaExecutionRoleArn": { 24 | "Type": "String", 25 | "Description": "Input parameter describing LambdaExecutionRoleArn attribute for function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch resource" 26 | }, 27 | "functionCalculateVectorEmbeddingForImagesInS3IndexInOpensearchMediaConvertExecutionRole": { 28 | "Type": "String", 29 | "Description": "Input parameter describing LambdaExecutionRole attribute for function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch resource" 30 | }, 31 | "storages3storageBucketName": { 32 | "Type": "String", 33 | "Description": "Input parameter describing BucketName attribute for storage/s3storage resource" 34 | }, 35 | "storages3storageRegion": { 36 | "Type": "String", 37 | "Description": "Input parameter describing Region attribute for storage/s3storage resource" 38 | } 39 | }, 40 | "Resources": { 41 | "MediaConvertExecutionRoleExecutionPolicy": { 42 | "Type": "AWS::IAM::Policy", 43 | "Properties": { 44 | "PolicyName": "clip-crunchers-media-convert-execution-policy", 45 | "PolicyDocument": { 46 | "Version": "2012-10-17", 47 | "Statement": [ 48 | { 49 | "Action": [ 50 | "s3:Get*", 51 | "s3:List*", 52 | "s3:Put*" 53 | ], 54 | "Resource": [ 55 | { 56 | "Fn::Sub": "arn:${AWS::Partition}:s3:::${storages3storageBucketName}/*" 57 | } 58 | ], 59 | "Effect": "Allow" 60 | } 61 | ] 62 | }, 63 | "Roles": [ 64 | { 65 | "Ref": "functionCalculateVectorEmbeddingForImagesInS3IndexInOpensearchMediaConvertExecutionRole" 66 | } 67 | ] 68 | } 69 | } 70 | }, 71 | "Description": "{\"createdOn\":\"Mac\",\"createdBy\":\"Amplify\",\"createdWith\":\"12.12.4\",\"stackType\":\"custom-customCloudformation\",\"metadata\":{}}" 72 | } -------------------------------------------------------------------------------- /amplify/backend/custom/vectordbaccess/CreateOpensearchIndexLambdaFunction.py: -------------------------------------------------------------------------------- 1 | import cfnresponse 2 | from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth 3 | import os 4 | import boto3 5 | import time 6 | 7 | 8 | 9 | def initialize_opensearch_client(): 10 | opensearch_endpoint = os.environ['OpensearchEndpoint'].replace('https://','',1) # serverless collection endpoint, without https:// 11 | print(f'Opensearch Endpoint: {opensearch_endpoint}') 12 | region = os.environ['AWS_REGION'] # e.g. us-east-1 13 | print(f'region: {region}') 14 | credentials = boto3.Session().get_credentials() 15 | identity = boto3.client('sts').get_caller_identity() 16 | print(f'Caller Identity: {identity}') 17 | 18 | auth = AWSV4SignerAuth(credentials, region, 'aoss') 19 | 20 | client = OpenSearch( 21 | hosts=[{'host': opensearch_endpoint, 'port': 443}], 22 | http_auth=auth, 23 | use_ssl=True, 24 | verify_certs=True, 25 | connection_class=RequestsHttpConnection, 26 | pool_maxsize=20, 27 | ) 28 | 29 | return client 30 | 31 | function_initialization_success = False 32 | try: 33 | print('initializing function') 34 | OPENSEARCH_CLIENT = initialize_opensearch_client() 35 | OPENSEARCH_INDEX_NAME = os.environ['OpensearchIndexName'] 36 | print(f'OPENSEARCH_INDEX_NAME: {OPENSEARCH_INDEX_NAME}') 37 | S3_BUCKET_NAME = os.environ['S3StorageBucketName'] 38 | print(f'S3_BUCKET_NAME: {S3_BUCKET_NAME}') 39 | MEDIA_CONVERT_JOB_TEMPLATE_NAME = os.environ['MediaConvertJobTemplateName'] 40 | print(f'MEDIA_CONVERT_JOB_TEMPLATE_NAME: {MEDIA_CONVERT_JOB_TEMPLATE_NAME}') 41 | MEDIA_CONVERT_FRAME_RATE_NUMERATOR = os.environ['MediaConvertFrameRateNumerator'] 42 | print(f'MEDIA_CONVERT_FRAME_RATE_NUMERATOR: {MEDIA_CONVERT_FRAME_RATE_NUMERATOR}') 43 | MEDIA_CONVERT_FRAME_RATE_DENOMINATOR = os.environ['MediaConvertFrameRateDenominator'] 44 | print(f'MEDIA_CONVERT_FRAME_RATE_DENOMINATOR: {MEDIA_CONVERT_FRAME_RATE_DENOMINATOR}') 45 | function_initialization_success = True 46 | except Exception as e: 47 | print('Exception occurred in function initialization') 48 | print(e) 49 | 50 | def create_mediaconvert_job_template(): 51 | template = { 52 | 'Description': 'Takes a video file from s3 and outputs the image frames to s3', 53 | 'Name': MEDIA_CONVERT_JOB_TEMPLATE_NAME, 54 | 'Settings': { 55 | 'TimecodeConfig': { 56 | 'Source': 'ZEROBASED' 57 | }, 58 | 'OutputGroups': [ 59 | { 60 | 'CustomName': 'Archive Video', 61 | 'Name': 'File Group', 62 | 'Outputs': [ 63 | { 64 | 'ContainerSettings': { 65 | 'Container': 'MP4', 66 | 'Mp4Settings': {} 67 | }, 68 | 'VideoDescription': { 69 | 'CodecSettings': { 70 | 'Codec': 'H_264', 71 | 'H264Settings': { 72 | 'MaxBitrate': 1000, 73 | 'RateControlMode': 'QVBR', 74 | 'SceneChangeDetect': 'TRANSITION_DETECTION' 75 | } 76 | } 77 | } 78 | } 79 | ], 80 | 'OutputGroupSettings': { 81 | 'Type': 'FILE_GROUP_SETTINGS', 82 | 'FileGroupSettings': { 83 | 'Destination': f's3://{S3_BUCKET_NAME}/mediaconvert-archive/', 84 | 'DestinationSettings': { 85 | 'S3Settings': { 86 | 'Encryption': { 87 | 'EncryptionType': 'SERVER_SIDE_ENCRYPTION_S3' 88 | } 89 | } 90 | } 91 | } 92 | } 93 | }, 94 | { 95 | 'CustomName': 'JPG Frames', 96 | 'Name': 'File Group', 97 | 'Outputs': [ 98 | { 99 | 'ContainerSettings': { 100 | 'Container': 'RAW' 101 | }, 102 | 'VideoDescription': { 103 | 'CodecSettings': { 104 | 'Codec': 'FRAME_CAPTURE', 105 | 'FrameCaptureSettings': { 106 | 'FramerateNumerator': int(MEDIA_CONVERT_FRAME_RATE_NUMERATOR), 107 | 'FramerateDenominator': int(MEDIA_CONVERT_FRAME_RATE_DENOMINATOR) 108 | } 109 | } 110 | }, 111 | 'Extension': '.jpg' 112 | } 113 | ], 114 | 'OutputGroupSettings': { 115 | 'Type': 'FILE_GROUP_SETTINGS', 116 | 'FileGroupSettings': { 117 | 'Destination': f's3://{S3_BUCKET_NAME}/public/mediaconvert-videos-to-image-frames/', 118 | 'DestinationSettings': { 119 | 'S3Settings': { 120 | 'Encryption': { 121 | 'EncryptionType': 'SERVER_SIDE_ENCRYPTION_S3' 122 | } 123 | } 124 | } 125 | } 126 | } 127 | } 128 | ], 129 | 'Inputs': [ 130 | { 131 | 'TimecodeSource': 'ZEROBASED' 132 | } 133 | ] 134 | }, 135 | 'AccelerationSettings': { 136 | 'Mode': 'DISABLED' 137 | }, 138 | 'StatusUpdateInterval': 'SECONDS_60', 139 | 'Priority': 0, 140 | 'HopDestinations': [] 141 | } 142 | 143 | MEDIACONVERT_CLIENT = boto3.client('mediaconvert') 144 | 145 | for retry_count in range(3): 146 | try: 147 | print(f'Creating Job Template using {template}') 148 | response = MEDIACONVERT_CLIENT.create_job_template(**template) 149 | print(f'MediaConvert Create Job Template Response: {response}') 150 | return cfnresponse.SUCCESS 151 | except Exception as e: 152 | print(f'Exception occurred while creating media convert job template: {template}') 153 | print(e) 154 | time.sleep(30) 155 | 156 | return cfnresponse.FAILED 157 | 158 | def delete_mediaconvert_job_template(): 159 | MEDIACONVERT_CLIENT = boto3.client('mediaconvert') 160 | 161 | for retry_count in range(3): 162 | try: 163 | print(f'Deleting Job Template with name {MEDIA_CONVERT_JOB_TEMPLATE_NAME}') 164 | response = MEDIACONVERT_CLIENT.delete_job_template(Name=MEDIA_CONVERT_JOB_TEMPLATE_NAME) 165 | print(f'MediaConvert Delete Job Template Response: {response}') 166 | return cfnresponse.SUCCESS 167 | except Exception as e: 168 | print(f'Exception occurred while deleting media convert job template name: {MEDIA_CONVERT_JOB_TEMPLATE_NAME}') 169 | print(e) 170 | time.sleep(30) 171 | 172 | return cfnresponse.FAILED 173 | 174 | def create_opensearch_index(): 175 | index_body = { 176 | 'settings': { 177 | 'index.knn': True 178 | }, 179 | 'mappings': { 180 | 'properties': { 181 | 'titan-embedding': { 182 | 'type': 'knn_vector', 183 | 'dimension': 1024, 184 | 'method': { 185 | 'engine': 'faiss', 186 | 'space_type': 'l2', 187 | 'name': 'hnsw', 188 | 'parameters': {} 189 | } 190 | }, 191 | 'fragment-number': { 192 | 'type': 'text' 193 | }, 194 | 's3-uri': { 195 | 'type': 'text' 196 | }, 197 | 'summary': { 198 | 'type': 'text' 199 | }, 200 | 'source': { 201 | 'type': 'text' 202 | }, 203 | 'custom-metadata': { 204 | 'type': 'text' 205 | }, 206 | 'timestamp': { 207 | 'type': 'date' 208 | } 209 | } 210 | } 211 | } 212 | 213 | for retry_count in range(3): 214 | try: 215 | response = OPENSEARCH_CLIENT.indices.create(index=OPENSEARCH_INDEX_NAME, body=index_body) 216 | print(f'Opensearch Index Create Response: {response}') 217 | return cfnresponse.SUCCESS 218 | except Exception as e: 219 | print(f'Exception occurred while creating index {OPENSEARCH_INDEX_NAME} with retry_count {retry_count}') 220 | print(e) 221 | time.sleep(30) 222 | 223 | return cfnresponse.FAILED 224 | def handler(event, context): 225 | print(f'event: {event}') 226 | response = cfnresponse.SUCCESS 227 | if not function_initialization_success: 228 | response = cfnresponse.FAILED 229 | elif event['RequestType'] == 'Delete': 230 | response = delete_mediaconvert_job_template() 231 | elif event['RequestType'] == 'Create': 232 | print('Create Request received') 233 | response = create_opensearch_index() 234 | 235 | if response != cfnresponse.FAILED: 236 | response = create_mediaconvert_job_template() 237 | 238 | cfnresponse.send(event, context, response, {}) -------------------------------------------------------------------------------- /amplify/backend/custom/vectordbaccess/convertPythonFileToCloudformationJSONZipfile.py: -------------------------------------------------------------------------------- 1 | #simple script used to prep CreateOpensearchIndexLambdaFunction.py for JSON Cloudformation zip file 2 | 3 | python_function_file = open("CreateOpensearchIndexLambdaFunction.py", "r") 4 | new_file = open("tempCloudformationJSONZipFile.txt", "w") 5 | for line in python_function_file: 6 | #print(line) 7 | new_line = f'"{line.rstrip()}",\n' 8 | #print(new_line) 9 | new_file.write(new_line) 10 | new_file.close() 11 | python_function_file.close() 12 | -------------------------------------------------------------------------------- /amplify/backend/function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch-cloudformation-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "{\"createdOn\":\"Mac\",\"createdBy\":\"Amplify\",\"createdWith\":\"12.12.4\",\"stackType\":\"function-Lambda\",\"metadata\":{}}", 4 | "Parameters": { 5 | "CloudWatchRule": { 6 | "Type": "String", 7 | "Default": "NONE", 8 | "Description": " Schedule Expression" 9 | }, 10 | "deploymentBucketName": { 11 | "Type": "String" 12 | }, 13 | "env": { 14 | "Type": "String" 15 | }, 16 | "s3Key": { 17 | "Type": "String" 18 | }, 19 | "functionfrontendClipCrunchersSharedArn": { 20 | "Type": "String", 21 | "Default": "functionfrontendClipCrunchersSharedArn" 22 | }, 23 | "vectordbopensearchCollectionEndpoint": { 24 | "Type": "String", 25 | "Description": "Input parameter describing CollectionEndpoint attribute for vectordb/opensearch resource" 26 | }, 27 | "vectordbopensearchopensearchServerlessCollectionARN": { 28 | "Type": "String", 29 | "Description": "Input parameter describing ARN of opensearchServerlessCollection for vectordb/opensearch resource" 30 | }, 31 | "vectordbopensearchopensearchServerlessCollectionName": { 32 | "Type": "String", 33 | "Description": "Input parameter describing opensearchServerlessCollectionName attribute for vectordb/opensearch resource" 34 | }, 35 | "vectordbopensearchopensearchServerlessIndexName": { 36 | "Type": "String", 37 | "Description": "Input parameter describing opensearchServerlessIndexName attribute for vectordb/opensearch resource" 38 | }, 39 | "bedrockEmbeddingModelId": { 40 | "Type": "String", 41 | "Default": "amazon.titan-embed-image-v1" 42 | }, 43 | "bedrockInferenceModelId": { 44 | "Type": "String", 45 | "Default": "anthropic.claude-3-sonnet-20240229-v1:0" 46 | }, 47 | "mediaConvertJobTemplatePrefix": { 48 | "Type": "String", 49 | "Default": "ClipCrunchersVideoToFramesJobTemplate" 50 | }, 51 | "cloudwatchNamespace": { 52 | "Type": "String", 53 | "Default": "clip-crunchers" 54 | } 55 | }, 56 | "Conditions": { 57 | "ShouldNotCreateEnvResources": { 58 | "Fn::Equals": [ 59 | { 60 | "Ref": "env" 61 | }, 62 | "NONE" 63 | ] 64 | } 65 | }, 66 | "Resources": { 67 | "LambdaFunction": { 68 | "Type": "AWS::Lambda::Function", 69 | "Metadata": { 70 | "aws:asset:path": "./src", 71 | "aws:asset:property": "Code" 72 | }, 73 | "Properties": { 74 | "Code": { 75 | "S3Bucket": { 76 | "Ref": "deploymentBucketName" 77 | }, 78 | "S3Key": { 79 | "Ref": "s3Key" 80 | } 81 | }, 82 | "Handler": "index.handler", 83 | "FunctionName": { 84 | "Fn::If": [ 85 | "ShouldNotCreateEnvResources", 86 | "CalculateVectorEmbeddInS3IndexInOpensearchb2ce577e0699", 87 | { 88 | "Fn::Join": [ 89 | "", 90 | [ 91 | "CalculateVectorEmbeddInS3IndexInOpensearchb2ce577e0699", 92 | "-", 93 | { 94 | "Ref": "env" 95 | } 96 | ] 97 | ] 98 | } 99 | ] 100 | }, 101 | "Environment": { 102 | "Variables": { 103 | "ENV": { 104 | "Ref": "env" 105 | }, 106 | "REGION": { 107 | "Ref": "AWS::Region" 108 | }, 109 | "OpensearchEndpoint": { 110 | "Ref": "vectordbopensearchCollectionEndpoint" 111 | }, 112 | "OpensearchIndexName": { 113 | "Ref": "vectordbopensearchopensearchServerlessIndexName" 114 | }, 115 | "BedrockEmbeddingModelId": { 116 | "Ref": "bedrockEmbeddingModelId" 117 | }, 118 | "BedrockInferenceModelId": { 119 | "Ref": "bedrockInferenceModelId" 120 | }, 121 | "MediaConvertExecutionRoleArn": { 122 | "Fn::GetAtt": [ 123 | "MediaConvertExecutionRole", 124 | "Arn" 125 | ] 126 | }, 127 | "MediaConvertJobTemplateName": { 128 | "Fn::Sub": ["${templatePrefix}-${env}", 129 | { 130 | "templatePrefix": { 131 | "Ref": "mediaConvertJobTemplatePrefix" 132 | }, 133 | "env": { 134 | "Ref": "env" 135 | } 136 | } 137 | ] 138 | }, 139 | "CLOUDWATCH_NAMESPACE": { 140 | "Ref": "cloudwatchNamespace" 141 | } 142 | } 143 | }, 144 | "Role": { 145 | "Fn::GetAtt": [ 146 | "LambdaExecutionRole", 147 | "Arn" 148 | ] 149 | }, 150 | "Runtime": "python3.11", 151 | "Layers": [ 152 | { 153 | "Ref": "functionfrontendClipCrunchersSharedArn" 154 | }, 155 | "arn:aws:lambda:us-east-1:336392948345:layer:AWSSDKPandas-Python311:4" 156 | ], 157 | "Timeout": 25 158 | } 159 | }, 160 | "LambdaExecutionRole": { 161 | "Type": "AWS::IAM::Role", 162 | "Properties": { 163 | "RoleName": { 164 | "Fn::If": [ 165 | "ShouldNotCreateEnvResources", 166 | "frontendLambdaRole4f8de6f0", 167 | { 168 | "Fn::Join": [ 169 | "", 170 | [ 171 | "frontendLambdaRole4f8de6f0", 172 | "-", 173 | { 174 | "Ref": "env" 175 | } 176 | ] 177 | ] 178 | } 179 | ] 180 | }, 181 | "AssumeRolePolicyDocument": { 182 | "Version": "2012-10-17", 183 | "Statement": [ 184 | { 185 | "Effect": "Allow", 186 | "Principal": { 187 | "Service": [ 188 | "lambda.amazonaws.com" 189 | ] 190 | }, 191 | "Action": [ 192 | "sts:AssumeRole" 193 | ] 194 | } 195 | ] 196 | } 197 | } 198 | }, 199 | "lambdaexecutionpolicy": { 200 | "DependsOn": [ 201 | "LambdaExecutionRole" 202 | ], 203 | "Type": "AWS::IAM::Policy", 204 | "Properties": { 205 | "PolicyName": "lambda-execution-policy", 206 | "Roles": [ 207 | { 208 | "Ref": "LambdaExecutionRole" 209 | } 210 | ], 211 | "PolicyDocument": { 212 | "Version": "2012-10-17", 213 | "Statement": [ 214 | { 215 | "Effect": "Allow", 216 | "Action": [ 217 | "logs:CreateLogGroup", 218 | "logs:CreateLogStream", 219 | "logs:PutLogEvents" 220 | ], 221 | "Resource": { 222 | "Fn::Sub": [ 223 | "arn:aws:logs:${region}:${account}:log-group:/aws/lambda/${lambda}:log-stream:*", 224 | { 225 | "region": { 226 | "Ref": "AWS::Region" 227 | }, 228 | "account": { 229 | "Ref": "AWS::AccountId" 230 | }, 231 | "lambda": { 232 | "Ref": "LambdaFunction" 233 | } 234 | } 235 | ] 236 | } 237 | } 238 | ] 239 | } 240 | } 241 | }, 242 | "CustomLambdaExecutionPolicy": { 243 | "Type": "AWS::IAM::Policy", 244 | "Properties": { 245 | "PolicyName": "custom-lambda-execution-policy", 246 | "PolicyDocument": { 247 | "Version": "2012-10-17", 248 | "Statement": [ 249 | { 250 | "Action": [ 251 | "bedrock:InvokeModel" 252 | ], 253 | "Resource": [ 254 | { 255 | "Fn::Sub": "arn:${AWS::Partition}:bedrock:${AWS::Region}::foundation-model/${bedrockEmbeddingModelId}" 256 | }, 257 | { 258 | "Fn::Sub": "arn:${AWS::Partition}:bedrock:${AWS::Region}::foundation-model/${bedrockInferenceModelId}" 259 | } 260 | ], 261 | "Effect": "Allow" 262 | }, 263 | { 264 | "Action": [ 265 | "cloudwatch:PutMetricData" 266 | ], 267 | "Resource": [ 268 | "*" 269 | ], 270 | "Condition": { 271 | "StringEquals": { 272 | "cloudwatch:namespace": { 273 | "Fn::Sub": "${cloudwatchNamespace}" 274 | } 275 | } 276 | }, 277 | "Effect": "Allow" 278 | }, 279 | { 280 | "Action": [ 281 | "aoss:APIAccessAll" 282 | ], 283 | "Resource": [ 284 | { 285 | "Ref": "vectordbopensearchopensearchServerlessCollectionARN" 286 | } 287 | ], 288 | "Effect": "Allow" 289 | }, 290 | { 291 | "Action": [ 292 | "MediaConvert:CreateJob" 293 | ], 294 | "Resource": [ 295 | { 296 | "Fn::Sub": "arn:${AWS::Partition}:mediaconvert:${AWS::Region}:${AWS::AccountId}:queues/*" 297 | }, 298 | { 299 | "Fn::Sub": "arn:${AWS::Partition}:mediaconvert:${AWS::Region}:${AWS::AccountId}:jobTemplates/${mediaConvertJobTemplateName}" 300 | } 301 | ], 302 | "Effect": "Allow" 303 | }, 304 | { 305 | "Action": [ 306 | "MediaConvert:GetJob" 307 | ], 308 | "Resource": [ 309 | { 310 | "Fn::Sub": "arn:${AWS::Partition}:mediaconvert:${AWS::Region}:${AWS::AccountId}:jobs/*" 311 | } 312 | ], 313 | "Effect": "Allow" 314 | }, 315 | { 316 | "Action": [ 317 | "iam:PassRole" 318 | ], 319 | "Resource": [ 320 | { 321 | "Fn::GetAtt": [ 322 | "MediaConvertExecutionRole", 323 | "Arn" 324 | ] 325 | } 326 | ], 327 | "Effect": "Allow" 328 | } 329 | ] 330 | }, 331 | "Roles": [ 332 | { 333 | "Ref": "LambdaExecutionRole" 334 | } 335 | ] 336 | }, 337 | "DependsOn": "LambdaExecutionRole" 338 | }, 339 | "MediaConvertExecutionRole": { 340 | "Type": "AWS::IAM::Role", 341 | "Properties": { 342 | "RoleName": { 343 | "Fn::If": [ 344 | "ShouldNotCreateEnvResources", 345 | "clipCrunchersMediaConvertExecutionRole", 346 | { 347 | "Fn::Join": [ 348 | "", 349 | [ 350 | "clipCrunchersMediaConvertExecutionRole", 351 | "-", 352 | { 353 | "Ref": "env" 354 | } 355 | ] 356 | ] 357 | } 358 | ] 359 | }, 360 | "AssumeRolePolicyDocument": { 361 | "Version": "2012-10-17", 362 | "Statement": [ 363 | { 364 | "Effect": "Allow", 365 | "Principal": { 366 | "Service": [ 367 | "mediaconvert.amazonaws.com" 368 | ] 369 | }, 370 | "Action": [ 371 | "sts:AssumeRole" 372 | ] 373 | } 374 | ] 375 | } 376 | } 377 | } 378 | }, 379 | "Outputs": { 380 | "Name": { 381 | "Value": { 382 | "Ref": "LambdaFunction" 383 | } 384 | }, 385 | "Arn": { 386 | "Value": { 387 | "Fn::GetAtt": [ 388 | "LambdaFunction", 389 | "Arn" 390 | ] 391 | } 392 | }, 393 | "Region": { 394 | "Value": { 395 | "Ref": "AWS::Region" 396 | } 397 | }, 398 | "LambdaExecutionRole": { 399 | "Value": { 400 | "Ref": "LambdaExecutionRole" 401 | } 402 | }, 403 | "LambdaExecutionRoleArn": { 404 | "Value": { 405 | "Fn::GetAtt": [ 406 | "LambdaExecutionRole", 407 | "Arn" 408 | ] 409 | } 410 | }, 411 | "MediaConvertExecutionRole": { 412 | "Value": { 413 | "Ref": "MediaConvertExecutionRole" 414 | } 415 | } 416 | } 417 | } -------------------------------------------------------------------------------- /amplify/backend/function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch/Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | 8 | [dev-packages] 9 | 10 | [requires] 11 | python_version = "3.11" 12 | -------------------------------------------------------------------------------- /amplify/backend/function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch/Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "4edb4c046dc5fce710f674ab69074bd8a4811ed87cb04e6c1fc78b1803e2e486" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.11" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "pillow": { 20 | "hashes": [ 21 | "sha256:02a2be69f9c9b8c1e97cf2713e789d4e398c751ecfd9967c18d0ce304efbf885", 22 | "sha256:030abdbe43ee02e0de642aee345efa443740aa4d828bfe8e2eb11922ea6a21ea", 23 | "sha256:06b2f7898047ae93fad74467ec3d28fe84f7831370e3c258afa533f81ef7f3df", 24 | "sha256:0755ffd4a0c6f267cccbae2e9903d95477ca2f77c4fcf3a3a09570001856c8a5", 25 | "sha256:0a9ec697746f268507404647e531e92889890a087e03681a3606d9b920fbee3c", 26 | "sha256:0ae24a547e8b711ccaaf99c9ae3cd975470e1a30caa80a6aaee9a2f19c05701d", 27 | "sha256:134ace6dc392116566980ee7436477d844520a26a4b1bd4053f6f47d096997fd", 28 | "sha256:166c1cd4d24309b30d61f79f4a9114b7b2313d7450912277855ff5dfd7cd4a06", 29 | "sha256:1b5dea9831a90e9d0721ec417a80d4cbd7022093ac38a568db2dd78363b00908", 30 | "sha256:1d846aea995ad352d4bdcc847535bd56e0fd88d36829d2c90be880ef1ee4668a", 31 | "sha256:1ef61f5dd14c300786318482456481463b9d6b91ebe5ef12f405afbba77ed0be", 32 | "sha256:297e388da6e248c98bc4a02e018966af0c5f92dfacf5a5ca22fa01cb3179bca0", 33 | "sha256:298478fe4f77a4408895605f3482b6cc6222c018b2ce565c2b6b9c354ac3229b", 34 | "sha256:29dbdc4207642ea6aad70fbde1a9338753d33fb23ed6956e706936706f52dd80", 35 | "sha256:2db98790afc70118bd0255c2eeb465e9767ecf1f3c25f9a1abb8ffc8cfd1fe0a", 36 | "sha256:32cda9e3d601a52baccb2856b8ea1fc213c90b340c542dcef77140dfa3278a9e", 37 | "sha256:37fb69d905be665f68f28a8bba3c6d3223c8efe1edf14cc4cfa06c241f8c81d9", 38 | "sha256:416d3a5d0e8cfe4f27f574362435bc9bae57f679a7158e0096ad2beb427b8696", 39 | "sha256:43efea75eb06b95d1631cb784aa40156177bf9dd5b4b03ff38979e048258bc6b", 40 | "sha256:4b35b21b819ac1dbd1233317adeecd63495f6babf21b7b2512d244ff6c6ce309", 41 | "sha256:4d9667937cfa347525b319ae34375c37b9ee6b525440f3ef48542fcf66f2731e", 42 | "sha256:5161eef006d335e46895297f642341111945e2c1c899eb406882a6c61a4357ab", 43 | "sha256:543f3dc61c18dafb755773efc89aae60d06b6596a63914107f75459cf984164d", 44 | "sha256:551d3fd6e9dc15e4c1eb6fc4ba2b39c0c7933fa113b220057a34f4bb3268a060", 45 | "sha256:59291fb29317122398786c2d44427bbd1a6d7ff54017075b22be9d21aa59bd8d", 46 | "sha256:5b001114dd152cfd6b23befeb28d7aee43553e2402c9f159807bf55f33af8a8d", 47 | "sha256:5b4815f2e65b30f5fbae9dfffa8636d992d49705723fe86a3661806e069352d4", 48 | "sha256:5dc6761a6efc781e6a1544206f22c80c3af4c8cf461206d46a1e6006e4429ff3", 49 | "sha256:5e84b6cc6a4a3d76c153a6b19270b3526a5a8ed6b09501d3af891daa2a9de7d6", 50 | "sha256:6209bb41dc692ddfee4942517c19ee81b86c864b626dbfca272ec0f7cff5d9fb", 51 | "sha256:673655af3eadf4df6b5457033f086e90299fdd7a47983a13827acf7459c15d94", 52 | "sha256:6c762a5b0997f5659a5ef2266abc1d8851ad7749ad9a6a5506eb23d314e4f46b", 53 | "sha256:7086cc1d5eebb91ad24ded9f58bec6c688e9f0ed7eb3dbbf1e4800280a896496", 54 | "sha256:73664fe514b34c8f02452ffb73b7a92c6774e39a647087f83d67f010eb9a0cf0", 55 | "sha256:76a911dfe51a36041f2e756b00f96ed84677cdeb75d25c767f296c1c1eda1319", 56 | "sha256:780c072c2e11c9b2c7ca37f9a2ee8ba66f44367ac3e5c7832afcfe5104fd6d1b", 57 | "sha256:7928ecbf1ece13956b95d9cbcfc77137652b02763ba384d9ab508099a2eca856", 58 | "sha256:7970285ab628a3779aecc35823296a7869f889b8329c16ad5a71e4901a3dc4ef", 59 | "sha256:7a8d4bade9952ea9a77d0c3e49cbd8b2890a399422258a77f357b9cc9be8d680", 60 | "sha256:7c1ee6f42250df403c5f103cbd2768a28fe1a0ea1f0f03fe151c8741e1469c8b", 61 | "sha256:7dfecdbad5c301d7b5bde160150b4db4c659cee2b69589705b6f8a0c509d9f42", 62 | "sha256:812f7342b0eee081eaec84d91423d1b4650bb9828eb53d8511bcef8ce5aecf1e", 63 | "sha256:866b6942a92f56300012f5fbac71f2d610312ee65e22f1aa2609e491284e5597", 64 | "sha256:86dcb5a1eb778d8b25659d5e4341269e8590ad6b4e8b44d9f4b07f8d136c414a", 65 | "sha256:87dd88ded2e6d74d31e1e0a99a726a6765cda32d00ba72dc37f0651f306daaa8", 66 | "sha256:8bc1a764ed8c957a2e9cacf97c8b2b053b70307cf2996aafd70e91a082e70df3", 67 | "sha256:8d4d5063501b6dd4024b8ac2f04962d661222d120381272deea52e3fc52d3736", 68 | "sha256:8f0aef4ef59694b12cadee839e2ba6afeab89c0f39a3adc02ed51d109117b8da", 69 | "sha256:930044bb7679ab003b14023138b50181899da3f25de50e9dbee23b61b4de2126", 70 | "sha256:950be4d8ba92aca4b2bb0741285a46bfae3ca699ef913ec8416c1b78eadd64cd", 71 | "sha256:961a7293b2457b405967af9c77dcaa43cc1a8cd50d23c532e62d48ab6cdd56f5", 72 | "sha256:9b885f89040bb8c4a1573566bbb2f44f5c505ef6e74cec7ab9068c900047f04b", 73 | "sha256:9f4727572e2918acaa9077c919cbbeb73bd2b3ebcfe033b72f858fc9fbef0026", 74 | "sha256:a02364621fe369e06200d4a16558e056fe2805d3468350df3aef21e00d26214b", 75 | "sha256:a985e028fc183bf12a77a8bbf36318db4238a3ded7fa9df1b9a133f1cb79f8fc", 76 | "sha256:ac1452d2fbe4978c2eec89fb5a23b8387aba707ac72810d9490118817d9c0b46", 77 | "sha256:b15e02e9bb4c21e39876698abf233c8c579127986f8207200bc8a8f6bb27acf2", 78 | "sha256:b2724fdb354a868ddf9a880cb84d102da914e99119211ef7ecbdc613b8c96b3c", 79 | "sha256:bbc527b519bd3aa9d7f429d152fea69f9ad37c95f0b02aebddff592688998abe", 80 | "sha256:bcd5e41a859bf2e84fdc42f4edb7d9aba0a13d29a2abadccafad99de3feff984", 81 | "sha256:bd2880a07482090a3bcb01f4265f1936a903d70bc740bfcb1fd4e8a2ffe5cf5a", 82 | "sha256:bee197b30783295d2eb680b311af15a20a8b24024a19c3a26431ff83eb8d1f70", 83 | "sha256:bf2342ac639c4cf38799a44950bbc2dfcb685f052b9e262f446482afaf4bffca", 84 | "sha256:c76e5786951e72ed3686e122d14c5d7012f16c8303a674d18cdcd6d89557fc5b", 85 | "sha256:cbed61494057c0f83b83eb3a310f0bf774b09513307c434d4366ed64f4128a91", 86 | "sha256:cfdd747216947628af7b259d274771d84db2268ca062dd5faf373639d00113a3", 87 | "sha256:d7480af14364494365e89d6fddc510a13e5a2c3584cb19ef65415ca57252fb84", 88 | "sha256:dbc6ae66518ab3c5847659e9988c3b60dc94ffb48ef9168656e0019a93dbf8a1", 89 | "sha256:dc3e2db6ba09ffd7d02ae9141cfa0ae23393ee7687248d46a7507b75d610f4f5", 90 | "sha256:dfe91cb65544a1321e631e696759491ae04a2ea11d36715eca01ce07284738be", 91 | "sha256:e4d49b85c4348ea0b31ea63bc75a9f3857869174e2bf17e7aba02945cd218e6f", 92 | "sha256:e4db64794ccdf6cb83a59d73405f63adbe2a1887012e308828596100a0b2f6cc", 93 | "sha256:e553cad5179a66ba15bb18b353a19020e73a7921296a7979c4a2b7f6a5cd57f9", 94 | "sha256:e88d5e6ad0d026fba7bdab8c3f225a69f063f116462c49892b0149e21b6c0a0e", 95 | "sha256:ecd85a8d3e79cd7158dec1c9e5808e821feea088e2f69a974db5edf84dc53141", 96 | "sha256:f5b92f4d70791b4a67157321c4e8225d60b119c5cc9aee8ecf153aace4aad4ef", 97 | "sha256:f5f0c3e969c8f12dd2bb7e0b15d5c468b51e5017e01e2e867335c81903046a22", 98 | "sha256:f7baece4ce06bade126fb84b8af1c33439a76d8a6fd818970215e0560ca28c27", 99 | "sha256:ff25afb18123cea58a591ea0244b92eb1e61a1fd497bf6d6384f09bc3262ec3e", 100 | "sha256:ff337c552345e95702c5fde3158acb0625111017d0e5f24bf3acdb9cc16b90d1" 101 | ], 102 | "index": "pypi", 103 | "markers": "python_version >= '3.8'", 104 | "version": "==10.4.0" 105 | } 106 | }, 107 | "develop": {} 108 | } 109 | -------------------------------------------------------------------------------- /amplify/backend/function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch/amplify.state: -------------------------------------------------------------------------------- 1 | { 2 | "pluginId": "amplify-nodejs-function-runtime-provider", 3 | "functionRuntime": "nodejs", 4 | "defaultEditorFile": "src/index.js", 5 | "useLegacyBuild": true 6 | } -------------------------------------------------------------------------------- /amplify/backend/function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch/custom-policies.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Action": ["bedrock:InvokeModel"], 4 | "Resource": [ 5 | {"Fn::Sub": "arn:${AWS::Partition}:bedrock:${AWS::Region}::foundation-model/${bedrockEmbeddingModelId}"}, 6 | {"Fn::Sub": "arn:${AWS::Partition}:bedrock:${AWS::Region}::foundation-model/${bedrockInferenceModelId}"} 7 | ] 8 | }, 9 | { 10 | "Action": ["cloudwatch:PutMetricData"], 11 | "Resource": ["*"], 12 | "Condition": { 13 | "StringEquals": { 14 | "cloudwatch:namespace": { "Fn::Sub": "${cloudwatchNamespace}" } 15 | } 16 | } 17 | }, 18 | { 19 | "Action": ["aoss:APIAccessAll"], 20 | "Resource": [{ "Ref": "vectordbopensearchopensearchServerlessCollectionARN" }] 21 | }, 22 | { 23 | "Action": ["MediaConvert:CreateJob"], 24 | "Resource": [ 25 | { "Fn::Sub": "arn:${AWS::Partition}:mediaconvert:${AWS::Region}:${AWS::AccountId}:queues/*" }, 26 | { "Fn::Sub": "arn:${AWS::Partition}:mediaconvert:${AWS::Region}:${AWS::AccountId}:jobTemplates/${mediaConvertJobTemplatePrefix}-${env}" } 27 | ] 28 | }, 29 | { 30 | "Action": ["MediaConvert:GetJob"], 31 | "Resource": [{ "Fn::Sub": "arn:${AWS::Partition}:mediaconvert:${AWS::Region}:${AWS::AccountId}:jobs/*" }] 32 | }, 33 | { 34 | "Action": ["iam:PassRole"], 35 | "Resource": [{ "Fn::GetAtt": ["MediaConvertExecutionRole", "Arn"] }] 36 | } 37 | ] -------------------------------------------------------------------------------- /amplify/backend/function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch/function-parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "permissions": {}, 3 | "lambdaLayers": [ 4 | { 5 | "type": "ProjectLayer", 6 | "resourceName": "frontendClipCrunchersShared", 7 | "version": "Always choose latest version", 8 | "isLatestVersionSelected": true, 9 | "env": "dev" 10 | } 11 | ] 12 | } -------------------------------------------------------------------------------- /amplify/backend/function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch/parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "vectordbopensearchCollectionEndpoint": { 3 | "Fn::GetAtt": [ 4 | "vectordbopensearch", 5 | "Outputs.CollectionEndpoint" 6 | ] 7 | }, 8 | "vectordbopensearchopensearchServerlessCollectionName": { 9 | "Fn::GetAtt": [ 10 | "vectordbopensearch", 11 | "Outputs.opensearchServerlessCollectionName" 12 | ] 13 | }, 14 | "vectordbopensearchopensearchServerlessIndexName": { 15 | "Fn::GetAtt": [ 16 | "vectordbopensearch", 17 | "Outputs.opensearchServerlessIndexName" 18 | ] 19 | } 20 | } -------------------------------------------------------------------------------- /amplify/backend/function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch/src/index.py: -------------------------------------------------------------------------------- 1 | import json 2 | import urllib.parse 3 | import boto3 4 | import numpy as np 5 | import base64 6 | from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth 7 | from datetime import datetime 8 | import os 9 | from botocore.exceptions import ClientError 10 | 11 | 12 | 13 | def initialize_opensearch_client(): 14 | opensearch_endpoint = os.environ['OpensearchEndpoint'].replace('https://','',1) # serverless collection endpoint, without https:// 15 | print(f"Opensearch Endpoint: {opensearch_endpoint}") 16 | print(f"region: {AWS_REGION}") 17 | credentials = boto3.Session().get_credentials() 18 | print(f"Caller Identity: {boto3.client('sts').get_caller_identity()}") 19 | 20 | auth = AWSV4SignerAuth(credentials, AWS_REGION, 'aoss') 21 | 22 | client = OpenSearch( 23 | hosts=[{'host': opensearch_endpoint, 'port': 443}], 24 | http_auth=auth, 25 | use_ssl=True, 26 | verify_certs=True, 27 | connection_class=RequestsHttpConnection, 28 | pool_maxsize=20, 29 | ) 30 | 31 | return client 32 | 33 | print('Intializing function') 34 | S3_RESOURCE = boto3.resource('s3') 35 | S3_CLIENT = boto3.client('s3') 36 | BEDROCK_CLIENT = boto3.client('bedrock-runtime') 37 | CLOUDWATCH_CLIENT = boto3.client('cloudwatch') 38 | 39 | AWS_REGION = os.environ['AWS_REGION'] # e.g. us-east-1 40 | 41 | OPENSEARCH_CLIENT = initialize_opensearch_client() 42 | OPENSEARCH_INDEX_NAME = os.environ['OpensearchIndexName'] 43 | print(f"OPENSEARCH_INDEX_NAME: {OPENSEARCH_INDEX_NAME}") 44 | 45 | BEDROCK_EMBEDDING_MODEL_ID = os.environ['BedrockEmbeddingModelId'] 46 | print(f"BEDROCK_EMBEDDING_MODEL_ID: {BEDROCK_EMBEDDING_MODEL_ID}") 47 | 48 | BEDROCK_INFERENCE_MODEL_ID = os.environ['BedrockInferenceModelId'] 49 | print(f"BEDROCK_INFERENCE_MODEL_ID: {BEDROCK_INFERENCE_MODEL_ID}") 50 | 51 | MEDIA_CONVERT_JOB_TEMPLATE_NAME = os.environ["MediaConvertJobTemplateName"] 52 | MEDIA_CONVERT_EXECUTION_ROLE_ARN = os.environ["MediaConvertExecutionRoleArn"] 53 | MEDIA_CONVERT_CLIENT = boto3.client('mediaconvert') 54 | 55 | # New environment variable for cosine similarity threshold 56 | COSINE_SIMILARITY_THRESHOLD = float(os.environ.get('COSINE_SIMILARITY_THRESHOLD', 0.95)) 57 | print(f"COSINE_SIMILARITY_THRESHOLD: {COSINE_SIMILARITY_THRESHOLD}") 58 | 59 | # Get the CloudWatch namespace from environment variable or use default 60 | CLOUDWATCH_NAMESPACE = os.environ.get('CLOUDWATCH_NAMESPACE', 'clip-crunchers') 61 | print(f"CloudWatch Namespace: {CLOUDWATCH_NAMESPACE}") 62 | 63 | # Get the CloudWatch namespace from environment variable or use default 64 | AMPLIFY_ENV = os.environ.get('ENV', 'Unknown Environment') 65 | print(f"Amplify Environment: {AMPLIFY_ENV}") 66 | 67 | def publish_metric(metric_name, value): 68 | try: 69 | CLOUDWATCH_CLIENT.put_metric_data( 70 | Namespace=CLOUDWATCH_NAMESPACE, 71 | MetricData=[ 72 | { 73 | 'MetricName': metric_name, 74 | 'Value': value, 75 | 'Unit': 'Count', 76 | 'Dimensions': [ 77 | { 78 | 'Name': 'Environment', 79 | 'Value': AMPLIFY_ENV 80 | }, 81 | { 82 | 'Name': 'Function', 83 | 'Value': 'CalculateVectorEmbeddingForImagesInS3IndexInOpensearch' 84 | }, 85 | ] 86 | }, 87 | ] 88 | ) 89 | print(f"Published metric: {metric_name} = {value} to namespace {CLOUDWATCH_NAMESPACE} with dimension Environment={AMPLIFY_ENV}") 90 | except ClientError as e: 91 | print(f"Error publishing metric {metric_name}: {e}") 92 | 93 | def bedrock_invoke_model(body, modelId): 94 | response = BEDROCK_CLIENT.invoke_model( 95 | body=body, modelId=modelId 96 | ) 97 | return json.loads(response.get("body").read()) 98 | 99 | def embedding_from_s3(bucket, key, normalize=True): 100 | print(f'Getting embedding for bucket:"{bucket}" key: "{key}"') 101 | 102 | bucket = S3_RESOURCE.Bucket(bucket) 103 | image = bucket.Object(key) 104 | img_data = image.get().get('Body').read() 105 | 106 | img_bytes = base64.b64encode(img_data).decode('utf8') 107 | 108 | body = json.dumps( 109 | { 110 | "inputText": None, 111 | "inputImage": img_bytes 112 | } 113 | ) 114 | 115 | embedding = np.array(bedrock_invoke_model(body, BEDROCK_EMBEDDING_MODEL_ID)['embedding']) 116 | if normalize: 117 | embedding /= np.linalg.norm(embedding) 118 | 119 | return embedding 120 | 121 | def add_document_to_opensearch(key, embedding, timestamp, fragment_number, summary, source, custom_metadata): 122 | document = { 123 | 's3-uri': key, 124 | 'titan-embedding': embedding.tolist(), 125 | 'timestamp': timestamp, 126 | 'fragment-number': fragment_number, 127 | 'summary': summary, # Add the summary to the document 128 | 'source': source, 129 | 'custom-metadata': custom_metadata 130 | } 131 | 132 | print(f"Opensearch document request: {document}") 133 | 134 | response = OPENSEARCH_CLIENT.index( 135 | index = OPENSEARCH_INDEX_NAME, 136 | body = document 137 | ) 138 | 139 | print(f"Opensearch response: {response}") 140 | 141 | def get_video_timestamp(data): 142 | 143 | timestamp = datetime.now().isoformat() + 'Z' 144 | 145 | try: 146 | if "Metadata" in data and "aws_kinesisvideo_producer_timestamp" in data["Metadata"]: 147 | dt = datetime.utcfromtimestamp(int(data["Metadata"]["aws_kinesisvideo_producer_timestamp"])/1000) 148 | timestamp = dt.isoformat() + 'Z' 149 | print(f"using aws_kinesisvideo_producer_timestamp as timestamp: {timestamp}") 150 | except Exception as e: 151 | print(e) 152 | print("Cannot determine timestamp from image metadata, using current system time") 153 | 154 | return timestamp 155 | 156 | def get_video_fragment_number(data): 157 | 158 | framgment = "" 159 | 160 | if "Metadata" in data and "aws_kinesisvideo_fragment_number" in data["Metadata"]: 161 | framgment = data["Metadata"]["aws_kinesisvideo_fragment_number"] 162 | print(f"using aws_kinesisvideo_fragment_number: {framgment}") 163 | else: 164 | print("aws_kinesisvideo_fragment_number NOT FOUND in metadata") 165 | 166 | return framgment 167 | 168 | def get_image_summary(bucket, key): 169 | """ 170 | Use Amazon Bedrock API with Claude 3 Sonnet model to generate a summary of the image 171 | using the multi-modal messages API. 172 | """ 173 | image_object = S3_RESOURCE.Object(bucket, key) 174 | image_data = image_object.get()['Body'].read() 175 | image_base64 = base64.b64encode(image_data).decode('utf-8') 176 | 177 | body = json.dumps({ 178 | "anthropic_version": "bedrock-2023-05-31", 179 | "max_tokens": 300, 180 | "messages": [ 181 | { 182 | "role": "user", 183 | "content": [ 184 | { 185 | "type": "image", 186 | "source": { 187 | "type": "base64", 188 | "media_type": "image/jpeg", 189 | "data": image_base64 190 | } 191 | }, 192 | { 193 | "type": "text", 194 | "text": "Analyze this image and provide a brief summary of its contents. Focus on the main subjects, actions, and any notable elements in the scene. Keep the summary concise, around 2-3 sentences." 195 | } 196 | ] 197 | } 198 | ], 199 | "temperature": 0 200 | }) 201 | 202 | response_body = bedrock_invoke_model(body, BEDROCK_INFERENCE_MODEL_ID) 203 | summary = response_body['content'][0]['text'] 204 | return summary.strip() 205 | 206 | 207 | def cosine_similarity(a, b): 208 | """Calculate cosine similarity between two vectors.""" 209 | return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)) 210 | 211 | 212 | def get_last_image_embedding(prefix): 213 | """Retrieve the embedding of the last processed image with the given prefix.""" 214 | query = { 215 | "size": 1, 216 | "sort": [{"timestamp": {"order": "desc"}}], 217 | "query": { 218 | "prefix": {"s3-uri": prefix} 219 | } 220 | } 221 | response = OPENSEARCH_CLIENT.search(index=OPENSEARCH_INDEX_NAME, body=query) 222 | 223 | if response['hits']['total']['value'] > 0: 224 | return np.array(response['hits']['hits'][0]['_source']['titan-embedding']) 225 | return None 226 | 227 | def get_s3_uri_input_from_mediaconvert_job_id(job_id): 228 | """Retrieve the S3 URI input from a MediaConvert job ID.""" 229 | response = MEDIA_CONVERT_CLIENT.get_job(Id=job_id) 230 | return response['Job']['Settings']['Inputs'][0]['FileInput'] 231 | 232 | def get_image_source(key, response): 233 | """Determine the source of the image based on the bucket and key.""" 234 | 235 | media_convert_job_id = "" 236 | if "Metadata" in response: 237 | if "mediaconvert-jobid" in response["Metadata"]: 238 | media_convert_job_id = response["Metadata"]["mediaconvert-jobid"] 239 | 240 | video_s3_uri = "" 241 | if media_convert_job_id != "": 242 | video_s3_uri = get_s3_uri_input_from_mediaconvert_job_id(media_convert_job_id) 243 | 244 | if "mediaconvert-videos-to-image-frames" in key and "/fileUploads/" in video_s3_uri: 245 | return "fileupload:video" 246 | elif "mediaconvert-videos-to-image-frames" in key and "/webcamUploads/" in video_s3_uri: 247 | return "webcam:video" 248 | elif "/fileUploads/" in key: 249 | return "fileupload:image" 250 | elif "/webcamUploads/" in key: 251 | return "webcam:image" 252 | else: 253 | return "unknown" 254 | 255 | def get_image_custom_metadata(response): 256 | custom_metadata = "" 257 | if "Metadata" in response: 258 | if "custom-metadata" in response["Metadata"]: 259 | custom_metadata = response["Metadata"]["custom-metadata"] 260 | return custom_metadata 261 | 262 | def process_new_image_upload(bucket, key): 263 | response = S3_CLIENT.get_object(Bucket=bucket, Key=key) 264 | print(f"S3 get object response for bucket:{bucket} key:{key} response:{response}") 265 | timestamp = get_video_timestamp(response) 266 | fragment_number = get_video_fragment_number(response) 267 | source = get_image_source(key, response) 268 | custom_metadata = get_image_custom_metadata(response) 269 | embedding = embedding_from_s3(bucket, key) 270 | print(f"Embedding for bucket:{bucket} key:{key} embedding:{embedding}") 271 | 272 | # Check if the image is from a video conversion 273 | if "mediaconvert-videos-to-image-frames" in key: 274 | prefix = '/'.join(key.split('/')[:-1]) + '/' 275 | last_embedding = get_last_image_embedding(prefix) 276 | 277 | if last_embedding is not None: 278 | similarity = cosine_similarity(embedding, last_embedding) 279 | print(f"Cosine similarity with last image: {similarity}") 280 | 281 | if similarity > COSINE_SIMILARITY_THRESHOLD: 282 | print(f"Skipping summary generation for {key} due to high similarity with previous image.") 283 | # Emit metric for discarded image 284 | publish_metric('ImagesDiscarded', 1) 285 | return 286 | 287 | # Generate summary for the image 288 | summary = get_image_summary(bucket, key) 289 | print(f"Generated summary for {key}: {summary}") 290 | 291 | # Add document to OpenSearch with the summary 292 | add_document_to_opensearch(key, embedding, timestamp, fragment_number, summary, source, custom_metadata) 293 | 294 | # Emit metric for processed image 295 | publish_metric('ImagesProcessed', 1) 296 | 297 | def process_new_video_upload(bucket, key): 298 | 299 | settings_input = {"Inputs": [ 300 | { 301 | "FileInput": f"s3://{bucket}/{key}" 302 | } 303 | ]} 304 | print(f"settings_input: {settings_input}") 305 | 306 | response = MEDIA_CONVERT_CLIENT.create_job( 307 | JobTemplate = MEDIA_CONVERT_JOB_TEMPLATE_NAME, 308 | Role = MEDIA_CONVERT_EXECUTION_ROLE_ARN, 309 | Settings = settings_input 310 | ) 311 | print(f"Media Convert Create Job Response: {response}") 312 | 313 | # function for lambda that processes events from s3 314 | def s3_notification_event_record_handler(record): 315 | print("Processing record: " + json.dumps(record, indent=2)) 316 | 317 | bucket = record['s3']['bucket']['name'] 318 | key = urllib.parse.unquote_plus(record['s3']['object']['key'], encoding='utf-8') 319 | 320 | supported_extensions = (".jpg", ".jpeg", ".mp4", ".webm") 321 | 322 | if not (key.endswith(supported_extensions)): 323 | publish_metric('ReceivedUnsupportedFile', 1) 324 | print("Skipping unsupported file: " + key) 325 | return None 326 | if not key.startswith("public"): 327 | publish_metric('ReceivedNonPublicFile', 1) 328 | print("Skipping file that is not public: " + key) 329 | return None 330 | 331 | if key.endswith((".jpg", ".jpeg")) and record['eventName'].startswith("ObjectCreated"): 332 | publish_metric('ReceivedImageFile', 1) 333 | process_new_image_upload(bucket, key) 334 | elif key.endswith((".mp4",".webm")) and record['eventName'].startswith("ObjectCreated"): 335 | publish_metric('ReceivedVideoFile', 1) 336 | process_new_video_upload(bucket, key) 337 | elif record['eventName'].startswith("ObjectRemoved"): 338 | publish_metric('ReceivedObjectRemovalEvent', 1) 339 | print("Object removal handling code is not available in this demo") 340 | else: 341 | publish_metric('ReceivedUnknownEvent', 1) 342 | print("Unknown event type: " + record['eventName']) 343 | 344 | def handler(event, context): 345 | print("Received event: " + json.dumps(event, indent=2)) 346 | 347 | for record in event['Records']: 348 | s3_notification_event_record_handler(record) 349 | 350 | 351 | if __name__ == "__main__": 352 | handler(0,0) -------------------------------------------------------------------------------- /amplify/backend/function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch/src/setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup(name='src', version='1.0') 4 | -------------------------------------------------------------------------------- /amplify/backend/function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch/src/src.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 2.1 2 | Name: src 3 | Version: 1.0 4 | -------------------------------------------------------------------------------- /amplify/backend/function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch/src/src.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | index.py 2 | setup.py 3 | src.egg-info/PKG-INFO 4 | src.egg-info/SOURCES.txt 5 | src.egg-info/dependency_links.txt 6 | src.egg-info/top_level.txt -------------------------------------------------------------------------------- /amplify/backend/function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch/src/src.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /amplify/backend/function/CalculateVectorEmbeddingForImagesInS3IndexInOpensearch/src/src.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | index 2 | -------------------------------------------------------------------------------- /amplify/backend/function/frontendClipCrunchersShared/frontendClipCrunchersShared-awscloudformation-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Lambda layer resource stack creation using Amplify CLI", 4 | "Parameters": { 5 | "env": { 6 | "Type": "String" 7 | }, 8 | "deploymentBucketName": { 9 | "Type": "String" 10 | }, 11 | "s3Key": { 12 | "Type": "String" 13 | }, 14 | "description": { 15 | "Type": "String", 16 | "Default": "" 17 | }, 18 | "runtimes": { 19 | "Type": "List" 20 | } 21 | }, 22 | "Resources": { 23 | "LambdaLayerVersiond4a62d61": { 24 | "Type": "AWS::Lambda::LayerVersion", 25 | "Properties": { 26 | "CompatibleRuntimes": [ 27 | "python3.11" 28 | ], 29 | "Content": { 30 | "S3Bucket": { 31 | "Ref": "deploymentBucketName" 32 | }, 33 | "S3Key": "amplify-builds/frontendClipCrunchersShared-LambdaLayerVersiond4a62d61-build.zip" 34 | }, 35 | "Description": "Updated layer version 2024-07-22T22:50:55.543Z", 36 | "LayerName": { 37 | "Fn::Sub": [ 38 | "frontendClipCrunchersShared-${env}", 39 | { 40 | "env": { 41 | "Ref": "env" 42 | } 43 | } 44 | ] 45 | } 46 | }, 47 | "DeletionPolicy": "Delete", 48 | "UpdateReplacePolicy": "Retain" 49 | }, 50 | "LambdaLayerPermissionPrivated4a62d61": { 51 | "Type": "AWS::Lambda::LayerVersionPermission", 52 | "Properties": { 53 | "Action": "lambda:GetLayerVersion", 54 | "LayerVersionArn": "arn:aws:lambda:us-east-1:749165261179:layer:frontendClipCrunchersShared-dev:5", 55 | "Principal": { 56 | "Ref": "AWS::AccountId" 57 | } 58 | } 59 | } 60 | }, 61 | "Outputs": { 62 | "Arn": { 63 | "Value": { 64 | "Ref": "LambdaLayerVersiond4a62d61" 65 | } 66 | } 67 | } 68 | } -------------------------------------------------------------------------------- /amplify/backend/function/frontendClipCrunchersShared/layer-configuration.json: -------------------------------------------------------------------------------- 1 | { 2 | "permissions": [ 3 | { 4 | "type": "Private" 5 | } 6 | ], 7 | "runtimes": [ 8 | { 9 | "value": "python", 10 | "name": "Python", 11 | "runtimePluginId": "amplify-python-function-runtime-provider", 12 | "layerExecutablePath": "python" 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /amplify/backend/function/frontendClipCrunchersShared/lib/python/Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [dev-packages] 7 | 8 | [packages] 9 | boto3 = "*" 10 | 11 | [requires] 12 | python_version = "3.11" -------------------------------------------------------------------------------- /amplify/backend/function/frontendClipCrunchersShared/lib/python/Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "9eb0ebd3499ed17c6a28bff85d645a08f0af6474ea186ccbd3471b3778b08811" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.11" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "boto3": { 20 | "hashes": [ 21 | "sha256:227487f9a40e7963aa108f4fabc81374d65e085891a2a442c190dfd976b86a9e", 22 | "sha256:47a7899af97960493ed58754c838be658650c8fb231c658866f491965ddfc94f" 23 | ], 24 | "index": "pypi", 25 | "markers": "python_version >= '3.8'", 26 | "version": "==1.34.78" 27 | }, 28 | "botocore": { 29 | "hashes": [ 30 | "sha256:889fcfd1813fad225a5a70940c58cd4bd7a6f5ba6c9769a1d41d0c670272b75d", 31 | "sha256:bc10738826a4970a6d3a40ac40b9799c02b1b661c0c741a67b915b500562ab3c" 32 | ], 33 | "markers": "python_version >= '3.8'", 34 | "version": "==1.34.78" 35 | }, 36 | "jmespath": { 37 | "hashes": [ 38 | "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", 39 | "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" 40 | ], 41 | "markers": "python_version >= '3.7'", 42 | "version": "==1.0.1" 43 | }, 44 | "python-dateutil": { 45 | "hashes": [ 46 | "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", 47 | "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" 48 | ], 49 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", 50 | "version": "==2.9.0.post0" 51 | }, 52 | "s3transfer": { 53 | "hashes": [ 54 | "sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19", 55 | "sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d" 56 | ], 57 | "markers": "python_version >= '3.8'", 58 | "version": "==0.10.1" 59 | }, 60 | "six": { 61 | "hashes": [ 62 | "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", 63 | "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" 64 | ], 65 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", 66 | "version": "==1.16.0" 67 | }, 68 | "urllib3": { 69 | "hashes": [ 70 | "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472", 71 | "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168" 72 | ], 73 | "index": "pypi", 74 | "markers": "python_version >= '3.8'", 75 | "version": "==2.2.2" 76 | } 77 | }, 78 | "develop": {} 79 | } 80 | -------------------------------------------------------------------------------- /amplify/backend/function/frontendClipCrunchersShared/parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "runtimes": [ 3 | "python3.11" 4 | ], 5 | "description": "Updated layer version 2024-11-04T16:43:14.784Z" 6 | } -------------------------------------------------------------------------------- /amplify/backend/function/frontendf6b68d3c/Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [dev-packages] 7 | 8 | [packages] 9 | src = {editable = true, path = "./src"} 10 | 11 | [requires] 12 | python_version = "3.11" 13 | -------------------------------------------------------------------------------- /amplify/backend/function/frontendf6b68d3c/Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "381c29f4cd76a2e5c615bdbed367c35cfafaf403dd297f1f772cc88d84eb62ce" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.11" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "src": { 20 | "editable": true, 21 | "path": "./src" 22 | } 23 | }, 24 | "develop": {} 25 | } 26 | -------------------------------------------------------------------------------- /amplify/backend/function/frontendf6b68d3c/amplify.state: -------------------------------------------------------------------------------- 1 | { 2 | "pluginId": "amplify-python-function-runtime-provider", 3 | "functionRuntime": "python", 4 | "useLegacyBuild": false, 5 | "defaultEditorFile": "src/index.py" 6 | } -------------------------------------------------------------------------------- /amplify/backend/function/frontendf6b68d3c/custom-policies.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Action": ["bedrock:InvokeModel"], 4 | "Resource": [ 5 | {"Fn::Sub": "arn:${AWS::Partition}:bedrock:${AWS::Region}::foundation-model/${bedrockEmbeddingModelId}"}, 6 | {"Fn::Sub": "arn:${AWS::Partition}:bedrock:${AWS::Region}::foundation-model/${bedrockInferenceModelId}"}] 7 | }, 8 | { 9 | "Action": ["aoss:APIAccessAll"], 10 | "Resource": [{ "Ref": "vectordbopensearchopensearchServerlessCollectionARN" }] 11 | }, 12 | { 13 | "Action": ["kinesisvideo:Describe*", "kinesisvideo:Get*", "kinesisvideo:List*"], 14 | "Resource": [{ "Fn::Sub": "arn:${AWS::Partition}:kinesisvideo:${AWS::Region}:${AWS::AccountId}:${customkinesisvideostreamkinesisVideoStreamName}" }] 15 | } 16 | ] -------------------------------------------------------------------------------- /amplify/backend/function/frontendf6b68d3c/frontendf6b68d3c-cloudformation-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "{\"createdOn\":\"Mac\",\"createdBy\":\"Amplify\",\"createdWith\":\"12.12.4\",\"stackType\":\"function-Lambda\",\"metadata\":{}}", 4 | "Parameters": { 5 | "CloudWatchRule": { 6 | "Type": "String", 7 | "Default": "NONE", 8 | "Description": " Schedule Expression" 9 | }, 10 | "deploymentBucketName": { 11 | "Type": "String" 12 | }, 13 | "env": { 14 | "Type": "String" 15 | }, 16 | "s3Key": { 17 | "Type": "String" 18 | }, 19 | "vectordbopensearchCollectionEndpoint": { 20 | "Type": "String", 21 | "Description": "Input parameter describing CollectionEndpoint attribute for vectordb/opensearch resource" 22 | }, 23 | "vectordbopensearchopensearchServerlessCollectionARN": { 24 | "Type": "String", 25 | "Description": "Input parameter describing ARN of opensearchServerlessCollection for vectordb/opensearch resource" 26 | }, 27 | "vectordbopensearchopensearchServerlessCollectionName": { 28 | "Type": "String", 29 | "Description": "Input parameter describing opensearchServerlessCollectionName attribute for vectordb/opensearch resource" 30 | }, 31 | "vectordbopensearchopensearchServerlessIndexName": { 32 | "Type": "String", 33 | "Description": "Input parameter describing opensearchServerlessIndexName attribute for vectordb/opensearch resource" 34 | }, 35 | "functionfrontendClipCrunchersSharedArn": { 36 | "Type": "String", 37 | "Default": "functionfrontendClipCrunchersSharedArn" 38 | }, 39 | "customkinesisvideostreamkinesisVideoStreamName": { 40 | "Type": "String", 41 | "Default": "clip-crunchers-video-stream" 42 | }, 43 | "bedrockEmbeddingModelId": { 44 | "Type": "String", 45 | "Default": "amazon.titan-embed-image-v1" 46 | }, 47 | "bedrockInferenceModelId": { 48 | "Type": "String", 49 | "Default": "anthropic.claude-3-sonnet-20240229-v1:0" 50 | }, 51 | "kinesisVideoStreamIntegration": { 52 | "Type": "String", 53 | "Default": "False", 54 | "AllowedValues": [ 55 | "True", 56 | "False" 57 | ] 58 | }, 59 | "storages3storageBucketName": { 60 | "Type": "String", 61 | "Default": "storages3storageBucketName" 62 | } 63 | }, 64 | "Conditions": { 65 | "ShouldNotCreateEnvResources": { 66 | "Fn::Equals": [ 67 | { 68 | "Ref": "env" 69 | }, 70 | "NONE" 71 | ] 72 | } 73 | }, 74 | "Resources": { 75 | "LambdaFunction": { 76 | "Type": "AWS::Lambda::Function", 77 | "Metadata": { 78 | "aws:asset:path": "./src", 79 | "aws:asset:property": "Code" 80 | }, 81 | "Properties": { 82 | "Code": { 83 | "S3Bucket": { 84 | "Ref": "deploymentBucketName" 85 | }, 86 | "S3Key": { 87 | "Ref": "s3Key" 88 | } 89 | }, 90 | "Handler": "index.handler", 91 | "FunctionName": { 92 | "Fn::If": [ 93 | "ShouldNotCreateEnvResources", 94 | "frontendf6b68d3c", 95 | { 96 | "Fn::Join": [ 97 | "", 98 | [ 99 | "frontendf6b68d3c", 100 | "-", 101 | { 102 | "Ref": "env" 103 | } 104 | ] 105 | ] 106 | } 107 | ] 108 | }, 109 | "Environment": { 110 | "Variables": { 111 | "ENV": { 112 | "Ref": "env" 113 | }, 114 | "REGION": { 115 | "Ref": "AWS::Region" 116 | }, 117 | "OpensearchEndpoint": { 118 | "Ref": "vectordbopensearchCollectionEndpoint" 119 | }, 120 | "OpensearchIndexName": { 121 | "Ref": "vectordbopensearchopensearchServerlessIndexName" 122 | }, 123 | "KinesisVideoStreamName": { 124 | "Ref": "customkinesisvideostreamkinesisVideoStreamName" 125 | }, 126 | "BedrockEmbeddingModelId": { 127 | "Ref": "bedrockEmbeddingModelId" 128 | }, 129 | "BedrockInferenceModelId": { 130 | "Ref": "bedrockInferenceModelId" 131 | }, 132 | "KinesisVideoStreamIntegration": { 133 | "Ref": "kinesisVideoStreamIntegration" 134 | }, 135 | "STORAGE_S3STORAGE_BUCKETNAME": { 136 | "Ref": "storages3storageBucketName" 137 | } 138 | } 139 | }, 140 | "Role": { 141 | "Fn::GetAtt": [ 142 | "LambdaExecutionRole", 143 | "Arn" 144 | ] 145 | }, 146 | "Runtime": "python3.11", 147 | "Layers": [ 148 | { 149 | "Ref": "functionfrontendClipCrunchersSharedArn" 150 | }, 151 | "arn:aws:lambda:us-east-1:336392948345:layer:AWSSDKPandas-Python311:4" 152 | ], 153 | "Timeout": 25 154 | } 155 | }, 156 | "LambdaExecutionRole": { 157 | "Type": "AWS::IAM::Role", 158 | "Properties": { 159 | "RoleName": { 160 | "Fn::If": [ 161 | "ShouldNotCreateEnvResources", 162 | "frontendLambdaRole8951cf29", 163 | { 164 | "Fn::Join": [ 165 | "", 166 | [ 167 | "frontendLambdaRole8951cf29", 168 | "-", 169 | { 170 | "Ref": "env" 171 | } 172 | ] 173 | ] 174 | } 175 | ] 176 | }, 177 | "AssumeRolePolicyDocument": { 178 | "Version": "2012-10-17", 179 | "Statement": [ 180 | { 181 | "Effect": "Allow", 182 | "Principal": { 183 | "Service": [ 184 | "lambda.amazonaws.com" 185 | ] 186 | }, 187 | "Action": [ 188 | "sts:AssumeRole" 189 | ] 190 | } 191 | ] 192 | } 193 | } 194 | }, 195 | "lambdaexecutionpolicy": { 196 | "DependsOn": [ 197 | "LambdaExecutionRole" 198 | ], 199 | "Type": "AWS::IAM::Policy", 200 | "Properties": { 201 | "PolicyName": "lambda-execution-policy", 202 | "Roles": [ 203 | { 204 | "Ref": "LambdaExecutionRole" 205 | } 206 | ], 207 | "PolicyDocument": { 208 | "Version": "2012-10-17", 209 | "Statement": [ 210 | { 211 | "Effect": "Allow", 212 | "Action": [ 213 | "logs:CreateLogGroup", 214 | "logs:CreateLogStream", 215 | "logs:PutLogEvents" 216 | ], 217 | "Resource": { 218 | "Fn::Sub": [ 219 | "arn:aws:logs:${region}:${account}:log-group:/aws/lambda/${lambda}:log-stream:*", 220 | { 221 | "region": { 222 | "Ref": "AWS::Region" 223 | }, 224 | "account": { 225 | "Ref": "AWS::AccountId" 226 | }, 227 | "lambda": { 228 | "Ref": "LambdaFunction" 229 | } 230 | } 231 | ] 232 | } 233 | } 234 | ] 235 | } 236 | } 237 | }, 238 | "CustomLambdaExecutionPolicy": { 239 | "Type": "AWS::IAM::Policy", 240 | "Properties": { 241 | "PolicyName": "custom-lambda-execution-policy", 242 | "PolicyDocument": { 243 | "Version": "2012-10-17", 244 | "Statement": [ 245 | { 246 | "Action": [ 247 | "bedrock:InvokeModel" 248 | ], 249 | "Resource": [ 250 | { 251 | "Fn::Sub": "arn:${AWS::Partition}:bedrock:${AWS::Region}::foundation-model/${bedrockEmbeddingModelId}" 252 | }, 253 | { 254 | "Fn::Sub": "arn:${AWS::Partition}:bedrock:${AWS::Region}::foundation-model/${bedrockInferenceModelId}" 255 | } 256 | ], 257 | "Effect": "Allow" 258 | }, 259 | { 260 | "Action": [ 261 | "aoss:APIAccessAll" 262 | ], 263 | "Resource": [ 264 | { 265 | "Ref": "vectordbopensearchopensearchServerlessCollectionARN" 266 | } 267 | ], 268 | "Effect": "Allow" 269 | }, 270 | { 271 | "Action": [ 272 | "kinesisvideo:Describe*", 273 | "kinesisvideo:Get*", 274 | "kinesisvideo:List*" 275 | ], 276 | "Resource": [ 277 | { 278 | "Fn::Sub": "arn:${AWS::Partition}:kinesisvideo:${AWS::Region}:${AWS::AccountId}:${customkinesisvideostreamkinesisVideoStreamName}" 279 | } 280 | ], 281 | "Effect": "Allow" 282 | } 283 | ] 284 | }, 285 | "Roles": [ 286 | { 287 | "Ref": "LambdaExecutionRole" 288 | } 289 | ] 290 | }, 291 | "DependsOn": "LambdaExecutionRole" 292 | }, 293 | "AmplifyResourcesPolicy": { 294 | "DependsOn": [ 295 | "LambdaExecutionRole" 296 | ], 297 | "Type": "AWS::IAM::Policy", 298 | "Properties": { 299 | "PolicyName": "amplify-lambda-execution-policy", 300 | "Roles": [ 301 | { 302 | "Ref": "LambdaExecutionRole" 303 | } 304 | ], 305 | "PolicyDocument": { 306 | "Version": "2012-10-17", 307 | "Statement": [ 308 | { 309 | "Effect": "Allow", 310 | "Action": "s3:ListBucket", 311 | "Resource": [ 312 | { 313 | "Fn::Join": [ 314 | "", 315 | [ 316 | "arn:aws:s3:::", 317 | { 318 | "Ref": "storages3storageBucketName" 319 | } 320 | ] 321 | ] 322 | } 323 | ] 324 | }, 325 | { 326 | "Effect": "Allow", 327 | "Action": [ 328 | "s3:GetObject" 329 | ], 330 | "Resource": [ 331 | { 332 | "Fn::Join": [ 333 | "", 334 | [ 335 | "arn:aws:s3:::", 336 | { 337 | "Ref": "storages3storageBucketName" 338 | }, 339 | "/*" 340 | ] 341 | ] 342 | } 343 | ] 344 | } 345 | ] 346 | } 347 | } 348 | } 349 | }, 350 | "Outputs": { 351 | "Name": { 352 | "Value": { 353 | "Ref": "LambdaFunction" 354 | } 355 | }, 356 | "Arn": { 357 | "Value": { 358 | "Fn::GetAtt": [ 359 | "LambdaFunction", 360 | "Arn" 361 | ] 362 | } 363 | }, 364 | "Region": { 365 | "Value": { 366 | "Ref": "AWS::Region" 367 | } 368 | }, 369 | "LambdaExecutionRole": { 370 | "Value": { 371 | "Ref": "LambdaExecutionRole" 372 | } 373 | }, 374 | "LambdaExecutionRoleArn": { 375 | "Value": { 376 | "Fn::GetAtt": [ 377 | "LambdaExecutionRole", 378 | "Arn" 379 | ] 380 | } 381 | } 382 | } 383 | } -------------------------------------------------------------------------------- /amplify/backend/function/frontendf6b68d3c/function-parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "lambdaLayers": [ 3 | { 4 | "type": "ProjectLayer", 5 | "resourceName": "frontendClipCrunchersShared", 6 | "version": "Always choose latest version", 7 | "isLatestVersionSelected": true, 8 | "env": "dev" 9 | } 10 | ], 11 | "permissions": { 12 | "storage": { 13 | "s3storage": [ 14 | "read" 15 | ] 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /amplify/backend/function/frontendf6b68d3c/parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "vectordbopensearchCollectionEndpoint": { 3 | "Fn::GetAtt": [ 4 | "vectordbopensearch", 5 | "Outputs.CollectionEndpoint" 6 | ] 7 | }, 8 | "vectordbopensearchopensearchServerlessCollectionName": { 9 | "Fn::GetAtt": [ 10 | "vectordbopensearch", 11 | "Outputs.opensearchServerlessCollectionName" 12 | ] 13 | }, 14 | "vectordbopensearchopensearchServerlessIndexName": { 15 | "Fn::GetAtt": [ 16 | "vectordbopensearch", 17 | "Outputs.opensearchServerlessIndexName" 18 | ] 19 | } 20 | } -------------------------------------------------------------------------------- /amplify/backend/function/frontendf6b68d3c/src/event.json: -------------------------------------------------------------------------------- 1 | { "test": "event" } 2 | -------------------------------------------------------------------------------- /amplify/backend/function/frontendf6b68d3c/src/index.py: -------------------------------------------------------------------------------- 1 | import json 2 | import boto3 3 | import numpy as np 4 | import base64 5 | from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth 6 | from dateutil import parser 7 | from datetime import datetime, timedelta 8 | import os 9 | import base64 10 | 11 | def initialize_opensearch_client(): 12 | opensearch_endpoint = os.environ['OpensearchEndpoint'].replace('https://','',1) # serverless collection endpoint, without https:// 13 | print(f"Opensearch Endpoint: {opensearch_endpoint}") 14 | region = os.environ['AWS_REGION'] # e.g. us-east-1 15 | print(f"region: {region}") 16 | credentials = boto3.Session().get_credentials() 17 | print(f"Caller Identity: {boto3.client('sts').get_caller_identity()}") 18 | 19 | auth = AWSV4SignerAuth(credentials, region, 'aoss') 20 | 21 | client = OpenSearch( 22 | hosts=[{'host': opensearch_endpoint, 'port': 443}], 23 | http_auth=auth, 24 | use_ssl=True, 25 | verify_certs=True, 26 | connection_class=RequestsHttpConnection, 27 | pool_maxsize=20, 28 | ) 29 | 30 | return client 31 | 32 | print('Intializing function') 33 | S3_RESOURCE = boto3.resource('s3') 34 | S3_CLIENT = boto3.client('s3') 35 | BEDROCK_CLIENT = boto3.client('bedrock-runtime') 36 | 37 | OPENSEARCH_CLIENT = initialize_opensearch_client() 38 | OPENSEARCH_INDEX_NAME = os.environ['OpensearchIndexName'] 39 | print(f"OPENSEARCH_INDEX_NAME: {OPENSEARCH_INDEX_NAME}") 40 | 41 | BEDROCK_EMBEDDING_MODEL_ID = os.environ['BedrockEmbeddingModelId'] 42 | print(f"BEDROCK_EMBEDDING_MODEL_ID: {BEDROCK_EMBEDDING_MODEL_ID}") 43 | 44 | BEDROCK_INFERENCE_MODEL_ID = os.environ['BedrockInferenceModelId'] 45 | print(f"BEDROCK_INFERENCE_MODEL_ID: {BEDROCK_INFERENCE_MODEL_ID}") 46 | 47 | KINESIS_VIDEO_STREAM_INTEGRATION = os.environ['KinesisVideoStreamIntegration'] == 'True' 48 | print(f"KINESIS_VIDEO_STREAM_INTEGRATION: {KINESIS_VIDEO_STREAM_INTEGRATION}") 49 | 50 | S3STORAGE_BUCKETNAME = os.environ.get('STORAGE_S3STORAGE_BUCKETNAME', '') 51 | print(f"S3STORAGE_BUCKETNAME: {S3STORAGE_BUCKETNAME}") 52 | 53 | if KINESIS_VIDEO_STREAM_INTEGRATION: 54 | KINESIS_VIDEO_STREAM_NAME = os.environ['KinesisVideoStreamName'] 55 | print(f"KINESIS_VIDEO_STREAM_NAME: {KINESIS_VIDEO_STREAM_NAME}") 56 | 57 | def get_kinesis_data_endpoint(): 58 | client = boto3.client("kinesisvideo") 59 | 60 | response = client.get_data_endpoint( 61 | StreamName=KINESIS_VIDEO_STREAM_NAME, 62 | APIName='GET_HLS_STREAMING_SESSION_URL' 63 | ) 64 | 65 | return response['DataEndpoint'] 66 | 67 | KINESIS_DATA_ENDPOINT = get_kinesis_data_endpoint() 68 | KINESIS_VIDEO_ARCHIVED_MEDIA_CLIENT = boto3.client("kinesis-video-archived-media", endpoint_url=KINESIS_DATA_ENDPOINT) 69 | 70 | def get_hls_streaming_session_live_url(): 71 | response = KINESIS_VIDEO_ARCHIVED_MEDIA_CLIENT.get_hls_streaming_session_url( 72 | StreamName=KINESIS_VIDEO_STREAM_NAME, 73 | PlaybackMode='LIVE', 74 | HLSFragmentSelector={ 75 | 'FragmentSelectorType': 'PRODUCER_TIMESTAMP', 76 | }, 77 | DiscontinuityMode='ON_DISCONTINUITY', 78 | DisplayFragmentTimestamp='ALWAYS', 79 | Expires=3600 80 | ) 81 | 82 | return response['HLSStreamingSessionURL'] 83 | 84 | def deduplicate_by_timestamp(data, window=30): 85 | 86 | data.sort(key=lambda x: x['timestamp']) 87 | 88 | deduplicated_data = [] 89 | last_timestamp = None 90 | 91 | for item in data: 92 | current_timestamp = parser.isoparse(item['timestamp']) 93 | if last_timestamp is None or (current_timestamp - last_timestamp) > timedelta(seconds=window): 94 | deduplicated_data.append(item) 95 | last_timestamp = current_timestamp 96 | 97 | return deduplicated_data 98 | 99 | def get_hls_streaming_session_url(timestamp): 100 | image_date = parser.isoparse(timestamp) 101 | print(image_date) 102 | start_time = image_date - timedelta(seconds=20) 103 | print(start_time) 104 | end_time = image_date + timedelta(seconds=10) 105 | print(end_time) 106 | 107 | response = KINESIS_VIDEO_ARCHIVED_MEDIA_CLIENT.get_hls_streaming_session_url( 108 | StreamName=KINESIS_VIDEO_STREAM_NAME, 109 | PlaybackMode='ON_DEMAND', 110 | HLSFragmentSelector={ 111 | 'FragmentSelectorType': 'PRODUCER_TIMESTAMP', 112 | 'TimestampRange': { 113 | 'StartTimestamp': start_time, 114 | 'EndTimestamp': end_time 115 | } 116 | }, 117 | DiscontinuityMode='ON_DISCONTINUITY', 118 | DisplayFragmentTimestamp='ALWAYS', 119 | Expires=3600 120 | ) 121 | 122 | print(response['HLSStreamingSessionURL']) 123 | 124 | return response['HLSStreamingSessionURL'] 125 | 126 | def bedrock_invoke_model(body): 127 | response = BEDROCK_CLIENT.invoke_model( 128 | body=body, modelId=BEDROCK_EMBEDDING_MODEL_ID 129 | ) 130 | return json.loads(response.get("body").read()) 131 | 132 | 133 | def embedding_from_text_and_s3(text, bucket, key, normalize=True): 134 | print(f'Getting embedding for text: "{text}" bucket: "{bucket}" key: "{key}"') 135 | 136 | img_bytes = None 137 | if key != "" and bucket != "": 138 | bucket = S3_RESOURCE.Bucket(bucket) 139 | image = bucket.Object(key) 140 | img_data = image.get().get('Body').read() 141 | 142 | img_bytes = base64.b64encode(img_data).decode('utf8') 143 | 144 | if text == "": 145 | text = None 146 | 147 | if text == None and img_bytes == None: 148 | raise ValueError("Either text or image must be provided") 149 | 150 | body = json.dumps( 151 | { 152 | "inputText": text, 153 | "inputImage": img_bytes 154 | } 155 | ) 156 | 157 | embedding = np.array(bedrock_invoke_model(body)['embedding']) 158 | if normalize: 159 | embedding /= np.linalg.norm(embedding) 160 | 161 | return embedding 162 | 163 | def get_date_range_filter(date_range): 164 | if date_range['type'] == 'relative': 165 | end_date = datetime.utcnow() 166 | if date_range['unit'] == 'year': 167 | start_date = end_date - timedelta(days=365 * date_range['amount']) 168 | elif date_range['unit'] == 'month': 169 | start_date = end_date - timedelta(days=30 * date_range['amount']) 170 | elif date_range['unit'] == 'week': 171 | start_date = end_date - timedelta(weeks=date_range['amount']) 172 | elif date_range['unit'] == 'day': 173 | start_date = end_date - timedelta(days=date_range['amount']) 174 | elif date_range['unit'] == 'hour': 175 | start_date = end_date - timedelta(hours=date_range['amount']) 176 | elif date_range['unit'] == 'minute': 177 | start_date = end_date - timedelta(minutes=date_range['amount']) 178 | elif date_range['unit'] == 'second': 179 | start_date = end_date - timedelta(seconds=date_range['amount']) 180 | else: 181 | raise ValueError(f"Unsupported unit: {date_range['unit']}") 182 | else: # absolute 183 | start_date = parser.isoparse(date_range['startDate']) 184 | end_date = parser.isoparse(date_range['endDate']) 185 | 186 | return { 187 | "range": { 188 | "timestamp": { 189 | "gte": start_date.isoformat(), 190 | "lte": end_date.isoformat() 191 | } 192 | } 193 | } 194 | 195 | def get_nearest_neighbors_from_opensearch(k_neighbors, embedding, date_range): 196 | date_filter = get_date_range_filter(date_range) 197 | 198 | query = { 199 | "size": k_neighbors, 200 | "query": { 201 | "knn": { 202 | "titan-embedding": { 203 | "vector": embedding.tolist(), 204 | "k": k_neighbors, 205 | "filter": date_filter 206 | } 207 | } 208 | }, 209 | } 210 | 211 | print(f"Sending query to opensearch: {query}") 212 | response = OPENSEARCH_CLIENT.search(body=query, index=OPENSEARCH_INDEX_NAME) 213 | print(f"Received response from opensearch: {response}") 214 | 215 | return response 216 | 217 | def process_nearest_neighbor_raw_response_for_image_search(response, baseline_embedding, confidenceThreshold, includeSimilarTimestamp=True): 218 | output_dict = {} 219 | 220 | for hit in response["hits"]["hits"]: 221 | s3_filename = hit["_source"]["s3-uri"] 222 | timestamp = hit["_source"]["timestamp"] 223 | 224 | text_summary = "No text summary available" 225 | if "summary" in hit["_source"]: 226 | print(f'Text summary: {hit["_source"]["summary"]}') 227 | text_summary = hit["_source"]["summary"] 228 | 229 | source = "unknown" 230 | if "source" in hit["_source"]: 231 | print(f'source: {hit["_source"]["source"]}') 232 | source = hit["_source"]["source"] 233 | 234 | custom_metadata = "" 235 | if "custom-metadata" in hit["_source"]: 236 | print(f'custom-metadata: {hit["_source"]["custom-metadata"]}') 237 | custom_metadata = hit["_source"]["custom-metadata"] 238 | if custom_metadata == "": 239 | custom_metadata = "No custom metadata available" 240 | 241 | if not str(timestamp).endswith("Z"): 242 | timestamp = f"{timestamp}Z" 243 | 244 | if s3_filename.startswith("public/"): 245 | s3_filename = s3_filename.replace("public/", "") 246 | 247 | target_embedding = np.array(hit["_source"]["titan-embedding"]).astype(float) 248 | dist = np.linalg.norm(target_embedding - baseline_embedding) 249 | confidence = (1 - dist / 2) * 100 250 | 251 | print(f"s3 uri: {s3_filename} - confidence: {confidence}") 252 | if round(confidence, 2) >= confidenceThreshold: 253 | output_dict[s3_filename] = {"confidence": round(confidence, 2), "timestamp": timestamp, "summary": text_summary, "source": source, "custom_metadata": custom_metadata} 254 | 255 | print(f"output dictionary:{output_dict}") 256 | images = [(lambda d: d.update(file=key) or d)(val) for (key, val) in output_dict.items()] 257 | 258 | if not includeSimilarTimestamp: 259 | images = deduplicate_by_timestamp(images) 260 | 261 | images = sorted(images, key=lambda x: x['confidence'], reverse=True) 262 | 263 | print(f"Images after all processing: {images}") 264 | 265 | return {"images": images} 266 | 267 | def get_s3_uris_from_opensearch(date_range, max_results=20): 268 | date_filter = get_date_range_filter(date_range) 269 | 270 | query = { 271 | "size": max_results, 272 | "query": date_filter, 273 | "_source": ["s3-uri"] 274 | } 275 | 276 | print(f"Sending query to opensearch: {query}") 277 | response = OPENSEARCH_CLIENT.search(body=query, index=OPENSEARCH_INDEX_NAME) 278 | print(f"Received response from opensearch: {response}") 279 | 280 | s3_uris = [] 281 | for hit in response["hits"]["hits"]: 282 | s3_uris.append(hit["_source"]["s3-uri"]) 283 | 284 | return s3_uris 285 | 286 | def get_image_data(bucket, uris): 287 | bucket = S3_RESOURCE.Bucket(bucket) 288 | images_content = [] 289 | for uri in uris: 290 | image = bucket.Object(uri) 291 | img_data = image.get().get('Body').read() 292 | image_content = { 293 | "type": "image", 294 | "source": { 295 | "type": "base64", 296 | "media_type": "image/jpeg", 297 | "data": base64.b64encode(img_data).decode('utf8') 298 | } 299 | } 300 | images_content.append(image_content) 301 | 302 | return images_content 303 | 304 | def bedrock_invoke_multimodal_model(images_content, custom_summary_prompt="Analyze these images and provide a brief summary of its contents. Focus on the main subjects, actions, and any notable elements in the scene. Keep the summary concise, around 2-3 sentences."): 305 | if len(images_content) == 0: 306 | return "No summary" 307 | 308 | text_content = { 309 | "type": "text", 310 | "text": custom_summary_prompt 311 | } 312 | multimodal_content = images_content 313 | multimodal_content.append(text_content) 314 | # print(f"multimodal content first: {multimodal_content[0]}") 315 | # print(f"multimodal content last: {multimodal_content[-1]}") 316 | 317 | body = json.dumps({ 318 | "anthropic_version": "bedrock-2023-05-31", 319 | "max_tokens": 300, 320 | "messages": [ 321 | { 322 | "role": "user", 323 | "content": multimodal_content 324 | } 325 | ], 326 | "temperature": 0 327 | }) 328 | 329 | response = BEDROCK_CLIENT.invoke_model( 330 | body=body, 331 | modelId=BEDROCK_INFERENCE_MODEL_ID, 332 | contentType="application/json", 333 | accept="application/json" 334 | ) 335 | 336 | response_body = json.loads(response['body'].read()) 337 | summary = response_body['content'][0]['text'].strip() 338 | return summary 339 | 340 | def handler(event, context): 341 | print("received event:") 342 | print(event) 343 | 344 | statusCode = 200 345 | output = {} 346 | 347 | if event["path"] == "/images/search" and event["httpMethod"] == "POST": 348 | eventJson = json.loads(event["body"]) 349 | 350 | searchText = eventJson["searchText"] 351 | searchImage = eventJson["searchImage"] 352 | date_range = eventJson["dateRange"] 353 | confidenceThreshold = 0.0 354 | maxResults = 50 355 | 356 | if "confidenceThreshold" in eventJson: 357 | confidenceThreshold = float(eventJson["confidenceThreshold"]) 358 | 359 | includeSimilarTimestamp = True 360 | if "includeSimilarTimestamp" in eventJson: 361 | includeSimilarTimestamp = eventJson["includeSimilarTimestamp"] 362 | 363 | if "maxResults" in eventJson: 364 | maxResults = float(eventJson["maxResults"]) 365 | 366 | print(f"searchText: {searchText} - searchImage: {searchImage} - date_range: {date_range} - confidence: {confidenceThreshold} - includeSimilarTimestamp: {includeSimilarTimestamp}") 367 | 368 | object_key = "" 369 | if searchImage != "": 370 | object_key = f'public/fileUploads/{searchImage}' 371 | 372 | user_query_embedding = embedding_from_text_and_s3(searchText, S3STORAGE_BUCKETNAME, object_key) 373 | 374 | print(f"embedding of user query: {user_query_embedding}") 375 | 376 | raw_nearest_neighbors_response = get_nearest_neighbors_from_opensearch(maxResults, user_query_embedding, date_range) 377 | 378 | output = process_nearest_neighbor_raw_response_for_image_search(raw_nearest_neighbors_response, user_query_embedding, confidenceThreshold, includeSimilarTimestamp) 379 | 380 | elif event["path"] == "/images/sessionURL" and event["httpMethod"] == "GET" and KINESIS_VIDEO_STREAM_INTEGRATION: 381 | timestamp = event["queryStringParameters"]["timestamp"] 382 | session_url = get_hls_streaming_session_url(timestamp) 383 | output = {"sessionURL": session_url} 384 | 385 | elif event["path"] == "/images/liveURL" and event["httpMethod"] == "GET" and KINESIS_VIDEO_STREAM_INTEGRATION: 386 | session_url = get_hls_streaming_session_live_url() 387 | output = {"sessionURL": session_url} 388 | 389 | elif event["path"] == "/images/summarize" and event["httpMethod"] == "POST": 390 | eventJson = json.loads(event["body"]) 391 | date_range = eventJson["dateRange"] 392 | print(f"date_range: {date_range}") 393 | custom_summary_prompt = eventJson["customSummaryPrompt"] 394 | print(f"custom_summary_prompt: {custom_summary_prompt}") 395 | 396 | uris = get_s3_uris_from_opensearch(date_range) 397 | images = get_image_data(S3STORAGE_BUCKETNAME, uris) 398 | output = {} 399 | output['summary'] = bedrock_invoke_multimodal_model(images, custom_summary_prompt) 400 | output['images'] = uris 401 | 402 | else: 403 | statusCode = 404 404 | output = {"message": "Path not found"} 405 | 406 | return { 407 | "statusCode": statusCode, 408 | "headers": { 409 | "Access-Control-Allow-Headers": "*", 410 | "Access-Control-Allow-Origin": "*", 411 | "Access-Control-Allow-Methods": "OPTIONS,POST,GET", 412 | }, 413 | "body": json.dumps(output), 414 | } -------------------------------------------------------------------------------- /amplify/backend/function/frontendf6b68d3c/src/setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup(name='src', version='1.0') 4 | -------------------------------------------------------------------------------- /amplify/backend/function/frontendf6b68d3c/src/src.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 2.1 2 | Name: src 3 | Version: 1.0 4 | -------------------------------------------------------------------------------- /amplify/backend/function/frontendf6b68d3c/src/src.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | index.py 2 | setup.py 3 | src.egg-info/PKG-INFO 4 | src.egg-info/SOURCES.txt 5 | src.egg-info/dependency_links.txt 6 | src.egg-info/top_level.txt -------------------------------------------------------------------------------- /amplify/backend/function/frontendf6b68d3c/src/src.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /amplify/backend/function/frontendf6b68d3c/src/src.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | index 2 | -------------------------------------------------------------------------------- /amplify/backend/storage/s3storage/cli-inputs.json: -------------------------------------------------------------------------------- 1 | { 2 | "resourceName": "s3storage", 3 | "policyUUID": "eeb13a3d", 4 | "bucketName": "clipcrunchobjects", 5 | "storageAccess": "auth", 6 | "guestAccess": [], 7 | "authAccess": [ 8 | "CREATE_AND_UPDATE", 9 | "READ", 10 | "DELETE" 11 | ], 12 | "triggerFunction": "CalculateVectorEmbeddingForImagesInS3IndexInOpensearch", 13 | "groupAccess": {} 14 | } -------------------------------------------------------------------------------- /amplify/backend/tags.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Key": "user:Stack", 4 | "Value": "{project-env}" 5 | }, 6 | { 7 | "Key": "user:Application", 8 | "Value": "{project-name}" 9 | } 10 | ] -------------------------------------------------------------------------------- /amplify/backend/types/amplify-dependent-resources-ref.d.ts: -------------------------------------------------------------------------------- 1 | export type AmplifyDependentResourcesAttributes = { 2 | "api": { 3 | "clipcrunchapi": { 4 | "ApiId": "string", 5 | "ApiName": "string", 6 | "RootUrl": "string" 7 | } 8 | }, 9 | "auth": { 10 | "frontend1790dc25": { 11 | "AppClientID": "string", 12 | "AppClientIDWeb": "string", 13 | "CreatedSNSRole": "string", 14 | "IdentityPoolId": "string", 15 | "IdentityPoolName": "string", 16 | "UserPoolArn": "string", 17 | "UserPoolId": "string", 18 | "UserPoolName": "string" 19 | } 20 | }, 21 | "custom": { 22 | "vectordbaccess": { 23 | "AccessPolicyName": "string" 24 | } 25 | }, 26 | "function": { 27 | "CalculateVectorEmbeddingForImagesInS3IndexInOpensearch": { 28 | "Arn": "string", 29 | "LambdaExecutionRole": "string", 30 | "LambdaExecutionRoleArn": "string", 31 | "MediaConvertExecutionRole": "string", 32 | "Name": "string", 33 | "Region": "string" 34 | }, 35 | "frontendClipCrunchersShared": { 36 | "Arn": "string" 37 | }, 38 | "frontendf6b68d3c": { 39 | "Arn": "string", 40 | "LambdaExecutionRole": "string", 41 | "LambdaExecutionRoleArn": "string", 42 | "Name": "string", 43 | "Region": "string" 44 | } 45 | }, 46 | "storage": { 47 | "s3storage": { 48 | "BucketName": "string", 49 | "Region": "string" 50 | } 51 | }, 52 | "vectordb": { 53 | "opensearch": { 54 | "CollectionEndpoint": "string", 55 | "opensearchServerlessCollectionARN": "string", 56 | "opensearchServerlessCollectionName": "string", 57 | "opensearchServerlessIndexName": "string" 58 | } 59 | } 60 | } -------------------------------------------------------------------------------- /amplify/backend/vectordb/opensearch/parameters.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /amplify/backend/vectordb/opensearch/template.json: -------------------------------------------------------------------------------- 1 | { 2 | "Description": "{\"createdOn\":\"Mac\",\"createdBy\":\"Amplify\",\"createdWith\":\"12.12.4\",\"stackType\":\"vectordb\",\"metadata\":{}}", 3 | "AWSTemplateFormatVersion": "2010-09-09", 4 | "Parameters": { 5 | "env": { 6 | "Type": "String" 7 | }, 8 | "opensearchServerlessCollectionName": { 9 | "Type": "String", 10 | "Default": "clip-crunchers" 11 | }, 12 | "opensearchServerlessIndexName": { 13 | "Type": "String", 14 | "Default": "clip-crunchers" 15 | }, 16 | "opensearchServerlessStandbyReplicas": { 17 | "Type": "String", 18 | "Default": "ENABLED" 19 | } 20 | }, 21 | "Resources": { 22 | "NetworkPolicy": { 23 | "Type": "AWS::OpenSearchServerless::SecurityPolicy", 24 | "Properties": { 25 | "Name": { 26 | "Fn::Sub": [ 27 | "${collection_name}-${env}-network", 28 | { 29 | "collection_name": { 30 | "Ref": "opensearchServerlessCollectionName" 31 | }, 32 | "env": { 33 | "Ref": "env" 34 | } 35 | } 36 | ] 37 | }, 38 | "Type": "network", 39 | "Description": { 40 | "Fn::Sub": [ 41 | "Network policy for ${collection_name}-${env} collection", 42 | { 43 | "collection_name": { 44 | "Ref": "opensearchServerlessCollectionName" 45 | }, 46 | "env": { 47 | "Ref": "env" 48 | } 49 | } 50 | ] 51 | }, 52 | "Policy": { 53 | "Fn::Sub": [ 54 | "[{\"Rules\":[{\"ResourceType\":\"collection\",\"Resource\":[\"collection/${collection_name}-${env}\"]}, {\"ResourceType\":\"dashboard\",\"Resource\":[\"collection/${collection_name}-${env}\"]}],\"AllowFromPublic\":true}]", 55 | { 56 | "collection_name": { 57 | "Ref": "opensearchServerlessCollectionName" 58 | }, 59 | "env": { 60 | "Ref": "env" 61 | } 62 | } 63 | ] 64 | } 65 | } 66 | }, 67 | "EncryptionPolicy": { 68 | "Type": "AWS::OpenSearchServerless::SecurityPolicy", 69 | "Properties": { 70 | "Name": { 71 | "Fn::Sub": [ 72 | "${collection_name}-${env}-security", 73 | { 74 | "collection_name": { 75 | "Ref": "opensearchServerlessCollectionName" 76 | }, 77 | "env": { 78 | "Ref": "env" 79 | } 80 | } 81 | ] 82 | }, 83 | "Type": "encryption", 84 | "Description": { 85 | "Fn::Sub": [ 86 | "Encryption policy for ${collection_name}-${env} collection", 87 | { 88 | "collection_name": { 89 | "Ref": "opensearchServerlessCollectionName" 90 | }, 91 | "env": { 92 | "Ref": "env" 93 | } 94 | } 95 | ] 96 | }, 97 | "Policy": { 98 | "Fn::Sub": [ 99 | "{\"Rules\":[{\"ResourceType\":\"collection\",\"Resource\":[\"collection/${collection_name}-${env}\"]}],\"AWSOwnedKey\":true}", 100 | { 101 | "collection_name": { 102 | "Ref": "opensearchServerlessCollectionName" 103 | }, 104 | "env": { 105 | "Ref": "env" 106 | } 107 | } 108 | ] 109 | } 110 | } 111 | }, 112 | "Collection": { 113 | "Type": "AWS::OpenSearchServerless::Collection", 114 | "Properties": { 115 | "Name": { 116 | "Fn::Sub": [ 117 | "${collection_name}-${env}", 118 | { 119 | "collection_name": { 120 | "Ref": "opensearchServerlessCollectionName" 121 | }, 122 | "env": { 123 | "Ref": "env" 124 | } 125 | } 126 | ] 127 | }, 128 | "Type": "VECTORSEARCH", 129 | "StandbyReplicas": { 130 | "Ref": "opensearchServerlessStandbyReplicas" 131 | }, 132 | "Description": "Collection for clip crunchers demo" 133 | }, 134 | "DependsOn": "EncryptionPolicy" 135 | } 136 | }, 137 | "Outputs": { 138 | "CollectionEndpoint": { 139 | "Value": { 140 | "Fn::GetAtt": [ 141 | "Collection", 142 | "CollectionEndpoint" 143 | ] 144 | } 145 | }, 146 | "opensearchServerlessCollectionARN": { 147 | "Value": { 148 | "Fn::GetAtt": [ 149 | "Collection", 150 | "Arn" 151 | ] 152 | } 153 | }, 154 | "opensearchServerlessCollectionName": { 155 | "Value": { 156 | "Fn::Sub": [ 157 | "${collection_name}-${env}", 158 | { 159 | "collection_name": { 160 | "Ref": "opensearchServerlessCollectionName" 161 | }, 162 | "env": { 163 | "Ref": "env" 164 | } 165 | } 166 | ] 167 | } 168 | }, 169 | "opensearchServerlessIndexName": { 170 | "Value": { 171 | "Ref": "opensearchServerlessIndexName" 172 | } 173 | } 174 | } 175 | } -------------------------------------------------------------------------------- /amplify/cli.json: -------------------------------------------------------------------------------- 1 | { 2 | "features": { 3 | "graphqltransformer": { 4 | "addmissingownerfields": false, 5 | "improvepluralization": false, 6 | "validatetypenamereservedwords": true, 7 | "useexperimentalpipelinedtransformer": false, 8 | "enableiterativegsiupdates": false, 9 | "secondarykeyasgsi": false, 10 | "skipoverridemutationinputtypes": false, 11 | "transformerversion": 1, 12 | "suppressschemamigrationprompt": true, 13 | "securityenhancementnotification": true, 14 | "showfieldauthnotification": true, 15 | "usesubusernamefordefaultidentityclaim": true, 16 | "usefieldnameforprimarykeyconnectionfield": false, 17 | "enableautoindexquerynames": false, 18 | "respectprimarykeyattributesonconnectionfield": false, 19 | "shoulddeepmergedirectiveconfigdefaults": false, 20 | "populateownerfieldforstaticgroupauth": false, 21 | "securityEnhancementNotification": false, 22 | "showFieldAuthNotification": false 23 | }, 24 | "frontend-ios": { 25 | "enablexcodeintegration": false 26 | }, 27 | "auth": { 28 | "enablecaseinsensitivity": false, 29 | "useinclusiveterminology": false, 30 | "breakcirculardependency": false, 31 | "forcealiasattributes": false, 32 | "useenabledmfas": false 33 | }, 34 | "codegen": { 35 | "useappsyncmodelgenplugin": false, 36 | "usedocsgeneratorplugin": false, 37 | "usetypesgeneratorplugin": false, 38 | "cleangeneratedmodelsdirectory": false, 39 | "retaincasestyle": false, 40 | "addtimestampfields": false, 41 | "handlelistnullabilitytransparently": false, 42 | "emitauthprovider": false, 43 | "generateindexrules": false, 44 | "enabledartnullsafety": false, 45 | "generatemodelsforlazyloadandcustomselectionset": false 46 | }, 47 | "appsync": { 48 | "generategraphqlpermissions": false 49 | }, 50 | "latestregionsupport": { 51 | "pinpoint": 0, 52 | "translate": 0, 53 | "transcribe": 0, 54 | "rekognition": 0, 55 | "textract": 0, 56 | "comprehend": 0 57 | }, 58 | "project": { 59 | "overrides": true 60 | } 61 | }, 62 | "debug": { 63 | "shareProjectConfig": true 64 | } 65 | } -------------------------------------------------------------------------------- /amplify/hooks/README.md: -------------------------------------------------------------------------------- 1 | # Command Hooks 2 | 3 | Command hooks can be used to run custom scripts upon Amplify CLI lifecycle events like pre-push, post-add-function, etc. 4 | 5 | To get started, add your script files based on the expected naming convention in this directory. 6 | 7 | Learn more about the script file naming convention, hook parameters, third party dependencies, and advanced configurations at https://docs.amplify.aws/cli/usage/command-hooks 8 | -------------------------------------------------------------------------------- /clip-crunchers-blog-code.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /img/solution-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/Serverless-Semantic-Video-Search-Vector-Database-and-a-Multi-Modal-Generative-Al-Embeddings-Model/effe9134813d3d0b896952595d655cb093c226a0/img/solution-architecture.png -------------------------------------------------------------------------------- /img/solution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/Serverless-Semantic-Video-Search-Vector-Database-and-a-Multi-Modal-Generative-Al-Embeddings-Model/effe9134813d3d0b896952595d655cb093c226a0/img/solution.png -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Clip Crunch 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "frontend", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "build": "vite build", 9 | "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0", 10 | "preview": "vite preview" 11 | }, 12 | "dependencies": { 13 | "@aws-amplify/ui-react": "^6.5.5", 14 | "@cloudscape-design/components": "^3.0.822", 15 | "@cloudscape-design/global-styles": "^1.0.32", 16 | "aws-amplify": "^6.8.0", 17 | "buffer": "^6.0.3", 18 | "react": "^18.3.1", 19 | "react-dom": "^18.3.1", 20 | "react-hls-player": "^3.0.7", 21 | "react-router-dom": "^6.27.0", 22 | "react-webcam": "^7.2.0" 23 | }, 24 | "devDependencies": { 25 | "@types/react": "^18.3.12", 26 | "@types/react-dom": "^18.3.1", 27 | "@vitejs/plugin-react": "^4.3.3", 28 | "eslint": "^9.14.0", 29 | "eslint-plugin-react": "^7.37.2", 30 | "eslint-plugin-react-hooks": "^5.0.0", 31 | "eslint-plugin-react-refresh": "^0.4.14", 32 | "vite": "^5.4.14" 33 | }, 34 | "overrides": { 35 | "react-hls-player": { 36 | "react": "^18.2.0", 37 | "react-dom": "^18.2.0" 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/Serverless-Semantic-Video-Search-Vector-Database-and-a-Multi-Modal-Generative-Al-Embeddings-Model/effe9134813d3d0b896952595d655cb093c226a0/public/favicon.ico -------------------------------------------------------------------------------- /src/App.jsx: -------------------------------------------------------------------------------- 1 | import '@aws-amplify/ui-react/styles.css'; 2 | 3 | import { useState, useEffect } from 'react' 4 | import { Amplify } from 'aws-amplify'; 5 | import { Hub } from 'aws-amplify/utils'; 6 | import { getCurrentUser } from 'aws-amplify/auth'; 7 | import AppLayout from "@cloudscape-design/components/app-layout" 8 | 9 | import NavBar from './components/NavBar'; 10 | import ToolsBar from './components/ToolsBar'; 11 | import SideBar from './components/SideBar'; 12 | import Routes from './Routes'; 13 | 14 | import amplifyConfig from "./aws-exports"; 15 | 16 | import { Authenticator } from '@aws-amplify/ui-react'; 17 | 18 | function App() { 19 | 20 | if (window.location.hostname.includes("localhost")) { 21 | amplifyConfig.oauth.redirectSignIn = "http://localhost:5173/"; 22 | amplifyConfig.oauth.redirectSignOut = "http://localhost:5173/"; 23 | } 24 | 25 | Amplify.configure(amplifyConfig); 26 | 27 | const [user, setUser] = useState(null); 28 | 29 | 30 | 31 | useEffect(() => { 32 | Hub.listen('auth', ({ payload: { event, data } }) => { 33 | console.log(event, data); 34 | switch (event) { 35 | case 'signIn': 36 | console.log(event) 37 | console.log(data) 38 | getUser().then(userData => setUser(userData)); 39 | break; 40 | case 'signOut': 41 | setUser(null); 42 | console.log('Sign out'); 43 | break; 44 | case 'signedOut': 45 | location.reload(); 46 | break; 47 | case 'signedIn': 48 | location.reload(); 49 | break; 50 | case 'signIn_failure': 51 | console.log('Sign in failure', data); 52 | break; 53 | } 54 | }); 55 | 56 | getUser().then(userData => setUser(userData)); 57 | }, []); 58 | 59 | function getUser() { 60 | return getCurrentUser() 61 | .then(userData => userData) 62 | .catch(() => console.log('Not signed in')); 63 | } 64 | 65 | 66 | return ( 67 | <> 68 | 69 | } 77 | tools={} 78 | content={user ? : } 79 | /> 80 | 81 | ) 82 | } 83 | 84 | export default App -------------------------------------------------------------------------------- /src/Routes.jsx: -------------------------------------------------------------------------------- 1 | import { 2 | createBrowserRouter, 3 | RouterProvider, 4 | } from "react-router-dom"; 5 | 6 | import SearchPage from "./pages/SearchPage"; 7 | import SummarizePage from "./pages/SummarizePage"; 8 | import LivePage from './pages/LivePage'; 9 | import WebcamUploadPage from './pages/WebcamUploadPage.jsx'; 10 | import FileUploadPage from './pages/FileUploadPage.jsx'; 11 | 12 | function Routes() { 13 | const router = createBrowserRouter([ 14 | { 15 | path: "/", 16 | element: , 17 | }, 18 | { 19 | path: "/live", 20 | element: , 21 | }, 22 | { 23 | path: "/webcamupload", 24 | element: , 25 | }, 26 | { 27 | path: "/fileupload", 28 | element: , 29 | }, 30 | { 31 | path: "/summarize", 32 | element: , 33 | } 34 | ]); 35 | 36 | return ( 37 | } /> 38 | ) 39 | } 40 | 41 | export default Routes 42 | -------------------------------------------------------------------------------- /src/components/NavBar.jsx: -------------------------------------------------------------------------------- 1 | import PropTypes from 'prop-types' 2 | import { signOut, signInWithRedirect } from 'aws-amplify/auth'; 3 | import TopNavigation from "@cloudscape-design/components/top-navigation"; 4 | 5 | const NavBar = (props) => { 6 | 7 | const loginlogout = async (isAuthenticated) => { 8 | if (isAuthenticated){ 9 | await signOut() 10 | } else { 11 | await signInWithRedirect({ 12 | provider: "AmazonFederate" 13 | }); 14 | } 15 | } 16 | 17 | return ( 18 | loginlogout(props.isAuthenticated) 29 | } 30 | ]} 31 | > 32 | 33 | 34 | ) 35 | } 36 | 37 | NavBar.propTypes = { 38 | title: PropTypes.string, 39 | isAuthenticated: PropTypes.bool 40 | } 41 | 42 | export default NavBar; 43 | -------------------------------------------------------------------------------- /src/components/SideBar.jsx: -------------------------------------------------------------------------------- 1 | import PropTypes from 'prop-types' 2 | import SideNavigation from "@cloudscape-design/components/side-navigation" 3 | 4 | const SideBar = (props) => { 5 | return ( 6 | 10 | ) 11 | } 12 | 13 | SideBar.propTypes = { 14 | title: PropTypes.string, 15 | items: PropTypes.array 16 | } 17 | 18 | export default SideBar; 19 | -------------------------------------------------------------------------------- /src/components/ToolsBar.jsx: -------------------------------------------------------------------------------- 1 | import PropTypes from 'prop-types' 2 | import HelpPanel from "@cloudscape-design/components/help-panel"; 3 | 4 | const ToolsBar = (props) => { 5 | return ( 6 | {props.title}} 8 | > 9 |
10 |

11 | {props.text} 12 |

13 |
14 | 15 | ) 16 | } 17 | 18 | ToolsBar.propTypes = { 19 | title: PropTypes.string, 20 | text: PropTypes.string 21 | } 22 | 23 | export default ToolsBar; 24 | -------------------------------------------------------------------------------- /src/main.jsx: -------------------------------------------------------------------------------- 1 | import "@cloudscape-design/global-styles/index.css" 2 | import React from 'react' 3 | import ReactDOM from 'react-dom/client' 4 | import App from './App.jsx' 5 | 6 | ReactDOM.createRoot(document.getElementById('root')).render( 7 | 8 | ) 9 | -------------------------------------------------------------------------------- /src/pages/FileUploadPage.jsx: -------------------------------------------------------------------------------- 1 | import { useCallback, useRef, useState } from 'react' 2 | import { get } from 'aws-amplify/api'; 3 | import Box from "@cloudscape-design/components/box" 4 | import Container from "@cloudscape-design/components/container" 5 | import SpaceBetween from "@cloudscape-design/components/space-between" 6 | import Alert from "@cloudscape-design/components/alert"; 7 | import Header from "@cloudscape-design/components/header" 8 | import Form from "@cloudscape-design/components/form" 9 | import Button from "@cloudscape-design/components/button" 10 | import Spinner from "@cloudscape-design/components/spinner" 11 | import Toggle from "@cloudscape-design/components/toggle"; 12 | import FileUpload from "@cloudscape-design/components/file-upload"; 13 | import { uploadData } from 'aws-amplify/storage'; 14 | import { Buffer } from "buffer" 15 | 16 | function FileUploadPage() { 17 | 18 | const [fileArray, setfileArray] = useState([]); 19 | 20 | const [alertText, setAlertText] = useState(""); 21 | const [alertType, setAlertType] = useState(""); 22 | const [alertVisible, setAlertVisible] = useState(false) 23 | const [isUploading, setIsUploading] = useState(false) 24 | 25 | const upload = async () => { 26 | setIsUploading(true) 27 | setAlertVisible(false) 28 | 29 | try { 30 | const result = await Promise.all(fileArray.map(async (file) => { 31 | var filename = "fileUploads/" + file.name; 32 | 33 | const result = await uploadData({ 34 | key: filename, 35 | data: file, 36 | }).result; 37 | console.log('Succeeded: ', result); 38 | }) 39 | ) 40 | 41 | setAlertText(`Successfully uploaded ${fileArray.length} files`) 42 | setAlertType("success") 43 | setAlertVisible(true) 44 | } catch (error) { 45 | console.log(fileArray) 46 | console.log('Error : ', error); 47 | setAlertText("File(s) failed to upload. Check console log for more information") 48 | setAlertType("error") 49 | setAlertVisible(true) 50 | } 51 | 52 | setIsUploading(false) 53 | setfileArray([]); 54 | }; 55 | 56 | return ( 57 | 58 |
File Upload}> 59 | 60 | 61 | 62 | { 63 | alertVisible ? 64 | 65 | { setAlertVisible(false) }} 68 | /> : <> 69 | } 70 | 71 | { !isUploading && 72 | { 74 | setfileArray(detail.value) 75 | setAlertVisible(false) 76 | } 77 | } 78 | value={fileArray} 79 | i18nStrings={{ 80 | uploadButtonText: e => 81 | e ? "Choose files" : "Choose file", 82 | dropzoneText: e => 83 | e 84 | ? "Drop files to upload" 85 | : "Drop file to upload", 86 | removeFileAriaLabel: e => 87 | `Remove file ${e + 1}`, 88 | limitShowFewer: "Show fewer files", 89 | limitShowMore: "Show more files", 90 | errorIconAriaLabel: "Error" 91 | }} 92 | multiple 93 | showFileLastModified 94 | showFileSize 95 | showFileThumbnail 96 | tokenLimit={3} 97 | /> 98 | } 99 | 100 | 101 | 106 | 107 | 108 |
109 | 110 | ) 111 | } 112 | 113 | export default FileUploadPage; 114 | -------------------------------------------------------------------------------- /src/pages/LivePage.jsx: -------------------------------------------------------------------------------- 1 | import { useState } from 'react' 2 | import { get } from 'aws-amplify/api'; 3 | import Box from "@cloudscape-design/components/box" 4 | import Container from "@cloudscape-design/components/container" 5 | import SpaceBetween from "@cloudscape-design/components/space-between" 6 | import Alert from "@cloudscape-design/components/alert"; 7 | import Header from "@cloudscape-design/components/header" 8 | import Form from "@cloudscape-design/components/form" 9 | import Button from "@cloudscape-design/components/button" 10 | import Spinner from "@cloudscape-design/components/spinner" 11 | import ReactHlsPlayer from 'react-hls-player'; 12 | 13 | function LivePage() { 14 | 15 | const [liveStreamURL, setLiveStreamURL] = useState(null); 16 | const [alertText, setAlertText] = useState(""); 17 | const [alertType, setAlertType] = useState(""); 18 | const [alertVisible, setAlertVisible] = useState(false) 19 | const [isConnecting, setIsConnecting] = useState(false) 20 | const [autoPlay, setAutoPlay] = useState(false) 21 | 22 | const GetLiveSession = async () => { 23 | 24 | try { 25 | setIsConnecting(true) 26 | setAlertText("Connecting to the live camera feed...") 27 | setAlertType("success") 28 | setAlertVisible(true) 29 | 30 | const restOperation = get({ 31 | apiName: 'clipcrunchapi', 32 | path: '/images/liveURL' 33 | }); 34 | 35 | const { body } = await restOperation.response; 36 | const response = await body.json(); 37 | 38 | console.log('API Response:', response) 39 | 40 | setAlertText("") 41 | setAlertType("") 42 | setAlertVisible(false) 43 | setIsConnecting(false) 44 | 45 | setLiveStreamURL(response.sessionURL) 46 | setAutoPlay(true) 47 | 48 | } catch (error) { 49 | setAlertText("Live streaming is not available. Please turn on the camera.") 50 | setAlertType("error") 51 | setAlertVisible(true) 52 | setIsConnecting(false) 53 | 54 | setLiveStreamURL(null) 55 | setAutoPlay(false) 56 | 57 | console.log('GET call failed: ', error); 58 | } 59 | } 60 | 61 | return ( 62 | 63 |
Live Feed}> 64 | 65 | 66 | 67 | { 68 | alertVisible ? 69 | 70 | { setAlertVisible(false) }} 73 | /> : <> 74 | } 75 | 76 | { 77 | liveStreamURL ? 78 | <> 79 | 80 | 87 | 88 | 89 | 90 | 91 | 92 | : 93 | 98 | } 99 | 100 | 101 |
102 | 103 | ) 104 | } 105 | 106 | export default LivePage; 107 | -------------------------------------------------------------------------------- /src/pages/SearchPage.css: -------------------------------------------------------------------------------- 1 | img:hover { 2 | cursor: pointer; 3 | } 4 | -------------------------------------------------------------------------------- /src/pages/SearchPage.jsx: -------------------------------------------------------------------------------- 1 | import { useState } from 'react' 2 | import { post, get } from 'aws-amplify/api'; 3 | import { getUrl } from 'aws-amplify/storage' 4 | import Header from "@cloudscape-design/components/header" 5 | import Form from "@cloudscape-design/components/form" 6 | import FormField from "@cloudscape-design/components/form-field" 7 | import Input from "@cloudscape-design/components/input" 8 | import Box from "@cloudscape-design/components/box" 9 | import Container from "@cloudscape-design/components/container" 10 | import SpaceBetween from "@cloudscape-design/components/space-between" 11 | import Spinner from "@cloudscape-design/components/spinner" 12 | import Alert from "@cloudscape-design/components/alert"; 13 | import Grid from "@cloudscape-design/components/grid"; 14 | import Select from "@cloudscape-design/components/select"; 15 | import Modal from "@cloudscape-design/components/modal"; 16 | import Toggle from "@cloudscape-design/components/toggle"; 17 | import DateRangePicker from "@cloudscape-design/components/date-range-picker"; 18 | import Multiselect from "@cloudscape-design/components/multiselect"; 19 | import ReactHlsPlayer from 'react-hls-player'; 20 | 21 | import "./SearchPage.css" 22 | 23 | function MainPage() { 24 | 25 | const [showModal, setShowModal] = useState(false) 26 | 27 | const [searchText, setSearchText] = useState(""); 28 | const [searchImage, setSearchImage] = useState(""); 29 | const [days, setDays] = useState({ label: "365", value: "365" }); 30 | const [confidenceThreshold, setConfidenceThreshold] = useState(0); 31 | const [maxResults, setMaxResults] = useState(50); 32 | const [includeSimilarTimestamp, setIncludeSimilarTimestamp] = useState(true) 33 | const [dateRange, setDateRange] = useState({ 34 | type: "relative", 35 | amount: 1, 36 | unit: "year" 37 | }); 38 | 39 | const allOptions = [ 40 | { 41 | label: "fileupload:video", 42 | value: "fileupload:video", 43 | }, 44 | { 45 | label: "webcam:video", 46 | value: "webcam:video", 47 | }, 48 | { 49 | label: "fileupload:image", 50 | value: "fileupload:image", 51 | }, 52 | { 53 | label: "webcam:image", 54 | value: "webcam:image", 55 | }, 56 | { 57 | label: "unknown", 58 | value: "unknown", 59 | }, 60 | ] 61 | const [ 62 | selectedOptions, 63 | setSelectedOptions 64 | ] = useState(allOptions); 65 | 66 | const [streamURL, setStreamURL] = useState(""); 67 | 68 | const [isProcessing, setIsProcessing] = useState(false) 69 | const [alertText, setAlertText] = useState(""); 70 | const [alertType, setAlertType] = useState(""); 71 | const [alertVisible, setAlertVisible] = useState(false) 72 | const [images, setImages] = useState([]); 73 | 74 | const setError = (message) => { 75 | setAlertText(message) 76 | setAlertType("error") 77 | setAlertVisible(true) 78 | } 79 | 80 | const setSuccess = (message) => { 81 | setAlertText(message) 82 | setAlertType("success") 83 | setAlertVisible(true) 84 | } 85 | 86 | const getVideoClipURL = async (timestamp) => { 87 | 88 | try { 89 | const restOperation = get({ 90 | apiName: 'clipcrunchapi', 91 | path: '/images/sessionURL', 92 | options: { 93 | queryParams: { 94 | timestamp: timestamp 95 | } 96 | } 97 | }); 98 | 99 | const { body } = await restOperation.response; 100 | const response = await body.json(); 101 | 102 | console.log('API Response:', response) 103 | 104 | return response.sessionURL 105 | 106 | } catch (error) { 107 | setIsProcessing(false) 108 | setError("An error occurred processing the search") 109 | console.log('GET call failed: ', error); 110 | } 111 | } 112 | 113 | const searchImages = async () => { 114 | 115 | setImages([]) 116 | 117 | try { 118 | if (searchText.trim() || searchImage.trim()) { 119 | setIsProcessing(true) 120 | 121 | //current doing client side filtering, can be modified to do server side filtering 122 | const imageSourceFilter = selectedOptions.map(item => item.value) 123 | 124 | const item = { 125 | searchText: searchText, 126 | searchImage: searchImage, 127 | dateRange: dateRange, 128 | confidenceThreshold: confidenceThreshold, 129 | maxResults: maxResults, 130 | includeSimilarTimestamp: includeSimilarTimestamp 131 | }; 132 | 133 | const restOperation = post({ 134 | apiName: 'clipcrunchapi', 135 | path: '/images/search', 136 | options: { 137 | body: item 138 | } 139 | }); 140 | 141 | const { body } = await restOperation.response; 142 | const response = await body.json(); 143 | 144 | console.log('API Response:', response) 145 | 146 | let newImages = [] 147 | for (let image of response.images) { 148 | const imageResponse = await getUrl({key: image.file}); 149 | image.url = imageResponse.url 150 | console.log(image) 151 | console.log(imageSourceFilter) 152 | if (imageSourceFilter.includes(image.source)) { 153 | newImages.push(image) 154 | } 155 | } 156 | 157 | setImages(newImages) 158 | setIsProcessing(false) 159 | 160 | if (newImages.length === 0) { 161 | setError("No images found") 162 | } else { 163 | setSuccess("Search completed. Found " + newImages.length + " images") 164 | } 165 | } 166 | 167 | } catch (error) { 168 | setIsProcessing(false) 169 | setError("An error occurred processing the search") 170 | console.log('POST call failed: ', error); 171 | } 172 | } 173 | 174 | return ( 175 | 176 |
Search Videos} 179 | > 180 | 181 | 182 | 183 | { 184 | alertVisible ? 185 | { setAlertVisible(false) }} 189 | /> : <> 190 | } 191 | 192 | 193 | { 195 | setSearchImage(detail.value) 196 | 197 | if (detail.value.length === 0) { 198 | setImages([]) 199 | } 200 | }} 201 | onKeyDown={({ detail }) => { 202 | if (detail.keyCode == 13) { 203 | searchImages() 204 | } 205 | }} 206 | /> 207 | 208 | 209 | { 211 | setSearchText(detail.value) 212 | 213 | if (detail.value.length === 0) { 214 | setImages([]) 215 | } 216 | }} 217 | onKeyDown={({ detail }) => { 218 | if (detail.keyCode == 13) { 219 | searchImages() 220 | } 221 | }} 222 | /> 223 | 224 | 225 | setSelectedOptions(detail.selectedOptions)} 228 | options={allOptions} 229 | placeholder="Choose options" 230 | selectedAriaLabel="Selected" 231 | /> 232 | 233 | 234 | setDateRange(detail.value)} 236 | value={dateRange} 237 | relativeOptions={[ 238 | { 239 | key: "previous-30-minutes", 240 | amount: 30, 241 | unit: "minute", 242 | type: "relative" 243 | }, 244 | { 245 | key: "previous-1-hour", 246 | amount: 1, 247 | unit: "hour", 248 | type: "relative" 249 | }, 250 | { key: "previous-1-day", amount: 1, unit: "day", type: "relative" }, 251 | { key: "previous-1-week", amount: 1, unit: "week", type: "relative" }, 252 | { key: "previous-1-month", amount: 1, unit: "month", type: "relative" }, 253 | { key: "previous-3-months", amount: 3, unit: "month", type: "relative" }, 254 | { key: "previous-1-year", amount: 1, unit: "year", type: "relative" }, 255 | ]} 256 | isValidRange={(range) => { 257 | if (range.type === "absolute") { 258 | const [startDateWithoutTime] = range.startDate.split("T"); 259 | const [endDateWithoutTime] = range.endDate.split("T"); 260 | if (!startDateWithoutTime || !endDateWithoutTime) { 261 | return { 262 | valid: false, 263 | errorMessage: "The selected date range is incomplete. Select a start and end date for the date range." 264 | }; 265 | } 266 | if (new Date(range.startDate) - new Date(range.endDate) > 0) { 267 | return { 268 | valid: false, 269 | errorMessage: "The selected date range is invalid. The start date must be before the end date." 270 | }; 271 | } 272 | } 273 | return { valid: true }; 274 | }} 275 | i18nStrings={{ 276 | relativeModeTitle: "Relative range", 277 | absoluteModeTitle: "Absolute range", 278 | relativeRangeSelectionHeading: "Choose a range", 279 | startDateLabel: "Start date", 280 | endDateLabel: "End date", 281 | startTimeLabel: "Start time", 282 | endTimeLabel: "End time", 283 | clearButtonLabel: "Clear and dismiss", 284 | cancelButtonLabel: "Cancel", 285 | applyButtonLabel: "Apply", 286 | formatRelativeRange: (e) => { 287 | const { amount, unit } = e; 288 | const unitString = amount === 1 ? unit : unit + "s"; 289 | return `Last ${amount} ${unitString}`; 290 | }, 291 | formatUnit: (unit, value) => { 292 | return value === 1 ? unit : unit + "s"; 293 | }, 294 | dateTimeConstraintText: "For date, use YYYY/MM/DD. For time, use 24-hour format.", 295 | errorIconAriaLabel: "Error", 296 | customRelativeRangeOptionLabel: "Custom range", 297 | customRelativeRangeOptionDescription: "Set a custom range in the past", 298 | customRelativeRangeUnitLabel: "Unit of time", 299 | customRelativeRangeDurationLabel: "Duration", 300 | todayAriaLabel: "Today", 301 | nextMonthAriaLabel: "Next month", 302 | previousMonthAriaLabel: "Previous month", 303 | nextYearAriaLabel: "Next year", 304 | previousYearAriaLabel: "Previous year", 305 | dropdownIconAriaLabel: "Show calendar", 306 | }} 307 | placeholder="Filter by a date and time range" 308 | /> 309 | 310 | 311 | { 313 | setConfidenceThreshold(detail.value) 314 | } 315 | } 316 | onKeyDown={({ detail }) => { 317 | if (detail.keyCode == 13) { 318 | searchImages() 319 | } 320 | }} 321 | value={confidenceThreshold} 322 | inputMode="numeric" 323 | type="number" 324 | /> 325 | 326 | 327 | { 329 | setMaxResults(detail.value) 330 | } 331 | } 332 | onKeyDown={({ detail }) => { 333 | if (detail.keyCode == 13) { 334 | searchImages() 335 | } 336 | }} 337 | value={maxResults} 338 | inputMode="numeric" 339 | type="number" 340 | /> 341 | 342 | 344 | setIncludeSimilarTimestamp(detail.checked) 345 | } 346 | checked={includeSimilarTimestamp} 347 | > 348 | Include images with similar timestamp 349 | 350 | 351 | 352 | 353 | {isProcessing ? : 354 | images.length > 0 ? 355 | <> 356 | ( 358 | { colspan: { default: 6, m: 6, l: 4 } } 359 | ))} 360 | > 361 | { 362 | images.map(image => ( 363 | 364 |
365 | { 370 | if (__KINESIS_VIDEO_STREAM_INTEGRATION__) { 371 | const u = await getVideoClipURL(image.timestamp) 372 | if (u) { 373 | setStreamURL(u) 374 | setShowModal(true) 375 | } 376 | } 377 | } 378 | } 379 | /> 380 |

381 | Timestamp: {new Date(image.timestamp).toLocaleString()} 382 |

383 |

Confidence: {image.confidence.toFixed(2)}

384 |

Text summary: {image.summary}

385 |

Source: {image.source}

386 | {image.custom_metadata !== "No custom metadata available" && ( 387 |

Custom metadata: {image.custom_metadata}

388 | )} 389 |
390 | 391 | )) 392 | } 393 | 394 | 395 | : <> 396 | } 397 | 398 | 399 | 400 |
401 | { 403 | setShowModal(false) 404 | setStreamURL("") 405 | }} 406 | visible={showModal} 407 | header="Video Clip" 408 | size='large' 409 | > 410 | 411 | 418 | 419 | 420 | 421 | ) 422 | } 423 | 424 | export default MainPage; 425 | -------------------------------------------------------------------------------- /src/pages/SummarizePage.jsx: -------------------------------------------------------------------------------- 1 | import { useState } from 'react' 2 | import { post, get } from 'aws-amplify/api'; 3 | import { getUrl } from 'aws-amplify/storage' 4 | import Header from "@cloudscape-design/components/header" 5 | import Form from "@cloudscape-design/components/form" 6 | import FormField from "@cloudscape-design/components/form-field" 7 | import Box from "@cloudscape-design/components/box" 8 | import Button from "@cloudscape-design/components/button" 9 | import Container from "@cloudscape-design/components/container" 10 | import SpaceBetween from "@cloudscape-design/components/space-between" 11 | import Spinner from "@cloudscape-design/components/spinner" 12 | import Alert from "@cloudscape-design/components/alert"; 13 | import Grid from "@cloudscape-design/components/grid"; 14 | import DateRangePicker from "@cloudscape-design/components/date-range-picker"; 15 | 16 | import "./SearchPage.css" 17 | import Textarea from "@cloudscape-design/components/textarea"; 18 | 19 | function MainPage() { 20 | 21 | const [customSummaryPrompt, setCustomSummaryPrompt] = useState("Analyze these images and provide a brief summary of its contents. Focus on the main subjects, actions, and any notable elements in the scene. Keep the summary concise, around 2-3 sentences.") 22 | const [dateRange, setDateRange] = useState({ 23 | type: "relative", 24 | amount: 1, 25 | unit: "year" 26 | }); 27 | 28 | const [isProcessing, setIsProcessing] = useState(false) 29 | const [alertText, setAlertText] = useState(""); 30 | const [alertType, setAlertType] = useState(""); 31 | const [alertVisible, setAlertVisible] = useState(false) 32 | const [images, setImages] = useState([]); 33 | const [summary, setSummary] = useState(""); 34 | 35 | const setError = (message) => { 36 | setAlertText(message) 37 | setAlertType("error") 38 | setAlertVisible(true) 39 | } 40 | 41 | const setSuccess = (message) => { 42 | setAlertText(message) 43 | setAlertType("success") 44 | setAlertVisible(true) 45 | } 46 | 47 | const summarizeImages = async () => { 48 | 49 | setImages([]) 50 | 51 | try { 52 | setIsProcessing(true) 53 | 54 | const item = { 55 | customSummaryPrompt: customSummaryPrompt, 56 | dateRange: dateRange 57 | }; 58 | 59 | const restOperation = post({ 60 | apiName: 'clipcrunchapi', 61 | path: '/images/summarize', 62 | options: { 63 | body: item 64 | } 65 | }); 66 | 67 | const { body } = await restOperation.response; 68 | const response = await body.json(); 69 | 70 | console.log('API Response:', response) 71 | 72 | setSummary(response.summary); 73 | 74 | let newImages = [] 75 | for (let image of response.images) { 76 | const imageResponse = await getUrl({key: image.slice(7)}); 77 | newImages.push(imageResponse.url); 78 | } 79 | 80 | setImages(newImages) 81 | setIsProcessing(false) 82 | 83 | if (response.summary === "") { 84 | setError("No summary found") 85 | } else if (newImages.length === 0) { 86 | setError("No images found") 87 | } else { 88 | setSuccess("Summary completed.") 89 | } 90 | 91 | } catch (error) { 92 | setIsProcessing(false) 93 | setError("An error occurred processing the summary") 94 | console.log('POST call failed: ', error); 95 | } 96 | } 97 | 98 | return ( 99 | 100 |
Summarize} 103 | > 104 | 105 | 106 | 107 | { 108 | alertVisible ? 109 | { setAlertVisible(false) }} 113 | /> : <> 114 | } 115 | 116 |