├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── assets ├── architecture.drawio ├── architecture.jpg ├── vus-chat-1.gif ├── vus-chat-2.gif ├── vus-chat-3.gif ├── vus-entities.gif ├── vus-search.gif └── vus-summary.gif ├── cdk.json ├── lib ├── amplify_deploy_lambda │ ├── index.py │ └── requirements.txt ├── app.py ├── cognito_user_setup │ └── index.py ├── db_setup_lambda │ ├── index.py │ └── requirements.txt ├── main_analyzer │ ├── Dockerfile │ ├── default_visual_extraction_system_prompt.txt │ ├── default_visual_extraction_task_prompt.txt │ ├── index.py │ └── requirements.txt ├── preprocessing_lambda │ ├── index.py │ └── requirements.txt ├── prompt_setup_lambda │ ├── index.py │ └── requirements.txt ├── video_understanding_solution_stack.py └── videos_search │ ├── lambda-handler.py │ └── requirements.txt ├── utils ├── bootstrap.sh ├── deploy.sh ├── destroy.sh ├── install_prerequisites_for_amazon_linux_2.sh ├── install_prerequisites_for_amazon_linux_2023.sh ├── install_prerequisites_for_macos_ventura.sh └── scan.sh └── webui ├── package.json ├── public ├── index.html ├── manifest.json └── robots.txt └── src ├── App.css ├── App.js ├── App.test.js ├── VideoTable ├── VideoTable.css └── VideoTable.js ├── VideoUpload ├── VideoUpload.css └── VideoUpload.js ├── aws-exports.js ├── index.css ├── index.js ├── reportWebVitals.js └── setupTests.js /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | package-lock.json 3 | webui/package-lock.json 4 | __pycache__ 5 | .pytest_cache 6 | .venv 7 | *.egg-info 8 | 9 | # CDK asset staging directory 10 | .cdk.staging 11 | cdk.out 12 | cdk.context.json 13 | 14 | .DS_STORE 15 | deployment-output.json 16 | .DEFAULT_REGION 17 | team-provider-info.json 18 | venv/ 19 | 20 | # Amplify 21 | build/ 22 | webui/build/ 23 | dist/ 24 | webui/dist/ 25 | node_modules/ 26 | webui/node_modules/ 27 | awsconfiguration.json 28 | amplifyconfiguration.json 29 | amplifyconfiguration.dart 30 | amplify-build-config.json 31 | amplify-gradle-config.json 32 | amplifytools.xcconfig 33 | .secret-* 34 | 35 | ui_repo*.zip 36 | banditreport.txt 37 | semgrepreport.txt 38 | semgreplog.txt 39 | npmauditreport.json 40 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | 6 | In addition, the acceptable use policy from the models used by the solution also applies. 7 | See: 8 | * https://www.anthropic.com/legal/aup 9 | * The EULA files for Anthropic Claude 3 Sonnet & Haiku models and Cohere Embed Multilingual model which can be seen from this page Amazon Bedrock model access page. Follow this instruction to go to the page https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html 10 | 11 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT No Attribution 2 | 3 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 13 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 15 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 16 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | 18 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | bootstrap: 2 | ./utils/bootstrap.sh 3 | 4 | prepare: 5 | OS=$$(uname -a | sed -n 's/^\([^ ]*\) .*/\1/p') ;\ 6 | if [ "$$OS" = "Darwin" ]; then ./utils/install_prerequisites_for_macos_ventura.sh ;\ 7 | elif [ "$$OS" = "Linux" ]; then @VER=$$(uname -a | sed -n 's/.*amzn\([0-9]*\).*/\1/p') ; if [ "$$VER" = "2" ]; then ./utils/install_prerequisites_for_amazon_linux_2.sh ; elif [ "$$VER" = "2023" ]; then ./utils/install_prerequisites_for_amazon_linux_2023.sh ; else echo "OS is not currently supported" ; fi;\ 8 | else echo "OS is not currently supported" ;\ 9 | fi 10 | 11 | deploy: 12 | OS=$$(uname -a | sed -n 's/^\([^ ]*\) .*/\1/p') ;\ 13 | if [ "$$OS" = "Darwin" ]; then ./utils/deploy.sh ;\ 14 | elif [ "$$OS" = "Linux" ]; then sg docker -c './utils/deploy.sh' ;\ 15 | else echo "OS not currently supported" ;\ 16 | fi 17 | 18 | destroy: 19 | ./utils/destroy.sh 20 | 21 | scan: 22 | ./utils/scan.sh 23 | 24 | bootstrap_and_deploy: bootstrap deploy -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Video Understanding Solution 2 | 3 | ## Introduction 4 | 5 | This is a deployable solution to help save your time in understanding videos without having to watch every video. You can upload videos and this solution can generate AI-powered summary and entities extraction for each video. It also supports Q&A about the video like "What is funny about the video?", "How does Jeff Bezos look like there?", and "What shirt did he wear?". You can also search for videos using semantic search e.g. "Amazon's culture and history". This solution extracts information from visual scenes, audio, visible texts, and detected celebrities or faces in the video. It leverages an LLM which can understand visual and describe the video frames. 6 | 7 | You can upload videos to your Amazon Simple Storage Service (S3) bucket bucket by using AWS console, CLI, SDK, or other means (e.g. via AWS Transfer Family). This solution will automatically trigger processes including call to Amazon Transcribe for voice transcription, call to Amazon Rekognition to extract the objects visible, and call to Amazon Bedrock with Claude 3 model to extract scenes and visually visible text. The LLM used can perform VQA (visual question answering) from images (video frames), which is used to extract the scene and text. This combined information is used to generate the summary and entities extraction as powered by generative AI with Amazon Bedrock. The UI chatbot also uses Amazon Bedrock for the Q&A chatbot. The summaries, entities, and combined extracted information are stored in S3 bucket, available to be used for further custom analytics. 8 | 9 | **Demo for summarization feature:** 10 | 11 | ![Summarization](./assets/vus-summary.gif) 12 | 13 | **Demo for entities and sentiment extraction feature:** 14 | 15 | ![Entities](./assets/vus-entities.gif) 16 | 17 | **Demo for Q&A 1:** 18 | 19 | ![Chat 1](./assets/vus-chat-1.gif) 20 | 21 | **Demo for Q&A 2:** 22 | 23 | ![Chat 2](./assets/vus-chat-2.gif) 24 | 25 | **Demo for Q&A 3:** 26 | 27 | ![Chat 3](./assets/vus-chat-3.gif) 28 | 29 | **Demo for search feature:** 30 | 31 | ![Search](./assets/vus-search.gif) 32 | 33 | 34 | The diagram below shows the architecture. This is extendable since the summaries, entities, and extracted information are stored in Amazon S3 bucket, which you can use for further purposes. 35 | 36 | ![Architecture](./assets/architecture.jpg) 37 | 38 | Note that the architecture diagram may not represent all the components and relations for readability. The Amazon Aurora PostgreSQL database is deployed in Amazon VPC with isolated subnet spanning across 3 availability zones. The Fargate task for analysis, the video search Lambda function, and the preprocessing Lambda function are all deployed in the VPC with private subnet. 39 | 40 | Refer to the Deployment section below on how to deploy it to your own AWS account. Refer to the Use section on how to use it. 41 | 42 | ## Deployment 43 | 44 | ### Prerequisites 45 | 46 | Here are prerequisites for deploying this solution with AWS CDK. Note that you can automatically install item 4-11 by running `make prepare`. 47 | 48 | The `make prepare` utility is currently supported only for MacOS Ventura, Amazon Linux 2, and Amazon Linux 2023 operating systems. 49 | 50 | 51 | 1. AWS account with Amazon Bedrock model access enabled for Anthropic - Claude, Anthropic - Claude Instant and Cohere - Embed Multilingual. Follow the steps in https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html#add-model-access to enable those models and make sure the region is correct. 52 | 2. Make 3.82 or above in your environment 53 | 3. AWS IAM credentials with sufficient permission for deploying this solution configured in your environment. See https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html for credentials setup. 54 | 4. AWS CLI 2.15.5 or above / 1.29.80 or above. Follow https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html for installation. 55 | 5. Python 3.8 or above, pip, and virtualenv. 56 | 6. Docker 20.10.25 or above 57 | 7. NodeJS 16 with version 16.20.2 above, or NodeJS 20 with version 20.10.0 or above, along with NPM. 58 | 8. jq, zip, unzip 59 | 9. CDK Toolkit 2.122.0 or above. Follow https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html#getting_started_install for installation. 60 | 10. Python CDK lib 2.122.0 or above with @aws-cdk/aws-amplify-alpha. Follow https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html#getting_started_concepts. This is for the Python CDK libraries since the CDK template is in Python. 61 | 11. CDK nag 2.28.16 or above. CDK nag is used to scan the CDK template for best practices violation. 62 | 63 | ### Deploy 64 | 65 | **IMPORTANT** 66 | 67 | At the moment,this solution can only be used in AWS regions where Amazon Bedrock is available. 68 | At the time of the writing of this document, you can deploy and use this in us-east-1 (N. Virginia) and us-west-2 (Oregon) regions. 69 | 70 | Follow the steps below for deploying this solution from your current environment 71 | 72 | 1. Make sure that all prerequisites are met. Please refer to the [Prerequisites](#prerequisites) section above. 73 | 74 | 2. Run `make bootstrap_and_deploy` and enter an email address and the region to deploy when prompted. Make sure the email address is valid to receive the login credentials. 75 | 76 | 3. [Optional] For second or later redeployment, run `make deploy` instead. Note that any change to the UI (under "webui" folder) is not redeployed with `make deploy`. For UI changes you can push them to the deployed CodeCommit repository directly or run `make destroy` followed by `make deploy`. 77 | 78 | 79 | ## Use 80 | 81 | ### First login 82 | After a successful deployment, the email address inputted during deployment should receive an email with temporary password and the web UI portal URL. It can take 15 minutes or so after you receive the email until the deployment completion, so just wait if the web UI isn't ready. Visit the web UI, use the email address as username, and input the temporary password. Then reset the password. When presented with a QR code, that is for you to set the MFA. Use authenticator software like Google Authenticator from your phone to set the MFA. Enter the MFA code back to the UI. After a successful sign-in, you should be able to see the page that will display your uploaded videos. 83 | 84 | ### Uploading videos and checking analysis outputs 85 | You can upload videos into the S3 bucket using the web UI. Or, you can upload them straight to the S3 bucket using other means, under folder "source" as also instructed in the web UI. Uploaded videos will automatically trigger asynchronous tasks. For each video wait for few minutes for the tasks to finish. The analysis will be displayed in the web UI. It will also be stored in S3 under folder "summary", "entities", "audio_transcript" (the formatted transcription result), and "video_timeline" (the combined extracted video raw information), which you can use for further custom analysis / machine learning by extending this solution. 86 | 87 | ### Using the web UI 88 | In the web UI, you can search for videos in your S3 bucket. For each video found, you can view the video, see its summary, check the extracted entities, and ask the chatbot about the video e.g. "What is funny or interesting about the video?". The chatbot is equipped with memory for the current conversation, so the user can have a context-aware conversation with the bot. You can also search videos using the name prefix, uploaded date range, and what the video is about. The latter is powered by semantic search using LLM. 89 | 90 | ## Configuration 91 | ### Enabling/disabling features 92 | You can enable/disable 2 features for now: 93 | 1. Transcription. For use cases where extracting information from the speech in the video won't be important, you can save cost by disabling this feature. 94 | 2. Face detection, and celebrity detection. For use case where these information won't be important, you can disable them. With these disabled, the solution will still extract the visual scene information and the text overlay (if any) of the videos. 95 | 96 | 97 | To enable/disable these, you can log in to your AWS Console and go to AWS SSM Parameter Store for a parameter VideoUnderstandingStack-configuration. For example you can go [here](https://us-west-2.console.aws.amazon.com/systems-manager/parameters/VideoUnderstandingStack-configuration/description?region=us-west-2&tab=Table) if you deploy the solution in Oregon (us-west-2) region. For other region, you can change any "us-west-2" in that hyperlink to your selected region. Then you can edit the parameter value. The value "1" means enabled and value "0" means disabled. The "label_detection_enabled" represents the face and celebrity detection while "transcription_enabled" represents the transcription. By default, both are enabled ("1"). 98 | 99 | ## Cost 100 | The cost of using this solution is determined by the pricing and usage of the components being deployed. This includes, but not being limited to (not the exhaustive list): 101 | 102 | 1. Less variable components: 103 | 104 | 1.1. Amazon Aurora PostgreSQL - Pricing is [here](https://aws.amazon.com/rds/aurora/pricing/). This solution by default uses 2 x 0.5 ACU of Aurora Serverless PostgreSQL. 105 | 106 | 1.2. NAT Gateway - Pricing is [here](https://aws.amazon.com/vpc/pricing/). This is used for egress access of Fargate task and some Lambda function. By default this solution uses 1 NAT Gateway. 107 | 108 | 2. More variable components (driven mainly by the total number of video minutes being processed and the activities e.g. chat): 109 | 110 | 2.1. Amazon Rekognition - Pricing is [here](https://aws.amazon.com/rekognition/pricing/). You may be eligible for its FREE TIER for image and video analysis. This solution uses StartLabelDetection and GetLabelDetection video APIs in addition to DetectFaces and RecognizeCelebrities image APIs. The cost can vary based on the number of detected faces in the whole videos. 111 | 112 | 2.2. Amazon Transcribe - Pricing is [here](https://aws.amazon.com/transcribe/pricing/). Youo may be eligible for its FREE TIER. This solution uses Standard Batch transcription. The cost can vary based on the video duration and the number of videos being processed. 113 | 114 | 2.3. Amazon Bedrock - Pricing is [here](https://aws.amazon.com/bedrock/pricing/). This solution uses Claude, Claude Instant and Cohere Embed Multilingual model. The cost can vary based on the amount of information extracted from the video and the usage frequency. 115 | 116 | 2.4. Amazon S3 - Pricing is [here](https://aws.amazon.com/s3/pricing/). You may be eligible for its FREE TIER. S3 is used for storing the raw videos and the extracted information. 117 | 118 | 2.5. AWS Fargate - Pricing is [here](https://aws.amazon.com/fargate/pricing/). Fargate task is where the extracted video information is being analyzed and where the calls to LLMs for summary and entities generation happen. It is provisioned on demand when a video is uploaded. 119 | 120 | 2.6. AWS Lambda - Pricing is [here](https://aws.amazon.com/lambda/pricing/). You may be eligible for its FREE TIER. Lambda function is used in several parts of the worklflow, including to handle video search requests. 121 | 122 | 2.7. Amazon CloudWatch - Pricing is [here](https://aws.amazon.com/cloudwatch/pricing/). You may be eligible for its FREE TIER. Amazon CloudWatch metrics and logs are used. 123 | 124 | It is recommended to test with smaller number of shorter videos first and observe the cost. It is recommended to monitor the cost with [AWS Cost Explorer](https://aws.amazon.com/aws-cost-management/aws-cost-explorer/) and set budget and alerts with [AWS Budget](https://aws.amazon.com/aws-cost-management/aws-budgets/). You can filter the cost by tagging with key = "Application" and value = "VideoUnderstandingSolution" to monitor the cost generated by this solution when deployed. Please note that some compnents may not be tagged or may not be taggable. 125 | 126 | 127 | ## Limitations 128 | 129 | 1. Currently only .mp4/.MP4 video files are supported 130 | 2. The language supported is limited by the language supported by Amazon Rekognition text detection, Amazon Transcribe, and the models used in Amazon Bedrock. 131 | 3. This works best for video under 15 minutes. It may have issue or show slow latency for very long video at the moment. 132 | 4. Video file names (when uploaded) must adhere to the [S3 object key pattern](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html). When uploaded via this solution's UI, it will automatically convert non-compliant characters to _ (underscore). 133 | 134 | ## Removal 135 | To remove the solution from your AWS account, follow these steps: 136 | 137 | 1. [ Conditional ] This is if you want to delete the database. In RDS console, find the Aurora PostgreSQL cluster provisioned by this solution, whose name starts with "videounderstandingstack". Disable the RDS deletion protection as appropriate. For disabling the protection, follow [these steps](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Aurora.Modifying.html#Aurora.Modifying.Cluster) and make sure you **uncheck** the "Enable deletion protection". You can choose to "Apply immediately" as appropriate. 138 | 2. [ Conditional ] This is if you want to delete the S3 bucket hosting the videos and the extracted outputs. In S3 console, find the S3 bucket provisioned by this solution, whose name starts with "videounderstandingstack". Empty the bucket. Follow this [documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/empty-bucket.html) to empty a bucket. Note that if the step (3) below failed because the S3 bucket is not empty while you have performed step (2), just redo step (2) and continue step (3). That situation might arise when some logs are still being delivered to the S3 bucket, so after it gets emptied the log comes in. 139 | 3. run `make destroy` and specify the region when asked. If you chose not to do step (1) and (2) above, then at some point the `make destroy` will fail, which is expected. You need to go to AWS CloudFormation console, find the stack named "VideoUnderstandingStack", and delete it. When asked whether to skip the deletion of the RDS and S3 bucket, then confirm. 140 | 141 | ## Security 142 | 143 | Run `make scan` before submitting any pull request to make sure that the introduced changes do not open a vulnerability. Make sure the generated banditreport.txt, semgrepreport.txt, semgreplog.txt, npmauditreport.txt, and the cdk_nag files in cdk.out folders all show **no high or critical finding**. 144 | 145 | See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. 146 | 147 | ## Extendability and customization 148 | This solution is architected to be extendable and customizable. 149 | 150 | ### Extending the solution for other purposes 151 | 1. The audio transcript, the combined visual and audio information extracted from LLM (with VQA) and AWS services, the summaries, and the extracted entities and sentiment are stored in S3 bucket. Navigate to the S3 bucket and find the relevant folders there. The file name for each of those category is based on the actual file name from the 'source' folder. You can further use this information for other purposes 152 | 153 | 2. The summaries, the combined visual and audio information extracted from LLM (with VQA) and AWS services, and the extracted entities and sentiment are stored in the Amazon Aurora Serverless v2 PostgreSQL database in addition to S3. The summaries and the combined extracted information are stored in the form of text and embedding (using pgVector). These are under "videos" database name. The summaries are stored in "videos" table. The entities are stored in "entities" table. The extracted information of the video (arranged in timestamps basis and may stored in chunks) is stored in "content" table. You can use the database with credentials from the AWS Secrets Manager. 154 | 155 | ### Customizing the solution 156 | To customize the solution, you can edit the code. Please refer to the Development section below. The possible customizations include: 157 | 1. Modifying prompts to add more data to be extracted from the video frames 158 | 2. Modifying prompts to add more features 159 | 3. Modifying the backend code 160 | 4. Modiyfing the UI 161 | 162 | ## Development 163 | 164 | This information is meant for the contributors of the repository or builders who build new solutions from this. 165 | 166 | ### Web UI development 167 | 168 | * You can first deploy the whole solution to deploy both storage, frontend, and backend. This should deploy an AWS CodeCommit repo. You can then clone from that repo, do development, and push the changes. Remember that running `make deploy` again will update the other CDK components, but it won't update or re-initialize the CodeCommit repo. 169 | 170 | * During development, you may want to test the UI changes live without having to wait for CodeCommit code update and Amplify build & deploy. Instead you can temporarily edit the /webui/src/aws-exports.js and replace the placeholders with actual value of the deployed components (e.g. Cognito user pool ID). Then `cd webui` and run `npm start`. This should run the frontend in your localhost, supported by the backend and storage in the cloud. 171 | 172 | ### Backend development 173 | 174 | * This solution uses CDK in Python. You can go [here](https://docs.aws.amazon.com/cdk/api/v2/python/) for references of the CDK components. 175 | 176 | * This solution uses cdk_nag to prevent the solution being deployed when some components are not adhering to the best practices. If your changes resulted in cdk_nag error, fix them. Sometimes, the cdk_nag finding could be a false alarm which you can suppress. 177 | 178 | 179 | ## License 180 | 181 | This library is licensed under the MIT-0 License. See the LICENSE file. 182 | 183 | ## Notes 184 | 185 | * The facial recognition in this solution is provided by Amazon Rekognition. 186 | * Please refer to the CODE_OF_CONDUCT.md file to see the terms of use and acceptable use policy. Some AI models have prohibition on what to use the model for. Refer to the architecture diagram above to see what models are being used by this solution and for which part. 187 | -------------------------------------------------------------------------------- /assets/architecture.drawio: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | -------------------------------------------------------------------------------- /assets/architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/video-understanding-solution/46fb6662c0dc547ac83746126cb61b85fe4e72bb/assets/architecture.jpg -------------------------------------------------------------------------------- /assets/vus-chat-1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/video-understanding-solution/46fb6662c0dc547ac83746126cb61b85fe4e72bb/assets/vus-chat-1.gif -------------------------------------------------------------------------------- /assets/vus-chat-2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/video-understanding-solution/46fb6662c0dc547ac83746126cb61b85fe4e72bb/assets/vus-chat-2.gif -------------------------------------------------------------------------------- /assets/vus-chat-3.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/video-understanding-solution/46fb6662c0dc547ac83746126cb61b85fe4e72bb/assets/vus-chat-3.gif -------------------------------------------------------------------------------- /assets/vus-entities.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/video-understanding-solution/46fb6662c0dc547ac83746126cb61b85fe4e72bb/assets/vus-entities.gif -------------------------------------------------------------------------------- /assets/vus-search.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/video-understanding-solution/46fb6662c0dc547ac83746126cb61b85fe4e72bb/assets/vus-search.gif -------------------------------------------------------------------------------- /assets/vus-summary.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/video-understanding-solution/46fb6662c0dc547ac83746126cb61b85fe4e72bb/assets/vus-summary.gif -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "python3 ./lib/app.py", 3 | "watch": { 4 | "include": [ 5 | "**" 6 | ], 7 | "exclude": [ 8 | "README.md", 9 | "cdk*.json", 10 | "requirements*.txt", 11 | "source.bat", 12 | "**/__init__.py", 13 | "**/__pycache__", 14 | "tests" 15 | ] 16 | }, 17 | "context": { 18 | "@aws-cdk/aws-lambda:recognizeLayerVersion": true, 19 | "@aws-cdk/core:checkSecretUsage": true, 20 | "@aws-cdk/core:target-partitions": [ 21 | "aws", 22 | "aws-cn" 23 | ], 24 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 25 | "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, 26 | "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, 27 | "@aws-cdk/aws-iam:minimizePolicies": true, 28 | "@aws-cdk/core:validateSnapshotRemovalPolicy": true, 29 | "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, 30 | "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, 31 | "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, 32 | "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, 33 | "@aws-cdk/core:enablePartitionLiterals": true, 34 | "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, 35 | "@aws-cdk/aws-iam:standardizedServicePrincipals": true, 36 | "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, 37 | "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, 38 | "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, 39 | "@aws-cdk/aws-route53-patters:useCertificate": true, 40 | "@aws-cdk/customresources:installLatestAwsSdkDefault": false, 41 | "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, 42 | "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, 43 | "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, 44 | "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, 45 | "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, 46 | "@aws-cdk/aws-redshift:columnId": true, 47 | "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, 48 | "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, 49 | "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, 50 | "@aws-cdk/aws-kms:aliasNameRef": true, 51 | "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, 52 | "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, 53 | "@aws-cdk/aws-efs:denyAnonymousAccess": true, 54 | "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, 55 | "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, 56 | "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true, 57 | "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true, 58 | "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true, 59 | "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /lib/amplify_deploy_lambda/index.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import os 3 | import json 4 | 5 | amplify = boto3.client('amplify') 6 | codepipeline = boto3.client('codepipeline') 7 | app_id = os.environ['AMPLIFY_APP_ID'] 8 | branch_name = os.environ['BRANCH_NAME'] 9 | 10 | # Notify CodePipeline of a successful job 11 | def put_job_success(job_id, message): 12 | codepipeline.put_job_success_result(jobId=job_id) 13 | 14 | # Notify CodePipeline of a failed job 15 | def put_job_failure(job_id, message): 16 | codepipeline.put_job_failure_result( 17 | jobId=job_id, 18 | failureDetails={ 19 | 'message': json.dumps(message), 20 | 'type': 'JobFailed', 21 | } 22 | ) 23 | 24 | def handler(event, context): 25 | job_id = event['CodePipeline.job']['id'] 26 | try: 27 | user_parameters = json.loads(event['CodePipeline.job']['data']['actionConfiguration']['configuration']['UserParameters']) 28 | bucket_name = user_parameters['bucket_name'] 29 | object_key = user_parameters['prefix'] 30 | 31 | # Then start the deployment with the created job 32 | start_response = amplify.start_deployment( 33 | appId=app_id, 34 | branchName=branch_name, 35 | sourceUrl=f"s3://{bucket_name}/{object_key}/", 36 | sourceUrlType="BUCKET_PREFIX" 37 | ) 38 | 39 | put_job_success(job_id, "Deployment is successful") 40 | 41 | return { 42 | 'statusCode': 200, 43 | 'body': json.dumps('Successfully triggered Amplify deployment') 44 | } 45 | 46 | except Exception as e: 47 | print(f"Error: {str(e)}") 48 | put_job_failure(job_id, str(e)) 49 | raise e -------------------------------------------------------------------------------- /lib/amplify_deploy_lambda/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3>=1.35.83 2 | botocore>=1.35.83 -------------------------------------------------------------------------------- /lib/app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | 4 | from aws_cdk import ( 5 | App, Environment, Tags 6 | ) 7 | 8 | from video_understanding_solution_stack import VideoUnderstandingSolutionStack 9 | 10 | 11 | account=os.environ["CDK_DEPLOY_ACCOUNT"] 12 | region=os.environ["CDK_DEPLOY_REGION"] 13 | allowed_regions = ["us-east-1", "us-west-2"] 14 | 15 | print(f"Account ID: {account}") 16 | print(f"Region: {region}") 17 | 18 | try: 19 | if region not in allowed_regions: 20 | raise AssertionError 21 | except AssertionError: 22 | print(f"Selected region is {region}. Please use only one of these regions {str(allowed_regions)}") 23 | 24 | 25 | app = App() 26 | vus_main_stack = VideoUnderstandingSolutionStack(app, "VideoUnderstandingStack", env=Environment( 27 | account=account, 28 | region=region 29 | )) 30 | Tags.of(vus_main_stack).add("Application", "VideoUnderstandingSolution") 31 | 32 | app.synth() -------------------------------------------------------------------------------- /lib/cognito_user_setup/index.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import time 5 | import boto3 6 | 7 | client = boto3.client("cognito-idp") 8 | 9 | def on_event(event, context): 10 | print(event) 11 | request_type = event["RequestType"].lower() 12 | if request_type == "create": 13 | return on_create(event) 14 | if request_type == "update": 15 | return on_update(event) 16 | if request_type == "delete": 17 | return on_delete(event) 18 | raise Exception(f"Invalid request type: {request_type}") 19 | 20 | 21 | def on_create(event): 22 | props = event["ResourceProperties"] 23 | print(f"create new resource with {props=}") 24 | 25 | user_pool = client.describe_user_pool(UserPoolId=props['user_pool_id'])['UserPool'] 26 | 27 | user_pool['AdminCreateUserConfig'] = {} 28 | 29 | user_pool['AdminCreateUserConfig']['AllowAdminCreateUserOnly'] = True 30 | 31 | user_pool['AdminCreateUserConfig']['InviteMessageTemplate']= {} 32 | user_pool['AdminCreateUserConfig']['InviteMessageTemplate']['SMSMessage'] = f"Hello {{username}}. Temporary password is {{####}} The video understanding portal will be at {props['url']}" 33 | user_pool['AdminCreateUserConfig']['InviteMessageTemplate']['EmailMessage'] = f"Hello {{username}}, welcome to video understanding solution. Your temporary password is {{####}} The video understanding portal will be at {props['url']}" 34 | user_pool['AdminCreateUserConfig']['InviteMessageTemplate']['EmailSubject'] = 'Welcome to the video understanding solution' 35 | 36 | response = client.update_user_pool( 37 | UserPoolId=props['user_pool_id'], 38 | Policies=user_pool["Policies"], 39 | DeletionProtection=user_pool["DeletionProtection"], 40 | MfaConfiguration=user_pool["MfaConfiguration"], 41 | AdminCreateUserConfig=user_pool['AdminCreateUserConfig'] 42 | ) 43 | 44 | response = client.admin_create_user( 45 | UserPoolId=props['user_pool_id'], 46 | Username=props['admin_email_address'], 47 | UserAttributes=[ 48 | { 49 | 'Name': 'email', 50 | 'Value': props['admin_email_address'] 51 | }, 52 | ], 53 | DesiredDeliveryMediums=['EMAIL'], 54 | ) 55 | 56 | physical_id = "admincognitouser" 57 | 58 | return {"PhysicalResourceId": physical_id} 59 | 60 | 61 | def on_update(event): 62 | props = event["ResourceProperties"] 63 | print(f"no op") 64 | 65 | return {"PhysicalResourceId": "admincognitouser"} 66 | 67 | def on_delete(event): 68 | props = event["ResourceProperties"] 69 | print(f"delete resource with {props=}") 70 | 71 | response = client.admin_delete_user( 72 | UserPoolId=props['user_pool_id'], 73 | Username=props['admin_email_address'] 74 | ) 75 | 76 | return {"PhysicalResourceId": None} 77 | -------------------------------------------------------------------------------- /lib/db_setup_lambda/index.py: -------------------------------------------------------------------------------- 1 | import json, os 2 | import boto3 3 | import psycopg2 4 | 5 | database_name = os.environ['DATABASE_NAME'] 6 | video_table_name = os.environ['VIDEO_TABLE_NAME'] 7 | entities_table_name = os.environ['ENTITIES_TABLE_NAME'] 8 | content_table_name = os.environ['CONTENT_TABLE_NAME'] 9 | secret_name = os.environ['SECRET_NAME'] 10 | embedding_dimension = os.environ["EMBEDDING_DIMENSION"] 11 | writer_endpoint = os.environ['DB_WRITER_ENDPOINT'] 12 | 13 | class Database(): 14 | def __init__(self, writer, database_name, embedding_dimension, port=5432): 15 | self.writer_endpoint = writer 16 | self.username = None 17 | self.password = None 18 | self.port = port 19 | self.database_name = database_name 20 | self.video_table_name = video_table_name 21 | self.entities_table_name = entities_table_name 22 | self.content_table_name = content_table_name 23 | self.secret_name = secret_name 24 | self.embedding_dimension = embedding_dimension 25 | self.conn = None 26 | 27 | def fetch_credentials(self): 28 | secrets_manager = boto3.client('secretsmanager') 29 | credentials = json.loads(secrets_manager.get_secret_value(SecretId=self.secret_name)["SecretString"]) 30 | self.username = credentials["username"] 31 | self.password = credentials["password"] 32 | 33 | def connect_for_writing(self): 34 | if self.username is None or self.password is None: self.fetch_credentials() 35 | 36 | conn = psycopg2.connect(host=self.writer_endpoint, port=self.port, user=self.username, password=self.password, database=self.database_name) 37 | self.conn = conn 38 | return conn 39 | 40 | def close_connection(self): 41 | self.conn.close() 42 | self.conn = None 43 | 44 | def setup_vector_db(self, embedding_dimension): 45 | if self.conn is None: 46 | self.connect_for_writing() 47 | 48 | cur = self.conn.cursor() 49 | 50 | # Create pgVector extension 51 | cur.execute("CREATE EXTENSION IF NOT EXISTS vector;") 52 | 53 | # For the statements below, semgrep rule for flagging formatted query is disabled as this Lambda is to be invoked in deployment phase by CloudFormation, not user facing. 54 | 55 | # Create videos table and set indexes 56 | # nosemgrep: python.lang.security.audit.formatted-sql-query.formatted-sql-query, python.sqlalchemy.security.sqlalchemy-execute-raw-query.sqlalchemy-execute-raw-query 57 | cur.execute(f"CREATE TABLE {self.video_table_name} (name varchar(200) PRIMARY KEY NOT NULL, uploaded_at timestamp without time zone NOT NULL DEFAULT (current_timestamp AT TIME ZONE 'UTC'), summary text, summary_embedding vector({str(embedding_dimension)}));") 58 | # nosemgrep: python.lang.security.audit.formatted-sql-query.formatted-sql-query, python.sqlalchemy.security.sqlalchemy-execute-raw-query.sqlalchemy-execute-raw-query 59 | cur.execute(f"CREATE INDEX name_index ON {self.video_table_name} (name);") 60 | # nosemgrep: python.lang.security.audit.formatted-sql-query.formatted-sql-query, python.sqlalchemy.security.sqlalchemy-execute-raw-query.sqlalchemy-execute-raw-query 61 | cur.execute(f"CREATE INDEX uploaded_at_index ON {self.video_table_name} (uploaded_at);") 62 | # nosemgrep: python.lang.security.audit.formatted-sql-query.formatted-sql-query, python.sqlalchemy.security.sqlalchemy-execute-raw-query.sqlalchemy-execute-raw-query 63 | cur.execute(f"CREATE INDEX name_and_uploaded_at_index ON {self.video_table_name} (name, uploaded_at);") 64 | 65 | # Create entities table 66 | # nosemgrep: python.lang.security.audit.formatted-sql-query.formatted-sql-query, python.sqlalchemy.security.sqlalchemy-execute-raw-query.sqlalchemy-execute-raw-query 67 | cur.execute(f"CREATE TABLE {self.entities_table_name} (id bigserial PRIMARY KEY NOT NULL, name VARCHAR(100) NOT NULL, sentiment VARCHAR(20), reason text, video_name varchar(200) NOT NULL REFERENCES {self.video_table_name}(name));") 68 | 69 | # Create content table 70 | # nosemgrep: python.lang.security.audit.formatted-sql-query.formatted-sql-query, python.sqlalchemy.security.sqlalchemy-execute-raw-query.sqlalchemy-execute-raw-query 71 | cur.execute(f"CREATE TABLE {self.content_table_name} (id bigserial PRIMARY KEY NOT NULL, chunk text NOT NULL, chunk_embedding vector({str(embedding_dimension)}), video_name varchar(200) NOT NULL REFERENCES {self.video_table_name}(name));") 72 | 73 | self.conn.commit() 74 | cur.close() 75 | return True 76 | 77 | def on_event(event, context): 78 | request_type = event['RequestType'].lower() 79 | if request_type == 'create': 80 | return on_create(event) 81 | if request_type == 'update': 82 | return on_update(event) 83 | if request_type == 'delete': 84 | return on_delete(event) 85 | raise Exception(f'Invalid request type: {request_type}') 86 | 87 | 88 | def on_create(event): 89 | try: 90 | db = Database(writer=writer_endpoint, database_name = database_name, embedding_dimension = embedding_dimension) 91 | db.setup_vector_db(embedding_dimension) 92 | db.close_connection() 93 | except Exception as e: 94 | print(e) 95 | 96 | return {'PhysicalResourceId': "VectorDBDatabaseSetup"} 97 | 98 | 99 | def on_update(event): 100 | physical_id = event["PhysicalResourceId"] 101 | print("no op") 102 | #return {'PhysicalResourceId': physical_id} 103 | return {'PhysicalResourceId': "VectorDBDatabaseSetup"} 104 | 105 | def on_delete(event): 106 | physical_id = event["PhysicalResourceId"] 107 | print("no op") 108 | #return {'PhysicalResourceId': physical_id} 109 | return {'PhysicalResourceId': "VectorDBDatabaseSetup"} 110 | -------------------------------------------------------------------------------- /lib/db_setup_lambda/requirements.txt: -------------------------------------------------------------------------------- 1 | psycopg2-binary>=2.9.9 -------------------------------------------------------------------------------- /lib/main_analyzer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM --platform=linux/amd64 python:3.12 2 | RUN apt-get update && apt-get install ffmpeg libsm6 libxext6 -y 3 | RUN addgroup --system vusgroup && adduser --system vususer --ingroup vusgroup --home /vus 4 | USER vususer 5 | WORKDIR /lib/main_analyzer 6 | ADD . /lib/main_analyzer 7 | RUN python3.12 -m pip install -r ./requirements.txt 8 | CMD python3.12 index.py -------------------------------------------------------------------------------- /lib/main_analyzer/default_visual_extraction_system_prompt.txt: -------------------------------------------------------------------------------- 1 | You are an expert in extracting information from video frames. Each video frame is an image. You will extract the scene, text, and caption. -------------------------------------------------------------------------------- /lib/main_analyzer/default_visual_extraction_task_prompt.txt: -------------------------------------------------------------------------------- 1 | Extract information from this image and output a JSON with this format: 2 | { 3 | "scene":"String", 4 | "caption":"String", 5 | "text":["String", "String" , . . .] 6 | } 7 | For "scene", look carefully, think hard, and describe what you see in the image in detail, yet succinct. 8 | For "caption", look carefully, think hard, and give a SHORT caption (3-8 words) that best describes what is happening in the image. This is intended for visually impaired ones. 9 | For "text", list the text you see in that image confidently. If nothing return empty list. -------------------------------------------------------------------------------- /lib/main_analyzer/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3>=1.35.84 2 | awscli>=1.36.25 3 | botocore>=1.35.84 4 | psycopg2-binary>=2.9.9 5 | SQLAlchemy>=2.0.31 6 | pgvector>=0.3.2 7 | opencv-python>=4.10.0.84 8 | pillow>=10.4.0 -------------------------------------------------------------------------------- /lib/preprocessing_lambda/index.py: -------------------------------------------------------------------------------- 1 | import json, os 2 | import boto3 3 | from datetime import datetime, timezone 4 | from sqlalchemy import create_engine, Column, Text, DateTime, String, func 5 | from sqlalchemy.ext.declarative import declarative_base 6 | from sqlalchemy.orm import mapped_column, sessionmaker 7 | from sqlalchemy.sql import bindparam 8 | from sqlalchemy.dialects.postgresql import insert as db_insert 9 | from pgvector.sqlalchemy import Vector 10 | 11 | database_name = os.environ['DATABASE_NAME'] 12 | video_table_name = os.environ['VIDEO_TABLE_NAME'] 13 | secret_name = os.environ['SECRET_NAME'] 14 | configuration_parameter_name = os.environ['CONFIGURATION_PARAMETER_NAME'] 15 | writer_endpoint = os.environ['DB_WRITER_ENDPOINT'] 16 | embedding_dimension = os.environ['EMBEDDING_DIMENSION'] 17 | CONFIG_LABEL_DETECTION_ENABLED = "label_detection_enabled" 18 | CONFIG_TRANSCRIPTION_ENABLED = "transcription_enabled" 19 | 20 | ssm = boto3.client('ssm') 21 | secrets_manager = boto3.client('secretsmanager') 22 | 23 | credentials = json.loads(secrets_manager.get_secret_value(SecretId=secret_name)["SecretString"]) 24 | username = credentials["username"] 25 | password = credentials["password"] 26 | 27 | engine = create_engine(f'postgresql://{username}:{password}@{writer_endpoint}:5432/{database_name}') 28 | Base = declarative_base() 29 | 30 | class Videos(Base): 31 | __tablename__ = video_table_name 32 | 33 | name = Column(String(200), primary_key=True, nullable=False) 34 | uploaded_at = Column(DateTime(timezone=True), nullable=False) 35 | summary = Column(Text) 36 | summary_embedding = mapped_column(Vector(int(embedding_dimension))) 37 | 38 | Session = sessionmaker(bind=engine) 39 | session = Session() 40 | 41 | def handler(event, context): 42 | video_s3_path = event["videoS3Path"] 43 | 44 | # Get the name of the video along with its folder location from the raw_folder. 45 | video_name = '/'.join(video_s3_path.split('/')[1:]) #os.path.basename(video_s3_path) 46 | 47 | # Validate video file extension. Only .mp4, .MP4, .mov, and .MOV are allowed. 48 | if video_name[-4:] not in [".mp4", ".MP4", ".mov", ".MOV"]: 49 | return { 50 | 'statusCode': 400, 51 | 'body': json.dumps({"preprocessing": "Unsupported video file extension. Only .mp4, .MP4, .mov, and .MOV are allowed."}) 52 | } 53 | 54 | # Parameterize 55 | video_name_param = bindparam('name') 56 | uploaded_at_param = bindparam('uploaded_at') 57 | 58 | upsert = db_insert(Videos).values( 59 | name=video_name_param, 60 | uploaded_at=uploaded_at_param 61 | ) 62 | 63 | upsert = upsert.on_conflict_do_update( 64 | constraint=f"{video_table_name}_pkey", 65 | set_={ 66 | Videos.uploaded_at: uploaded_at_param 67 | } 68 | ) 69 | 70 | 71 | configuration_parameter_json = ssm.get_parameter( 72 | Name=configuration_parameter_name 73 | )['Parameter']['Value'] 74 | configuration_parameter = json.loads(configuration_parameter_json) 75 | 76 | 77 | with session.begin(): 78 | date_now = datetime.now(timezone.utc) 79 | session.execute(upsert, { 80 | "name": video_name, 81 | "uploaded_at": date_now 82 | }) 83 | session.commit() 84 | 85 | return { 86 | 'statusCode': 200, 87 | 'body': { 88 | "preprocessing": "success", 89 | CONFIG_LABEL_DETECTION_ENABLED:configuration_parameter[CONFIG_LABEL_DETECTION_ENABLED], 90 | CONFIG_TRANSCRIPTION_ENABLED:configuration_parameter[CONFIG_TRANSCRIPTION_ENABLED] 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /lib/preprocessing_lambda/requirements.txt: -------------------------------------------------------------------------------- 1 | psycopg2-binary>=2.9.9 2 | SQLAlchemy>=2.0.31 3 | pgvector>=0.3.2 -------------------------------------------------------------------------------- /lib/prompt_setup_lambda/index.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | import base64 4 | 5 | def on_event(event, context): 6 | print(event) 7 | request_type = event['RequestType'].lower() 8 | 9 | if request_type == 'create' or request_type == 'update': 10 | return on_create(event) 11 | if request_type == 'delete': 12 | return on_delete(event) 13 | raise Exception(f"Invalid request type: {request_type}") 14 | 15 | def on_create(event): 16 | props = event["ResourceProperties"] 17 | bedrock = boto3.client('bedrock-agent') 18 | ssm = boto3.client('ssm') 19 | 20 | # Base64 encode the prompts to safely handle special characters 21 | system_prompt = base64.b64decode(props['visual_extraction_system_prompt_text']).decode('utf-8') 22 | task_prompt = base64.b64decode(props['visual_extraction_task_prompt_text']).decode('utf-8') 23 | 24 | # Create prompt 25 | prompt = bedrock.create_prompt( 26 | name=props['visual_extraction_prompt_name'], 27 | description=props['visual_extraction_prompt_description'], 28 | defaultVariant=props['visual_extraction_prompt_variant_name'], 29 | variants=[{ 30 | 'name': props['visual_extraction_prompt_variant_name'], 31 | 'templateConfiguration': { 32 | 'chat': { 33 | 'messages': [ 34 | { 35 | 'content': [ 36 | { 37 | 'text': task_prompt 38 | }, 39 | ], 40 | 'role': 'user' 41 | }, 42 | ], 43 | 'system': [ 44 | { 45 | 'text': system_prompt 46 | }, 47 | ] 48 | } 49 | }, 50 | 'templateType': 'CHAT' 51 | }] 52 | ) 53 | 54 | # Create prompt version 55 | prompt_version = bedrock.create_prompt_version( 56 | promptIdentifier=prompt['id'], 57 | description=props['visual_extraction_prompt_version_description'] 58 | )['version'] 59 | 60 | # Get current parameter store value 61 | try: 62 | parameter = ssm.get_parameter(Name=props['configuration_parameter_name']) 63 | config = json.loads(parameter['Parameter']['Value']) 64 | except ssm.exceptions.ParameterNotFound: 65 | config = {} 66 | 67 | # Update prompt configuration 68 | config['visual_extraction_prompt'] = { 69 | "prompt_id": prompt['id'], 70 | "variant_name": props['visual_extraction_prompt_variant_name'], 71 | "version_id": prompt_version 72 | } 73 | 74 | # Update parameter store 75 | ssm.put_parameter( 76 | Name=props['configuration_parameter_name'], 77 | Value=json.dumps(config), 78 | Type='String', 79 | Overwrite=True 80 | ) 81 | 82 | return { 83 | 'PhysicalResourceId': f"{prompt['id']}|{prompt_version}", 84 | 'Data': { 85 | 'PromptId': prompt['id'], 86 | 'VersionId': prompt_version 87 | } 88 | } 89 | 90 | def on_delete(event): 91 | physical_id = event['PhysicalResourceId'] 92 | prompt_id, version_id = physical_id.split('|') 93 | bedrock = boto3.client('bedrock-agent') 94 | 95 | try: 96 | bedrock.delete_prompt(promptIdentifier=prompt_id) 97 | except: 98 | pass 99 | 100 | return { 101 | 'PhysicalResourceId': physical_id 102 | } -------------------------------------------------------------------------------- /lib/prompt_setup_lambda/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3>=1.35.84 2 | botocore>=1.35.84 3 | awscli>=1.36.25 -------------------------------------------------------------------------------- /lib/videos_search/lambda-handler.py: -------------------------------------------------------------------------------- 1 | import os, json 2 | import boto3 3 | import urllib.parse 4 | from sqlalchemy import create_engine, Column, DateTime, String, Text 5 | from sqlalchemy.ext.declarative import declarative_base 6 | from sqlalchemy.orm import sessionmaker, mapped_column 7 | from sqlalchemy.sql import bindparam 8 | from pgvector.sqlalchemy import Vector 9 | from datetime import datetime 10 | 11 | bedrock = boto3.client("bedrock-runtime") 12 | secrets_manager = boto3.client('secretsmanager') 13 | 14 | reader_endpoint = os.environ['DB_READER_ENDPOINT'] 15 | database_name = os.environ['DATABASE_NAME'] 16 | video_table_name = os.environ['VIDEO_TABLE_NAME'] 17 | secret_name = os.environ['SECRET_NAME'] 18 | embedding_model_id = os.environ["EMBEDDING_MODEL_ID"] 19 | embedding_dimension = int(os.environ['EMBEDDING_DIMENSION']) 20 | acceptable_embedding_distance = float(os.environ['ACCEPTABLE_EMBEDDING_DISTANCE']) 21 | display_page_size = int(os.environ['DISPLAY_PAGE_SIZE']) 22 | 23 | credentials = json.loads(secrets_manager.get_secret_value(SecretId=secret_name)["SecretString"]) 24 | username = credentials["username"] 25 | password = credentials["password"] 26 | 27 | engine = create_engine(f'postgresql://{username}:{password}@{reader_endpoint}:5432/{database_name}') 28 | Base = declarative_base() 29 | 30 | Session = sessionmaker(bind=engine) 31 | session = Session() 32 | 33 | class Videos(Base): 34 | __tablename__ = video_table_name 35 | 36 | name = Column(String(200), primary_key=True, nullable=False) 37 | uploaded_at = Column(DateTime(timezone=True), nullable=False) 38 | summary = Column(Text) 39 | summary_embedding = mapped_column(Vector(int(embedding_dimension))) 40 | 41 | 42 | def handler(event, context): 43 | params = event["queryStringParameters"] 44 | 45 | page = int(params["page"]) 46 | video_name_contains = urllib.parse.unquote(params["videoNameContains"]) if "videoNameContains" in params else None 47 | uploaded_between= urllib.parse.unquote(params["uploadedBetween"]) if "uploadedBetween" in params else None 48 | about = urllib.parse.unquote(params["about"]) if "about" in params else None 49 | 50 | # Use SQLAlchemy to search videos with the 3 filters above. 51 | videos = session.query(Videos.name) 52 | if video_name_contains is not None: 53 | video_name_contains_param = bindparam("name") 54 | videos = videos.filter(Videos.name.like(video_name_contains_param)) 55 | if uploaded_between is not None: 56 | # Assume uploaded_between is like 2024-02-07T16:00:00.000Z|2024-02-15T16:00:00.000Z 57 | start, stop = uploaded_between.split("|") 58 | start = datetime.strptime(start[:-5], "%Y-%m-%dT%H:%M:%S") 59 | stop = datetime.strptime(stop[:-5], "%Y-%m-%dT%H:%M:%S") 60 | start_param = bindparam("start") 61 | stop_param = bindparam("stop") 62 | videos = videos.filter(Videos.uploaded_at.between(start_param, stop_param)) 63 | if about is not None: 64 | # Get the embedding for the video topic 65 | body = json.dumps({ 66 | "texts":[about], 67 | "input_type": "search_query", 68 | }) 69 | call_done = False 70 | while(not call_done): 71 | try: 72 | response = bedrock.invoke_model(body=body, modelId=embedding_model_id) 73 | call_done = True 74 | except ThrottlingException: 75 | print("Amazon Bedrock throttling exception") 76 | time.sleep(60) 77 | except Exception as e: 78 | raise e 79 | 80 | # Disabling semgrep rule for checking data size to be loaded to JSON as the source is from Amazon Bedrock 81 | # nosemgrep: python.aws-lambda.deserialization.tainted-json-aws-lambda.tainted-json-aws-lambda 82 | about_embedding = json.loads(response.get("body").read().decode())['embeddings'][0] 83 | 84 | videos = videos.filter(Videos.summary_embedding.cosine_distance(about_embedding) < acceptable_embedding_distance) 85 | videos = videos.order_by(Videos.summary_embedding.cosine_distance(about_embedding)) 86 | 87 | videos = videos.order_by(Videos.uploaded_at.desc()) 88 | 89 | if video_name_contains is not None: 90 | videos = videos.params(name=f"%{video_name_contains}%") 91 | if uploaded_between is not None: 92 | videos = videos.params(start=start, stop=stop) 93 | 94 | videos = videos.offset(page*display_page_size).limit(display_page_size+1) 95 | 96 | video_names = videos.all() 97 | video_names = [v.name for v in video_names] 98 | 99 | next_page = None 100 | if len(video_names) > display_page_size: 101 | video_names = video_names[:display_page_size] 102 | next_page = page + 1 103 | 104 | response_payload = { 105 | "videos": video_names, 106 | "pageSize": display_page_size, 107 | } 108 | if next_page is not None: response_payload['nextPage'] = next_page 109 | 110 | response = { 111 | "statusCode": 200, 112 | "headers": { 113 | "Content-Type": "application/json", 114 | "Access-Control-Allow-Origin": "*" 115 | }, 116 | "body": json.dumps(response_payload) 117 | } 118 | 119 | return response -------------------------------------------------------------------------------- /lib/videos_search/requirements.txt: -------------------------------------------------------------------------------- 1 | psycopg2-binary>=2.9.9 2 | SQLAlchemy>=2.0.31 3 | boto3>=1.34.150 4 | awscli>=1.33.32 5 | botocore>=1.34.150 6 | pgvector>=0.3.2 -------------------------------------------------------------------------------- /utils/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -d "venv" ]; then 4 | source venv/bin/activate 5 | fi 6 | 7 | export NVM_DIR="$HOME/.nvm" 8 | [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" 9 | 10 | nvm use 20.10.0 || nvm use 16.20.2 11 | node -e "console.log('Currently running Node.js ' + process.version)" 12 | 13 | account=$(aws sts get-caller-identity | jq -r '.Account') 14 | export CDK_DEPLOY_ACCOUNT=$account 15 | 16 | environment="dev" # This is a default value 17 | echo "Bootstrap is using default value for environment with value = dev" 18 | 19 | default_region="us-west-2" # This is a default value which is expected to be overridden by user input. 20 | echo "" 21 | read -p "Which AWS region to deploy this solution to? (default: $default_region) : " region 22 | region=${region:-$default_region} 23 | export CDK_DEPLOY_REGION=$region 24 | echo $region > .DEFAULT_REGION 25 | 26 | npm --prefix ./webui install ./webui 27 | 28 | cd webui && find . -name 'ui_repo*.zip' -exec rm {} \; && zip -r "ui_repo$(date +%s).zip" src package.json package-lock.json public && cd .. 29 | 30 | cdk bootstrap aws://${account}/${region} --context environment=$environment 31 | 32 | if [ -d "venv" ]; then 33 | deactivate 34 | fi -------------------------------------------------------------------------------- /utils/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -d "venv" ]; then 4 | source venv/bin/activate 5 | fi 6 | 7 | export NVM_DIR="$HOME/.nvm" 8 | [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" 9 | 10 | nvm use 20.10.0 || nvm use 16.20.2 11 | node -e "console.log('Currently running Node.js ' + process.version)" 12 | 13 | account=$(aws sts get-caller-identity | jq -r '.Account') 14 | export CDK_DEPLOY_ACCOUNT=$account 15 | 16 | default_region="us-west-2" # This is a default value which is expected to be overridden by user input. 17 | if [ -f .DEFAULT_REGION ] 18 | then 19 | default_region=$(cat .DEFAULT_REGION) 20 | fi 21 | echo "" 22 | email="" # This is a default value 23 | read -p "The admin email for the video management: " email 24 | email=${email:-$email} 25 | 26 | read -p "Which AWS region to deploy this solution to? (default: $default_region) : " region 27 | region=${region:-$default_region} 28 | export CDK_DEPLOY_REGION=$region 29 | 30 | npm --prefix ./webui install ./webui 31 | 32 | cd webui && find . -name 'ui_repo*.zip' -exec rm {} \; && zip -r "ui_repo$(date +%s).zip" src package.json package-lock.json public && cd .. 33 | 34 | cdk deploy --outputs-file ./deployment-output.json --context email=$email 35 | 36 | if [ -d "venv" ]; then 37 | deactivate 38 | fi 39 | -------------------------------------------------------------------------------- /utils/destroy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -d "venv" ]; then 4 | source venv/bin/activate 5 | fi 6 | 7 | export NVM_DIR="$HOME/.nvm" 8 | [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" 9 | 10 | nvm use 20.10.0 || nvm use 16.20.2 11 | node -e "console.log('Currently running Node.js ' + process.version)" 12 | 13 | account=$(aws sts get-caller-identity | jq -r '.Account') 14 | export CDK_DEPLOY_ACCOUNT=$account 15 | 16 | default_region="us-west-2" # This is a default value which is expected to be overridden by user input. 17 | if [ -f .DEFAULT_REGION ] 18 | then 19 | default_region=$(cat .DEFAULT_REGION) 20 | fi 21 | echo "" 22 | 23 | read -p "Which AWS region is the solution to be destroyed currently in? (default: $default_region) : " region 24 | region=${region:-$default_region} 25 | export CDK_DEPLOY_REGION=$region 26 | 27 | cdk destroy --context email="placeholder" 28 | 29 | if [ -d "venv" ]; then 30 | deactivate 31 | fi 32 | -------------------------------------------------------------------------------- /utils/install_prerequisites_for_amazon_linux_2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Install Python if not installed or if the version is below the required one 4 | needPython=false 5 | version=$(python3 -V 2>&1 | grep -Po '(?<=Python )(.+)') 6 | parsedVersion=$(echo "${version//./}") 7 | if [[ -z "$version" ]] 8 | then 9 | needPython=true 10 | else 11 | if { echo $version; echo "3.8.0"; } | sort --version-sort --check 12 | then 13 | needPython=true 14 | fi 15 | fi 16 | 17 | if [[ "$needPython" = true ]] 18 | then 19 | sudo yum install -y gcc openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel 20 | mkdir packages 21 | cd packages 22 | curl https://www.python.org/ftp/python/3.9.18/Python-3.9.18.tgz -o Python-3.9.18.tgz 23 | tar xzf Python-3.9.18.tgz && cd Python-3.9.18 24 | ./configure --enable-optimizations 25 | sudo make altinstall 26 | cd .. && sudo rm ./Python-3.9.18.tgz && sudo rm -rf ./Python-3.9.18 27 | cd .. && rm -rf ./packages 28 | fi 29 | 30 | # Install pip 31 | python3 -m ensurepip 32 | 33 | # Install VirtualEnv if not installed 34 | isVirtualEnvInstalled=$(python3 -m pip list | grep "virtualenv") 35 | if [[ -z "$isVirtualEnvInstalled" ]] 36 | then 37 | sudo python3 -m pip install virtualenv 38 | fi 39 | 40 | if [[ "$needPython" = true ]] 41 | then 42 | virtualenv -p $(which python3.9) venv 43 | else 44 | virtualenv -p $(which python3) venv 45 | fi 46 | 47 | # Activate virtual environment 48 | source venv/bin/activate 49 | 50 | # Install zip if not installed 51 | if ! command -v zip &> /dev/null 52 | then 53 | sudo yum update -y 54 | sudo yum install zip -y 55 | fi 56 | 57 | # Install unzip if not installed 58 | if ! command -v unzip &> /dev/null 59 | then 60 | sudo yum update -y 61 | sudo yum install unzip -y 62 | fi 63 | 64 | # Install NPM 65 | curl -o /tmp/nvm_install.sh https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh 66 | /bin/bash /tmp/nvm_install.sh 67 | . ~/.nvm/nvm.sh 68 | 69 | export NVM_DIR="$HOME/.nvm" 70 | [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" 71 | 72 | nvm install v16.20.2 73 | nvm use 16.20.2 74 | rm /tmp/nvm_install.sh 75 | 76 | node -e "console.log('Currently running Node.js ' + process.version)" 77 | npm install -g aws-cdk@">=2.173.1" 78 | npm install -g @aws-cdk/aws-amplify-alpha@">=2.173.1-alpha.0" 79 | 80 | python3 -m pip install --upgrade pip 81 | python3 -m pip install --upgrade "aws-cdk-lib>=2.173.1" 82 | python3 -m pip install --upgrade "aws-cdk.aws-amplify-alpha" 83 | python3 -m pip install --upgrade "cdk-nag>=2.34.8" 84 | 85 | # Install Docker if not installed. It will be needed to build the code for the AWS Lambda functions during CDK deployment. 86 | if ! command -v docker &> /dev/null 87 | then 88 | sudo yum update -y 89 | sudo yum install -y docker 90 | fi 91 | 92 | # Give users permissions to access docker 93 | sudo groupadd docker 94 | sudo usermod -a -G docker ec2-user 95 | sudo usermod -a -G docker ssm-user 96 | # IMPORTANT # 97 | # If you are using OS user other than ec2-user and ssm-user, also add them to the `docker` group below. Edit appropriately and uncomment below line. You can run `whoami` to get your OS user name. 98 | # sudo usermod -a -G docker 99 | # --------- # 100 | 101 | # Start Docker service 102 | sudo service docker start 103 | 104 | # Install JQ if not installed 105 | if ! command -v jq &> /dev/null 106 | then 107 | sudo yum update -y 108 | sudo yum install jq -y 109 | fi 110 | 111 | # Install AWS CLI if not installed 112 | if ! command -v aws &> /dev/null 113 | then 114 | curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" 115 | unzip awscliv2.zip 116 | sudo ./aws/install && sudo rm awscliv2.zip 117 | fi 118 | 119 | # Deactivate virtual environment 120 | deactivate -------------------------------------------------------------------------------- /utils/install_prerequisites_for_amazon_linux_2023.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Install Python if not installed or if the version is below the required one 4 | needPython=false 5 | version=$(python3 -V 2>&1 | grep -Po '(?<=Python )(.+)') 6 | parsedVersion=$(echo "${version//./}") 7 | if [[ -z "$version" ]] 8 | then 9 | needPython=true 10 | else 11 | if { echo $version; echo "3.8.0"; } | sort --version-sort --check 12 | then 13 | needPython=true 14 | fi 15 | fi 16 | 17 | if [[ "$needPython" = true ]] 18 | then 19 | sudo yum install -y gcc openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel 20 | mkdir packages 21 | cd packages 22 | curl https://www.python.org/ftp/python/3.9.18/Python-3.9.18.tgz -o Python-3.9.18.tgz 23 | tar xzf Python-3.9.18.tgz && cd Python-3.9.18 24 | ./configure --enable-optimizations 25 | sudo make altinstall 26 | cd .. && sudo rm ./Python-3.9.18.tgz && sudo rm -rf ./Python-3.9.18 && cd .. 27 | cd .. && rm -rf ./packages 28 | fi 29 | 30 | # Install pip 31 | python3 -m ensurepip 32 | 33 | # Install VirtualEnv if not installed 34 | isVirtualEnvInstalled=$(python3 -m pip list | grep "virtualenv") 35 | if [[ -z "$isVirtualEnvInstalled" ]] 36 | then 37 | python3 -m pip install --user virtualenv 38 | fi 39 | 40 | if [[ "$needPython" = true ]] 41 | then 42 | virtualenv -p $(which python3.9) venv 43 | else 44 | virtualenv -p $(which python3) venv 45 | fi 46 | 47 | # Activate virtual environment 48 | source venv/bin/activate 49 | 50 | # Install zip if not installed 51 | if ! command -v zip &> /dev/null 52 | then 53 | sudo yum update -y 54 | sudo yum install zip -y 55 | fi 56 | 57 | # Install unzip if not installed 58 | if ! command -v unzip &> /dev/null 59 | then 60 | sudo yum update -y 61 | sudo yum install unzip -y 62 | fi 63 | 64 | # Install NPM 65 | curl -o /tmp/nvm_install.sh https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh 66 | /bin/bash /tmp/nvm_install.sh 67 | . ~/.nvm/nvm.sh 68 | 69 | export NVM_DIR="$HOME/.nvm" 70 | [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" 71 | 72 | nvm install v20.10.0 73 | nvm use 20.10.0 74 | rm /tmp/nvm_install.sh 75 | 76 | node -e "console.log('Currently running Node.js ' + process.version)" 77 | npm install -g aws-cdk@">=2.173.1" 78 | npm install -g @aws-cdk/aws-amplify-alpha@">=2.173.1-alpha.0" 79 | 80 | python3 -m pip install --upgrade pip 81 | python3 -m pip install --upgrade "aws-cdk-lib>=2.173.1" 82 | python3 -m pip install --upgrade "aws-cdk.aws-amplify-alpha" 83 | python3 -m pip install --upgrade "cdk-nag>=2.34.8" 84 | 85 | # Install Docker if not installed. It will be needed to build the code for the AWS Lambda functions during CDK deployment. 86 | if ! command -v docker &> /dev/null 87 | then 88 | sudo yum update -y 89 | sudo yum install -y docker 90 | fi 91 | 92 | # Give users permissions to access docker 93 | sudo groupadd docker 94 | sudo usermod -a -G docker ec2-user 95 | sudo usermod -a -G docker ssm-user 96 | # IMPORTANT # 97 | # If you are using OS user other than ec2-user and ssm-user, also add them to the `docker` group below. Edit appropriately and uncomment below line. You can run `whoami` to get your OS user name. 98 | # sudo usermod -a -G docker 99 | # --------- # 100 | 101 | 102 | # Start Docker service 103 | sudo service docker start 104 | 105 | # Install JQ if not installed 106 | if ! command -v jq &> /dev/null 107 | then 108 | sudo yum update -y 109 | sudo yum install jq -y 110 | fi 111 | 112 | # Install AWS CLI if not installed 113 | if ! command -v aws &> /dev/null 114 | then 115 | curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" 116 | unzip awscliv2.zip 117 | sudo ./aws/install && sudo rm awscliv2.zip 118 | fi 119 | 120 | # Deactivate virtual environment 121 | deactivate -------------------------------------------------------------------------------- /utils/install_prerequisites_for_macos_ventura.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Install brew if not there 4 | if ! command -v brew &> /dev/null 5 | then 6 | curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh -o /tmp/brew_install.sh 7 | /bin/bash /tmp/brew_install.sh 8 | brew update 9 | rm /tmp/brew_install.sh 10 | fi 11 | 12 | # Install Python if not installed or if the version is below the required one 13 | needPython=false 14 | version=$(python3 --version | sed 's/.* \([0-9\.]*\).*/\1/') 15 | parsedVersion=$(echo "${version//./}") 16 | if [[ -z "$version" ]] 17 | then 18 | needPython=true 19 | else 20 | if { echo $version; echo "3.8.0"; } | sort --version-sort --check 21 | then 22 | needPython=true 23 | fi 24 | fi 25 | 26 | if [[ "$needPython" = true ]] 27 | then 28 | brew install python@3.9 29 | fi 30 | 31 | # Install pip 32 | python3 -m ensurepip 33 | 34 | # Install VirtualEnv if not installed 35 | isVirtualEnvInstalled=$(python3 -m pip list | grep "virtualenv") 36 | if [[ -z "$isVirtualEnvInstalled" ]] 37 | then 38 | python3 -m pip install --user virtualenv 39 | fi 40 | 41 | if [[ "$needPython" = true ]] 42 | then 43 | virtualenv -p $(which python3.9) venv 44 | else 45 | virtualenv -p $(which python3) venv 46 | fi 47 | 48 | # Activate virtual environment 49 | source venv/bin/activate 50 | 51 | # Install zip if not installed 52 | if ! command -v zip &> /dev/null 53 | then 54 | brew install zip 55 | fi 56 | 57 | # Install unzip if not installed 58 | if ! command -v unzip &> /dev/null 59 | then 60 | brew install unzip 61 | fi 62 | 63 | # Install NPM if not installed 64 | brew install nvm 65 | 66 | export NVM_DIR="$HOME/.nvm" 67 | [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" 68 | 69 | nvm install v20.10.0 70 | nvm use 20.10.0 71 | 72 | node -e "console.log('Currently running Node.js ' + process.version)" 73 | sudo npm install -g aws-cdk@">=2.173.1" 74 | sudo npm install -g @aws-cdk/aws-amplify-alpha@">=2.173.1-alpha.0" 75 | 76 | python3 -m pip install --upgrade pip 77 | python3 -m pip install --upgrade "aws-cdk-lib>=2.173.1" 78 | python3 -m pip install --upgrade "aws-cdk.aws-amplify-alpha" 79 | python3 -m pip install --upgrade "cdk-nag>=2.34.8" 80 | 81 | # Install Docker if not installed. It will be needed to build the code for the AWS Lambda functions during CDK deployment. 82 | if ! command -v docker &> /dev/null 83 | then 84 | brew install --cask docker 85 | fi 86 | 87 | # Install JQ if not installed 88 | if ! command -v jq &> /dev/null 89 | then 90 | brew install jq 91 | fi 92 | 93 | # Install AWS CLI if not installed 94 | if ! command -v aws &> /dev/null 95 | then 96 | curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg" 97 | sudo installer -pkg AWSCLIV2.pkg -target / 98 | fi 99 | 100 | # Open docker 101 | open /Applications/Docker.app 102 | 103 | # Deactivate virtual environment 104 | deactivate 105 | -------------------------------------------------------------------------------- /utils/scan.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python3 -m pip install bandit 4 | bandit -r . -o banditreport.txt -f txt --exclude "./cdk.out/,./webui/node_modules/,./venv/" 5 | 6 | python3 -m pip install semgrep 7 | semgrep login 8 | semgrep scan --exclude "./cdk.out/" -o semgrepreport.txt &> semgreplog.txt 9 | 10 | cd webui && npm i --package-lock-only && npm audit --json > ../npmauditreport.json 11 | cd .. -------------------------------------------------------------------------------- /webui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "video-understanding-solution", 3 | "version": "1.2.2", 4 | "private": true, 5 | "dependencies": { 6 | "@aws-amplify/ui-react": "^6.7.2", 7 | "@aws-cdk/aws-amplify-alpha": "^2.173.2-alpha.0", 8 | "@aws-sdk/client-bedrock-runtime": "^3.714.0", 9 | "@aws-sdk/client-s3": "^3.715.0", 10 | "@aws-sdk/s3-request-presigner": "^3.715.0", 11 | "@aws-sdk/credential-providers": "^3.715.0", 12 | "@aws-sdk/client-cognito-identity": "^3.714.0", 13 | "aws-amplify": "^6.10.3", 14 | "bootstrap": "^5.3.3", 15 | "react": "^18.3.1", 16 | "react-bootstrap": "^2.10.7", 17 | "react-dom": "^18.3.1", 18 | "react-scripts": "5.0.1", 19 | "web-vitals": "^4.2.4", 20 | "@svgr/webpack": "^8.1.0", 21 | "postcss": "^8.4.49", 22 | "react-datepicker": "^7.5.0" 23 | }, 24 | "overrides": { 25 | "@svgr/webpack": "$@svgr/webpack", 26 | "postcss": "$postcss" 27 | }, 28 | "scripts": { 29 | "start": "react-scripts start", 30 | "build": "react-scripts build", 31 | "test": "react-scripts test", 32 | "eject": "react-scripts eject" 33 | }, 34 | "eslintConfig": { 35 | "extends": [ 36 | "react-app", 37 | "react-app/jest" 38 | ] 39 | }, 40 | "browserslist": { 41 | "production": [ 42 | ">0.2%", 43 | "not dead", 44 | "not op_mini all" 45 | ], 46 | "development": [ 47 | "last 1 chrome version", 48 | "last 1 firefox version", 49 | "last 1 safari version" 50 | ] 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /webui/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | 17 | 18 | 27 | Video Understanding Solution 28 | 29 | 30 | 31 |
32 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /webui/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | }, 10 | { 11 | "src": "logo192.png", 12 | "type": "image/png", 13 | "sizes": "192x192" 14 | }, 15 | { 16 | "src": "logo512.png", 17 | "type": "image/png", 18 | "sizes": "512x512" 19 | } 20 | ], 21 | "start_url": ".", 22 | "display": "standalone", 23 | "theme_color": "#000000", 24 | "background_color": "#ffffff" 25 | } 26 | -------------------------------------------------------------------------------- /webui/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /webui/src/App.css: -------------------------------------------------------------------------------- 1 | .App { 2 | text-align: center; 3 | } 4 | 5 | .App-logo { 6 | height: 40vmin; 7 | pointer-events: none; 8 | } 9 | 10 | @media (prefers-reduced-motion: no-preference) { 11 | .App-logo { 12 | animation: App-logo-spin infinite 20s linear; 13 | } 14 | } 15 | 16 | .App-header { 17 | background-color: #282c34; 18 | min-height: 100vh; 19 | display: flex; 20 | flex-direction: column; 21 | align-items: center; 22 | justify-content: center; 23 | font-size: calc(10px + 2vmin); 24 | color: white; 25 | } 26 | 27 | .App-link { 28 | color: #61dafb; 29 | } 30 | 31 | @keyframes App-logo-spin { 32 | from { 33 | transform: rotate(0deg); 34 | } 35 | to { 36 | transform: rotate(360deg); 37 | } 38 | } 39 | 40 | #VideoTableContainer { 41 | margin-top: 20px; 42 | } 43 | -------------------------------------------------------------------------------- /webui/src/App.js: -------------------------------------------------------------------------------- 1 | import "./App.css"; 2 | import "@aws-amplify/ui-react/styles.css"; 3 | import { VideoTable } from "./VideoTable/VideoTable"; 4 | import { VideoUpload } from "./VideoUpload/VideoUpload"; 5 | import awsExports from "./aws-exports"; 6 | import { Amplify } from "aws-amplify"; 7 | import { withAuthenticator } from "@aws-amplify/ui-react"; 8 | import type { WithAuthenticatorProps } from "@aws-amplify/ui-react"; 9 | import { fetchAuthSession } from "aws-amplify/auth"; 10 | import { fromCognitoIdentityPool } from "@aws-sdk/credential-providers"; 11 | import { 12 | CognitoIdentityClient, 13 | GetIdCommand, 14 | } from "@aws-sdk/client-cognito-identity"; 15 | import { S3Client } from "@aws-sdk/client-s3"; 16 | import { BedrockRuntimeClient } from "@aws-sdk/client-bedrock-runtime"; 17 | 18 | import Container from "react-bootstrap/Container"; 19 | import Row from "react-bootstrap/Row"; 20 | import Col from "react-bootstrap/Col"; 21 | import Navbar from "react-bootstrap/Navbar"; 22 | import { useEffect, useState } from "react"; 23 | 24 | Amplify.configure({ 25 | Auth: { 26 | Cognito: { 27 | userPoolId: awsExports.aws_user_pools_id, 28 | userPoolClientId: awsExports.aws_user_pools_web_client_id, 29 | }, 30 | }, 31 | }); 32 | 33 | const REGION = awsExports.aws_cognito_region; 34 | const COGNITO_ID = `cognito-idp.${REGION}.amazonaws.com/${awsExports.aws_user_pools_id}`; 35 | 36 | function App({ signOut, user }: WithAuthenticatorProps) { 37 | const [s3Client, setS3Client] = useState(null); 38 | const [bedrockClient, setBedrockClient] = useState(null); 39 | const [tokens, setTokens] = useState(null); 40 | 41 | const getTs = async () => { 42 | return (await fetchAuthSession()).tokens; 43 | } 44 | 45 | const getCredentials = async (ts) => { 46 | let idT = (await ts).idToken.toString(); 47 | 48 | const cognitoidentity = new CognitoIdentityClient({ 49 | credentials: fromCognitoIdentityPool({ 50 | clientConfig: { region: awsExports.aws_cognito_region }, 51 | identityPoolId: awsExports.aws_cognito_identity_pool_id, 52 | logins: { 53 | [COGNITO_ID]: idT, 54 | }, 55 | }), 56 | }); 57 | return await cognitoidentity.config.credentials(); 58 | }; 59 | 60 | useEffect(() => { 61 | const initializeClients = async () => { 62 | const ts = await getTs(); 63 | const crs = await getCredentials(ts); 64 | 65 | setTokens(ts); 66 | 67 | setS3Client(new S3Client({ 68 | region: REGION, 69 | credentials: crs, 70 | })); 71 | 72 | setBedrockClient(new BedrockRuntimeClient({ 73 | region: REGION, 74 | credentials: crs, 75 | })); 76 | }; 77 | 78 | initializeClients(); 79 | const interval = setInterval(initializeClients, 3300000); // Refresh every 55 minutes 80 | 81 | return () => clearInterval(interval); 82 | }, []); 83 | 84 | if (!s3Client || !bedrockClient || !tokens) return (
Loading...
) 85 | 86 | return ( 87 |
88 | 89 | 90 | Video Understanding Solution 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 104 | 105 | 106 | 107 | 108 |
109 | 110 | 111 | 112 | 113 | 129 | 130 | 131 | 132 | 133 | 134 | 135 |
136 | ); 137 | } 138 | 139 | export default withAuthenticator(App, { hideSignUp: true }); 140 | -------------------------------------------------------------------------------- /webui/src/App.test.js: -------------------------------------------------------------------------------- 1 | import { render, screen } from '@testing-library/react'; 2 | import App from './App'; 3 | 4 | test('renders learn react link', () => { 5 | render(); 6 | const linkElement = screen.getByText(/learn react/i); 7 | expect(linkElement).toBeInTheDocument(); 8 | }); 9 | -------------------------------------------------------------------------------- /webui/src/VideoTable/VideoTable.css: -------------------------------------------------------------------------------- 1 | .preserve-line-breaks { 2 | white-space: pre-wrap; 3 | } 4 | .page-spinner{ 5 | margin-top: 100px; 6 | } 7 | .chat-spinner{ 8 | margin-bottom: 5px; 9 | } 10 | .paragraph-with-newlines{ 11 | white-space: pre-line 12 | } 13 | .left-align-no-bottom-margin{ 14 | text-align: left; 15 | margin-bottom: 0px; 16 | } 17 | .search-button{ 18 | margin-top: 15px; 19 | } 20 | .find-part-result{ 21 | text-align:left; 22 | margin-top:5px 23 | } -------------------------------------------------------------------------------- /webui/src/VideoTable/VideoTable.js: -------------------------------------------------------------------------------- 1 | import React, { Component, useState } from 'react'; 2 | import { Buffer } from "buffer"; 3 | import Accordion from 'react-bootstrap/Accordion'; 4 | import Collapse from 'react-bootstrap/Collapse'; 5 | import Button from 'react-bootstrap/Button'; 6 | import Form from 'react-bootstrap/Form'; 7 | import InputGroup from 'react-bootstrap/InputGroup'; 8 | import Dropdown from 'react-bootstrap/Dropdown'; 9 | import DropdownButton from 'react-bootstrap/DropdownButton'; 10 | import Pagination from 'react-bootstrap/Pagination'; 11 | import Spinner from 'react-bootstrap/Spinner'; 12 | import DatePicker from "react-datepicker"; 13 | 14 | import {ListObjectsV2Command, GetObjectCommand } from "@aws-sdk/client-s3"; 15 | import { InvokeModelWithResponseStreamCommand, InvokeModelCommand } from "@aws-sdk/client-bedrock-runtime"; 16 | import {getSignedUrl} from "@aws-sdk/s3-request-presigner" 17 | 18 | import Row from 'react-bootstrap/Row'; 19 | import Col from 'react-bootstrap/Col'; 20 | 21 | import './VideoTable.css'; 22 | import "react-datepicker/dist/react-datepicker.css"; 23 | 24 | export class VideoTable extends Component { 25 | constructor(props) { 26 | super(props); 27 | 28 | this.fastModelId= props.fastModelId 29 | this.balancedModelId = props.balancedModelId 30 | 31 | this.state = { 32 | videos: [], 33 | pages: {}, 34 | firstFetch: false, 35 | pageLoading: false, 36 | searchByNameText: "", 37 | searchByStartDate: null, 38 | searchByEndDate: null, 39 | searchByAboutText: "", 40 | chatModelChoice: "Fastest" // as default 41 | }; 42 | 43 | this.s3Client = props.s3Client 44 | this.bedrockClient = props.bedrockClient 45 | this.bucketName = props.bucketName 46 | this.rawFolder = props.rawFolder 47 | this.summaryFolder = props.summaryFolder 48 | this.transcriptionFolder = props.transcriptionFolder 49 | this.videoScriptFolder = props.videoScriptFolder 50 | this.videoCaptionFolder = props.videoCaptionFolder 51 | this.entitySentimentFolder = props.entitySentimentFolder 52 | this.restApiUrl = props.restApiUrl 53 | this.videosApiResource = props.videosApiResource 54 | this.cognitoTs = props.cognitoTs 55 | this.maxCharactersForChat = 5000 56 | this.maxCharactersForVideoScript = 300000 57 | this.maxCharactersForSingleLLMCall = 100000 58 | this.estimatedCharactersPerToken = 4 59 | 60 | this.renderVideoTableBody = this.renderVideoTableBody.bind(this) 61 | this.render = this.render.bind(this) 62 | this.renderChat = this.renderChat.bind(this) 63 | 64 | } 65 | 66 | async componentDidMount() { 67 | var [videos, pages] = await this.fetchVideos(0) 68 | this.setState({videos: videos, pages: pages, firstFetch: true}) 69 | } 70 | 71 | async fetchVideos(page){ 72 | var videos = [] 73 | 74 | this.setState({ 75 | pageLoading: true 76 | }) 77 | 78 | // Construct filter 79 | var params = `?page=${page.toString()}&` 80 | if(this.state.searchByNameText != ""){ 81 | params += `videoNameContains=${encodeURI(this.state.searchByNameText)}&` 82 | } 83 | if(this.state.searchByStartDate != null && this.state.searchByStartDate != null ){ 84 | const startDateString = this.state.searchByStartDate.toISOString() 85 | const endDateString = this.state.searchByEndDate.toISOString() 86 | params += `uploadedBetween=${encodeURI(startDateString + "|" + endDateString)}&` 87 | } 88 | if(this.state.searchByAboutText != ""){ 89 | params += `about=${encodeURI(this.state.searchByAboutText)}` 90 | } 91 | 92 | // Call API Gateway to fetch the video names 93 | var ts = await this.cognitoTs; 94 | 95 | const response = await fetch(`${this.restApiUrl}/${this.videosApiResource}${params}`, { 96 | method: 'GET', 97 | headers: { 98 | 'Content-Type': 'application/json', 99 | 'Authorization': ts.idToken.toString(), 100 | } 101 | }) 102 | 103 | const responseBody = await(response.json()) 104 | 105 | this.setState({ 106 | pageLoading: false 107 | }) 108 | 109 | if(responseBody.videos.length == 0) return [videos, this.state.pages]; 110 | 111 | // Add the video representation in the UI 112 | for (const i in responseBody.videos){ 113 | const videoName = responseBody.videos[i] 114 | 115 | videos.push({ 116 | index: videos.length, 117 | name: videoName, 118 | loaded: false, 119 | summary: undefined, 120 | rawEntities: undefined, 121 | entities: undefined, 122 | videoScript: undefined, 123 | videoCaption: undefined, 124 | videoShown: false, 125 | chats: [], 126 | chatSummary: "", 127 | chatLength: 0, 128 | chatWaiting: false, 129 | findPartWaiting: false, 130 | findPartResult: undefined, 131 | url: undefined 132 | }) 133 | } 134 | 135 | // Activate the right page on the pagination bar 136 | var pages = this.state.pages 137 | var pageFound = false 138 | for (const pi in pages){ 139 | if(pages[pi].index == page){ 140 | pageFound = true 141 | pages[pi] = { 142 | displayName: (page + 1).toString(), 143 | active: true, 144 | index: page 145 | } 146 | }else{ 147 | pages[pi].active = false 148 | } 149 | } 150 | // If the page is not found, add it to the pagination bar 151 | if(!pageFound){ 152 | pages[page] = { 153 | displayName: (page + 1).toString(), 154 | active: true, 155 | index: page 156 | } 157 | } 158 | 159 | // If there is a next page indicated in the API call return, then either add a new page in pagination bar or just update existing page metadata 160 | if ("nextPage" in responseBody){ 161 | pages[page+1] = { 162 | displayName: (page + 2).toString(), 163 | active: false, 164 | index: page + 1 165 | } 166 | } 167 | 168 | return [videos, pages] 169 | } 170 | 171 | async fetchVideoDetails(video){ 172 | const name = video.name 173 | 174 | // Try to find the summary 175 | var command = new GetObjectCommand({ 176 | Bucket: this.bucketName, 177 | Key: this.summaryFolder + "/" + name + ".txt", 178 | }); 179 | try { 180 | const response = await this.s3Client.send(command); 181 | video.summary = await response.Body.transformToString(); 182 | } catch (err) { 183 | console.log(err) 184 | } 185 | 186 | // Try to find the entities and sentiment 187 | var command = new GetObjectCommand({ 188 | Bucket: this.bucketName, 189 | Key: this.entitySentimentFolder + "/" + name + ".txt", 190 | }); 191 | try { 192 | const response = await this.s3Client.send(command); 193 | const rawEntities = await response.Body.transformToString(); 194 | video.rawEntities = rawEntities 195 | video.entities = [] 196 | rawEntities.split("\n").forEach(entity => { 197 | const data = entity.split("|") 198 | video.entities.push(data) 199 | }) 200 | } catch (err) { 201 | console.log(err) 202 | } 203 | 204 | 205 | // Try to get the video script 206 | var command = new GetObjectCommand({ 207 | Bucket: this.bucketName, 208 | Key: this.videoScriptFolder + "/" + name + ".txt", 209 | }); 210 | try { 211 | const response = await this.s3Client.send(command); 212 | video.videoScript = await response.Body.transformToString(); 213 | } catch (err) {} 214 | 215 | // Try to get the video per frame caption 216 | var command = new GetObjectCommand({ 217 | Bucket: this.bucketName, 218 | Key: this.videoCaptionFolder + "/" + name + ".txt", 219 | }); 220 | try { 221 | const response = await this.s3Client.send(command); 222 | video.videoCaption = await response.Body.transformToString(); 223 | } catch (err) {} 224 | 225 | video.loaded = true 226 | 227 | return video 228 | } 229 | 230 | async showVideo(video){ 231 | // Generate presigned URL for video 232 | const getObjectParams = { 233 | Bucket: this.bucketName, 234 | Key: this.rawFolder + "/" + video.name 235 | } 236 | const command = new GetObjectCommand(getObjectParams); 237 | video.url = await getSignedUrl(this.s3Client, command, { expiresIn: 180 }); 238 | video.videoShown = true 239 | this.setState({videos: this.state.videos}) 240 | } 241 | 242 | async videoClicked(video) { 243 | if(!video.loaded) { 244 | video = await this.fetchVideoDetails(video) 245 | this.setState({videos: this.state.videos}) 246 | } 247 | } 248 | 249 | handleChatInputChange(video, event) { 250 | if(event.keyCode == 13 && !event.shiftKey) { // When Enter is pressed without Shift 251 | const chatText = event.target.value 252 | video.chats.push({ 253 | actor: "You", 254 | text: chatText 255 | }) 256 | video.chatLength += chatText.length // Increment the length of the chat 257 | this.setState({videos: this.state.videos}) 258 | event.target.value = "" // Clear input field 259 | this.handleChat(video.index) 260 | } 261 | } 262 | 263 | async handleFindPartInputChange(video, event) { 264 | if(event.keyCode == 13 && !event.shiftKey) { // When Enter is pressed without Shift 265 | video.findPartWaiting = true 266 | this.setState({videos: this.state.videos}) 267 | 268 | const findPartText = event.target.value 269 | const part = await this.handleFindPart(video.index, findPartText) 270 | if("error" in part){ 271 | video.findPartResult = "Part not found" 272 | video.findPartWaiting = false 273 | this.setState({videos: this.state.videos}) 274 | }else{ 275 | video.findPartResult = `Start at second ${part.start} and stop at second ${part.stop}` 276 | video.findPartWaiting = false 277 | this.setState({videos: this.state.videos}) 278 | 279 | if(!video.videoShown){ 280 | await this.showVideo(video) 281 | await new Promise(r => setTimeout(r, 5000)); 282 | } 283 | const videoElement = document.getElementById(`video-${video.index}-canvas`); 284 | videoElement.currentTime = parseInt(part.start); 285 | await new Promise(r => setTimeout(r, 3000)); 286 | videoElement.scrollIntoView() 287 | } 288 | 289 | event.target.value = "" // Clear input field 290 | } 291 | } 292 | 293 | async handleFindPart(videoIndex, findPartText){ 294 | var video = this.state.videos[videoIndex] 295 | var systemPrompt = "You can find relevant part of a video given the text presentation of the video called ." 296 | 297 | var prompt = "" 298 | prompt += "Below is the with information about the voice heard in the video, visual scenes seen, visual text visible, and any face or celebrity visible.\n" 299 | prompt += "For voice, the same speaker number ALWAYS indicates the same person throughout the video.\n" 300 | prompt += "For faces, while the estimated age may differ across timestamp, if they are very close, they can refer to the same person.\n" 301 | prompt += "The numbers on the left represents the seconds into the video where the information was extracted.\n" 302 | prompt += `\n${video.videoScript}\n\n` 303 | prompt += "Now your job is to find the part of the video based on the .\n" 304 | prompt += "The answer MUST be expressed as the start and stop second of the relevant part.\n" 305 | prompt += "The \"start\" MUST represents the earliest point of where the relevant part is. Start from the BEGINNING of the context that leads to the relevant part. It is okay to have some buffer for the \"start\" point.\n" 306 | prompt += "The \"stop\" MUST represents the latest point of where the relevant part is. Do not put the \"stop\" in a way that it will stop the clip abruptly. Add some buffer so that the clip has full context.\n" 307 | prompt += "ALWAYS answer in JSON format ONLY. For example:{\"start\":2.0,\"stop\":15.5}\n" 308 | prompt += "If there is no relevant video part, return {\"error\":404}\n\n" 309 | prompt += `${findPartText}\n\n` 310 | prompt += "Your JSON ONLY answer:" 311 | 312 | const input = { 313 | body: JSON.stringify({ 314 | anthropic_version: "bedrock-2023-05-31", 315 | system: systemPrompt, 316 | messages: [ 317 | { role: "user", content: prompt}, 318 | { role: "assistant", content: "Here is my JSON only answer:"} 319 | ], 320 | temperature: 0.1, 321 | max_tokens: 50 322 | }), 323 | modelId: this.balancedModelId, 324 | contentType: "application/json", 325 | accept: "application/json" 326 | }; 327 | const response = await this.bedrockClient.send(new InvokeModelCommand(input)); 328 | const part = JSON.parse(JSON.parse(new TextDecoder().decode(response.body)).content[0].text.trim()) 329 | 330 | return part 331 | } 332 | 333 | renderChat(video) { 334 | return video.chats.map((chat, i) => { 335 | return ( 336 |

{(chat.actor + " : ")}

{chat.text}

 337 | ) 338 | }) 339 | } 340 | 341 | async retrieveAnswerForShortVideo(video, systemPrompt){ 342 | var [videoScriptPrompt, chatPrompt] = ["",""] 343 | 344 | videoScriptPrompt += "To help you, here is the summary of the whole video that you previously wrote:\n" 345 | videoScriptPrompt += `${video.summary}\n\n` 346 | videoScriptPrompt += "Also, here are the entities and sentiment that you previously extracted. Each row is entity|sentiment|sentiment's reason:\n" 347 | videoScriptPrompt += `${video.rawEntities}\n\n` 348 | 349 | videoScriptPrompt += "Below is the video timeline with information about the voice heard in the video, visual scenes seen, visual text visible, and any face or celebrity visible.\n" 350 | videoScriptPrompt += "For voice, the same speaker number ALWAYS indicates the same person throughout the video.\n" 351 | videoScriptPrompt += "For faces, while the estimated age may differ across timestamp, if they are very close, they can refer to the same person.\n" 352 | videoScriptPrompt += "The numbers on the left represents the seconds into the video where the information was extracted.\n" 353 | videoScriptPrompt += `\n${video.videoScript}\n\n` 354 | 355 | // If the chat history is not too long, then include the whole history in the prompt 356 | // Otherwise, use the summary + last chat instead. 357 | // However, when the summary is not yet generated (First generation is when characters exceed 5000), then use the whole history first 358 | // The last scenario mentioned above may happen during the transition of <5000 characters to >5000 characters for the history 359 | if (video.chatLength <= this.maxCharactersForChat || video.chatSummary == ""){ 360 | for (const i in video.chats){ 361 | if (video.chats[i].actor == "You"){ 362 | chatPrompt += "\nUser: " + video.chats[i].text 363 | }else{ 364 | chatPrompt += "\nYou: " + video.chats[i].text 365 | } 366 | } 367 | }else{ // If the chat history is too long, so that we include the summary of the chats + last chat only. 368 | chatPrompt += "You had a conversation with a user about that video as summarized below.\n" 369 | chatPrompt += `\n${video.chatSummary}\n\n` 370 | chatPrompt += "Now the user comes back with a reply below.\n" 371 | chatPrompt += `\n${video.chats[video.chats.length - 1].text}\n\n` 372 | chatPrompt += "Answer the user's reply above directly, without any XML tag.\n" 373 | } 374 | 375 | const prompt = videoScriptPrompt + chatPrompt 376 | 377 | const input = { 378 | body: JSON.stringify({ 379 | anthropic_version: "bedrock-2023-05-31", 380 | system: systemPrompt, 381 | messages: [ 382 | { role: "user", content: prompt}, 383 | { role: "assistant", content: "Here is my succinct answer:"} 384 | ], 385 | temperature: 0.1, 386 | max_tokens: 1000, 387 | stop_sequences: ["\nUser:"] 388 | }), 389 | modelId: this.state.chatModelChoice == "Fastest" ? this.fastModelId : this.balancedModelId, 390 | contentType: "application/json", 391 | accept: "application/json" 392 | }; 393 | const response = await this.bedrockClient.send(new InvokeModelWithResponseStreamCommand(input)); 394 | 395 | await this.displayNewBotAnswerWithStreaming(video, response) 396 | } 397 | 398 | async retrieveAnswerForLongVideo(video, systemPrompt){ 399 | const numberOfFragments = parseInt(video.videoScript.length/this.maxCharactersForVideoScript) + 1 400 | 401 | const estimatedTime = (numberOfFragments * 25/60).toFixed(1) // Assuming one LLM call is 25 seconds. 402 | await this.displayNewBotAnswer(video, `Long video is detected. Estimated answer time is ${estimatedTime} minutes or sooner.`) 403 | 404 | var [currentFragment, rollingAnswer] = [1,""] 405 | // For every video fragment, try to find the answer to the question, if and not found just write partial answer and go to next iteration. 406 | while(currentFragment <= numberOfFragments){ 407 | var [videoScriptPrompt, chatPrompt, instructionPrompt, partialAnswerPrompt] = ["","","",""] 408 | 409 | videoScriptPrompt += "To help you, here is the summary of the whole video that you previously wrote:\n" 410 | videoScriptPrompt += `${video.summary}\n\n` 411 | videoScriptPrompt += "Also, here are the entities and sentiment that you previously extracted. Each row is entity|sentiment|sentiment's reason:\n" 412 | videoScriptPrompt += `${video.rawEntities}\n\n` 413 | 414 | // Construct the prompt containing the video script (the text representation of the current part of the video) 415 | videoScriptPrompt += "The video timeline has information about the voice heard in the video, visual scenes seen, visual texts visible, and any face or celebrity visible.\n" 416 | videoScriptPrompt += "For voice, the same speaker number ALWAYS indicates the same person throughout the video.\n" 417 | videoScriptPrompt += "For faces, while the estimated age may differ across timestamp, if they are very close, they can refer to the same person.\n" 418 | videoScriptPrompt += "The numbers on the left represents the seconds into the video where the information was extracted.\n" 419 | videoScriptPrompt += `Because the video is long, the video timeline is split into ${numberOfFragments} parts. ` 420 | if (currentFragment == numberOfFragments){ 421 | videoScriptPrompt += "Below is the last part of the video timeline.\n" 422 | }else if (currentFragment == 1){ 423 | videoScriptPrompt += "Below is the first part of the video timeline.\n" 424 | }else{ 425 | videoScriptPrompt += `Below is the part ${currentFragment.toString()} out of ${numberOfFragments} of the video timeline.\n` 426 | } 427 | 428 | // The video script part 429 | const currentVideoScriptFragment = video.videoScript.substring((currentFragment - 1)*this.maxCharactersForVideoScript, currentFragment*this.maxCharactersForVideoScript) 430 | videoScriptPrompt += `\n${currentVideoScriptFragment}\n\n\n` 431 | 432 | // Add the conversation between user and bot for context 433 | if (video.chatLength <= this.maxCharactersForChat || video.chatSummary == ""){ 434 | chatPrompt += "You had a conversation with a user as below. Your previous answers may be based on the knowledge of the whole video timeline, not only part of it.\n" 435 | chatPrompt += "\n" 436 | for (const i in video.chats){ 437 | if (video.chats[i].actor == "You"){ 438 | chatPrompt += `\nUser: ${video.chats[i].text}` 439 | }else{ 440 | chatPrompt += `\nYou: ${video.chats[i].text}` 441 | } 442 | } 443 | chatPrompt += "\n\n\n" 444 | }else{ // If the chat history is too long, so that we include the summary of the chats + last chat only. 445 | chatPrompt += "You had a conversation with a user as summarized below. Your previous answers may be based on the knowledge of the whole video timeline, not only part of it.\n" 446 | chatPrompt += `\n${video.chatSummary}\n` 447 | chatPrompt += "Now the user comes back with a reply below.\n" 448 | chatPrompt += `\n${video.chats[video.chats.length - 1].text}\n\n\n` 449 | } 450 | 451 | // Add previous partial answer from previous parts of the videos to gather information so far 452 | if (rollingAnswer != ""){ 453 | partialAnswerPrompt += "You also saved some partial answer from the previous video parts below. You can use this to provide the final answer or the next partial answer.\n" 454 | partialAnswerPrompt += `\n${rollingAnswer}\n\n\n` 455 | } 456 | 457 | if (currentFragment == (numberOfFragments + 1)){ 458 | instructionPrompt += "Now, since there is no more video part left, provide your final answer directly without XML tag. Answer succinctly. Use well-formatted language and do not quote information from video timeline as is, unless asked." 459 | }else{ 460 | instructionPrompt += "When answering the question, do not use XML tags and do not mention about video timeline, chat summary, or partial answer.\n" 461 | instructionPrompt += "If you have the final answer already, WRITE |BREAK| at the very end of your answer.\n" 462 | instructionPrompt += "If you stil need the next part to come with final answer, then write any partial answer based on the current , , if any, and .\n" 463 | instructionPrompt += "This partial answer will be included when we ask you again right after this by giving the video timeline for the next part, so that you can use the current partial answer.\n" 464 | instructionPrompt += `For question that needs you to see the whole video such as "is there any?", "does it have", "is it mentioned", DO NOT provide final answer until we give you all ${numberOfFragments} parts of the video timeline.\n` 465 | } 466 | 467 | const prompt = videoScriptPrompt + chatPrompt + partialAnswerPrompt + instructionPrompt 468 | 469 | const input = { 470 | body: JSON.stringify({ 471 | anthropic_version: "bedrock-2023-05-31", 472 | system: systemPrompt, 473 | messages: [ 474 | { role: "user", content: prompt}, 475 | { role: "assistant", content: "Here is my succinct answer:"} 476 | ], 477 | temperature: 0.1, 478 | max_tokens: 1000, 479 | stop_sequences: ["\nUser:"] 480 | }), 481 | modelId: this.state.chatModelChoice == "Fastest" ? this.fastModelId : this.balancedModelId, 482 | contentType: "application/json", 483 | accept: "application/json" 484 | }; 485 | const response = await this.bedrockClient.send(new InvokeModelCommand(input)); 486 | rollingAnswer = JSON.parse(new TextDecoder().decode(response.body)).content[0].text 487 | 488 | // When the answer is found without having to go through all video fragments, then break and make it as a final answer. 489 | if (rollingAnswer.includes("|BREAK|")) { 490 | rollingAnswer = rollingAnswer.replace("|BREAK|","") 491 | break 492 | } 493 | 494 | currentFragment += 1 495 | } 496 | 497 | await this.displayNewBotAnswer(video, rollingAnswer) 498 | } 499 | 500 | async handleChat(videoIndex, target){ 501 | var video = this.state.videos[videoIndex] 502 | 503 | if(video.chats.length <= 0) return // Ignore if no chats are there 504 | if(video.chats[video.chats.length - 1].actor != "You") return // Ignore if last message was from bot 505 | 506 | this.addChatWaitingSpinner(video) 507 | 508 | var systemPrompt = "You are an expert in analyzing video and you can answer questions about the video given the video timeline.\n" 509 | systemPrompt += "Answer in the same language as the question from user.\n" 510 | systemPrompt += "If you do not know, say do not know. DO NOT make up wrong answers.\n" 511 | systemPrompt += "The Human who asks can watch the video and they are not aware of the 'video timeline' which will be copied below. So, when answering, DO NOT indicate the presence of 'video timeline'. DO NOT quote raw information as is from the 'video timeline'\n" 512 | systemPrompt += "Use well-formatted language, sentences, and paragraphs. Use correct grammar and writing rule.\n\n" 513 | systemPrompt += "When answering, answer SUCCINCTLY and DO NOT provide extra information unless asked.\n" 514 | systemPrompt += "Answer SUCCINCTLY like examples below:\n" 515 | systemPrompt += "Question: Who is the first narrator and how does the person look like?\n" 516 | systemPrompt += "Answer: The first narrator is likely Tom Bush. He is around 25-30 years old man with beard and mustache. He appears to be happy throughout the video.\n" 517 | systemPrompt += "Question: Did Oba smile at the end of the video?\n" 518 | systemPrompt += "Answer: Yes he did.\n" 519 | systemPrompt += "Question: When did he smile?\n" 520 | systemPrompt += "Answer: He smiled at 5.5 and 7 second marks.\n" 521 | 522 | if(video.videoScript.length < this.maxCharactersForSingleLLMCall){ 523 | await this.retrieveAnswerForShortVideo(video, systemPrompt) 524 | } else { 525 | // Long video gets split into fragments for the video script. For each fragment, it goes to LLM to find the partial answer. 526 | await this.retrieveAnswerForLongVideo(video, systemPrompt) 527 | } 528 | 529 | await this.removeChatWaitingSpinner(video) 530 | this.updateChatSummary(video) 531 | 532 | } 533 | 534 | async displayNewBotAnswer(video, text){ 535 | video.chats.push({actor: "Bot", text: text}) 536 | video.chatLength += text.length 537 | this.setState({videos: this.state.videos}) 538 | } 539 | 540 | async displayNewBotAnswerWithStreaming(video, response){ 541 | video.chats.push({actor: "Bot", text: ""}) 542 | for await (const event of response.body) { 543 | if (event.chunk && event.chunk.bytes) { 544 | const chunk = JSON.parse(Buffer.from(event.chunk.bytes).toString("utf-8")); 545 | if(chunk.type == "content_block_delta"){ 546 | video.chats[video.chats.length - 1].text += chunk.delta.text 547 | video.chatLength += chunk.delta.text.length // Increment the length of the chat 548 | this.setState({videos: this.state.videos}) 549 | } 550 | } 551 | }; 552 | } 553 | 554 | async handleChatModelPicker(event){ 555 | this.setState({chatModelChoice: event}) 556 | } 557 | 558 | async removeChatWaitingSpinner(video){ 559 | video.chatWaiting = false 560 | this.setState({videos: this.state.videos}) 561 | } 562 | 563 | async addChatWaitingSpinner(video){ 564 | video.chatWaiting = true 565 | this.setState({videos: this.state.videos}) 566 | } 567 | 568 | async updateChatSummary(video){ 569 | // If the chats between bot and human is roughly more than 5000 characters (1250 tokens) then summarize the chats, otherwise ignore and return. 570 | if (video.chatLength < this.maxCharactersForChat) return; 571 | 572 | var systemPrompt = "You are an expert in summarizing chat. The chat is between user and a bot. The bot answers questions about a video.\n" 573 | var prompt = "" 574 | 575 | if (video.chatSummary == ""){ 576 | prompt += "Find the chat below.\n\n" 577 | for (const i in video.chats){ 578 | if (video.chats[i].actor == "You"){ 579 | prompt += `\nUser: ${video.chats[i].text}` 580 | }else{ 581 | prompt += `\nYou: ${video.chats[i].text}` 582 | } 583 | } 584 | prompt += "\n\nSummarize the chat in no longer than 1 page." 585 | }else{ 586 | prompt += "Below is the summary of the chat so far.\n" 587 | prompt += `\n${video.chatSummary}\n\n` 588 | prompt += "You recently replied to the User as below.\n" 589 | prompt += `\n${video.chats[video.chats.length - 1].text}\n\n` 590 | prompt += "Summarize the whole chat including your newest reply in no longer than 1 page." 591 | } 592 | 593 | const input = { 594 | body: JSON.stringify({ 595 | anthropic_version: "bedrock-2023-05-31", 596 | system: systemPrompt, 597 | messages: [ 598 | { role: "user", content: prompt}, 599 | { role: "assistant", content: "Here is my succinct answer:"} 600 | ], 601 | temperature: 0.1, 602 | max_tokens: Math.round(this.maxCharactersForChat/this.estimatedCharactersPerToken), 603 | stop_sequences: ["\nUser:", "\nYou:"] 604 | }), 605 | modelId: this.fastModelId, 606 | contentType: "application/json", 607 | accept: "application/json" 608 | }; 609 | const response = await this.bedrockClient.send(new InvokeModelCommand(input)); 610 | video.chatSummary = JSON.parse(new TextDecoder().decode(response.body)).content[0].text 611 | this.setState({videos: this.state.videos}) 612 | } 613 | 614 | async handleSearchByNameTextChange(event){ 615 | this.setState({searchByNameText: event.target.value }) 616 | } 617 | 618 | async handleSearchByAboutTextChange(event){ 619 | this.setState({searchByAboutText: event.target.value }) 620 | } 621 | 622 | async handleSearchByDateChange(event){ 623 | const startDate = event[0] 624 | const endDate = event[1] 625 | this.setState({ 626 | searchByStartDate: startDate, 627 | searchByEndDate: endDate 628 | }) 629 | } 630 | 631 | async handleSearch(){ 632 | await this.setState({pages: []}) 633 | var [videos, pages] = await this.fetchVideos(0) 634 | this.setState({videos: videos, pages: pages}) 635 | } 636 | 637 | renderEntities(video, videoIndex){ 638 | return video.entities.map((entity, i) => { 639 | return ( 640 | 641 | {entity[0]} 642 | {entity[1]} 643 | {entity[2]} 644 | 645 | ) 646 | }) 647 | } 648 | 649 | renderVideoTableBody() { 650 | // Get which page is active to make sure Accordion keeps open for same page and will auto close for page switch. 651 | var activePage = 0 652 | Object.keys(this.state.pages).forEach((i) => {if(this.state.pages[i].active) activePage = i}) 653 | 654 | // Render the video names as Accordion headers. 655 | return this.state.videos.map((video, i) => { 656 | return ( 657 | 658 | {video.name} 659 | 660 | 661 | 662 | { !video.videoShown ? : } 663 | 664 | 665 | 666 | 667 |
Summary:
 668 |

{typeof video.summary === "undefined" ? "Summary is not ready yet" : video.summary }

 669 | 670 | 671 | 672 | 673 |
Entities:
 674 | { typeof video.entities === "undefined" ?

Entity list is not ready yet

 : EntitySentimentReason } 675 | { typeof video.entities !== "undefined" ? this.renderEntities(video, i) : "" } 676 |
677 | 678 | 679 | 680 | 681 |
Ask about the video:
 682 | {this.renderChat(video)} 683 | { video.chatWaiting ? Thinking ... : "" } 684 | 685 | 686 | 693 | 694 | 700 | Fastest 701 | Balanced 702 | Smartest 703 | 704 | 705 |
706 | 707 | 708 | 709 | 710 | 711 |
Find video part:
 712 | 713 | 714 | 715 | Show me where 716 | 723 | 724 | 725 | 726 | {(typeof video.findPartResult !== "undefined") ? {video.findPartResult} : "" } 727 | 728 | { video.findPartWaiting ? Thinking ... : "" } 729 | 730 | 731 | 732 | 733 | ) 734 | }) 735 | } 736 | 737 | async handlePageClick(index){ 738 | var [videos, pages] = await this.fetchVideos(parseInt(index)) 739 | this.setState({videos: videos, pages: pages}) 740 | } 741 | 742 | renderPagination(index){ 743 | var page = this.state.pages[index] 744 | return ( 745 | {page.displayName} 746 | ) 747 | } 748 | 749 | render() { 750 | return ( 751 | 752 | 753 | 754 | 755 |

756 | Search videos with filters 757 |

758 | 759 | 766 | { 771 | this.handleSearchByDateChange(update); 772 | }} 773 | isClearable={true} 774 | showIcon={true} 775 | placeholderText="Video uploaded between . . ." 776 | /> 777 | 778 | 785 |
786 | 789 |
790 | 791 | 792 | {/* */} 793 | 794 |
795 | 796 | 797 | { Object.keys(this.state.pages).map((i) => { return this.renderPagination(i) })} 798 | 799 | 800 | 801 | 802 | { this.state.pageLoading ? Fetching videos ... : '' } 803 | { Object.keys(this.state.pages).length > 0 && this.state.videos.length > 0 ? this.renderVideoTableBody() : ""} 804 | { this.state.firstFetch && Object.keys(this.state.pages).length == 0 && this.state.videos.length == 0 ?

Found no relevant video in S3 "{this.rawFolder}" folder

: ""} 805 | 806 | 807 | 808 | 809 | ) 810 | } 811 | } -------------------------------------------------------------------------------- /webui/src/VideoUpload/VideoUpload.css: -------------------------------------------------------------------------------- 1 | #video-upload-text{ 2 | background-color: white; 3 | } 4 | .no-margin-bottom{ 5 | margin-bottom: 0px; 6 | } -------------------------------------------------------------------------------- /webui/src/VideoUpload/VideoUpload.js: -------------------------------------------------------------------------------- 1 | import React, { Component, useState } from "react"; 2 | import { CreateMultipartUploadCommand, 3 | UploadPartCommand, 4 | CompleteMultipartUploadCommand, 5 | AbortMultipartUploadCommand, 6 | PutObjectCommand, 7 | } from "@aws-sdk/client-s3"; 8 | 9 | import Form from "react-bootstrap/Form"; 10 | import InputGroup from 'react-bootstrap/InputGroup'; 11 | import Row from "react-bootstrap/Row"; 12 | import Col from "react-bootstrap/Col"; 13 | import Button from "react-bootstrap/Button"; 14 | import ProgressBar from 'react-bootstrap/ProgressBar'; 15 | 16 | import './VideoUpload.css'; 17 | 18 | export class VideoUpload extends Component { 19 | constructor(props) { 20 | super(props); 21 | this.state = { 22 | selectedVideoFiles: null, 23 | isUploading: false, 24 | currentUploadSize: 0, 25 | totalUploadSize: 0.001, 26 | }; 27 | 28 | this.s3Client = props.s3Client; 29 | this.bucketName = props.bucketName; 30 | this.rawFolder = props.rawFolder; 31 | } 32 | 33 | cleanFileName = (rawFileName) => { 34 | return rawFileName.replace(/[^a-zA-Z0-9\-\.]/g, '_'); 35 | } 36 | 37 | onFileChange = (event) => { 38 | this.setState({ 39 | selectedVideoFiles: event.target.files, 40 | }); 41 | }; 42 | 43 | onFilesUpload = async () => { 44 | this.setState({ 45 | isUploading: true 46 | }); 47 | 48 | var totalSizeToUpload = 0.0 49 | for (let i = 0; i < this.state.selectedVideoFiles.length; i++) { 50 | totalSizeToUpload += this.state.selectedVideoFiles[i].size 51 | } 52 | this.setState({ 53 | totalUploadSize: this.state.totalUploadSize + totalSizeToUpload 54 | }); 55 | 56 | for (let i = 0; i < this.state.selectedVideoFiles.length; i++) { 57 | const fileSize = this.state.selectedVideoFiles[i].size 58 | if(fileSize >=25e6){ // User Multipart upload 59 | let uploadId; 60 | try { 61 | const multipartUpload = await this.s3Client.send( 62 | new CreateMultipartUploadCommand({ 63 | Bucket: this.bucketName, 64 | Key: `${this.rawFolder}/${this.cleanFileName(this.state.selectedVideoFiles[i].name)}`, 65 | }), 66 | ); 67 | 68 | uploadId = multipartUpload.UploadId; 69 | 70 | const uploadPromises = []; 71 | // Multipart uploads require a minimum size of 5 MB per part. 72 | const partSize = 10e6 73 | const numberOfParts = Math.floor(fileSize / partSize) 74 | const lastPartSize = partSize + fileSize % partSize 75 | 76 | // Upload each part. 77 | for (let j = 0; j < numberOfParts; j++) { 78 | const start = j * partSize; 79 | const end = (j === numberOfParts - 1 ) ? (start + lastPartSize) : (start + partSize) 80 | uploadPromises.push( 81 | this.s3Client 82 | .send( 83 | new UploadPartCommand({ 84 | Bucket: this.bucketName, 85 | Key: `${this.rawFolder}/${this.cleanFileName(this.state.selectedVideoFiles[i].name)}`, 86 | UploadId: uploadId, 87 | Body: this.state.selectedVideoFiles[i].slice(start, end), 88 | PartNumber: j + 1, 89 | }), 90 | ).then(singlePartUploadResult => { 91 | this.setState({ 92 | currentUploadSize: this.state.currentUploadSize + ((j === numberOfParts - 1 ) ? lastPartSize : partSize) 93 | }); 94 | return singlePartUploadResult 95 | }) 96 | ); 97 | } 98 | 99 | const uploadResults = await Promise.all(uploadPromises); 100 | 101 | await this.s3Client.send( 102 | new CompleteMultipartUploadCommand({ 103 | Bucket: this.bucketName, 104 | Key: `${this.rawFolder}/${this.cleanFileName(this.state.selectedVideoFiles[i].name)}`, 105 | UploadId: uploadId, 106 | MultipartUpload: { 107 | Parts: uploadResults.map(({ ETag }, i) => ({ 108 | ETag, 109 | PartNumber: i + 1, 110 | })), 111 | }, 112 | }), 113 | ); 114 | } catch (err) { 115 | console.error(err); 116 | 117 | if (uploadId) { 118 | const abortCommand = new AbortMultipartUploadCommand({ 119 | Bucket: this.bucketName, 120 | Key: `${this.rawFolder}/${this.cleanFileName(this.state.selectedVideoFiles[i].name)}`, 121 | UploadId: uploadId, 122 | }); 123 | 124 | await this.s3Client.send(abortCommand); 125 | } 126 | } 127 | 128 | }else{ // Use simple upload 129 | const command = new PutObjectCommand({ 130 | Bucket: this.bucketName, 131 | Key: `${this.rawFolder}/${this.cleanFileName(this.state.selectedVideoFiles[i].name)}`, 132 | Body: this.state.selectedVideoFiles[i], 133 | }); 134 | 135 | // Upload file 136 | try { 137 | await this.s3Client.send(command); 138 | this.setState({ 139 | currentUploadSize: this.state.currentUploadSize + parseInt(this.state.selectedVideoFiles[i].size) 140 | }); 141 | } catch (error) { 142 | console.log(error) 143 | } 144 | } 145 | } 146 | 147 | this.setState({ 148 | isUploading: false, 149 | selectedVideoFiles: null, 150 | currentUploadSize: 0, 151 | totalUploadSize: 0.001, 152 | }); 153 | }; 154 | 155 | render() { 156 | return ( 157 | <> 158 | 159 | 160 | 161 | 162 | Upload video files 163 | 169 | 177 | 178 | 179 | 180 | 181 |

Alternatively, upload to Amazon S3 bucket "{this.bucketName}" inside folder "{this.rawFolder}".

182 | 183 | 184 | { this.state.isUploading ? : "" } 185 | { this.state.isUploading && Math.floor(this.state.currentUploadSize / parseInt(this.state.totalUploadSize) * 100) > 50 ?

When upload finishes, you can reload page or re-search the videos to refresh the video list.

 : "" } 186 | 187 | ); 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /webui/src/aws-exports.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | 3 | export default { 4 | "aws_project_region": "PLACEHOLDER_REGION", 5 | "aws_cognito_identity_pool_id": "PLACEHOLDER_IDENTITY_POOL_ID", 6 | "aws_cognito_region": "PLACEHOLDER_COGNITO_REGION", 7 | "aws_user_pools_id": "PLACEHOLDER_USER_POOL_ID", 8 | "aws_user_pools_web_client_id": "PLACEHOLDER_USER_POOL_WEB_CLIENT_ID", 9 | "bucket_name": "PLACEHOLDER_BUCKET_NAME", 10 | "balanced_model_id": "PLACEHOLDER_BALANCED_MODEL_ID", 11 | "fast_model_id": "PLACEHOLDER_FAST_MODEL_ID", 12 | "raw_folder": "PLACEHOLDER_RAW_FOLDER", 13 | "video_script_folder": "PLACEHOLDER_VIDEO_SCRIPT_FOLDER", 14 | "video_caption_folder": "PLACEHOLDER_VIDEO_CAPTION_FOLDER", 15 | "transcription_folder": "PLACEHOLDER_TRANSCRIPTION_FOLDER", 16 | "entity_sentiment_folder": "PLACEHOLDER_ENTITY_SENTIMENT_FOLDER", 17 | "summary_folder": "PLACEHOLDER_SUMMARY_FOLDER", 18 | "rest_api_url": "PLACEHOLDER_REST_API_URL", 19 | "videos_api_resource": "PLACEHOLDER_VIDEOS_API_RESOURCE" 20 | } -------------------------------------------------------------------------------- /webui/src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /webui/src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom/client'; 3 | import './index.css'; 4 | import App from './App'; 5 | import reportWebVitals from './reportWebVitals'; 6 | import 'bootstrap/dist/css/bootstrap.min.css'; 7 | 8 | 9 | const root = ReactDOM.createRoot(document.getElementById('root')); 10 | root.render( 11 | 12 | 13 | 14 | ); 15 | 16 | // If you want to start measuring performance in your app, pass a function 17 | // to log results (for example: reportWebVitals(console.log)) 18 | // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals 19 | reportWebVitals(); 20 | -------------------------------------------------------------------------------- /webui/src/reportWebVitals.js: -------------------------------------------------------------------------------- 1 | const reportWebVitals = onPerfEntry => { 2 | if (onPerfEntry && onPerfEntry instanceof Function) { 3 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 4 | getCLS(onPerfEntry); 5 | getFID(onPerfEntry); 6 | getFCP(onPerfEntry); 7 | getLCP(onPerfEntry); 8 | getTTFB(onPerfEntry); 9 | }); 10 | } 11 | }; 12 | 13 | export default reportWebVitals; 14 | -------------------------------------------------------------------------------- /webui/src/setupTests.js: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom'; 6 | --------------------------------------------------------------------------------