├── packages ├── types │ ├── src │ │ ├── index.d.ts │ │ └── protocol.d.ts │ └── package.json ├── web │ ├── src │ │ ├── @types │ │ │ ├── common.d.ts │ │ │ └── react-i18next.d.ts │ │ ├── index.css │ │ ├── models │ │ │ └── sample-avatar.glb │ │ ├── vite-env.d.ts │ │ ├── main.tsx │ │ ├── i18n │ │ │ ├── zh │ │ │ │ └── index.ts │ │ │ ├── ja │ │ │ │ └── index.ts │ │ │ ├── ko │ │ │ │ └── index.ts │ │ │ ├── en │ │ │ │ └── index.ts │ │ │ ├── vi │ │ │ │ └── index.ts │ │ │ └── index.ts │ │ ├── hooks │ │ │ ├── useHttp.ts │ │ │ ├── useQuestionApi.ts │ │ │ ├── useQuestion.ts │ │ │ ├── useTranscribeStreaming.ts │ │ │ └── useAvatar.ts │ │ ├── components │ │ │ ├── Avatar.tsx │ │ │ ├── ButtonIcon.tsx │ │ │ ├── Select.tsx │ │ │ └── InputQuestion.tsx │ │ ├── assets │ │ │ └── react.svg │ │ └── App.tsx │ ├── .env │ ├── postcss.config.js │ ├── vite.config.ts │ ├── tsconfig.node.json │ ├── .gitignore │ ├── index.html │ ├── tailwind.config.js │ ├── .eslintrc.cjs │ ├── tsconfig.json │ ├── README.md │ ├── public │ │ └── vite.svg │ └── package.json └── cdk │ ├── .npmignore │ ├── docs │ └── FAQ.pdf │ ├── .gitignore │ ├── lib │ ├── constructs │ │ ├── index.ts │ │ ├── kendra-index.ts │ │ ├── s3-data-source.ts │ │ ├── backend-api.ts │ │ └── frontend.ts │ └── rag-avatar-stack.ts │ ├── jest.config.js │ ├── .eslintrc.cjs │ ├── lambda │ ├── utils │ │ ├── translateApi.ts │ │ ├── kendraApi.ts │ │ └── bedrockApi.ts │ ├── prompts │ │ └── ragPrompt.ts │ └── streamQuestion.ts │ ├── README.md │ ├── test │ └── cdk.test.ts │ ├── tsconfig.json │ ├── bin │ └── cdk.ts │ ├── package.json │ └── cdk.json ├── .prettierignore ├── docs └── picture │ ├── ui_sample_en.png │ ├── ui_sample_ja.png │ └── architecture_v4.png ├── .prettierrc.json ├── .gitignore ├── CODE_OF_CONDUCT.md ├── package.json ├── LICENSE ├── CONTRIBUTING.md ├── README.md └── README_en.md /packages/types/src/index.d.ts: -------------------------------------------------------------------------------- 1 | export * from './protocol'; 2 | -------------------------------------------------------------------------------- /packages/web/src/@types/common.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'stream-browserify'; 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | **/*.json 2 | **/*.md 3 | **/*.txt 4 | **/node_modules 5 | **/dist 6 | 7 | cdk.out -------------------------------------------------------------------------------- /packages/web/src/index.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | -------------------------------------------------------------------------------- /packages/cdk/.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /packages/cdk/docs/FAQ.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/generative-ai-avatar-chat/HEAD/packages/cdk/docs/FAQ.pdf -------------------------------------------------------------------------------- /docs/picture/ui_sample_en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/generative-ai-avatar-chat/HEAD/docs/picture/ui_sample_en.png -------------------------------------------------------------------------------- /docs/picture/ui_sample_ja.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/generative-ai-avatar-chat/HEAD/docs/picture/ui_sample_ja.png -------------------------------------------------------------------------------- /docs/picture/architecture_v4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/generative-ai-avatar-chat/HEAD/docs/picture/architecture_v4.png -------------------------------------------------------------------------------- /packages/web/.env: -------------------------------------------------------------------------------- 1 | VITE_APP_REGION= 2 | VITE_APP_IDENTITY_POOL_ID= 3 | VITE_APP_API_ENDPOINT= 4 | VITE_APP_QUESTION_STREAM_FUNCTION_ARN= -------------------------------------------------------------------------------- /packages/web/postcss.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /packages/types/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@types/rag-avatar-demo", 3 | "version": "1.0.0", 4 | "types": "src/index.d.ts", 5 | "private": true 6 | } -------------------------------------------------------------------------------- /packages/web/src/models/sample-avatar.glb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/generative-ai-avatar-chat/HEAD/packages/web/src/models/sample-avatar.glb -------------------------------------------------------------------------------- /packages/types/src/protocol.d.ts: -------------------------------------------------------------------------------- 1 | export type QuestionRequest = { 2 | question: string; 3 | questionLang: string; 4 | questionLangCode: string; 5 | }; 6 | -------------------------------------------------------------------------------- /packages/cdk/.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | !jest.config.js 3 | *.d.ts 4 | node_modules 5 | 6 | # CDK asset staging directory 7 | .cdk.staging 8 | cdk.out 9 | 10 | output.json -------------------------------------------------------------------------------- /packages/cdk/lib/constructs/index.ts: -------------------------------------------------------------------------------- 1 | export * from './backend-api'; 2 | export * from './s3-data-source'; 3 | export * from './kendra-index'; 4 | export * from './frontend'; 5 | -------------------------------------------------------------------------------- /packages/cdk/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | testEnvironment: 'node', 3 | roots: ['/test'], 4 | testMatch: ['**/*.test.ts'], 5 | transform: { 6 | '^.+\\.tsx?$': 'ts-jest', 7 | }, 8 | }; 9 | -------------------------------------------------------------------------------- /packages/web/src/@types/react-i18next.d.ts: -------------------------------------------------------------------------------- 1 | import 'react-i18next'; 2 | import ja from '../i18n/ja'; 3 | 4 | declare module 'i18next' { 5 | interface CustomTypeOptions { 6 | resources: typeof ja; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /packages/web/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite'; 2 | import react from '@vitejs/plugin-react'; 3 | 4 | // https://vitejs.dev/config/ 5 | export default defineConfig({ 6 | plugins: [react()], 7 | }); 8 | -------------------------------------------------------------------------------- /packages/web/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | interface ImportMetaEnv { 4 | readonly VITE_APP_API_ENDPOINT: string; 5 | } 6 | 7 | interface ImportMeta { 8 | readonly env: ImportMetaEnv; 9 | } 10 | -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "es5", 3 | "tabWidth": 2, 4 | "semi": true, 5 | "singleQuote": true, 6 | "bracketSpacing": true, 7 | "bracketSameLine": true, 8 | "arrowParens": "always", 9 | "plugins": ["prettier-plugin-tailwindcss"] 10 | } 11 | -------------------------------------------------------------------------------- /packages/web/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "composite": true, 4 | "skipLibCheck": true, 5 | "module": "ESNext", 6 | "moduleResolution": "bundler", 7 | "allowSyntheticDefaultImports": true 8 | }, 9 | "include": ["vite.config.ts"] 10 | } 11 | -------------------------------------------------------------------------------- /packages/web/src/main.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom/client'; 3 | import App from './App.tsx'; 4 | import './index.css'; 5 | 6 | ReactDOM.createRoot(document.getElementById('root')!).render( 7 | 8 | 9 | 10 | ); 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | node_modules/ 3 | 4 | # Environment variables 5 | .env.local 6 | .env 7 | 8 | # Build outputs 9 | dist/ 10 | build/ 11 | cdk.out/ 12 | 13 | # CDK outputs 14 | output.json 15 | cdk.context.json 16 | 17 | # OS files 18 | .DS_Store 19 | 20 | # IDE 21 | .vscode/ 22 | .idea/ 23 | 24 | # Logs 25 | *.log -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /packages/web/src/i18n/zh/index.ts: -------------------------------------------------------------------------------- 1 | const translation = { 2 | translation: { 3 | message: { 4 | initial: '如果对工作有任何不明白的地方\n请随时提出问题。', 5 | thinking: '思考中...', 6 | transcribing: '请跟我说话', 7 | apiError: '连接 API 时发生错误。', 8 | }, 9 | inputPlaceholder: '在这里输入您的问题', 10 | }, 11 | }; 12 | 13 | export default translation; -------------------------------------------------------------------------------- /packages/web/src/i18n/ja/index.ts: -------------------------------------------------------------------------------- 1 | const translation = { 2 | translation: { 3 | message: { 4 | initial: '勤務についてわからないことがあれば\nお気軽にご質問ください。', 5 | thinking: '考え中…', 6 | transcribing: '話しかけてください', 7 | apiError: 'API 接続でエラーが発生しました。', 8 | }, 9 | inputPlaceholder: 'こちらに質問を入力してください', 10 | }, 11 | }; 12 | 13 | export default translation; 14 | -------------------------------------------------------------------------------- /packages/web/src/i18n/ko/index.ts: -------------------------------------------------------------------------------- 1 | const translation = { 2 | translation: { 3 | message: { 4 | initial: '근무에 대해 모르는 것이 있으면\n저에게 자유롭게 질문해 주세요.', 5 | thinking: '생각 중...', 6 | transcribing: '말을 걸어주세요', 7 | apiError: 'API 연결에서 오류가 발생했습니다.', 8 | }, 9 | inputPlaceholder: '여기에 질문을 입력해 주세요', 10 | }, 11 | }; 12 | 13 | export default translation; 14 | -------------------------------------------------------------------------------- /packages/web/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /packages/web/src/i18n/en/index.ts: -------------------------------------------------------------------------------- 1 | const translation = { 2 | translation: { 3 | message: { 4 | initial: 'Feel free to ask any questions you have\nabout your work.', 5 | thinking: 'Thinking...', 6 | transcribing: 'Please speak to me', 7 | apiError: 'An error occurred connecting to the API.', 8 | }, 9 | inputPlaceholder: 'Type your question here', 10 | }, 11 | }; 12 | 13 | export default translation; -------------------------------------------------------------------------------- /packages/web/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | RAG Avatar Sample 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /packages/web/src/i18n/vi/index.ts: -------------------------------------------------------------------------------- 1 | const translation = { 2 | translation: { 3 | message: { 4 | initial: 5 | 'Nếu bạn có bất kỳ câu hỏi nào về công việc,\nxin hãy hỏi tôi thoải mái.', 6 | thinking: 'Đang suy nghĩ...', 7 | transcribing: 'Xin hãy nói chuyện với tôi', 8 | apiError: 'Có lỗi khi kết nối API.', 9 | }, 10 | inputPlaceholder: 'Hãy nhập câu hỏi của bạn ở đây', 11 | }, 12 | }; 13 | export default translation; 14 | -------------------------------------------------------------------------------- /packages/web/tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | export default { 3 | content: ['./index.html', './src/**/*.{js,ts,jsx,tsx}'], 4 | theme: { 5 | extend: {}, 6 | colors: { 7 | primary: '#484870', 8 | 'text-white': '#EDEDED', 9 | 'text-black': '#334155', 10 | 'background-white': '#F2F2F2', 11 | white: '#FFFFFF', 12 | transparent: 'transparent', 13 | }, 14 | }, 15 | plugins: [], 16 | }; 17 | -------------------------------------------------------------------------------- /packages/cdk/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | extends: [ 4 | 'eslint:recommended', 5 | 'plugin:@typescript-eslint/eslint-recommended', 6 | 'plugin:@typescript-eslint/recommended', 7 | ], 8 | ignorePatterns: ['cdk.out'], 9 | parser: '@typescript-eslint/parser', 10 | parserOptions: { 11 | project: './tsconfig.json', 12 | }, 13 | plugins: ['@typescript-eslint'], 14 | rules: { 15 | '@typescript-eslint/no-namespace': 'off', 16 | }, 17 | }; 18 | -------------------------------------------------------------------------------- /packages/web/src/hooks/useHttp.ts: -------------------------------------------------------------------------------- 1 | const URL = import.meta.env.VITE_APP_API_ENDPOINT; 2 | 3 | const useHttp = () => { 4 | return { 5 | post: (path: string, body: BODY) => { 6 | return fetch(URL + path, { 7 | method: 'POST', 8 | headers: { 9 | 'Content-Type': 'application/json', 10 | Authorization: 'Bearer temp', 11 | }, 12 | body: JSON.stringify(body), 13 | }); 14 | }, 15 | }; 16 | }; 17 | 18 | export default useHttp; 19 | -------------------------------------------------------------------------------- /packages/web/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | env: { browser: true, es2020: true }, 4 | extends: [ 5 | 'eslint:recommended', 6 | 'plugin:@typescript-eslint/recommended', 7 | 'plugin:react-hooks/recommended', 8 | 'prettier', 9 | ], 10 | ignorePatterns: ['dist', '.eslintrc.cjs'], 11 | parser: '@typescript-eslint/parser', 12 | plugins: ['react-refresh'], 13 | rules: { 14 | 'react-refresh/only-export-components': [ 15 | 'warn', 16 | { allowConstantExport: true }, 17 | ], 18 | }, 19 | }; 20 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rag-avatar-demo", 3 | "version": "1.0.0", 4 | "description": "", 5 | "scripts": { 6 | "lint": "npx prettier --write .", 7 | "cdk:deploy": "npm exec -w cdk -- cdk deploy --all -O output.json", 8 | "cdk:destroy": "npm exec -w cdk -- cdk destroy --all", 9 | "web:dev": "npm run dev -w web", 10 | "web:build": "npm run build -w web" 11 | }, 12 | "workspaces": [ 13 | "packages/*" 14 | ], 15 | "devDependencies": { 16 | "prettier": "^3.1.1", 17 | "prettier-plugin-tailwindcss": "^0.5.9" 18 | } 19 | } -------------------------------------------------------------------------------- /packages/cdk/lambda/utils/translateApi.ts: -------------------------------------------------------------------------------- 1 | import { 2 | TranslateClient, 3 | TranslateTextCommand, 4 | } from '@aws-sdk/client-translate'; 5 | 6 | const translate = new TranslateClient({}); 7 | 8 | const translateApi = { 9 | translateText: (text: string, source: string, target: string) => { 10 | const command = new TranslateTextCommand({ 11 | SourceLanguageCode: source, 12 | TargetLanguageCode: target, 13 | Text: text, 14 | }); 15 | 16 | return translate.send(command); 17 | }, 18 | }; 19 | 20 | export default translateApi; 21 | -------------------------------------------------------------------------------- /packages/cdk/README.md: -------------------------------------------------------------------------------- 1 | # Welcome to your CDK TypeScript project 2 | 3 | This is a blank project for CDK development with TypeScript. 4 | 5 | The `cdk.json` file tells the CDK Toolkit how to execute your app. 6 | 7 | ## Useful commands 8 | 9 | - `npm run build` compile typescript to js 10 | - `npm run watch` watch for changes and compile 11 | - `npm run test` perform the jest unit tests 12 | - `cdk deploy` deploy this stack to your default AWS account/region 13 | - `cdk diff` compare deployed stack with current state 14 | - `cdk synth` emits the synthesized CloudFormation template 15 | -------------------------------------------------------------------------------- /packages/cdk/test/cdk.test.ts: -------------------------------------------------------------------------------- 1 | // import * as cdk from 'aws-cdk-lib'; 2 | // import { Template } from 'aws-cdk-lib/assertions'; 3 | // import * as Cdk from '../lib/cdk-stack'; 4 | 5 | // example test. To run these tests, uncomment this file along with the 6 | // example resource in lib/cdk-stack.ts 7 | test('SQS Queue Created', () => { 8 | // const app = new cdk.App(); 9 | // // WHEN 10 | // const stack = new Cdk.RagAvatarStack(app, 'MyTestStack'); 11 | // // THEN 12 | // const template = Template.fromStack(stack); 13 | // template.hasResourceProperties('AWS::SQS::Queue', { 14 | // VisibilityTimeout: 300 15 | // }); 16 | }); 17 | -------------------------------------------------------------------------------- /packages/web/src/components/Avatar.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import '@babylonjs/loaders/glTF'; 3 | 4 | import glbModel from '../models/sample-avatar.glb?url'; 5 | import { useSceneLoader } from 'react-babylonjs'; 6 | import useAvatar from '../hooks/useAvatar'; 7 | 8 | const folderName = glbModel.split('/').slice(0, -1).join('/').concat('/'); 9 | const fileName = glbModel.split('/').slice(-1)[0]; 10 | 11 | const Avatar: React.FC = () => { 12 | const { setModel } = useAvatar(); 13 | 14 | useSceneLoader(folderName, fileName, undefined, { 15 | onModelLoaded: (model) => { 16 | setModel(model); 17 | }, 18 | }); 19 | 20 | return null; 21 | }; 22 | 23 | export default Avatar; 24 | -------------------------------------------------------------------------------- /packages/web/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "useDefineForClassFields": true, 5 | "lib": ["ES2020", "DOM", "DOM.Iterable"], 6 | "module": "ESNext", 7 | "skipLibCheck": true, 8 | 9 | /* Bundler mode */ 10 | "moduleResolution": "bundler", 11 | "allowImportingTsExtensions": true, 12 | "resolveJsonModule": true, 13 | "isolatedModules": true, 14 | "noEmit": true, 15 | "jsx": "react-jsx", 16 | 17 | /* Linting */ 18 | "strict": true, 19 | "noUnusedLocals": true, 20 | "noUnusedParameters": true, 21 | "noFallthroughCasesInSwitch": true 22 | }, 23 | "include": ["src"], 24 | "references": [{ "path": "./tsconfig.node.json" }] 25 | } 26 | -------------------------------------------------------------------------------- /packages/cdk/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "module": "commonjs", 5 | "lib": ["es2020", "dom"], 6 | "declaration": true, 7 | "strict": true, 8 | "noImplicitAny": true, 9 | "strictNullChecks": true, 10 | "noImplicitThis": true, 11 | "alwaysStrict": true, 12 | "noUnusedLocals": false, 13 | "noUnusedParameters": false, 14 | "noImplicitReturns": true, 15 | "noFallthroughCasesInSwitch": false, 16 | "inlineSourceMap": true, 17 | "inlineSources": true, 18 | "experimentalDecorators": true, 19 | "strictPropertyInitialization": false, 20 | "typeRoots": ["./node_modules/@types"], 21 | "types": ["node"] 22 | }, 23 | "exclude": ["node_modules", "cdk.out"] 24 | } 25 | -------------------------------------------------------------------------------- /packages/web/src/components/ButtonIcon.tsx: -------------------------------------------------------------------------------- 1 | import React, { ReactNode } from 'react'; 2 | import { twMerge } from 'tailwind-merge'; 3 | 4 | type Props = { 5 | className?: string; 6 | square?: boolean; 7 | disabled?: boolean; 8 | children: ReactNode; 9 | onClick: () => void; 10 | }; 11 | 12 | const ButtonIcon: React.FC = (props) => { 13 | return ( 14 | 24 | ); 25 | }; 26 | 27 | export default ButtonIcon; 28 | -------------------------------------------------------------------------------- /packages/cdk/lambda/utils/kendraApi.ts: -------------------------------------------------------------------------------- 1 | import { 2 | AttributeFilter, 3 | KendraClient, 4 | RetrieveCommand, 5 | } from '@aws-sdk/client-kendra'; 6 | 7 | const kendra = new KendraClient({}); 8 | 9 | const INDEX_ID = process.env.KENDRA_INDEX_ID; 10 | 11 | // デフォルト言語が英語なので、言語設定は必ず行う 12 | const attributeFilter: AttributeFilter = { 13 | AndAllFilters: [ 14 | { 15 | EqualsTo: { 16 | Key: '_language_code', 17 | Value: { 18 | StringValue: 'ja', 19 | }, 20 | }, 21 | }, 22 | ], 23 | }; 24 | 25 | const kendraApi = { 26 | retrieve: (query: string) => { 27 | const retrieveCommand = new RetrieveCommand({ 28 | IndexId: INDEX_ID, 29 | QueryText: query, 30 | AttributeFilter: attributeFilter, 31 | }); 32 | 33 | return kendra.send(retrieveCommand); 34 | }, 35 | }; 36 | 37 | export default kendraApi; 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT No Attribution 2 | 3 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 13 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 15 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 16 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | 18 | -------------------------------------------------------------------------------- /packages/cdk/bin/cdk.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import 'source-map-support/register'; 3 | import * as cdk from 'aws-cdk-lib'; 4 | import { RagAvatarStack } from '../lib/rag-avatar-stack'; 5 | 6 | const app = new cdk.App(); 7 | new RagAvatarStack(app, 'RagAvatarStack', { 8 | /* If you don't specify 'env', this stack will be environment-agnostic. 9 | * Account/Region-dependent features and context lookups will not work, 10 | * but a single synthesized template can be deployed anywhere. */ 11 | /* Uncomment the next line to specialize this stack for the AWS Account 12 | * and Region that are implied by the current CLI configuration. */ 13 | // env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION }, 14 | /* Uncomment the next line if you know exactly what Account and Region you 15 | * want to deploy the stack to. */ 16 | // env: { account: '123456789012', region: 'us-east-1' }, 17 | /* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */ 18 | }); 19 | -------------------------------------------------------------------------------- /packages/cdk/lib/constructs/kendra-index.ts: -------------------------------------------------------------------------------- 1 | import { aws_kendra, aws_iam } from 'aws-cdk-lib'; 2 | import { Construct } from 'constructs'; 3 | 4 | export class KendraIndex extends Construct { 5 | public readonly index: aws_kendra.CfnIndex; 6 | constructor(scope: Construct, id: string) { 7 | super(scope, id); 8 | 9 | const indexRole = new aws_iam.Role(this, 'KendraIndexRole', { 10 | assumedBy: new aws_iam.ServicePrincipal('kendra.amazonaws.com'), 11 | }); 12 | 13 | indexRole.addToPolicy( 14 | new aws_iam.PolicyStatement({ 15 | effect: aws_iam.Effect.ALLOW, 16 | resources: ['*'], 17 | actions: ['s3:GetObject'], 18 | }) 19 | ); 20 | 21 | indexRole.addManagedPolicy( 22 | aws_iam.ManagedPolicy.fromAwsManagedPolicyName('CloudWatchLogsFullAccess') 23 | ); 24 | 25 | const index = new aws_kendra.CfnIndex(this, 'KendraIndex', { 26 | name: 'rag-avatar-index', 27 | edition: 'DEVELOPER_EDITION', 28 | roleArn: indexRole.roleArn, 29 | }); 30 | 31 | this.index = index; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /packages/cdk/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cdk", 3 | "version": "0.1.0", 4 | "scripts": { 5 | "build": "tsc", 6 | "watch": "tsc -w", 7 | "test": "jest", 8 | "cdk": "cdk" 9 | }, 10 | "devDependencies": { 11 | "@types/aws-lambda": "^8.10.129", 12 | "@types/jest": "^29.5.5", 13 | "@types/node": "^20.10.5", 14 | "@typescript-eslint/eslint-plugin": "^6.12.0", 15 | "@typescript-eslint/parser": "^6.12.0", 16 | "aws-cdk": "^2.1025.0", 17 | "eslint": "^8.54.0", 18 | "jest": "^29.7.0", 19 | "prettier": "^3.1.0", 20 | "ts-jest": "^29.1.1", 21 | "ts-node": "^10.9.1", 22 | "typescript": "~5.2.2" 23 | }, 24 | "dependencies": { 25 | "@aws-sdk/client-bedrock-runtime": "^3.454.0", 26 | "@aws-sdk/client-kendra": "^3.454.0", 27 | "@aws-sdk/client-translate": "^3.458.0", 28 | "@types/rag-avatar-demo": "^1.0.0", 29 | "aws-cdk-lib": "^2.212.0", 30 | "constructs": "^10.0.0", 31 | "deploy-time-build": "^0.3.46", 32 | "langchain": "^0.2.19", 33 | "source-map-support": "^0.5.21" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /packages/cdk/lambda/prompts/ragPrompt.ts: -------------------------------------------------------------------------------- 1 | import { RetrieveResultItem } from '@aws-sdk/client-kendra'; 2 | 3 | const ragPrompt = { 4 | qaPrompt: ( 5 | documents: RetrieveResultItem[], 6 | question: string, 7 | language: string 8 | ) => { 9 | return `Instructions: あなたは社内の情報を知り尽くしたエキスパートです。 10 | 従業員から質問されるので、ですます調で丁寧に答えてください。 11 | 12 | Human: 以下ので質問に答えてください。 13 | 14 | 15 | 1. が質問内容です。質問内容を正しく理解してください。 16 | 2. に回答の参考となるドキュメントが設定されているので、全て理解してください。 17 | 3. の内容をもとに、の回答を生成してください。 18 | 4. に回答に必要な情報がない場合は、「資料に該当の情報がありません」を回答とします。回答を創造しないでください。 19 | 5. 回答内容をで指定された言語に翻訳してください。指定された言語以外は出力しないでください。 20 | 6. 生成した回答だけを出力してください。それ以外の文言は一切出力禁止です。回答の過程なども一切出力しないでください。 21 | 22 | 23 | 24 | ${question} 25 | 26 | 27 | 28 | ${JSON.stringify( 29 | documents.map((doc) => ({ 30 | DocumentTitle: doc.DocumentTitle, 31 | Content: doc.Content, 32 | })) 33 | )} 34 | 35 | 36 | 37 | ${language} 38 | 39 | 40 | Assistant: `; 41 | }, 42 | }; 43 | 44 | export default ragPrompt; 45 | -------------------------------------------------------------------------------- /packages/cdk/lambda/utils/bedrockApi.ts: -------------------------------------------------------------------------------- 1 | import { 2 | BedrockRuntimeClient, 3 | ConversationRole, 4 | ConverseStreamCommand, 5 | InferenceConfiguration 6 | } from '@aws-sdk/client-bedrock-runtime'; 7 | 8 | const client = new BedrockRuntimeClient({ 9 | region: process.env.BEDROCK_REGION, 10 | }); 11 | 12 | const modelId = process.env.BEDROCK_MODELID; 13 | 14 | const inferenceConfig: InferenceConfiguration = { 15 | maxTokens: 300, 16 | temperature: 0, 17 | topP: 0.999 18 | }; 19 | 20 | const bedrockApi = { 21 | invokeStream: async function* (prompt: string) { 22 | const conversation = [ 23 | { 24 | role: ConversationRole.USER, 25 | content: [{text: prompt}], 26 | }, 27 | ]; 28 | const command = new ConverseStreamCommand({ 29 | modelId: modelId, 30 | messages: conversation, 31 | inferenceConfig, 32 | }); 33 | const res = await client.send(command); 34 | 35 | if (!res.stream) { 36 | return; 37 | } 38 | 39 | for await (const item of res.stream) { 40 | if (item.contentBlockDelta) { 41 | yield item.contentBlockDelta.delta?.text || ''; 42 | } 43 | } 44 | }, 45 | }; 46 | 47 | export default bedrockApi; 48 | -------------------------------------------------------------------------------- /packages/cdk/lib/rag-avatar-stack.ts: -------------------------------------------------------------------------------- 1 | import * as cdk from 'aws-cdk-lib'; 2 | import { Construct } from 'constructs'; 3 | import { Api, KendraIndex, S3DataSource } from './constructs'; 4 | import { Frontend } from './constructs/frontend'; 5 | 6 | export class RagAvatarStack extends cdk.Stack { 7 | constructor(scope: Construct, id: string, props?: cdk.StackProps) { 8 | super(scope, id, props); 9 | 10 | const bedrockRegion: string = 11 | this.node.tryGetContext('bedrock-region') || 'ap-northeast-1'; 12 | const bedrockModelId: string = 13 | this.node.tryGetContext('bedrock-model-id') || 14 | 'anthropic.claude-instant-v1'; 15 | 16 | const kendraIndex = new KendraIndex(this, 'KendraIndex'); 17 | 18 | const dataSource = new S3DataSource(this, 'S3DataSource', { 19 | index: kendraIndex.index, 20 | }); 21 | 22 | dataSource.node.addDependency(kendraIndex); 23 | 24 | const api = new Api(this, 'Api', { 25 | bedrockRegion, 26 | bedrockModelId, 27 | index: kendraIndex.index, 28 | }); 29 | 30 | new Frontend(this, 'Frontend', { 31 | questionStreamFunctionArn: api.questionStreamFunction.functionArn, 32 | idPoolId: api.idPool.identityPoolId, 33 | }); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /packages/cdk/lambda/streamQuestion.ts: -------------------------------------------------------------------------------- 1 | import { Handler } from 'aws-lambda'; 2 | import api from './utils/bedrockApi'; 3 | import kendraApi from './utils/kendraApi'; 4 | import ragPrompt from './prompts/ragPrompt'; 5 | import translateApi from './utils/translateApi'; 6 | import { QuestionRequest } from 'rag-avatar-demo'; 7 | 8 | declare global { 9 | namespace awslambda { 10 | function streamifyResponse( 11 | f: ( 12 | event: QuestionRequest, 13 | responseStream: NodeJS.WritableStream 14 | ) => Promise 15 | ): Handler; 16 | } 17 | } 18 | 19 | export const handler = awslambda.streamifyResponse( 20 | async (event, responseStream) => { 21 | let question = event.question; 22 | if (event.questionLangCode !== 'ja') { 23 | const { TranslatedText } = await translateApi.translateText( 24 | event.question, 25 | event.questionLangCode, 26 | 'ja' 27 | ); 28 | question = TranslatedText ?? ''; 29 | } 30 | 31 | const documents = (await kendraApi.retrieve(question)).ResultItems ?? []; 32 | 33 | const prompt = ragPrompt.qaPrompt(documents, question, event.questionLang); 34 | for await (const token of api.invokeStream(prompt)) { 35 | responseStream.write(token); 36 | } 37 | responseStream.end(); 38 | } 39 | ); 40 | -------------------------------------------------------------------------------- /packages/web/README.md: -------------------------------------------------------------------------------- 1 | # React + TypeScript + Vite 2 | 3 | This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules. 4 | 5 | Currently, two official plugins are available: 6 | 7 | - [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh 8 | - [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh 9 | 10 | ## Expanding the ESLint configuration 11 | 12 | If you are developing a production application, we recommend updating the configuration to enable type aware lint rules: 13 | 14 | - Configure the top-level `parserOptions` property like this: 15 | 16 | ```js 17 | export default { 18 | // other rules... 19 | parserOptions: { 20 | ecmaVersion: 'latest', 21 | sourceType: 'module', 22 | project: ['./tsconfig.json', './tsconfig.node.json'], 23 | tsconfigRootDir: __dirname, 24 | }, 25 | }; 26 | ``` 27 | 28 | - Replace `plugin:@typescript-eslint/recommended` to `plugin:@typescript-eslint/recommended-type-checked` or `plugin:@typescript-eslint/strict-type-checked` 29 | - Optionally add `plugin:@typescript-eslint/stylistic-type-checked` 30 | - Install [eslint-plugin-react](https://github.com/jsx-eslint/eslint-plugin-react) and add `plugin:react/recommended` & `plugin:react/jsx-runtime` to the `extends` list 31 | -------------------------------------------------------------------------------- /packages/web/public/vite.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/web/src/i18n/index.ts: -------------------------------------------------------------------------------- 1 | import i18next, { Resource } from 'i18next'; 2 | import { initReactI18next } from 'react-i18next'; 3 | import ja from './ja'; 4 | import ko from './ko'; 5 | import vi from './vi'; 6 | import zh from './zh'; 7 | import en from './en'; 8 | 9 | export const LANGUAGE_OPTIONS = [ 10 | { 11 | label: '日本語', 12 | value: '日本語', 13 | code: 'ja', 14 | transcribeCode: 'ja-JP', 15 | }, 16 | { 17 | label: '한국어', 18 | value: '韓国語', 19 | code: 'ko', 20 | transcribeCode: 'ko-KR', 21 | }, 22 | { 23 | label: 'Tiếng Việt', 24 | value: 'ベトナム語', 25 | code: 'vi', 26 | // "vi-VN" is NOT supported stream transcription. 27 | // https://docs.aws.amazon.com/ja_jp/transcribe/latest/dg/supported-languages.html 28 | transcribeCode: '', 29 | }, 30 | { 31 | label: '简体中文', 32 | value: '中国語（簡体字）', 33 | code: 'zh', 34 | transcribeCode: 'zh-CN', 35 | }, 36 | { 37 | label: 'English', 38 | value: '英語', 39 | code: 'en', 40 | transcribeCode: 'en-US', 41 | }, 42 | ]; 43 | export const resources: Resource = { 44 | ja, 45 | ko, 46 | vi, 47 | zh, 48 | en, 49 | }; 50 | 51 | // Settings i18n 52 | const i18n = i18next.use(initReactI18next).init({ 53 | resources, 54 | fallbackLng: 'ja', 55 | interpolation: { 56 | escapeValue: false, // react already safes from xss => https://www.i18next.com/translation-function/interpolation#unescape 57 | }, 58 | }); 59 | 60 | export default i18n; 61 | -------------------------------------------------------------------------------- /packages/web/src/hooks/useQuestionApi.ts: -------------------------------------------------------------------------------- 1 | import { CognitoIdentityClient } from '@aws-sdk/client-cognito-identity'; 2 | import { 3 | InvokeWithResponseStreamCommand, 4 | LambdaClient, 5 | } from '@aws-sdk/client-lambda'; 6 | import { fromCognitoIdentityPool } from '@aws-sdk/credential-provider-cognito-identity'; 7 | import { QuestionRequest } from 'rag-avatar-demo'; 8 | 9 | const useQuestionApi = () => { 10 | return { 11 | // Streaming Response 12 | questionStream: async function* (req: QuestionRequest) { 13 | const region = import.meta.env.VITE_APP_REGION; 14 | const idPoolId = import.meta.env.VITE_APP_IDENTITY_POOL_ID; 15 | 16 | const cognito = new CognitoIdentityClient({ region }); 17 | 18 | const lambda = new LambdaClient({ 19 | region, 20 | credentials: await fromCognitoIdentityPool({ 21 | client: cognito, 22 | identityPoolId: idPoolId, 23 | }), 24 | }); 25 | 26 | const res = await lambda.send( 27 | new InvokeWithResponseStreamCommand({ 28 | FunctionName: import.meta.env.VITE_APP_QUESTION_STREAM_FUNCTION_ARN, 29 | Payload: JSON.stringify(req), 30 | }) 31 | ); 32 | const events = res.EventStream!; 33 | 34 | for await (const event of events) { 35 | if (event.PayloadChunk) { 36 | yield new TextDecoder('utf-8').decode(event.PayloadChunk.Payload); 37 | } 38 | 39 | if (event.InvokeComplete) { 40 | break; 41 | } 42 | } 43 | }, 44 | }; 45 | }; 46 | 47 | export default useQuestionApi; 48 | -------------------------------------------------------------------------------- /packages/web/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "web", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "build": "tsc && vite build", 9 | "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", 10 | "preview": "vite preview" 11 | }, 12 | "dependencies": { 13 | "@aws-sdk/client-cognito-identity": "^3.700.0", 14 | "@aws-sdk/client-lambda": "^3.700.0", 15 | "@aws-sdk/client-transcribe-streaming": "^3.700.0", 16 | "@aws-sdk/credential-provider-cognito-identity": "^3.700.0", 17 | "@babylonjs/core": "^7.0.0", 18 | "@babylonjs/gui": "^7.0.0", 19 | "@babylonjs/loaders": "^7.0.0", 20 | "@headlessui/react": "^2.0.0", 21 | "babylonjs-loaders": "^7.0.0", 22 | "i18next": "^25.4.1", 23 | "immer": "^10.1.0", 24 | "microphone-stream": "^6.0.1", 25 | "react": "^19.0.0", 26 | "react-babylonjs": "^3.2.4", 27 | "react-dom": "^19.0.0", 28 | "react-i18next": "^15.0.0", 29 | "react-icons": "^5.0.0", 30 | "readable-stream": "^4.4.2", 31 | "stream-browserify": "^3.0.0", 32 | "tailwind-merge": "^2.5.0", 33 | "zustand": "^5.0.0" 34 | }, 35 | "devDependencies": { 36 | "@types/react": "^19.0.0", 37 | "@types/react-dom": "^19.0.0", 38 | "@types/readable-stream": "^4.0.9", 39 | "@typescript-eslint/eslint-plugin": "^8.0.0", 40 | "@typescript-eslint/parser": "^8.0.0", 41 | "@vitejs/plugin-react": "^5.0.0", 42 | "autoprefixer": "^10.4.16", 43 | "eslint": "^9.0.0", 44 | "eslint-config-prettier": "^9.1.0", 45 | "eslint-plugin-react-hooks": "^5.0.0", 46 | "eslint-plugin-react-refresh": "^0.4.14", 47 | "postcss": "^8.5.0", 48 | "tailwindcss": "^3.4.0", 49 | "typescript": "^5.7.0", 50 | "vite": "^6.0.0" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /packages/web/src/hooks/useQuestion.ts: -------------------------------------------------------------------------------- 1 | import { create } from 'zustand'; 2 | import useQuestionApi from './useQuestionApi'; 3 | import { useTranslation } from 'react-i18next'; 4 | 5 | const STREAMING_TEXT_POSTFIX = '▍'; 6 | 7 | const useQuestionState = create<{ 8 | answerText: string; 9 | setAnswerText: (s: string) => void; 10 | appendAnswerText: (s: string) => void; 11 | removeAnswerTextPostfix: () => void; 12 | }>((set, get) => { 13 | return { 14 | answerText: '', 15 | setAnswerText: (s) => { 16 | set({ 17 | answerText: s, 18 | }); 19 | }, 20 | appendAnswerText: (s) => { 21 | set(() => { 22 | return { 23 | answerText: get().answerText.endsWith(STREAMING_TEXT_POSTFIX) 24 | ? get().answerText.slice(0, -1) + s 25 | : get().answerText + s, 26 | }; 27 | }); 28 | }, 29 | removeAnswerTextPostfix: () => { 30 | set(() => { 31 | return { 32 | answerText: get().answerText.endsWith(STREAMING_TEXT_POSTFIX) 33 | ? get().answerText.slice(0, -1) 34 | : get().answerText, 35 | }; 36 | }); 37 | }, 38 | }; 39 | }); 40 | 41 | const useQuestion = () => { 42 | const { 43 | answerText, 44 | setAnswerText, 45 | appendAnswerText, 46 | removeAnswerTextPostfix, 47 | } = useQuestionState(); 48 | 49 | const { questionStream } = useQuestionApi(); 50 | 51 | const { t } = useTranslation(); 52 | 53 | return { 54 | answerText, 55 | question: async ( 56 | content: string, 57 | language: string, 58 | languageCode: string, 59 | speechAction: () => void 60 | ) => { 61 | try { 62 | setAnswerText(t('message.thinking')); 63 | 64 | const stream = questionStream({ 65 | question: content, 66 | questionLang: language, 67 | questionLangCode: languageCode, 68 | }); 69 | 70 | let isFirstChunk = true; 71 | 72 | // 発言を更新 73 | for await (const chunk of stream) { 74 | if (isFirstChunk) { 75 | setAnswerText(STREAMING_TEXT_POSTFIX); 76 | isFirstChunk = false; 77 | speechAction(); 78 | } 79 | appendAnswerText(chunk + STREAMING_TEXT_POSTFIX); 80 | } 81 | removeAnswerTextPostfix(); 82 | } catch (e) { 83 | console.error(e); 84 | setAnswerText(t('message.apiError')); 85 | throw e; 86 | } 87 | }, 88 | }; 89 | }; 90 | 91 | export default useQuestion; 92 | -------------------------------------------------------------------------------- /packages/cdk/lib/constructs/s3-data-source.ts: -------------------------------------------------------------------------------- 1 | import { 2 | RemovalPolicy, 3 | Token, 4 | CfnOutput, 5 | aws_s3, 6 | aws_s3_deployment, 7 | aws_iam, 8 | aws_kendra, 9 | } from 'aws-cdk-lib'; 10 | import { Construct } from 'constructs'; 11 | import * as path from 'path'; 12 | 13 | export interface S3DataSourceProps { 14 | index: aws_kendra.CfnIndex; 15 | } 16 | 17 | export class S3DataSource extends Construct { 18 | constructor(scope: Construct, id: string, props: S3DataSourceProps) { 19 | super(scope, id); 20 | 21 | // S3 Document Bucket 22 | const docsBucket = new aws_s3.Bucket(this, 'DocsBucket', { 23 | versioned: true, 24 | removalPolicy: RemovalPolicy.DESTROY, 25 | blockPublicAccess: aws_s3.BlockPublicAccess.BLOCK_ALL, 26 | }); 27 | 28 | // Upload contents of docs folder to S3 29 | new aws_s3_deployment.BucketDeployment(this, 'DeployWebsite', { 30 | sources: [ 31 | aws_s3_deployment.Source.asset( 32 | path.join(__dirname, '../../../cdk/docs') 33 | ), 34 | ], 35 | destinationBucket: docsBucket, 36 | }); 37 | 38 | // Kendra S3 Data Source IAM Role 39 | const s3DataSourceRole = new aws_iam.Role(this, 'DataSourceRole', { 40 | assumedBy: new aws_iam.ServicePrincipal('kendra.amazonaws.com'), 41 | }); 42 | 43 | s3DataSourceRole.addToPolicy( 44 | new aws_iam.PolicyStatement({ 45 | effect: aws_iam.Effect.ALLOW, 46 | resources: [`arn:aws:s3:::${docsBucket.bucketName}`], 47 | actions: ['s3:ListBucket'], 48 | }) 49 | ); 50 | 51 | s3DataSourceRole.addToPolicy( 52 | new aws_iam.PolicyStatement({ 53 | effect: aws_iam.Effect.ALLOW, 54 | resources: [`arn:aws:s3:::${docsBucket.bucketName}/*`], 55 | actions: ['s3:GetObject'], 56 | }) 57 | ); 58 | 59 | s3DataSourceRole.addToPolicy( 60 | new aws_iam.PolicyStatement({ 61 | effect: aws_iam.Effect.ALLOW, 62 | resources: [Token.asString(props.index.getAtt('Arn'))], 63 | actions: ['kendra:BatchPutDocument', 'kendra:BatchDeleteDocument'], 64 | }) 65 | ); 66 | 67 | const dataSource = new aws_kendra.CfnDataSource(this, 'S3DataSource', { 68 | indexId: props.index.ref, 69 | type: 'S3', 70 | name: 's3-data-source', 71 | roleArn: s3DataSourceRole.roleArn, 72 | languageCode: 'ja', 73 | dataSourceConfiguration: { 74 | s3Configuration: { 75 | bucketName: docsBucket.bucketName, 76 | }, 77 | }, 78 | }); 79 | 80 | new CfnOutput(this, 'KendraIndexId', { 81 | value: dataSource.indexId, 82 | }); 83 | 84 | new CfnOutput(this, 'KendraS3DataSourceId', { 85 | value: dataSource.attrId, 86 | }); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /packages/cdk/lib/constructs/backend-api.ts: -------------------------------------------------------------------------------- 1 | import { Duration, aws_kendra, CfnOutput, Token } from 'aws-cdk-lib'; 2 | import { Runtime } from 'aws-cdk-lib/aws-lambda'; 3 | import { NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs'; 4 | import * as idPool from 'aws-cdk-lib/aws-cognito-identitypool'; 5 | import * as iam from 'aws-cdk-lib/aws-iam'; 6 | import { Construct } from 'constructs'; 7 | 8 | export interface ApiProps { 9 | bedrockRegion: string; 10 | bedrockModelId: string; 11 | index: aws_kendra.CfnIndex; 12 | } 13 | 14 | export class Api extends Construct { 15 | public readonly questionStreamFunction: NodejsFunction; 16 | public readonly idPool: idPool.IIdentityPool; 17 | 18 | constructor(scope: Construct, id: string, props: ApiProps) { 19 | super(scope, id); 20 | // ----- 21 | // Identity Pool の作成 22 | // ----- 23 | 24 | const identityPool = new idPool.IdentityPool( 25 | this, 26 | 'IdentityPoolForStreamingLambda', 27 | { 28 | allowUnauthenticatedIdentities: true, 29 | } 30 | ); 31 | 32 | identityPool.unauthenticatedRole.addToPrincipalPolicy( 33 | new iam.PolicyStatement({ 34 | effect: iam.Effect.ALLOW, 35 | actions: ['transcribe:*'], 36 | resources: ['*'], 37 | }) 38 | ); 39 | 40 | const questionStreamFunction = new NodejsFunction(this, 'StreamQuestion', { 41 | runtime: Runtime.NODEJS_22_X, 42 | entry: './lambda/streamQuestion.ts', 43 | timeout: Duration.minutes(15), 44 | environment: { 45 | BEDROCK_REGION: props.bedrockRegion, 46 | BEDROCK_MODELID: props.bedrockModelId, 47 | KENDRA_INDEX_ID: props.index.attrId, 48 | }, 49 | bundling: { 50 | externalModules: [], 51 | // nodeModules: ['@aws-sdk/client-bedrock-runtime'], 52 | }, 53 | }); 54 | questionStreamFunction.role?.addToPrincipalPolicy( 55 | new iam.PolicyStatement({ 56 | effect: iam.Effect.ALLOW, 57 | resources: [Token.asString(props.index.getAtt('Arn'))], 58 | actions: ['kendra:Retrieve'], 59 | }) 60 | ); 61 | questionStreamFunction.role?.addToPrincipalPolicy( 62 | new iam.PolicyStatement({ 63 | effect: iam.Effect.ALLOW, 64 | resources: ['*'], 65 | actions: ['bedrock:*', 'logs:*', 'translate:*'], 66 | }) 67 | ); 68 | questionStreamFunction.grantInvoke(identityPool.unauthenticatedRole); 69 | 70 | new CfnOutput(this, 'IdPoolId', { 71 | value: identityPool.identityPoolId, 72 | }); 73 | new CfnOutput(this, 'QuestionStreamFunctionARN', { 74 | value: questionStreamFunction.functionArn, 75 | }); 76 | 77 | this.questionStreamFunction = questionStreamFunction; 78 | this.idPool = identityPool; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /packages/cdk/lib/constructs/frontend.ts: -------------------------------------------------------------------------------- 1 | import { CfnOutput, RemovalPolicy, Stack, Duration } from 'aws-cdk-lib'; 2 | import * as s3 from 'aws-cdk-lib/aws-s3'; 3 | import * as cloudfront from 'aws-cdk-lib/aws-cloudfront'; 4 | import * as cloudfront_origins from 'aws-cdk-lib/aws-cloudfront-origins'; 5 | import { NodejsBuild } from 'deploy-time-build'; 6 | import { Construct } from 'constructs'; 7 | 8 | export interface FrontendProps { 9 | readonly questionStreamFunctionArn: string; 10 | readonly idPoolId: string; 11 | } 12 | 13 | export class Frontend extends Construct { 14 | readonly distribution: cloudfront.Distribution; 15 | 16 | constructor(scope: Construct, id: string, props: FrontendProps) { 17 | super(scope, id); 18 | 19 | const assetBucket = new s3.Bucket(this, 'AssetBucket', { 20 | encryption: s3.BucketEncryption.S3_MANAGED, 21 | blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, 22 | enforceSSL: true, 23 | removalPolicy: RemovalPolicy.DESTROY, 24 | autoDeleteObjects: true, 25 | }); 26 | const s3Origin = cloudfront_origins.S3BucketOrigin.withOriginAccessIdentity(assetBucket); 27 | 28 | this.distribution = new cloudfront.Distribution(this, 'Distribution', { 29 | defaultBehavior: { 30 | origin: s3Origin, 31 | }, 32 | defaultRootObject: 'index.html', 33 | errorResponses: [ 34 | { 35 | httpStatus: 404, 36 | responseHttpStatus: 200, 37 | responsePagePath: '/index.html', 38 | ttl: Duration.seconds(0), 39 | }, 40 | { 41 | httpStatus: 403, 42 | responseHttpStatus: 200, 43 | responsePagePath: '/index.html', 44 | ttl: Duration.seconds(0), 45 | }, 46 | ], 47 | }); 48 | 49 | new NodejsBuild(this, 'ReactBuild', { 50 | assets: [ 51 | { 52 | path: '../../', 53 | exclude: [ 54 | '.git', 55 | '.gitignore', 56 | '*.md', 57 | 'LICENSE', 58 | 'node_modules', 59 | 'packages/cdk/**/*', 60 | 'packages/web/dist', 61 | 'packages/web/node_modules', 62 | ], 63 | }, 64 | ], 65 | buildCommands: ['npm ci', 'npm run web:build'], 66 | buildEnvironment: { 67 | VITE_APP_REGION: Stack.of(this).region, 68 | VITE_APP_IDENTITY_POOL_ID: props.idPoolId, 69 | VITE_APP_QUESTION_STREAM_FUNCTION_ARN: props.questionStreamFunctionArn, 70 | }, 71 | outputSourceDirectory: './packages/web/dist', 72 | destinationBucket: assetBucket, 73 | distribution: this.distribution, 74 | nodejsVersion: 22, 75 | }); 76 | 77 | new CfnOutput(this, 'CloudFrontURL', { 78 | description: 'CloudFrontURL', 79 | value: `https://${this.distribution.distributionDomainName}`, 80 | }); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /packages/cdk/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node --prefer-ts-exts bin/cdk.ts", 3 | "watch": { 4 | "include": [ 5 | "**" 6 | ], 7 | "exclude": [ 8 | "README.md", 9 | "cdk*.json", 10 | "**/*.d.ts", 11 | "**/*.js", 12 | "tsconfig.json", 13 | "package*.json", 14 | "yarn.lock", 15 | "node_modules", 16 | "test" 17 | ] 18 | }, 19 | "context": { 20 | "bedrock-region": "ap-northeast-1", 21 | "bedrock-model-id": "anthropic.claude-3-5-sonnet-20240620-v1:0", 22 | "@aws-cdk/aws-lambda:recognizeLayerVersion": true, 23 | "@aws-cdk/core:checkSecretUsage": true, 24 | "@aws-cdk/core:target-partitions": [ 25 | "aws", 26 | "aws-cn" 27 | ], 28 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 29 | "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, 30 | "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, 31 | "@aws-cdk/aws-iam:minimizePolicies": true, 32 | "@aws-cdk/core:validateSnapshotRemovalPolicy": true, 33 | "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, 34 | "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, 35 | "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, 36 | "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, 37 | "@aws-cdk/core:enablePartitionLiterals": true, 38 | "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, 39 | "@aws-cdk/aws-iam:standardizedServicePrincipals": true, 40 | "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, 41 | "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, 42 | "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, 43 | "@aws-cdk/aws-route53-patters:useCertificate": true, 44 | "@aws-cdk/customresources:installLatestAwsSdkDefault": false, 45 | "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, 46 | "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, 47 | "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, 48 | "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, 49 | "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, 50 | "@aws-cdk/aws-redshift:columnId": true, 51 | "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, 52 | "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, 53 | "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, 54 | "@aws-cdk/aws-kms:aliasNameRef": true, 55 | "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, 56 | "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, 57 | "@aws-cdk/aws-efs:denyAnonymousAccess": true, 58 | "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, 59 | "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, 60 | "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true, 61 | "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true, 62 | "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /packages/web/src/components/Select.tsx: -------------------------------------------------------------------------------- 1 | import { Fragment, useCallback, useMemo } from 'react'; 2 | import { Listbox, Transition } from '@headlessui/react'; 3 | import { PiCaretUpDown, PiCheck, PiX } from 'react-icons/pi'; 4 | import { twMerge } from 'tailwind-merge'; 5 | 6 | type Props = { 7 | className?: string; 8 | label?: string; 9 | value: string; 10 | disabled?: boolean; 11 | options: { 12 | value: string; 13 | label: string; 14 | }[]; 15 | clearable?: boolean; 16 | onChange: (value: string) => void; 17 | }; 18 | 19 | const Select: React.FC = (props) => { 20 | const selectedLabel = useMemo(() => { 21 | return props.value === '' 22 | ? '' 23 | : props.options.filter((o) => o.value === props.value)[0].label; 24 | }, [props.options, props.value]); 25 | 26 | const onClear = useCallback(() => { 27 | props.onChange(''); 28 | }, [props]); 29 | 30 | return ( 31 | <> 32 | {props.label && } 33 | 37 |
38 | 39 | {selectedLabel} 40 | 41 | 42 | 43 | 44 | 45 | {props.clearable && props.value !== '' && ( 46 | 47 | 50 | 51 | )} 52 | 57 | 58 | {props.options.map((option, idx) => ( 59 | 62 | `relative cursor-default select-none py-2 pl-10 pr-4 ${ 63 | active ? 'text-aws-primary bg-primary/10' : '' 64 | }` 65 | } 66 | value={option.value}> 67 | {({ selected }) => ( 68 | <> 69 | 73 | {option.label} 74 | 75 | {selected ? ( 76 | 77 | 78 | 79 | ) : null} 80 | 81 | )} 82 | 83 | ))} 84 | 85 | 86 |
87 | 88 | 89 | ); 90 | }; 91 | 92 | export default Select; 93 | -------------------------------------------------------------------------------- /packages/web/src/assets/react.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/web/src/components/InputQuestion.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useMemo, useRef, useState } from 'react'; 2 | import { useTranslation } from 'react-i18next'; 3 | import { 4 | FaEllipsis, 5 | FaMicrophone, 6 | FaMicrophoneSlash, 7 | FaPaperPlane, 8 | } from 'react-icons/fa6'; 9 | import useTranscribeStreaming from '../hooks/useTranscribeStreaming'; 10 | import ButtonIcon from './ButtonIcon'; 11 | 12 | const IDENTITY_POOL_ID = import.meta.env.VITE_APP_IDENTITY_POOL_ID!; 13 | const REGION = import.meta.env.VITE_APP_REGION!; 14 | 15 | type Props = { 16 | className?: string; 17 | content: string; 18 | transcribeLanguageCode?: string; 19 | disabled?: boolean; 20 | onChange: (s: string) => void; 21 | onSend: (content: string) => void; 22 | }; 23 | 24 | const InputQuestion: React.FC = (props) => { 25 | const { t } = useTranslation(); 26 | const disabledSend = useMemo(() => { 27 | return props.content === '' || props.disabled; 28 | }, [props.content, props.disabled]); 29 | 30 | const inputRef = useRef(null); 31 | useEffect(() => { 32 | const listener = (e: DocumentEventMap['keypress']) => { 33 | if (e.key === 'Enter' && !e.shiftKey) { 34 | e.preventDefault(); 35 | 36 | if (!disabledSend) { 37 | props.onSend(props.content); 38 | } 39 | } 40 | }; 41 | const element = inputRef.current; 42 | element?.addEventListener('keypress', listener); 43 | 44 | return () => { 45 | element?.removeEventListener('keypress', listener); 46 | }; 47 | }); 48 | 49 | const { transcripts, recording, startRecording, stopRecording } = 50 | useTranscribeStreaming({ 51 | languageCode: props.transcribeLanguageCode ?? '', 52 | identityPoolId: IDENTITY_POOL_ID, 53 | region: REGION, 54 | }); 55 | 56 | const [transcript, setTranscript] = useState(''); 57 | 58 | useEffect(() => { 59 | for (const t of transcripts) { 60 | if (!t.isPartial) { 61 | props.onSend(t.transcripts.join(' ')); 62 | setTranscript(''); 63 | } else { 64 | setTranscript(t.transcripts.join(' ')); 65 | } 66 | } 67 | // eslint-disable-next-line react-hooks/exhaustive-deps 68 | }, [transcripts]); 69 | 70 | return ( 71 |
72 |
73 |
74 | {recording ? ( 75 | 76 | 77 | 78 | ) : ( 79 |
80 | 83 | 84 | 85 |
86 | )} 87 |
88 | 89 | {recording ? ( 90 |
91 | {transcript === '' ? ( 92 | <> 93 | {t('message.transcribing')} 94 | 95 | 96 | ) : ( 97 |
{transcript}
98 | )} 99 |
100 | ) : ( 101 | { 108 | props.onChange(e.target.value); 109 | }} 110 | /> 111 | )} 112 | 113 | { 118 | props.onSend(props.content); 119 | }}> 120 | 121 | 122 |
123 |
124 | ); 125 | }; 126 | 127 | export default InputQuestion; 128 | -------------------------------------------------------------------------------- /packages/web/src/hooks/useTranscribeStreaming.ts: -------------------------------------------------------------------------------- 1 | import { useState } from 'react'; 2 | import { 3 | TranscribeStreamingClient, 4 | StartStreamTranscriptionCommand, 5 | LanguageCode, 6 | } from '@aws-sdk/client-transcribe-streaming'; 7 | import { CognitoIdentityClient } from '@aws-sdk/client-cognito-identity'; 8 | import { fromCognitoIdentityPool } from '@aws-sdk/credential-provider-cognito-identity'; 9 | import MicrophoneStream from 'microphone-stream'; 10 | import { Readable } from 'readable-stream'; 11 | import { PassThrough } from 'stream-browserify'; 12 | import { Buffer } from 'buffer'; 13 | import * as process from 'process'; 14 | import { create } from 'zustand'; 15 | 16 | window.process = process; 17 | window.Buffer = Buffer; 18 | 19 | export interface Transcripts { 20 | isPartial: boolean; 21 | transcripts: string[]; 22 | } 23 | 24 | export const useTranscribeStreamingState = create<{ 25 | recording: boolean; 26 | setRecording: (b: boolean) => void; 27 | }>((set) => { 28 | return { 29 | recording: false, 30 | setRecording: (b) => { 31 | set({ 32 | recording: b, 33 | }); 34 | }, 35 | }; 36 | }); 37 | 38 | interface UseTranscribeStreamingProps { 39 | languageCode: string; 40 | identityPoolId: string; 41 | region: string; 42 | } 43 | 44 | function useTranscribeStreaming(props: UseTranscribeStreamingProps) { 45 | const { recording, setRecording } = useTranscribeStreamingState(); 46 | const [transcripts, setTranscripts] = useState([]); 47 | const [micStream, setMicStream] = useState(null); 48 | 49 | const cognito = new CognitoIdentityClient({ region: props.region }); 50 | 51 | const client = new TranscribeStreamingClient({ 52 | region: props.region, 53 | credentials: fromCognitoIdentityPool({ 54 | client: cognito, 55 | identityPoolId: props.identityPoolId, 56 | }), 57 | }); 58 | 59 | const stopRecording = () => { 60 | if (micStream) { 61 | micStream.stop(); 62 | } 63 | 64 | setMicStream(null); 65 | setRecording(false); 66 | }; 67 | 68 | const startRecording = async () => { 69 | try { 70 | const micStream = new MicrophoneStream(); 71 | 72 | setRecording(true); 73 | setMicStream(micStream); 74 | 75 | const pcmEncodeChunk = (chunk: Buffer) => { 76 | const input = MicrophoneStream.toRaw(chunk); 77 | let offset = 0; 78 | const buffer = new ArrayBuffer(input.length * 2); 79 | const view = new DataView(buffer); 80 | for (let i = 0; i < input.length; i++, offset += 2) { 81 | const s = Math.max(-1, Math.min(1, input[i])); 82 | view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true); 83 | } 84 | return Buffer.from(buffer); 85 | }; 86 | 87 | const audioPayloadStream = new PassThrough({ highWaterMark: 1 * 1024 }); 88 | 89 | micStream.setStream( 90 | await window.navigator.mediaDevices.getUserMedia({ 91 | video: false, 92 | audio: true, 93 | }) 94 | ); 95 | 96 | (micStream as Readable).pipe(audioPayloadStream); 97 | 98 | const audioStream = async function* () { 99 | for await (const chunk of audioPayloadStream) { 100 | yield { AudioEvent: { AudioChunk: pcmEncodeChunk(chunk) } }; 101 | } 102 | }; 103 | 104 | const command = new StartStreamTranscriptionCommand({ 105 | LanguageCode: props.languageCode as LanguageCode, 106 | MediaEncoding: 'pcm', 107 | MediaSampleRateHertz: 44100, 108 | AudioStream: audioStream(), 109 | }); 110 | 111 | const response = await client.send(command); 112 | 113 | for await (const event of response.TranscriptResultStream!) { 114 | if (event.TranscriptEvent) { 115 | const results = event!.TranscriptEvent!.Transcript!.Results!.map( 116 | (r) => { 117 | return { 118 | isPartial: r.IsPartial!, 119 | transcripts: r.Alternatives!.map((a) => a.Transcript!), 120 | }; 121 | } 122 | ); 123 | 124 | if (results.length > 0) { 125 | setTranscripts(results); 126 | } 127 | } 128 | } 129 | } catch (e) { 130 | console.error(e); 131 | stopRecording(); 132 | } 133 | }; 134 | 135 | return { 136 | transcripts, 137 | recording, 138 | startRecording, 139 | stopRecording, 140 | }; 141 | } 142 | 143 | export default useTranscribeStreaming; 144 | -------------------------------------------------------------------------------- /packages/web/src/hooks/useAvatar.ts: -------------------------------------------------------------------------------- 1 | import { AbstractMesh, Animation, AnimationGroup } from '@babylonjs/core'; 2 | import { ILoadedModel } from 'react-babylonjs'; 3 | import { create } from 'zustand'; 4 | 5 | const useAvatarState = create<{ 6 | rootMesh: AbstractMesh | undefined; 7 | bodyMesh: AbstractMesh | undefined; 8 | faceMesh: AbstractMesh | undefined; 9 | idleAnimation: AnimationGroup | undefined; 10 | raisingHandAnimation: AnimationGroup | undefined; 11 | raisingHandToIdleAnimation: AnimationGroup | undefined; 12 | thinkingAnimation: AnimationGroup | undefined; 13 | setModel: (model: ILoadedModel) => void; 14 | }>((set) => { 15 | return { 16 | rootMesh: undefined, 17 | bodyMesh: undefined, 18 | faceMesh: undefined, 19 | idleAnimation: undefined, 20 | raisingHandAnimation: undefined, 21 | raisingHandToIdleAnimation: undefined, 22 | thinkingAnimation: undefined, 23 | setModel: (model) => { 24 | const rootMesh = model.meshes?.find((m) => m.name === '__root__'); 25 | rootMesh?.position.set(0, -2.6, 0); 26 | rootMesh?.scaling.set(1.8, 1.8, 1.8); 27 | rootMesh?.rotation.set(0, Math.PI / 2, 0); 28 | 29 | set({ 30 | ...(model.animationGroups 31 | ? { 32 | idleAnimation: model.animationGroups.find(anim => anim.name === 'Idle'), 33 | raisingHandAnimation: model.animationGroups.find(anim => anim.name === 'RaiseHand'), 34 | raisingHandToIdleAnimation: model.animationGroups.find(anim => anim.name === 'RaiseToIdle'), 35 | thinkingAnimation: model.animationGroups.find(anim => anim.name === 'Thinking'), 36 | } 37 | : { 38 | idleAnimation: undefined, 39 | raisingHandAnimation: undefined, 40 | raisingHandToIdleAnimation: undefined, 41 | thinkingAnimation: undefined, 42 | }), 43 | rootMesh, 44 | faceMesh: model.meshes?.find((m) => m.name === 'Face'), 45 | bodyMesh: model.meshes?.find((m) => m.name === 'Body'), 46 | }); 47 | }, 48 | }; 49 | }); 50 | 51 | const useAvatar = () => { 52 | const { 53 | faceMesh, 54 | idleAnimation, 55 | raisingHandAnimation, 56 | raisingHandToIdleAnimation, 57 | thinkingAnimation, 58 | setModel, 59 | } = useAvatarState(); 60 | 61 | const speech = () => { 62 | thinkingAnimation?.stop(); 63 | raisingHandAnimation?.start(true); 64 | if (faceMesh) { 65 | const morphTarget = faceMesh.morphTargetManager?.getTarget(33); 66 | if (morphTarget) { 67 | const animationDuration = 0.7; 68 | const maxInfluence = 0.6; // Maximum Mouth Opening Size 69 | const totalAnimationTime = 5; 70 | 71 | // Round up loopCount to look natural 72 | const loopCount = Math.ceil(totalAnimationTime / animationDuration); 73 | 74 | const animation = new Animation( 75 | 'morphAnim', 76 | 'influence', 77 | 30, 78 | Animation.ANIMATIONTYPE_FLOAT, 79 | Animation.ANIMATIONLOOPMODE_CYCLE 80 | ); 81 | 82 | // Key for animations 83 | const keys = []; 84 | 85 | // Transition from 0 to maxInfluence and back to 0 over animationDuration seconds, 86 | // repeating for loopCount times 87 | for (let i = 0; i < loopCount; i++) { 88 | keys.push({ frame: 30 * animationDuration * i, value: 0 }); 89 | // Reach maxInfluence in half the time 90 | keys.push({ 91 | frame: 30 * animationDuration * i + (animationDuration / 2) * 30, 92 | value: maxInfluence, 93 | }); 94 | // Return to 0 before starting the next cycle 95 | keys.push({ frame: 30 * animationDuration * (i + 1), value: 0 }); 96 | } 97 | 98 | animation.setKeys(keys); 99 | 100 | // Apply animation to target 101 | morphTarget.animations = []; 102 | morphTarget.animations.push(animation); 103 | 104 | // Start Animation 105 | const animatable = faceMesh 106 | .getScene() 107 | .beginAnimation( 108 | morphTarget, 109 | 0, 110 | 30 * animationDuration * loopCount, 111 | false 112 | ); 113 | 114 | // Set shape key to 0 at the end of animation 115 | animatable.onAnimationEnd = () => { 116 | raisingHandAnimation?.stop(); 117 | raisingHandToIdleAnimation?.start(false); 118 | }; 119 | } 120 | } 121 | }; 122 | 123 | return { 124 | startThinking: () => { 125 | idleAnimation?.stop(); 126 | thinkingAnimation?.start(true); 127 | }, 128 | stopThinking: () => { 129 | thinkingAnimation?.stop(); 130 | idleAnimation?.start(true); 131 | }, 132 | startSpeech: () => { 133 | speech(); 134 | }, 135 | setModel, 136 | }; 137 | }; 138 | 139 | export default useAvatar; 140 | -------------------------------------------------------------------------------- /packages/web/src/App.tsx: -------------------------------------------------------------------------------- 1 | import React, { useCallback, useState } from 'react'; 2 | 3 | import { Engine, Scene } from 'react-babylonjs'; 4 | import { Color4, Vector3 } from '@babylonjs/core'; 5 | import Avatar from './components/Avatar'; 6 | import useAvatar from './hooks/useAvatar'; 7 | import useQuestion from './hooks/useQuestion'; 8 | import InputQuestion from './components/InputQuestion'; 9 | import { produce } from 'immer'; 10 | import Select from './components/Select'; 11 | import { PiGlobe } from 'react-icons/pi'; 12 | import './i18n'; 13 | import { LANGUAGE_OPTIONS } from './i18n'; 14 | import { useTranslation } from 'react-i18next'; 15 | import { useTranscribeStreamingState } from './hooks/useTranscribeStreaming'; 16 | 17 | const App: React.FC = () => { 18 | const { t, i18n } = useTranslation(); 19 | const [content, setContent] = useState(''); 20 | const [language, setLanguage] = useState('日本語'); 21 | const [questionedContents, setQuestionedContents] = useState(['']); 22 | const [isLoading, setIsLoading] = useState(false); 23 | 24 | const { recording } = useTranscribeStreamingState(); 25 | 26 | const { startThinking, stopThinking, startSpeech } = useAvatar(); 27 | const { answerText, question } = useQuestion(); 28 | 29 | const onSendQuestion = useCallback( 30 | (questionContent: string) => { 31 | setIsLoading(true); 32 | setContent(''); 33 | setQuestionedContents( 34 | produce(questionedContents, (draft) => { 35 | draft[draft.length - 1] = questionContent; 36 | draft.push(''); 37 | }) 38 | ); 39 | startThinking(); 40 | try { 41 | question( 42 | questionContent, 43 | language, 44 | LANGUAGE_OPTIONS.filter((l) => l.value === language)[0].code, 45 | startSpeech 46 | ).finally(() => { 47 | setIsLoading(false); 48 | }); 49 | } catch (e) { 50 | console.log(e); 51 | stopThinking(); 52 | } 53 | }, 54 | [ 55 | language, 56 | question, 57 | questionedContents, 58 | startSpeech, 59 | startThinking, 60 | stopThinking, 61 | ] 62 | ); 63 | const onChangeLanguage = useCallback( 64 | (lang: string) => { 65 | setLanguage(lang); 66 | i18n.changeLanguage( 67 | LANGUAGE_OPTIONS.filter((l) => l.value === lang)[0].code 68 | ); 69 | }, 70 | [i18n] 71 | ); 72 | 73 | return ( 74 |
75 |
76 |
77 | 78 |