├── .gitignore ├── README.md ├── jsconfig.json ├── next.config.mjs ├── package.json ├── postcss.config.js ├── public ├── images │ ├── Nanami.jpg │ └── Naoki.jpg ├── models │ ├── Nanami.fbx │ ├── Naoki.fbx │ ├── Teacher_Nanami.glb │ ├── Teacher_Naoki.glb │ ├── animations_Nanami.glb │ ├── animations_Naoki.glb │ ├── classroom_alternative.glb │ └── classroom_default.glb ├── next.svg └── vercel.svg ├── src ├── app │ ├── api │ │ ├── ai │ │ │ └── route.js │ │ └── tts │ │ │ └── route.js │ ├── favicon.ico │ ├── globals.css │ ├── layout.js │ └── page.js ├── components │ ├── BoardSettings.jsx │ ├── Experience.jsx │ ├── MessagesList.jsx │ ├── Teacher.jsx │ └── TypingBox.jsx └── hooks │ └── useAITeacher.js ├── tailwind.config.js └── yarn.lock /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | .yarn/install-state.gz 8 | 9 | # testing 10 | /coverage 11 | 12 | # next.js 13 | /.next/ 14 | /out/ 15 | 16 | # production 17 | /build 18 | 19 | # misc 20 | .DS_Store 21 | *.pem 22 | 23 | # debug 24 | npm-debug.log* 25 | yarn-debug.log* 26 | yarn-error.log* 27 | 28 | # local env files 29 | .env*.local 30 | 31 | # vercel 32 | .vercel 33 | 34 | # typescript 35 | *.tsbuildinfo 36 | next-env.d.ts 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Video Thumbnail](https://img.youtube.com/vi/_bi4Ol0QEL4/maxresdefault.jpg) 2 | 3 | [Video tutorial](https://youtu.be/_bi4Ol0QEL4) 4 | 5 | 6 | ## Deploy on Elestio 7 | 8 | The easiest way to deploy your Next.js app is to use the [Elestio Platform](https://ellest.io). 9 | 10 | -------------------------------------------------------------------------------- /jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "paths": { 4 | "@/*": ["./src/*"] 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /next.config.mjs: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const nextConfig = {}; 3 | 4 | export default nextConfig; 5 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "r3f-ai-language-teacher", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "@react-three/drei": "^9.99.3", 13 | "@react-three/fiber": "^8.15.16", 14 | "leva": "^0.9.35", 15 | "microsoft-cognitiveservices-speech-sdk": "^1.35.0", 16 | "next": "14.1.0", 17 | "openai": "^4.28.0", 18 | "react": "^18", 19 | "react-dom": "^18", 20 | "three": "^0.161.0", 21 | "zustand": "^4.5.1" 22 | }, 23 | "devDependencies": { 24 | "autoprefixer": "^10.0.1", 25 | "eslint": "^8", 26 | "eslint-config-next": "14.1.0", 27 | "postcss": "^8", 28 | "tailwindcss": "^3.3.0" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /public/images/Nanami.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wass08/r3f-ai-language-teacher/08d7957e389390d41f72a27acc8d5ec6e56b81d3/public/images/Nanami.jpg -------------------------------------------------------------------------------- /public/images/Naoki.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wass08/r3f-ai-language-teacher/08d7957e389390d41f72a27acc8d5ec6e56b81d3/public/images/Naoki.jpg -------------------------------------------------------------------------------- /public/models/Nanami.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wass08/r3f-ai-language-teacher/08d7957e389390d41f72a27acc8d5ec6e56b81d3/public/models/Nanami.fbx -------------------------------------------------------------------------------- /public/models/Naoki.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wass08/r3f-ai-language-teacher/08d7957e389390d41f72a27acc8d5ec6e56b81d3/public/models/Naoki.fbx -------------------------------------------------------------------------------- /public/models/Teacher_Nanami.glb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wass08/r3f-ai-language-teacher/08d7957e389390d41f72a27acc8d5ec6e56b81d3/public/models/Teacher_Nanami.glb -------------------------------------------------------------------------------- /public/models/Teacher_Naoki.glb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wass08/r3f-ai-language-teacher/08d7957e389390d41f72a27acc8d5ec6e56b81d3/public/models/Teacher_Naoki.glb -------------------------------------------------------------------------------- /public/models/animations_Nanami.glb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wass08/r3f-ai-language-teacher/08d7957e389390d41f72a27acc8d5ec6e56b81d3/public/models/animations_Nanami.glb -------------------------------------------------------------------------------- /public/models/animations_Naoki.glb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wass08/r3f-ai-language-teacher/08d7957e389390d41f72a27acc8d5ec6e56b81d3/public/models/animations_Naoki.glb -------------------------------------------------------------------------------- /public/models/classroom_alternative.glb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wass08/r3f-ai-language-teacher/08d7957e389390d41f72a27acc8d5ec6e56b81d3/public/models/classroom_alternative.glb -------------------------------------------------------------------------------- /public/models/classroom_default.glb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wass08/r3f-ai-language-teacher/08d7957e389390d41f72a27acc8d5ec6e56b81d3/public/models/classroom_default.glb -------------------------------------------------------------------------------- /public/next.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /public/vercel.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/app/api/ai/route.js: -------------------------------------------------------------------------------- 1 | import OpenAI from "openai"; 2 | 3 | const openai = new OpenAI({ 4 | apiKey: process.env["OPENAI_API_KEY"], // This is the default and can be omitted 5 | }); 6 | 7 | const formalExample = { 8 | japanese: [ 9 | { word: "日本", reading: "にほん" }, 10 | { word: "に" }, 11 | { word: "住んで", reading: "すんで" }, 12 | { word: "います" }, 13 | { word: "か" }, 14 | { word: "?" }, 15 | ], 16 | grammarBreakdown: [ 17 | { 18 | english: "Do you live in Japan?", 19 | japanese: [ 20 | { word: "日本", reading: "にほん" }, 21 | { word: "に" }, 22 | { word: "住んで", reading: "すんで" }, 23 | { word: "います" }, 24 | { word: "か" }, 25 | { word: "?" }, 26 | ], 27 | chunks: [ 28 | { 29 | japanese: [{ word: "日本", reading: "にほん" }], 30 | meaning: "Japan", 31 | grammar: "Noun", 32 | }, 33 | { 34 | japanese: [{ word: "に" }], 35 | meaning: "in", 36 | grammar: "Particle", 37 | }, 38 | { 39 | japanese: [{ word: "住んで", reading: "すんで" }, { word: "います" }], 40 | meaning: "live", 41 | grammar: "Verb + て form + います", 42 | }, 43 | { 44 | japanese: [{ word: "か" }], 45 | meaning: "question", 46 | grammar: "Particle", 47 | }, 48 | { 49 | japanese: [{ word: "?" }], 50 | meaning: "question", 51 | grammar: "Punctuation", 52 | }, 53 | ], 54 | }, 55 | ], 56 | }; 57 | 58 | const casualExample = { 59 | japanese: [ 60 | { word: "日本", reading: "にほん" }, 61 | { word: "に" }, 62 | { word: "住んで", reading: "すんで" }, 63 | { word: "いる" }, 64 | { word: "の" }, 65 | { word: "?" }, 66 | ], 67 | grammarBreakdown: [ 68 | { 69 | english: "Do you live in Japan?", 70 | japanese: [ 71 | { word: "日本", reading: "にほん" }, 72 | { word: "に" }, 73 | { word: "住んで", reading: "すんで" }, 74 | { word: "いる" }, 75 | { word: "の" }, 76 | { word: "?" }, 77 | ], 78 | chunks: [ 79 | { 80 | japanese: [{ word: "日本", reading: "にほん" }], 81 | meaning: "Japan", 82 | grammar: "Noun", 83 | }, 84 | { 85 | japanese: [{ word: "に" }], 86 | meaning: "in", 87 | grammar: "Particle", 88 | }, 89 | { 90 | japanese: [{ word: "住んで", reading: "すんで" }, { word: "いる" }], 91 | meaning: "live", 92 | grammar: "Verb + て form + いる", 93 | }, 94 | { 95 | japanese: [{ word: "の" }], 96 | meaning: "question", 97 | grammar: "Particle", 98 | }, 99 | { 100 | japanese: [{ word: "?" }], 101 | meaning: "question", 102 | grammar: "Punctuation", 103 | }, 104 | ], 105 | }, 106 | ], 107 | }; 108 | 109 | export async function GET(req) { 110 | // WARNING: Do not expose your keys 111 | // WARNING: If you host publicly your project, add an authentication layer to limit the consumption of ChatGPT resources 112 | 113 | const speech = req.nextUrl.searchParams.get("speech") || "formal"; 114 | const speechExample = speech === "formal" ? formalExample : casualExample; 115 | 116 | const chatCompletion = await openai.chat.completions.create({ 117 | messages: [ 118 | { 119 | role: "system", 120 | content: `You are a Japanese language teacher. 121 | Your student asks you how to say something from english to japanese. 122 | You should respond with: 123 | - english: the english version ex: "Do you live in Japan?" 124 | - japanese: the japanese translation in split into words ex: ${JSON.stringify( 125 | speechExample.japanese 126 | )} 127 | - grammarBreakdown: an explanation of the grammar structure per sentence ex: ${JSON.stringify( 128 | speechExample.grammarBreakdown 129 | )} 130 | `, 131 | }, 132 | { 133 | role: "system", 134 | content: `You always respond with a JSON object with the following format: 135 | { 136 | "english": "", 137 | "japanese": [{ 138 | "word": "", 139 | "reading": "" 140 | }], 141 | "grammarBreakdown": [{ 142 | "english": "", 143 | "japanese": [{ 144 | "word": "", 145 | "reading": "" 146 | }], 147 | "chunks": [{ 148 | "japanese": [{ 149 | "word": "", 150 | "reading": "" 151 | }], 152 | "meaning": "", 153 | "grammar": "" 154 | }] 155 | }] 156 | }`, 157 | }, 158 | { 159 | role: "user", 160 | content: `How to say ${ 161 | req.nextUrl.searchParams.get("question") || 162 | "Have you ever been to Japan?" 163 | } in Japanese in ${speech} speech?`, 164 | }, 165 | ], 166 | // model: "gpt-4-turbo-preview", // https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo 167 | model: "gpt-3.5-turbo", // https://help.openai.com/en/articles/7102672-how-can-i-access-gpt-4 168 | response_format: { 169 | type: "json_object", 170 | }, 171 | }); 172 | console.log(chatCompletion.choices[0].message.content); 173 | return Response.json(JSON.parse(chatCompletion.choices[0].message.content)); 174 | } 175 | -------------------------------------------------------------------------------- /src/app/api/tts/route.js: -------------------------------------------------------------------------------- 1 | import * as sdk from "microsoft-cognitiveservices-speech-sdk"; 2 | import { PassThrough } from "stream"; 3 | 4 | export async function GET(req) { 5 | // WARNING: Do not expose your keys 6 | // WARNING: If you host publicly your project, add an authentication layer to limit the consumption of Azure resources 7 | 8 | const speechConfig = sdk.SpeechConfig.fromSubscription( 9 | process.env["SPEECH_KEY"], 10 | process.env["SPEECH_REGION"] 11 | ); 12 | 13 | // https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts 14 | const teacher = req.nextUrl.searchParams.get("teacher") || "Nanami"; 15 | speechConfig.speechSynthesisVoiceName = `ja-JP-${teacher}Neural`; 16 | 17 | const speechSynthesizer = new sdk.SpeechSynthesizer(speechConfig); 18 | const visemes = []; 19 | speechSynthesizer.visemeReceived = function (s, e) { 20 | // console.log( 21 | // "(Viseme), Audio offset: " + 22 | // e.audioOffset / 10000 + 23 | // "ms. Viseme ID: " + 24 | // e.visemeId 25 | // ); 26 | visemes.push([e.audioOffset / 10000, e.visemeId]); 27 | }; 28 | const audioStream = await new Promise((resolve, reject) => { 29 | speechSynthesizer.speakTextAsync( 30 | req.nextUrl.searchParams.get("text") || 31 | "I'm excited to try text to speech", 32 | (result) => { 33 | const { audioData } = result; 34 | 35 | speechSynthesizer.close(); 36 | 37 | // convert arrayBuffer to stream 38 | const bufferStream = new PassThrough(); 39 | bufferStream.end(Buffer.from(audioData)); 40 | resolve(bufferStream); 41 | }, 42 | (error) => { 43 | console.log(error); 44 | speechSynthesizer.close(); 45 | reject(error); 46 | } 47 | ); 48 | }); 49 | const response = new Response(audioStream, { 50 | headers: { 51 | "Content-Type": "audio/mpeg", 52 | "Content-Disposition": `inline; filename=tts.mp3`, 53 | Visemes: JSON.stringify(visemes), 54 | }, 55 | }); 56 | // audioStream.pipe(response); 57 | return response; 58 | } 59 | -------------------------------------------------------------------------------- /src/app/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wass08/r3f-ai-language-teacher/08d7957e389390d41f72a27acc8d5ec6e56b81d3/src/app/favicon.ico -------------------------------------------------------------------------------- /src/app/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | :root { 6 | --foreground-rgb: 0, 0, 0; 7 | --background-start-rgb: 214, 219, 220; 8 | --background-end-rgb: 255, 255, 255; 9 | } 10 | 11 | @media (prefers-color-scheme: dark) { 12 | :root { 13 | --foreground-rgb: 255, 255, 255; 14 | --background-start-rgb: 0, 0, 0; 15 | --background-end-rgb: 0, 0, 0; 16 | } 17 | } 18 | 19 | body { 20 | color: rgb(var(--foreground-rgb)); 21 | background: linear-gradient( 22 | to bottom, 23 | transparent, 24 | rgb(var(--background-end-rgb)) 25 | ) 26 | rgb(var(--background-start-rgb)); 27 | } 28 | 29 | @layer utilities { 30 | .text-balance { 31 | text-wrap: balance; 32 | } 33 | } 34 | 35 | ::-webkit-scrollbar { 36 | width: 24px; 37 | background-color: rgba(255, 255, 255, 0.3); 38 | } 39 | 40 | ::-webkit-scrollbar-thumb { 41 | background-color: white; 42 | } 43 | -------------------------------------------------------------------------------- /src/app/layout.js: -------------------------------------------------------------------------------- 1 | import { Noto_Sans_JP, Roboto } from "next/font/google"; 2 | import "./globals.css"; 3 | 4 | const roboto = Roboto({ 5 | subsets: ["latin"], 6 | display: "swap", 7 | weight: ["400", "700"], 8 | variable: "--font-roboto", 9 | }); 10 | 11 | export const notoSansJP = Noto_Sans_JP({ 12 | display: "swap", 13 | subsets: ["latin"], 14 | variable: "--font-noto-sans-jp", 15 | }); 16 | 17 | export const metadata = { 18 | title: "AI Sensei", 19 | description: "Learn Japanese with AI Sensei", 20 | }; 21 | 22 | export default function RootLayout({ children }) { 23 | return ( 24 | 25 | {children} 26 | 27 | ); 28 | } 29 | -------------------------------------------------------------------------------- /src/app/page.js: -------------------------------------------------------------------------------- 1 | import { Experience } from "@/components/Experience"; 2 | 3 | export default function Home() { 4 | return ( 5 |
6 | 7 |
8 | ); 9 | } 10 | -------------------------------------------------------------------------------- /src/components/BoardSettings.jsx: -------------------------------------------------------------------------------- 1 | import { teachers, useAITeacher } from "@/hooks/useAITeacher"; 2 | 3 | export const BoardSettings = () => { 4 | const furigana = useAITeacher((state) => state.furigana); 5 | const setFurigana = useAITeacher((state) => state.setFurigana); 6 | 7 | const english = useAITeacher((state) => state.english); 8 | const setEnglish = useAITeacher((state) => state.setEnglish); 9 | 10 | const teacher = useAITeacher((state) => state.teacher); 11 | const setTeacher = useAITeacher((state) => state.setTeacher); 12 | 13 | const speech = useAITeacher((state) => state.speech); 14 | const setSpeech = useAITeacher((state) => state.setSpeech); 15 | 16 | const classroom = useAITeacher((state) => state.classroom); 17 | const setClassroom = useAITeacher((state) => state.setClassroom); 18 | 19 | return ( 20 | <> 21 |
22 | {teachers.map((sensei, idx) => ( 23 |
29 |
setTeacher(sensei)}> 30 | {sensei} 35 |
36 |

{sensei}

37 |
38 | ))} 39 |
40 |
41 | 51 | 61 |
62 |
63 | 73 | 83 |
84 |
85 | 95 | 105 |
106 | 107 | ); 108 | }; 109 | -------------------------------------------------------------------------------- /src/components/Experience.jsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | import { useAITeacher } from "@/hooks/useAITeacher"; 3 | import { 4 | CameraControls, 5 | Environment, 6 | Float, 7 | Gltf, 8 | Html, 9 | Loader, 10 | useGLTF, 11 | } from "@react-three/drei"; 12 | import { Canvas } from "@react-three/fiber"; 13 | import { Leva, button, useControls } from "leva"; 14 | import { Suspense, useEffect, useRef } from "react"; 15 | import { degToRad } from "three/src/math/MathUtils"; 16 | import { BoardSettings } from "./BoardSettings"; 17 | import { MessagesList } from "./MessagesList"; 18 | import { Teacher } from "./Teacher"; 19 | import { TypingBox } from "./TypingBox"; 20 | 21 | const itemPlacement = { 22 | default: { 23 | classroom: { 24 | position: [0.2, -1.7, -2], 25 | }, 26 | teacher: { 27 | position: [-1, -1.7, -3], 28 | }, 29 | board: { 30 | position: [0.45, 0.382, -6], 31 | }, 32 | }, 33 | alternative: { 34 | classroom: { 35 | position: [0.3, -1.7, -1.5], 36 | rotation: [0, degToRad(-90), 0], 37 | scale: 0.4, 38 | }, 39 | teacher: { position: [-1, -1.7, -3] }, 40 | board: { position: [1.4, 0.84, -8] }, 41 | }, 42 | }; 43 | 44 | export const Experience = () => { 45 | const teacher = useAITeacher((state) => state.teacher); 46 | const classroom = useAITeacher((state) => state.classroom); 47 | 48 | return ( 49 | <> 50 |
51 | 52 |
53 |