├── frontend ├── public │ ├── robots.txt │ ├── logo.png │ ├── favicon.ico │ ├── logo192.png │ ├── logo512.png │ ├── manifest.json │ └── index.html ├── src │ ├── components │ │ ├── common │ │ │ ├── Icon.css │ │ │ ├── Icon.js │ │ │ ├── Button.js │ │ │ └── Button.css │ │ ├── RolePlayPage │ │ │ ├── SpeechWave.js │ │ │ ├── ControlButtons.js │ │ │ ├── TranslationDisplay.css │ │ │ ├── TranslationDisplay.js │ │ │ ├── RolePlayPage.css │ │ │ └── RolePlayPage.js │ │ ├── Conversation │ │ │ ├── Conversation.css │ │ │ └── Conversation.js │ │ ├── ScenarioSelector │ │ │ ├── ScenarioSelector.css │ │ │ └── ScenarioSelector.js │ │ └── AudioRecorder.js │ ├── App.css │ ├── setupTests.js │ ├── App.js │ ├── App.test.js │ ├── index.js │ ├── index.css │ ├── reportWebVitals.js │ ├── services │ │ ├── speechRecognitionService.js │ │ ├── audioService.js │ │ └── apiService.js │ ├── hooks │ │ ├── useAudio.js │ │ ├── useAudioManager.js │ │ └── useSpeechRecognition.js │ └── logo.svg ├── .gitignore ├── package.json └── README.md ├── backend ├── src │ ├── scenarios │ │ ├── Scenario.ts │ │ ├── ScenarioFactory.ts │ │ ├── Restaurant.ts │ │ ├── TrainStation.ts │ │ └── Supermarket.ts │ ├── types.ts │ ├── startup │ │ ├── createResources.ts │ │ └── createRoutes.ts │ ├── app.ts │ ├── clients │ │ ├── deepseek.ts │ │ ├── gTTS.ts │ │ └── pollyClient.ts │ └── services │ │ ├── TranslationService.ts │ │ ├── ConversationService.ts │ │ └── AudioToTextService.ts ├── tsconfig.json ├── .gitignore └── package.json └── README.md /frontend/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /frontend/src/components/common/Icon.css: -------------------------------------------------------------------------------- 1 | .icon { 2 | font-size: 24px; 3 | margin-bottom: 10px; 4 | color: #FF4081; 5 | } -------------------------------------------------------------------------------- /frontend/public/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rosadiaznewyork/Lex-interactive-German-Language-Learning-Assistant/HEAD/frontend/public/logo.png -------------------------------------------------------------------------------- /frontend/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rosadiaznewyork/Lex-interactive-German-Language-Learning-Assistant/HEAD/frontend/public/favicon.ico -------------------------------------------------------------------------------- /frontend/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rosadiaznewyork/Lex-interactive-German-Language-Learning-Assistant/HEAD/frontend/public/logo192.png -------------------------------------------------------------------------------- /frontend/public/logo512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rosadiaznewyork/Lex-interactive-German-Language-Learning-Assistant/HEAD/frontend/public/logo512.png -------------------------------------------------------------------------------- /frontend/src/App.css: -------------------------------------------------------------------------------- 1 | .App { 2 | text-align: center; 3 | padding: 20px; 4 | font-family: 'Your Font', sans-serif; 5 | background-color: #f8f9fa; 6 | min-height: 100vh; 7 | } -------------------------------------------------------------------------------- /frontend/src/components/common/Icon.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | const Icon = ({ icon, color }) => ( 4 | {icon} 5 | ); 6 | 7 | export default Icon; -------------------------------------------------------------------------------- /frontend/src/setupTests.js: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom'; 6 | -------------------------------------------------------------------------------- /frontend/src/App.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ScenarioSelector from './components/ScenarioSelector/ScenarioSelector'; 3 | 4 | function App() { 5 | return ( 6 |
7 | 8 |
9 | ); 10 | } 11 | 12 | export default App; -------------------------------------------------------------------------------- /frontend/src/App.test.js: -------------------------------------------------------------------------------- 1 | import { render, screen } from '@testing-library/react'; 2 | import App from './App'; 3 | 4 | test('renders learn react link', () => { 5 | render(); 6 | const linkElement = screen.getByText(/learn react/i); 7 | expect(linkElement).toBeInTheDocument(); 8 | }); 9 | -------------------------------------------------------------------------------- /frontend/src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom/client'; 3 | import './index.css'; 4 | import App from './App'; 5 | 6 | const root = ReactDOM.createRoot(document.getElementById('root')); 7 | root.render( 8 | 9 | 10 | 11 | ); 12 | -------------------------------------------------------------------------------- /frontend/src/components/common/Button.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import './Button.css'; 3 | 4 | const Button = ({ children, onClick, disabled, className }) => ( 5 | 8 | ); 9 | 10 | export default Button; -------------------------------------------------------------------------------- /backend/src/scenarios/Scenario.ts: -------------------------------------------------------------------------------- 1 | import { ScenarioStates } from "../types" 2 | 3 | export abstract class Scenario { 4 | protected role: string 5 | protected tone: string 6 | 7 | constructor(role: string, tone: string) { 8 | this.role = role 9 | this.tone = tone 10 | } 11 | 12 | abstract getSystemPrompt(state: ScenarioStates): string 13 | } -------------------------------------------------------------------------------- /backend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2022", 4 | "module": "ESNext", 5 | "moduleResolution": "node", 6 | "strict": true, 7 | "esModuleInterop": true, 8 | "skipLibCheck": true, 9 | "forceConsistentCasingInFileNames": true, 10 | "outDir": "./dist" 11 | }, 12 | "include": [ 13 | "src/**/*.ts" 14 | ], 15 | "exclude": [ 16 | "node_modules" 17 | ] 18 | } -------------------------------------------------------------------------------- /frontend/src/components/RolePlayPage/SpeechWave.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | const SpeechWave = ({ isPlaying }) => ( 4 |
5 |
6 | {[1, 2, 3].map((wave) => ( 7 |
8 | ))} 9 |
10 |
11 | ); 12 | 13 | export default SpeechWave; -------------------------------------------------------------------------------- /backend/src/types.ts: -------------------------------------------------------------------------------- 1 | import { DeepseekClient } from "./clients/deepseek" 2 | import { GTTSClient } from "./clients/gTTS" 3 | import { PollyClient } from "./clients/pollyClient" 4 | 5 | export interface Clients { 6 | deepseek: DeepseekClient 7 | polly?: PollyClient 8 | gTTS: GTTSClient 9 | } 10 | 11 | export enum ScenarioStates { 12 | START = 'START_CONVERSATION', 13 | CONTINUE = 'CONTINUE_CONVERSATION' 14 | } -------------------------------------------------------------------------------- /frontend/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* -------------------------------------------------------------------------------- /frontend/src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /frontend/src/reportWebVitals.js: -------------------------------------------------------------------------------- 1 | const reportWebVitals = onPerfEntry => { 2 | if (onPerfEntry && onPerfEntry instanceof Function) { 3 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 4 | getCLS(onPerfEntry); 5 | getFID(onPerfEntry); 6 | getFCP(onPerfEntry); 7 | getLCP(onPerfEntry); 8 | getTTFB(onPerfEntry); 9 | }); 10 | } 11 | }; 12 | 13 | export default reportWebVitals; 14 | -------------------------------------------------------------------------------- /frontend/src/services/speechRecognitionService.js: -------------------------------------------------------------------------------- 1 | export const initializeSpeechRecognition = (lang = 'de-DE') => { 2 | if (!('webkitSpeechRecognition' in window)) { 3 | throw new Error('Speech recognition not supported'); 4 | } 5 | 6 | const recognition = new window.webkitSpeechRecognition(); 7 | recognition.continuous = true; 8 | recognition.interimResults = true; 9 | recognition.lang = lang; 10 | recognition.maxAlternatives = 1; 11 | 12 | return recognition; 13 | }; -------------------------------------------------------------------------------- /backend/src/startup/createResources.ts: -------------------------------------------------------------------------------- 1 | import { DeepseekClient } from "../clients/deepseek" 2 | import { GTTSClient } from "../clients/gTTS" 3 | import { PollyClient } from "../clients/pollyClient" 4 | import { Clients } from "../types" 5 | 6 | export async function createResources(): Promise { 7 | 8 | const deepseekClient = new DeepseekClient() 9 | const gttsClient = new GTTSClient() 10 | 11 | return { 12 | deepseek: deepseekClient, 13 | gTTS: gttsClient 14 | } 15 | } -------------------------------------------------------------------------------- /frontend/src/hooks/useAudio.js: -------------------------------------------------------------------------------- 1 | import { useEffect, useRef } from 'react'; 2 | 3 | export const useAudio = (audioSrc) => { 4 | const audioRef = useRef(null); 5 | 6 | useEffect(() => { 7 | const audio = audioRef.current; 8 | if (audio && audioSrc) { 9 | audio.src = audioSrc; 10 | audio.play().catch((error) => { 11 | console.log("Autoplay was prevented. Please interact with the page to play audio."); 12 | }); 13 | } 14 | }, [audioSrc]); 15 | 16 | return audioRef; 17 | }; -------------------------------------------------------------------------------- /frontend/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | }, 10 | { 11 | "src": "logo192.png", 12 | "type": "image/png", 13 | "sizes": "192x192" 14 | }, 15 | { 16 | "src": "logo512.png", 17 | "type": "image/png", 18 | "sizes": "512x512" 19 | } 20 | ], 21 | "start_url": ".", 22 | "display": "standalone", 23 | "theme_color": "#000000", 24 | "background_color": "#ffffff" 25 | } 26 | -------------------------------------------------------------------------------- /backend/src/scenarios/ScenarioFactory.ts: -------------------------------------------------------------------------------- 1 | import { SupermarketScenario } from './Supermarket' 2 | import { RestaurantScenario } from './Restaurant' 3 | import { TrainStationScenario } from './TrainStation' 4 | import { Scenario } from './Scenario' 5 | 6 | export class ScenarioFactory { 7 | static createScenario(scenario: string): Scenario { 8 | switch (scenario) { 9 | case 'supermarket': 10 | return new SupermarketScenario() 11 | case 'restaurant': 12 | return new RestaurantScenario() 13 | case 'train station': 14 | return new TrainStationScenario() 15 | default: 16 | throw new Error(`Scenario "${scenario}" not found.`) 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /frontend/src/services/audioService.js: -------------------------------------------------------------------------------- 1 | export const saveAudioLocally = (recordedFile) => { 2 | if (!recordedFile) return; 3 | 4 | const fileName = `recording_${new Date().toISOString().replace(/:/g, '-')}.webm`; 5 | const downloadLink = document.createElement('a'); 6 | downloadLink.href = URL.createObjectURL(recordedFile); 7 | downloadLink.download = fileName; 8 | downloadLink.click(); 9 | 10 | setTimeout(() => { 11 | URL.revokeObjectURL(downloadLink.href); 12 | }, 100); 13 | }; 14 | 15 | export const initializeMicrophone = async () => { 16 | try { 17 | await navigator.mediaDevices.getUserMedia({ audio: true }); 18 | return true; 19 | } catch (err) { 20 | return false; 21 | } 22 | }; -------------------------------------------------------------------------------- /backend/.gitignore: -------------------------------------------------------------------------------- 1 | # Node modules 2 | /node_modules/ 3 | 4 | # TypeScript output (compiled JavaScript files) 5 | /dist/ 6 | 7 | # Environment variables 8 | .env 9 | 10 | # Logs 11 | *.log 12 | npm-debug.log* 13 | yarn-debug.log* 14 | yarn-error.log* 15 | 16 | # OS-specific files 17 | .DS_Store # macOS 18 | Thumbs.db # Windows 19 | 20 | # IDE/editor files 21 | .vscode/ 22 | .idea/ 23 | 24 | # TypeScript & TSC output 25 | *.tsbuildinfo 26 | *.d.ts 27 | 28 | # Debug 29 | *.vscode/ 30 | *.idea/ 31 | 32 | # Dependency directories 33 | /pnpm-lock.yaml 34 | package-lock.json 35 | 36 | # Optional .env files 37 | .env.*.local 38 | 39 | # Ignore coverage reports generated by test frameworks 40 | /coverage/ 41 | -------------------------------------------------------------------------------- /frontend/src/components/common/Button.css: -------------------------------------------------------------------------------- 1 | .button { 2 | padding: 10px 20px; 3 | font-size: 16px; 4 | color: #ffffff; 5 | background-color: #007bff; 6 | border: none; 7 | border-radius: 5px; 8 | cursor: pointer; 9 | transition: background-color 0.3s; 10 | } 11 | 12 | .button:hover { 13 | background-color: #0056b3; 14 | } 15 | 16 | .button:disabled { 17 | opacity: 0.6; 18 | cursor: not-allowed; 19 | } 20 | 21 | .button.microphone-button { 22 | background-color: #E8618CFF; 23 | } 24 | 25 | .button.microphone-button:hover:not(:disabled) { 26 | background-color: #E8618CFF; 27 | } 28 | 29 | .button.start-button { 30 | background-color: #FF4081; 31 | } 32 | 33 | .button.start-button:hover { 34 | background-color: #e91e63; 35 | } -------------------------------------------------------------------------------- /frontend/src/hooks/useAudioManager.js: -------------------------------------------------------------------------------- 1 | import { useState, useCallback } from 'react'; 2 | import { useAudio } from './useAudio'; 3 | 4 | export const useAudioManager = () => { 5 | const audioRef = useAudio(); 6 | const [isPlaying, setIsPlaying] = useState(false); 7 | 8 | const playAudio = useCallback(async (audioSrc) => { 9 | if (!audioSrc || !audioRef.current) return; 10 | 11 | try { 12 | audioRef.current.pause(); 13 | audioRef.current.src = ''; 14 | audioRef.current.src = audioSrc; 15 | await audioRef.current.play(); 16 | } catch (err) { 17 | throw err; 18 | } 19 | }, [audioRef]); 20 | 21 | return { 22 | audioRef, 23 | isPlaying, 24 | setIsPlaying, 25 | playAudio 26 | }; 27 | }; -------------------------------------------------------------------------------- /backend/src/app.ts: -------------------------------------------------------------------------------- 1 | import express from 'express' 2 | import dotenv from 'dotenv' 3 | import { Clients } from './types' 4 | import { createResources } from './startup/createResources' 5 | import { createRoutes } from './startup/createRoutes' 6 | import cors from 'cors' 7 | 8 | dotenv.config() 9 | 10 | const run = async () => { 11 | 12 | const app = express() 13 | 14 | app.use(cors({ 15 | origin: 'http://localhost:3000', // Allow only this origin 16 | methods: ['GET', 'POST', 'PUT', 'DELETE'], // Allowed HTTP methods 17 | credentials: true, // Allow cookies and credentials 18 | })) 19 | 20 | const clients: Clients = await createResources() 21 | 22 | app.use(express.json()) 23 | app.use(createRoutes(clients)) 24 | 25 | const PORT = process.env.PORT || 3000 26 | app.listen(PORT, () => { 27 | console.log(`🚀 Server running on port ${PORT}`) 28 | }) 29 | } 30 | 31 | run() 32 | -------------------------------------------------------------------------------- /frontend/src/components/RolePlayPage/ControlButtons.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Button from '../common/Button'; 3 | import { faStop, faMicrophone, faLanguage } from '@fortawesome/free-solid-svg-icons'; 4 | import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; 5 | 6 | const ControlButtons = ({ 7 | isRecording, 8 | isPlaying, 9 | recordedFile, 10 | onToggleRecording, 11 | onSaveLocally, 12 | onToggleTranslation 13 | }) => ( 14 |
15 | 22 | 28 |
29 | ); 30 | 31 | export default ControlButtons; -------------------------------------------------------------------------------- /backend/src/clients/deepseek.ts: -------------------------------------------------------------------------------- 1 | import dotenv from 'dotenv' 2 | import OpenAI from "openai" 3 | import { ChatCompletion, ChatCompletionMessageParam } from 'openai/resources' 4 | 5 | dotenv.config() 6 | 7 | export class DeepseekClient { 8 | private client: OpenAI 9 | 10 | constructor() { 11 | this.client = new OpenAI({ 12 | baseURL: 'https://openrouter.ai/api/v1', 13 | apiKey: process.env.DEEPSEEK_KEY 14 | }) 15 | } 16 | 17 | public async completion(messages: ChatCompletionMessageParam[]): Promise { 18 | try { 19 | const response = await this.client.chat.completions.create({ 20 | messages, 21 | model: "deepseek/deepseek-chat", 22 | temperature: 0.7, 23 | max_tokens: 100 24 | }) 25 | console.log('Connected to deepseek!') 26 | return response 27 | } catch (error) { 28 | console.error('Error connecting to deepseek:', error) 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /frontend/src/components/RolePlayPage/TranslationDisplay.css: -------------------------------------------------------------------------------- 1 | .translation-display { 2 | width: 100%; 3 | max-width: 600px; 4 | margin-top: 2rem; 5 | } 6 | 7 | .translation-content { 8 | background: white; 9 | border-radius: 16px; 10 | padding: 1.5rem; 11 | box-shadow: 0 4px 6px rgba(0, 0, 0, 0.05); 12 | margin-bottom: 1rem; 13 | } 14 | 15 | .translation-text { 16 | font-size: 1.125rem; 17 | line-height: 1.6; 18 | color: #212529; 19 | margin: 0; 20 | } 21 | 22 | .language-toggle { 23 | display: flex; 24 | justify-content: center; 25 | gap: 0.5rem; 26 | } 27 | 28 | .toggle-button { 29 | padding: 0.5rem 1rem; 30 | border: none; 31 | border-radius: 8px; 32 | background: white; 33 | color: #495057; 34 | font-weight: 500; 35 | cursor: pointer; 36 | transition: all 0.2s ease; 37 | } 38 | 39 | .toggle-button:hover { 40 | background: #e9ecef; 41 | } 42 | 43 | .toggle-button.active { 44 | background: #228be6; 45 | color: white; 46 | } -------------------------------------------------------------------------------- /frontend/src/components/Conversation/Conversation.css: -------------------------------------------------------------------------------- 1 | .conversation { 2 | padding: 20px; 3 | border: 1px solid #dee2e6; 4 | border-radius: 10px; 5 | background-color: #ffffff; 6 | box-shadow: 0 4px 10px rgba(0, 0, 0, 0.1); 7 | max-width: 600px; 8 | margin: 0 auto; 9 | } 10 | 11 | .conversation h2 { 12 | margin-bottom: 20px; 13 | font-size: 24px; 14 | color: #343a40; 15 | text-align: center; 16 | } 17 | 18 | .conversation div { 19 | margin-bottom: 20px; 20 | } 21 | 22 | .conversation div strong { 23 | color: #007bff; 24 | } 25 | 26 | .conversation button { 27 | margin-right: 10px; 28 | padding: 10px 20px; 29 | font-size: 16px; 30 | color: #ffffff; 31 | background-color: #007bff; 32 | border: none; 33 | border-radius: 5px; 34 | cursor: pointer; 35 | transition: background-color 0.3s; 36 | } 37 | 38 | .conversation button:hover { 39 | background-color: #0056b3; 40 | } 41 | 42 | .conversation button:disabled { 43 | opacity: 0.6; 44 | cursor: not-allowed; 45 | } -------------------------------------------------------------------------------- /backend/src/services/TranslationService.ts: -------------------------------------------------------------------------------- 1 | // src/services/TranslationService.ts 2 | import { ChatCompletionMessageParam } from 'openai/resources' 3 | import { Clients } from '../types' 4 | 5 | export class TranslationService { 6 | private clients: Clients 7 | 8 | constructor(clients: Clients) { 9 | this.clients = clients 10 | } 11 | 12 | public async translateToEnglish(germanText: string): Promise { 13 | const systemPrompt = "You are a German to English translator. Translate the following German text to English. Only respond with the English translation, nothing else." 14 | const messages: ChatCompletionMessageParam[] = [ 15 | { role: 'system', content: systemPrompt }, 16 | { role: 'user', content: germanText } 17 | ] 18 | 19 | try { 20 | const response = await this.clients.deepseek.completion(messages) 21 | return response?.choices[0]?.message?.content || '' 22 | } catch (error) { 23 | console.error('Error translating text:', error) 24 | throw new Error('Failed to translate text') 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /frontend/src/components/RolePlayPage/TranslationDisplay.js: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | import './TranslationDisplay.css'; 3 | 4 | const TranslationDisplay = ({ germanText, englishText }) => { 5 | const [selectedLanguage, setSelectedLanguage] = useState('german'); 6 | 7 | return ( 8 |
9 |
10 |

11 | {selectedLanguage === 'german' ? germanText : englishText} 12 |

13 |
14 | 15 |
16 | 22 | 28 |
29 |
30 | ); 31 | }; 32 | 33 | export default TranslationDisplay; -------------------------------------------------------------------------------- /backend/src/clients/gTTS.ts: -------------------------------------------------------------------------------- 1 | import { existsSync, mkdirSync } from 'fs' 2 | // @ts-ignore 3 | import gTTS from 'gtts' 4 | import { join } from 'path' 5 | 6 | export class GTTSClient { 7 | private language: string 8 | 9 | constructor(language: string = 'de') { 10 | this.language = language 11 | } 12 | 13 | public async convertTextToAudio(text: string): Promise { 14 | return new Promise((resolve, reject) => { 15 | const gtts = new gTTS(text, this.language) 16 | const audioDir = join(__dirname, 'audio') // Use a valid directory path 17 | const audioFilePath = join(audioDir, `${Date.now()}.mp3`) // Unique file name 18 | 19 | // Create the audio directory if it doesn't exist 20 | if (!existsSync(audioDir)) { 21 | mkdirSync(audioDir, { recursive: true }) 22 | } 23 | 24 | gtts.save(audioFilePath, (err: any) => { 25 | if (err) { 26 | console.error('Error converting text to audio:', err) 27 | reject(err) 28 | } else { 29 | console.log('Audio file saved:', audioFilePath) 30 | resolve(audioFilePath) 31 | } 32 | }) 33 | }) 34 | } 35 | } -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lex-frontend", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@ffmpeg/core": "^0.11.0", 7 | "@ffmpeg/ffmpeg": "^0.11.6", 8 | "@fortawesome/free-solid-svg-icons": "^6.7.2", 9 | "@fortawesome/react-fontawesome": "^0.2.2", 10 | "axios": "^1.7.9", 11 | "cors": "^2.8.5", 12 | "cra-template": "1.2.0", 13 | "lucide-react": "^0.474.0", 14 | "react": "^19.0.0", 15 | "react-dom": "^19.0.0", 16 | "react-icons": "^5.4.0", 17 | "react-router-dom": "^7.1.5", 18 | "react-scripts": "5.0.1", 19 | "recordrtc": "^5.6.2" 20 | }, 21 | "scripts": { 22 | "start": "react-scripts start", 23 | "build": "react-scripts build", 24 | "test": "react-scripts test", 25 | "eject": "react-scripts eject" 26 | }, 27 | "eslintConfig": { 28 | "extends": [ 29 | "react-app", 30 | "react-app/jest" 31 | ] 32 | }, 33 | "browserslist": { 34 | "production": [ 35 | ">0.2%", 36 | "not dead", 37 | "not op_mini all" 38 | ], 39 | "development": [ 40 | "last 1 chrome version", 41 | "last 1 firefox version", 42 | "last 1 safari version" 43 | ] 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /frontend/src/components/Conversation/Conversation.js: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | import { startConversationWithText } from '../../services/apiService'; 3 | import Button from '../common/Button'; 4 | import './Conversation.css'; 5 | 6 | const Conversation = ({ scenario, messages }) => { 7 | const [isListening, setIsListening] = useState(false); 8 | 9 | const handleStartRolePlay = async () => { 10 | try { 11 | const { audio: audioBlob } = await startConversationWithText(scenario); 12 | const audio = new Audio(URL.createObjectURL(audioBlob)); 13 | audio.play(); 14 | } catch (error) { 15 | console.error('Error starting role play:', error); 16 | } 17 | }; 18 | 19 | return ( 20 |
21 |

Conversation

22 |
23 | {messages.map((msg, index) => ( 24 |
25 | {msg.role}: {msg.content} 26 |
27 | ))} 28 |
29 | 30 | 33 |
34 | ); 35 | }; 36 | 37 | export default Conversation; -------------------------------------------------------------------------------- /backend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "langfuse-use-case", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "tsx src/app.ts", 8 | "dev": "tsx app.ts", 9 | "build": "tsc", 10 | "start:prod": "node dist/app.js", 11 | "lint": "eslint . --ext .ts", 12 | "test": "echo \"Error: no test specified\" && exit 1" 13 | }, 14 | "keywords": [], 15 | "author": "", 16 | "license": "ISC", 17 | "dependencies": { 18 | "@apla/clickhouse": "^1.6.4", 19 | "@aws-sdk/client-polly": "^3.738.0", 20 | "@clickhouse/client": "^1.10.1", 21 | "@google-cloud/speech": "^6.7.0", 22 | "axios": "^1.7.9", 23 | "cors": "^2.8.5", 24 | "dotenv": "^16.4.7", 25 | "express": "^4.21.2", 26 | "form-data": "^4.0.1", 27 | "fs": "^0.0.1-security", 28 | "gtts": "^0.2.1", 29 | "multer": "^1.4.5-lts.1", 30 | "openai": "^4.82.0", 31 | "pg": "^8.13.1", 32 | "ref-napi": "^3.0.3", 33 | "speech-to-text": "^2.9.1", 34 | "tsx": "^4.19.2", 35 | "whisper-node": "^1.1.1" 36 | }, 37 | "devDependencies": { 38 | "@types/cors": "^2.8.17", 39 | "@types/express": "^5.0.0", 40 | "@types/multer": "^1.4.12", 41 | "@types/node": "^22.12.0", 42 | "@types/pg": "^8.11.11", 43 | "typescript": "^5.7.3" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /backend/src/clients/pollyClient.ts: -------------------------------------------------------------------------------- 1 | import dotenv from 'dotenv' 2 | import { PollyClient as Client, SynthesizeSpeechCommand, VoiceId } from '@aws-sdk/client-polly' 3 | import { writeFileSync } from 'fs' 4 | 5 | dotenv.config() 6 | 7 | export class PollyClient { 8 | private client: Client 9 | 10 | constructor() { 11 | this.client = new Client({ 12 | region: process.env.AWS_REGION, 13 | credentials: { 14 | accessKeyId: process.env.AWS_ACCESS_KEY_ID!, 15 | secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!, 16 | }, 17 | }) 18 | } 19 | 20 | public async convertTextToAudio(text: string, voiceId: string = 'Hans'): Promise { 21 | const audioFilePath = `./audio/${Date.now()}.mp3` 22 | 23 | try { 24 | const command = new SynthesizeSpeechCommand({ 25 | OutputFormat: 'mp3', 26 | Text: text, 27 | VoiceId: voiceId as VoiceId, 28 | Engine: 'neural', 29 | }) 30 | 31 | const response = await this.client.send(command) 32 | 33 | if (response.AudioStream) { 34 | const audioStream = await response.AudioStream.transformToByteArray() 35 | writeFileSync(audioFilePath, Buffer.from(audioStream)) 36 | console.log('Audio file saved:', audioFilePath) 37 | return audioFilePath 38 | } else { 39 | throw new Error('No audio stream returned from Polly') 40 | } 41 | } catch (error) { 42 | console.error('Error converting text to audio:', error) 43 | throw error 44 | } 45 | } 46 | } -------------------------------------------------------------------------------- /frontend/src/components/ScenarioSelector/ScenarioSelector.css: -------------------------------------------------------------------------------- 1 | .scenario-selector { 2 | display: flex; 3 | flex-direction: column; 4 | align-items: center; 5 | justify-content: center; 6 | height: 100vh; 7 | background-color: #ffffff; 8 | font-family: 'Your Font', sans-serif; 9 | } 10 | 11 | .logo { 12 | width: 100px; 13 | margin-bottom: 20px; 14 | } 15 | 16 | .scenario-selector h2 { 17 | margin-bottom: 20px; 18 | font-size: 24px; 19 | color: #343a40; 20 | } 21 | 22 | .scenario-buttons { 23 | display: flex; 24 | gap: 15px; 25 | margin-bottom: 30px; 26 | } 27 | 28 | .scenario-buttons button { 29 | display: flex; 30 | flex-direction: column; 31 | align-items: center; 32 | padding: 20px; 33 | font-size: 16px; 34 | color: #495057; 35 | background-color: #ffffff; 36 | border: 1px solid #dee2e6; 37 | border-radius: 10px; 38 | cursor: pointer; 39 | transition: background-color 0.3s, transform 0.3s; 40 | } 41 | 42 | .scenario-buttons button:hover { 43 | background-color: #e9ecef; 44 | transform: translateY(-5px); 45 | } 46 | 47 | .scenario-buttons button.selected { 48 | border-color: #FF4081; 49 | } 50 | 51 | .scenario-buttons button svg { 52 | font-size: 24px; 53 | margin-bottom: 10px; 54 | color: #FF4081; 55 | } 56 | 57 | .start-button { 58 | padding: 15px 30px; 59 | font-size: 16px; 60 | color: #ffffff; 61 | background-color: #FF4081; 62 | border: none; 63 | border-radius: 10px; 64 | cursor: pointer; 65 | transition: background-color 0.3s; 66 | } 67 | 68 | .start-button:hover { 69 | background-color: #e91e63; 70 | } 71 | 72 | .start-button:disabled { 73 | opacity: 0.6; 74 | cursor: not-allowed; 75 | } -------------------------------------------------------------------------------- /frontend/src/services/apiService.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | 3 | const API_BASE_URL = 'http://localhost:8002'; 4 | 5 | export const startConversationWithAudio = async (scenarioName, audioFile) => { 6 | try { 7 | const formData = new FormData(); 8 | formData.append('scenarioName', scenarioName); 9 | formData.append('audio', audioFile, audioFile.name); 10 | 11 | const response = await axios.post( 12 | `${API_BASE_URL}/converse/audio`, 13 | formData, 14 | { 15 | headers: { 16 | 'Content-Type': 'multipart/form-data', 17 | }, 18 | responseType: 'json', 19 | } 20 | ); 21 | 22 | const { audio, germanText, englishText } = response.data; 23 | const audioBlob = new Blob( 24 | [Uint8Array.from(atob(audio), c => c.charCodeAt(0))], 25 | { type: 'audio/mpeg' } 26 | ); 27 | 28 | return { 29 | audio: audioBlob, 30 | germanText, 31 | englishText 32 | }; 33 | } catch (error) { 34 | throw error; 35 | } 36 | }; 37 | 38 | // Function for starting a conversation with text input 39 | export const startConversationWithText = async (scenarioName) => { 40 | try { 41 | const response = await axios.post( 42 | `${API_BASE_URL}/converse/text`, 43 | { scenarioName }, 44 | { responseType: 'json' } // Expecting an audio response 45 | ); 46 | 47 | const { audio, germanText, englishText } = response.data; 48 | const audioBlob = new Blob( 49 | [Uint8Array.from(atob(audio), c => c.charCodeAt(0))], 50 | { type: 'audio/mpeg' } 51 | ); 52 | 53 | return { 54 | audio: audioBlob, 55 | germanText, 56 | englishText 57 | }; 58 | } catch (error) { 59 | throw error; 60 | } 61 | }; 62 | -------------------------------------------------------------------------------- /frontend/src/hooks/useSpeechRecognition.js: -------------------------------------------------------------------------------- 1 | import { useState, useRef } from "react"; 2 | 3 | export const useSpeechRecognition = (onRecordingComplete) => { 4 | const [isRecording, setIsRecording] = useState(false); 5 | const mediaRecorderRef = useRef(null); 6 | const audioChunksRef = useRef([]); 7 | const streamRef = useRef(null); 8 | 9 | const toggleRecording = async () => { 10 | if (isRecording) { 11 | mediaRecorderRef.current.stop(); 12 | streamRef.current.getTracks().forEach((track) => track.stop()); 13 | setIsRecording(false); 14 | } else { 15 | try { 16 | const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); 17 | streamRef.current = stream; 18 | const tracks = stream.getAudioTracks(); 19 | tracks.forEach(track => track.getSettings()); 20 | 21 | const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); 22 | const supportedType = isSafari ? "audio/mp4" : "audio/webm;codecs=opus"; 23 | const mediaRecorder = new MediaRecorder(stream, { mimeType: supportedType }); 24 | 25 | mediaRecorderRef.current = mediaRecorder; 26 | audioChunksRef.current = []; 27 | 28 | mediaRecorder.ondataavailable = (event) => { 29 | if (event.data.size > 0) { 30 | audioChunksRef.current.push(event.data); 31 | } 32 | }; 33 | 34 | mediaRecorder.onstop = () => { 35 | const audioBlob = new Blob(audioChunksRef.current, { type: supportedType }); 36 | if (audioBlob.size > 0) { 37 | const file = new File([audioBlob], "recording.webm", { type: supportedType }); 38 | onRecordingComplete(file); 39 | } 40 | }; 41 | 42 | mediaRecorder.start(); 43 | setIsRecording(true); 44 | } catch (error) { 45 | setIsRecording(false); 46 | } 47 | } 48 | }; 49 | 50 | return { isRecording, toggleRecording }; 51 | }; 52 | -------------------------------------------------------------------------------- /backend/src/services/ConversationService.ts: -------------------------------------------------------------------------------- 1 | import { ScenarioFactory } from '../scenarios/ScenarioFactory' 2 | import { ChatCompletionMessageParam } from 'openai/resources' 3 | import { Clients, ScenarioStates } from '../types' 4 | import { TranslationService } from './TranslationService' 5 | 6 | export class ConversationService { 7 | private clients: Clients 8 | private translationService: TranslationService 9 | 10 | constructor(clients: Clients) { 11 | this.clients = clients 12 | this.translationService = new TranslationService(clients) 13 | } 14 | 15 | public async converse(scenario: string, messages: ChatCompletionMessageParam[]) { 16 | const { text } = await this.conversationText(scenario, messages) 17 | const audioFilePath = await this.clients.gTTS.convertTextToAudio(text) 18 | const translation = await this.translationService.translateToEnglish(text) 19 | return { text, translation, audioFilePath } 20 | } 21 | 22 | private conversationText = async (scenario: string, messages: ChatCompletionMessageParam[]) => { 23 | const scenarioInstance = ScenarioFactory.createScenario(scenario) 24 | const isConversationNew = messages.length === 0 25 | const state = isConversationNew ? ScenarioStates.START : ScenarioStates.CONTINUE 26 | 27 | const systemPrompt = scenarioInstance.getSystemPrompt(state) 28 | const systemMessage: ChatCompletionMessageParam = { role: 'system', content: systemPrompt } 29 | 30 | const defaultUserMessage: ChatCompletionMessageParam = { role: 'user', content: 'Hallo!' } 31 | const conversationMessages = isConversationNew ? [systemMessage, defaultUserMessage] : [systemMessage, ...messages] 32 | 33 | try { 34 | const response = await this.clients.deepseek.completion(conversationMessages) 35 | const generatedText = response?.choices[0]?.message?.content || '' 36 | return { text: generatedText } 37 | } catch (error) { 38 | throw new Error('Failed to generate response') 39 | } 40 | } 41 | } -------------------------------------------------------------------------------- /backend/src/scenarios/Restaurant.ts: -------------------------------------------------------------------------------- 1 | import { ScenarioStates } from '../types' 2 | import { Scenario } from './Scenario' 3 | 4 | const START_CONVERSATION = ` 5 | You are Lex, a waiter at a German restaurant. Your role is to take orders and assist customers in German. 6 | Always respond in German and maintain a professional yet friendly tone. 7 | 8 | Start the conversation by greeting the customer warmly and asking if they are ready to order. 9 | For example: 10 | - "Guten Tag! Willkommen in unserem Restaurant. Sind Sie bereit zu bestellen?" 11 | - "Hallo! Schön, Sie bei uns zu haben. Möchten Sie bereits bestellen?" 12 | 13 | Do not wait for a user message. Initiate the conversation as if the customer has just walked in. 14 | 15 | Keep the response to less than 50 characters. 16 | ` 17 | 18 | const CONTINUE_CONVERSATION = ` 19 | You are Lex, a waiter at a German restaurant. Your role is to take orders and assist customers in German. 20 | Always respond in German and maintain a professional yet friendly tone. 21 | 22 | Continue the conversation based on the customer's previous messages. For example: 23 | - If the customer is ready to order, ask for their choices or suggest popular dishes. 24 | - If the customer has questions about the menu, provide clear and helpful answers. 25 | - If the customer seems unsure, offer recommendations or ask clarifying questions. 26 | 27 | Keep the conversation natural and engaging, and ensure the customer feels well taken care of. 28 | 29 | Keep the response to less than 50 characters. 30 | ` 31 | 32 | export class RestaurantScenario extends Scenario { 33 | constructor() { 34 | super('waiter', 'professional yet friendly') 35 | } 36 | 37 | getSystemPrompt(state: ScenarioStates): string { 38 | switch (state) { 39 | case ScenarioStates.START: 40 | return START_CONVERSATION 41 | case ScenarioStates.CONTINUE: 42 | return CONTINUE_CONVERSATION 43 | default: 44 | throw new Error(`Invalid scenario state: ${state}`) 45 | } 46 | } 47 | } -------------------------------------------------------------------------------- /backend/src/scenarios/TrainStation.ts: -------------------------------------------------------------------------------- 1 | import { ScenarioStates } from '../types' 2 | import { Scenario } from './Scenario' 3 | 4 | const START_CONVERSATION = ` 5 | You are Lex, a ticket inspector at a German train station. Your role is to check tickets and assist passengers in German. 6 | Always respond in German and maintain a polite but authoritative tone. 7 | 8 | Start the conversation by greeting the passenger and asking to see their ticket. 9 | For example: 10 | - "Guten Tag! Darf ich bitte Ihr Ticket sehen?" 11 | - "Hallo! Haben Sie Ihr Ticket zur Hand?" 12 | 13 | Do not wait for a user message. Initiate the conversation as if the passenger has just boarded the train. 14 | 15 | Keep the response to less than 50 characters. 16 | ` 17 | 18 | const CONTINUE_CONVERSATION = ` 19 | You are Lex, a ticket inspector at a German train station. Your role is to check tickets and assist passengers in German. 20 | Always respond in German and maintain a polite but authoritative tone. 21 | 22 | Continue the conversation based on the passenger's previous messages. For example: 23 | - If the passenger has a valid ticket, thank them and wish them a pleasant journey. 24 | - If the passenger has an issue with their ticket, explain the problem and guide them on how to resolve it. 25 | - If the passenger asks for directions or assistance, provide clear and helpful information. 26 | 27 | Keep the conversation professional and ensure the passenger feels respected and assisted. 28 | 29 | Keep the response to less than 50 characters. 30 | ` 31 | 32 | export class TrainStationScenario extends Scenario { 33 | constructor() { 34 | super('ticket inspector', 'polite but authoritative') 35 | } 36 | 37 | getSystemPrompt(state: ScenarioStates): string { 38 | switch (state) { 39 | case ScenarioStates.START: 40 | return START_CONVERSATION 41 | case ScenarioStates.CONTINUE: 42 | return CONTINUE_CONVERSATION 43 | default: 44 | throw new Error(`Invalid scenario state: ${state}`) 45 | } 46 | } 47 | } -------------------------------------------------------------------------------- /backend/src/scenarios/Supermarket.ts: -------------------------------------------------------------------------------- 1 | import { ScenarioStates } from '../types' 2 | import { Scenario } from './Scenario' 3 | 4 | const START_CONVERSATION = ` 5 | You are Lex, a friendly supermarket cashier. Your role is to assist customers in German. 6 | Always respond in German and maintain a polite and helpful tone. 7 | 8 | Start the conversation by greeting the customer warmly and asking if they need any assistance. 9 | For example: 10 | - "Guten Tag! Willkommen in unserem Supermarkt. Kann ich Ihnen bei etwas helfen?" 11 | - "Hallo! Schön, Sie bei uns zu sehen. Brauchen Sie Hilfe beim Finden von Produkten?" 12 | 13 | Do not wait for a user message. Initiate the conversation as if the customer has just approached the counter. 14 | 15 | Keep the response to less than 50 characters. 16 | ` 17 | 18 | const CONTINUE_CONVERSATION = ` 19 | You are Lex, a friendly supermarket cashier. Your role is to assist customers in German. 20 | Always respond in German and maintain a polite and helpful tone. 21 | 22 | Continue the conversation based on the customer's previous messages. For example: 23 | - If the customer is looking for a specific product, guide them to the correct aisle or shelf. 24 | - If the customer is ready to check out, ask if they have a loyalty card or need a bag. 25 | - If the customer has questions about prices or promotions, provide clear and accurate information. 26 | 27 | Keep the conversation natural and engaging, and ensure the customer feels well taken care of. 28 | 29 | Keep the response to less than 50 characters. 30 | ` 31 | 32 | export class SupermarketScenario extends Scenario { 33 | constructor() { 34 | super('supermarket cashier', 'polite and helpful') 35 | } 36 | 37 | getSystemPrompt(state: ScenarioStates): string { 38 | switch (state) { 39 | case ScenarioStates.START: 40 | return START_CONVERSATION 41 | case ScenarioStates.CONTINUE: 42 | return CONTINUE_CONVERSATION 43 | default: 44 | throw new Error(`Invalid scenario state: ${state}`) 45 | } 46 | } 47 | } -------------------------------------------------------------------------------- /frontend/src/components/RolePlayPage/RolePlayPage.css: -------------------------------------------------------------------------------- 1 | .role-play-page { 2 | display: flex; 3 | flex-direction: column; 4 | align-items: center; 5 | justify-content: center; 6 | padding: 5px; 7 | background-color: #ffffff; 8 | border-radius: 10px; 9 | max-width: 600px; 10 | margin: 0 auto; 11 | } 12 | 13 | .speech-wave-container { 14 | display: flex; 15 | justify-content: center; 16 | align-items: center; 17 | width: 200px; 18 | height: 200px; 19 | border-radius: 50%; 20 | background-color: #ffffff; 21 | box-shadow: 0 4px 10px rgba(0, 0, 0, 0.1); 22 | margin-top: 20px; 23 | } 24 | 25 | .speech-wave { 26 | display: flex; 27 | align-items: center; 28 | justify-content: center; 29 | gap: 8px; 30 | } 31 | 32 | .wave { 33 | width: 10px; 34 | height: 40px; 35 | background-color: #007bff; 36 | border-radius: 5px; 37 | animation: wave 1.2s infinite ease-in-out; 38 | animation-play-state: paused; 39 | } 40 | 41 | .speech-wave.active .wave { 42 | animation-play-state: running; 43 | } 44 | 45 | @keyframes wave { 46 | 47 | 0%, 48 | 60%, 49 | 100% { 50 | transform: scaleY(0.4); 51 | } 52 | 53 | 20% { 54 | transform: scaleY(1); 55 | } 56 | } 57 | 58 | .button-container { 59 | display: flex; 60 | justify-content: center; 61 | gap: 20px; 62 | margin-top: 20px; 63 | } 64 | 65 | .control-button { 66 | width: 60px; 67 | height: 60px; 68 | border-radius: 50%; 69 | border: none; 70 | background-color: #007bff; 71 | color: white; 72 | font-size: 14px; 73 | cursor: pointer; 74 | display: flex; 75 | align-items: center; 76 | justify-content: center; 77 | transition: background-color 0.3s ease; 78 | } 79 | 80 | .control-button:disabled { 81 | background-color: #cccccc; 82 | cursor: not-allowed; 83 | } 84 | 85 | .microphone-button { 86 | background-color: #E8618CFF; 87 | } 88 | 89 | .microphone-button:hover:not(:disabled) { 90 | background-color: #E8618CFF; 91 | } 92 | 93 | .role-play-page p { 94 | margin-top: 20px; 95 | font-size: 16px; 96 | color: #343a40; 97 | } -------------------------------------------------------------------------------- /frontend/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 17 | 18 | 27 | React App 28 | 29 | 30 | 31 | 32 |
33 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /frontend/src/components/AudioRecorder.js: -------------------------------------------------------------------------------- 1 | import React, { useState, useRef } from 'react'; 2 | import RecordRTC from 'recordrtc'; 3 | 4 | const AudioRecorder = () => { 5 | const [audioUrl, setAudioUrl] = useState(null); 6 | const [recording, setRecording] = useState(false); 7 | const recorder = useRef(null); 8 | 9 | // Start recording 10 | const startRecording = async () => { 11 | const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); 12 | recorder.current = new RecordRTC(stream, { 13 | type: 'audio', 14 | mimeType: 'audio/wav', 15 | recorderType: RecordRTC.AudioRecorder, // Use Mono instead of Stereo 16 | }); 17 | recorder.current.startRecording(); 18 | setRecording(true); 19 | }; 20 | 21 | // Stop recording with a delay to ensure the Blob is properly generated 22 | const stopRecording = async () => { 23 | await recorder.current.stopRecording(); 24 | 25 | // Add a slight delay before checking the Blob 26 | setTimeout(() => { 27 | const audioBlob = recorder.current.getBlob(); 28 | 29 | // Log the audioBlob to check what it's returning 30 | console.log("Audio Blob:", audioBlob); 31 | 32 | // Check if the Blob is valid 33 | if (audioBlob && audioBlob.size > 0) { 34 | const audioUrl = URL.createObjectURL(audioBlob); 35 | setAudioUrl(audioUrl); 36 | } else { 37 | console.error('Failed to record audio, no valid blob or Blob is empty.'); 38 | } 39 | setRecording(false); 40 | }, 500); // Delay of 500ms 41 | }; 42 | 43 | return ( 44 |
45 | 51 | 52 | {audioUrl && ( 53 | 59 | )} 60 |
61 | ); 62 | }; 63 | 64 | export default AudioRecorder; -------------------------------------------------------------------------------- /frontend/src/logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /backend/src/services/AudioToTextService.ts: -------------------------------------------------------------------------------- 1 | import axios from 'axios' 2 | import fs from 'fs' 3 | 4 | export class AudioToTextService { 5 | private apiKey: string 6 | 7 | constructor(apiKey: string) { 8 | if (!apiKey) { 9 | throw new Error('AssemblyAI API key is required') 10 | } 11 | this.apiKey = apiKey 12 | } 13 | 14 | public async convertAudioToText(audioFilePath: string): Promise { 15 | try { 16 | if (!fs.existsSync(audioFilePath)) { 17 | throw new Error(`Audio file not found at ${audioFilePath}`) 18 | } 19 | 20 | const uploadResponse = await this.uploadAudio(audioFilePath) 21 | const transcriptResponse = await this.createTranscriptionJob(uploadResponse.upload_url) 22 | const finalTranscript = await this.pollTranscriptionStatus(transcriptResponse.id) 23 | 24 | return finalTranscript 25 | } catch (error) { 26 | throw error 27 | } 28 | } 29 | 30 | private async uploadAudio(filePath: string) { 31 | const fileStream = fs.createReadStream(filePath) 32 | 33 | const response = await axios.post('https://api.assemblyai.com/v2/upload', fileStream, { 34 | headers: { 35 | 'Authorization': this.apiKey, 36 | 'Content-Type': 'application/octet-stream' 37 | } 38 | }) 39 | 40 | return response.data 41 | } 42 | 43 | private async createTranscriptionJob(audioUrl: string) { 44 | const response = await axios.post('https://api.assemblyai.com/v2/transcript', 45 | { audio_url: audioUrl, language_code: 'de' }, 46 | { 47 | headers: { 48 | 'Authorization': this.apiKey, 49 | 'Content-Type': 'application/json' 50 | } 51 | } 52 | ) 53 | 54 | return response.data 55 | } 56 | 57 | private async pollTranscriptionStatus(transcriptId: string): Promise { 58 | const maxAttempts = 30 59 | const intervalMs = 3000 60 | 61 | for (let attempt = 0;attempt < maxAttempts;attempt++) { 62 | const response = await axios.get(`https://api.assemblyai.com/v2/transcript/${transcriptId}`, { 63 | headers: { 64 | 'Authorization': this.apiKey 65 | } 66 | }) 67 | 68 | const status = response.data.status 69 | 70 | switch (status) { 71 | case 'completed': 72 | return response.data.text 73 | case 'error': 74 | throw new Error(`Transcription failed: ${response.data.error}`) 75 | case 'queued': 76 | case 'processing': 77 | await new Promise(resolve => setTimeout(resolve, intervalMs)) 78 | break 79 | default: 80 | throw new Error(`Unexpected status: ${status}`) 81 | } 82 | } 83 | 84 | throw new Error('Transcription timed out') 85 | } 86 | } -------------------------------------------------------------------------------- /backend/src/startup/createRoutes.ts: -------------------------------------------------------------------------------- 1 | import { Router, Request, Response } from 'express' 2 | import multer from 'multer' 3 | import { AudioToTextService } from '../services/AudioToTextService' 4 | import { ConversationService } from '../services/ConversationService' 5 | import { readFileSync } from 'fs' 6 | import path from 'path' 7 | 8 | // FOR DEBUGGING 9 | const storage = multer.diskStorage({ 10 | destination: 'uploads/', 11 | filename: (req, file, cb) => { 12 | const ext = path.extname(file.originalname) || '.mp3' // Get original extension or default to .mp3 13 | const filename = `${Date.now()}-${file.originalname.replace(/\s+/g, '_')}` // Generate unique filename 14 | cb(null, filename) 15 | }, 16 | }) 17 | 18 | const upload = multer({ storage }) 19 | 20 | export const createRoutes = (clients: any) => { 21 | const router = Router() 22 | const apiKey = process.env.AUDIO_API_KEY || '' 23 | const audioToTextService = new AudioToTextService(apiKey) 24 | const conversationService = new ConversationService(clients) 25 | 26 | // Endpoint for handling audio file input 27 | // @ts-ignore 28 | router.post('/converse/audio', upload.single('audio'), async (req: Request, res: Response) => { 29 | if (!req.file) { 30 | return res.status(400).send({ error: 'No audio file uploaded' }) 31 | } 32 | 33 | try { 34 | const audioFilePath = req.file.path 35 | const scenarioName = req.body.scenarioName 36 | 37 | const transcribedText = await audioToTextService.convertAudioToText(audioFilePath) 38 | 39 | const { audioFilePath: responseAudioPath, text, translation } = await conversationService.converse(scenarioName, [ 40 | { role: 'user', content: transcribedText }, 41 | ]) 42 | 43 | const audioFile = readFileSync(responseAudioPath) 44 | 45 | res.setHeader('Content-Type', 'application/json') 46 | res.send({ 47 | audio: audioFile.toString('base64'), // Base64 encode the audio data 48 | germanText: text, 49 | englishText: translation 50 | }) 51 | } catch (error) { 52 | console.error('Error in /converse/audio:', error) 53 | res.status(500).send({ error: 'Failed to process audio' }) 54 | } 55 | }) 56 | 57 | // Endpoint for handling direct text input 58 | router.post('/converse/text', async (req: Request, res: Response) => { 59 | try { 60 | const scenarioName = req.body.scenarioName 61 | const userText = req.body.text || 'hello' 62 | 63 | const { audioFilePath: responseAudioPath, text, translation } = await conversationService.converse(scenarioName, [ 64 | { role: 'user', content: userText }, 65 | ]) 66 | 67 | const audioFile = readFileSync(responseAudioPath) 68 | 69 | res.setHeader('Content-Type', 'application/json') 70 | res.send({ 71 | audio: audioFile.toString('base64'), 72 | germanText: text, 73 | englishText: translation 74 | }) 75 | } catch (error) { 76 | console.error('Error in /converse/text:', error) 77 | res.status(500).send({ error: 'Failed to generate response' }) 78 | } 79 | }) 80 | 81 | return router 82 | } 83 | -------------------------------------------------------------------------------- /frontend/src/components/ScenarioSelector/ScenarioSelector.js: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | import { FaShoppingCart, FaUtensils, FaTrain } from 'react-icons/fa'; 3 | import { startConversationWithText } from '../../services/apiService'; 4 | import RolePlayPage from '../RolePlayPage/RolePlayPage'; 5 | import Button from '../common/Button'; 6 | import Icon from '../common/Icon'; 7 | import './ScenarioSelector.css'; 8 | 9 | const ScenarioSelector = () => { 10 | const [selectedScenario, setSelectedScenario] = useState(null); 11 | const [audioSrc, setAudioSrc] = useState(null); 12 | const [germanText, setGermanText] = useState(''); 13 | const [englishText, setEnglishText] = useState(''); 14 | const [isLoading, setIsLoading] = useState(false); 15 | const [showRolePlayPage, setShowRolePlayPage] = useState(false); 16 | 17 | const scenarios = [ 18 | { name: 'supermarket', icon: }, 19 | { name: 'restaurant', icon: }, 20 | { name: 'train station', icon: }, 21 | ]; 22 | 23 | const handleScenarioSelect = (scenario) => { 24 | setSelectedScenario(scenario); 25 | }; 26 | 27 | const handleStartRolePlay = async () => { 28 | if (!selectedScenario) { 29 | alert('Please select a scenario first.'); 30 | return; 31 | } 32 | 33 | setIsLoading(true); 34 | 35 | try { 36 | // Get audio and texts from the API 37 | const { audio, germanText, englishText } = await startConversationWithText(selectedScenario); 38 | const audioUrl = URL.createObjectURL(audio); 39 | 40 | // Store texts in state 41 | setAudioSrc(audioUrl); 42 | setGermanText(germanText); 43 | setEnglishText(englishText); 44 | 45 | setShowRolePlayPage(true); 46 | } catch (error) { 47 | console.error('Error starting role play:', error); 48 | alert('Failed to start role play. Please try again.'); 49 | } finally { 50 | setIsLoading(false); 51 | } 52 | }; 53 | 54 | return ( 55 |
56 |
57 | Logo 58 |
59 |

Choose your scenario

60 |
61 | {scenarios.map((scenario, index) => ( 62 | 70 | ))} 71 |
72 | 73 | 76 | 77 | {showRolePlayPage && ( 78 | 84 | )} 85 |
86 | ); 87 | }; 88 | 89 | export default ScenarioSelector; 90 | -------------------------------------------------------------------------------- /frontend/README.md: -------------------------------------------------------------------------------- 1 | # Getting Started with Create React App 2 | 3 | This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). 4 | 5 | ## Available Scripts 6 | 7 | In the project directory, you can run: 8 | 9 | ### `npm start` 10 | 11 | Runs the app in the development mode.\ 12 | Open [http://localhost:3000](http://localhost:3000) to view it in your browser. 13 | 14 | The page will reload when you make changes.\ 15 | You may also see any lint errors in the console. 16 | 17 | ### `npm test` 18 | 19 | Launches the test runner in the interactive watch mode.\ 20 | See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information. 21 | 22 | ### `npm run build` 23 | 24 | Builds the app for production to the `build` folder.\ 25 | It correctly bundles React in production mode and optimizes the build for the best performance. 26 | 27 | The build is minified and the filenames include the hashes.\ 28 | Your app is ready to be deployed! 29 | 30 | See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information. 31 | 32 | ### `npm run eject` 33 | 34 | **Note: this is a one-way operation. Once you `eject`, you can't go back!** 35 | 36 | If you aren't satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project. 37 | 38 | Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you're on your own. 39 | 40 | You don't have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn't feel obligated to use this feature. However we understand that this tool wouldn't be useful if you couldn't customize it when you are ready for it. 41 | 42 | ## Learn More 43 | 44 | You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started). 45 | 46 | To learn React, check out the [React documentation](https://reactjs.org/). 47 | 48 | ### Code Splitting 49 | 50 | This section has moved here: [https://facebook.github.io/create-react-app/docs/code-splitting](https://facebook.github.io/create-react-app/docs/code-splitting) 51 | 52 | ### Analyzing the Bundle Size 53 | 54 | This section has moved here: [https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size](https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size) 55 | 56 | ### Making a Progressive Web App 57 | 58 | This section has moved here: [https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app](https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app) 59 | 60 | ### Advanced Configuration 61 | 62 | This section has moved here: [https://facebook.github.io/create-react-app/docs/advanced-configuration](https://facebook.github.io/create-react-app/docs/advanced-configuration) 63 | 64 | ### Deployment 65 | 66 | This section has moved here: [https://facebook.github.io/create-react-app/docs/deployment](https://facebook.github.io/create-react-app/docs/deployment) 67 | 68 | ### `npm run build` fails to minify 69 | 70 | This section has moved here: [https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify](https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify) 71 | -------------------------------------------------------------------------------- /frontend/src/components/RolePlayPage/RolePlayPage.js: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useCallback, useState } from 'react'; 2 | import { startConversationWithAudio } from '../../services/apiService'; 3 | import { useSpeechRecognition } from '../../hooks/useSpeechRecognition'; 4 | import { useAudioManager } from '../../hooks/useAudioManager'; 5 | import { saveAudioLocally, initializeMicrophone } from '../../services/audioService'; 6 | import SpeechWave from './SpeechWave'; 7 | import ControlButtons from './ControlButtons'; 8 | import './RolePlayPage.css'; 9 | import TranslationDisplay from './TranslationDisplay'; 10 | 11 | const RolePlayPage = ({ selectedScenario, audioSrc, german, english }) => { 12 | const { audioRef, isPlaying, setIsPlaying, playAudio } = useAudioManager(); 13 | const [recordedFile, setRecordingFile] = useState(null); 14 | const [germanText, setGermanText] = useState(german); 15 | const [englishText, setEnglishText] = useState(english); 16 | const [toggleTranslation, setToggleTranslation] = useState(false); 17 | 18 | useEffect(() => { 19 | setGermanText(german); 20 | }, [german]) 21 | 22 | useEffect(() => { 23 | setEnglishText(english); 24 | }, [english]) 25 | 26 | const { isRecording, toggleRecording } = useSpeechRecognition((file) => { 27 | if (file && file.size > 0) { 28 | setRecordingFile(file); 29 | } else { 30 | console.error('Error: Recorded file is empty.'); 31 | } 32 | }); 33 | 34 | const handleSaveLocally = useCallback(() => { 35 | saveAudioLocally(recordedFile); 36 | }, [recordedFile]); 37 | 38 | // Send recorded audio file to endpoint and receive response 39 | const sendMessageToEndpoint = useCallback(async (file) => { 40 | if (!file || file.size === 0) { 41 | console.error('Error: Attempted to send an empty file.'); 42 | return; 43 | } 44 | 45 | try { 46 | const { audio, germanText, englishText } = await startConversationWithAudio(selectedScenario, file); 47 | const newAudioUrl = URL.createObjectURL(audio); 48 | await playAudio(newAudioUrl); 49 | 50 | // Set the texts in the state 51 | setGermanText(germanText); 52 | setEnglishText(englishText); 53 | } catch (error) { 54 | console.error('Error sending message to endpoint:', error); 55 | } 56 | }, [selectedScenario, playAudio]); 57 | 58 | useEffect(() => { 59 | initializeMicrophone(); // Initialize the microphone when the component mounts 60 | }, []); 61 | 62 | useEffect(() => { 63 | if (audioSrc) { 64 | playAudio(audioSrc); // If an audio source is provided, play it 65 | } 66 | }, [audioSrc, playAudio]); 67 | 68 | // Send the recorded file when recording stops 69 | useEffect(() => { 70 | if (recordedFile && !isRecording) { 71 | console.log('FILE IS RECORDED') 72 | sendMessageToEndpoint(recordedFile); 73 | } 74 | }, [recordedFile, isRecording, sendMessageToEndpoint]); 75 | 76 | return ( 77 |
78 | 79 | 80 | 88 | 89 | setToggleTranslation(!toggleTranslation)} 96 | /> 97 | 98 | {toggleTranslation && ( 99 | 103 | )} 104 |
105 | ); 106 | }; 107 | 108 | export default RolePlayPage; 109 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # German Language Learning Application 2 | 3 | An interactive web application designed to help users practice German language skills through realistic scenario-based conversations. The application features speech recognition, text-to-speech capabilities, and real-time translations. 4 | 5 | ## Features 6 | 7 | - 🎭 **Scenario-based Learning**: Practice German in real-life situations: 8 | - Supermarket conversations 9 | - Restaurant interactions 10 | - Train station dialogues 11 | 12 | - 🎙️ **Speech Recognition**: Record your responses in German using your device's microphone 13 | 14 | - 🔊 **Text-to-Speech**: Listen to native German pronunciations 15 | 16 | - 🔄 **Real-time Translation**: Toggle between German and English translations 17 | 18 | - 🤖 **AI-Powered Conversations**: Natural dialogue flow using advanced language models 19 | 20 | ## Technology Stack 21 | 22 | ### Frontend 23 | - React.js 24 | - RecordRTC for audio recording 25 | - Web Speech API 26 | - CSS for animations and styling 27 | - FontAwesome icons 28 | 29 | ### Backend 30 | - Node.js with Express 31 | - TypeScript 32 | - Integration with: 33 | - AssemblyAI for speech-to-text conversion 34 | - Deepseek AI for natural language processing 35 | - gTTS (Google Text-to-Speech) for audio generation 36 | 37 | ## Getting Started 38 | 39 | ### Prerequisites 40 | - Node.js (v14 or higher) 41 | - npm or yarn 42 | - Modern web browser with microphone support 43 | 44 | ### Installation 45 | 46 | 1. Clone the repository: 47 | ```bash 48 | git clone https://github.com/yourusername/german-language-learning.git 49 | cd german-language-learning 50 | ``` 51 | 52 | 2. Install dependencies for both frontend and backend: 53 | ```bash 54 | # Install backend dependencies 55 | cd backend 56 | npm install 57 | 58 | # Install frontend dependencies 59 | cd ../frontend 60 | npm install 61 | ``` 62 | 63 | 3. Set up environment variables: 64 | ```bash 65 | # In backend directory 66 | cp .env.example .env 67 | ``` 68 | Edit the `.env` file with your API keys and configuration: 69 | ``` 70 | ASSEMBLYAI_API_KEY=your_key_here 71 | DEEPSEEK_API_KEY=your_key_here 72 | PORT=3000 73 | ``` 74 | 75 | ### Running the Application 76 | 77 | 1. Start the backend server: 78 | ```bash 79 | # In backend directory 80 | npm run dev 81 | ``` 82 | 83 | 2. Start the frontend development server: 84 | ```bash 85 | # In frontend directory 86 | npm start 87 | ``` 88 | 89 | 3. Open your browser and navigate to `http://localhost:3000` 90 | 91 | ## Usage 92 | 93 | 1. Select a scenario from the available options on the home page 94 | 2. Click the microphone icon to start recording your response 95 | 3. Speak your response in German 96 | 4. The application will: 97 | - Convert your speech to text 98 | - Provide feedback on pronunciation 99 | - Offer suggestions for improvement 100 | - Generate an appropriate AI response 101 | 102 | ## Contributing 103 | 104 | We welcome contributions! Please follow these steps: 105 | 106 | 1. Fork the repository 107 | 2. Create a new branch (`git checkout -b feature/improvement`) 108 | 3. Make your changes 109 | 4. Commit your changes (`git commit -am 'Add new feature'`) 110 | 5. Push to the branch (`git push origin feature/improvement`) 111 | 6. Create a Pull Request 112 | 113 | ## License 114 | 115 | This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. 116 | 117 | ## Acknowledgments 118 | 119 | - Thanks to AssemblyAI for speech recognition capabilities 120 | - Thanks to Deepseek AI for natural language processing 121 | - Thanks to all contributors who have helped shape this project 122 | 123 | ## Support 124 | 125 | If you encounter any issues or have questions, please: 126 | - fix it yourself. this is just for fun. 127 | 128 | ## Roadmap 129 | 130 | - [ ] Add more conversation scenarios 131 | - [ ] Implement progress tracking 132 | - [ ] Add gamification elements 133 | - [ ] Support for different German dialects 134 | - [ ] Mobile app development --------------------------------------------------------------------------------