├── README.md
├── requirements.txt
├── microservice
    ├── RAG
    │   ├── README.md
    │   ├── test.py
    │   ├── templates
    │   │   └── chat.html
    │   ├── static
    │   │   └── style.css
    │   └── app.py
    ├── ASR
    │   ├── test.py
    │   ├── README.md
    │   └── app.py
    └── LLM
    │   ├── test.py
    │   └── README.md
├── app.py
└── templates
    └── index.html


/README.md:
--------------------------------------------------------------------------------
1 | https://youtu.be/uyUsmA7xosw?si=0GfhygY_oveeb6iC


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | Flask
2 | Flask-SocketIO
3 | llama-index-core
4 | llama-index-readers-file
5 | llama-index-llms-ollama
6 | llama-index-embeddings-huggingface


--------------------------------------------------------------------------------
/microservice/RAG/README.md:
--------------------------------------------------------------------------------
 1 | 
 2 | 
 3 | 
 4 | ```sh
 5 | pip3 install --no-cache-dir --verbose \
 6 |         llama-index-core \
 7 |         llama-index-readers-file \
 8 |         llama-index-llms-ollama \
 9 |         llama-index-embeddings-huggingface
10 | ```


--------------------------------------------------------------------------------
/microservice/ASR/test.py:
--------------------------------------------------------------------------------
 1 | import requests
 2 | 
 3 | # Flask 服务器的 URL
 4 | url = 'http://127.0.0.1:7771//api/get-text'
 5 | 
 6 | # 发送 GET 请求
 7 | response = requests.get(url)
 8 | 
 9 | # 检查响应状态码
10 | if response.status_code == 200:
11 |     # 获取并打印 JSON 响应内容
12 |     data = response.json()
13 |     print(data)
14 | else:
15 |     print('Failed to retrieve data:', response.status_code)
16 | 


--------------------------------------------------------------------------------
/microservice/RAG/test.py:
--------------------------------------------------------------------------------
 1 | import requests
 2 | 
 3 | # Flask 应用的 URL
 4 | url = 'http://127.0.0.1:7774/api/rag-query'
 5 | 
 6 | # 发送请求的数据
 7 | params = {'prompt': 'What wish did Leo make in the end, and why did he choose that particular wish?'}
 8 | 
 9 | # 发起 GET 请求
10 | response = requests.get(url, params=params)
11 | 
12 | # 检查响应状态码
13 | if response.status_code == 200:
14 |     # 获取并打印 JSON 响应内容
15 |     data = response.json()
16 |     print(data)
17 | else:
18 |     print('Failed to retrieve data:', response.status_code)
19 | 
20 | 


--------------------------------------------------------------------------------
/microservice/ASR/README.md:
--------------------------------------------------------------------------------
 1 | # Automatic Speech Recognition
 2 | 
 3 | In this project, we use [Nvidia Riva](https://www.nvidia.com/en-us/ai-data-science/products/riva/) to implement ASR functionality, which can perform real-time audio transcription.
 4 | 
 5 | 
 6 | First, please refer to [this document](https://wiki.seeedstudio.com/Local_Voice_Chatbot/#install-riva-server) to install Nvidia Riva Server.
 7 | 
 8 | Then run the following script to start Riva microservice:
 9 | 
10 | ``` shell
11 | python3 app.py  # microservice/ASR/app.py
12 | ```
13 | 
14 | 
15 | 


--------------------------------------------------------------------------------
/microservice/LLM/test.py:
--------------------------------------------------------------------------------
 1 | import requests
 2 | import json
 3 | 
 4 | 
 5 | url = 'http://localhost:11434/api/generate'
 6 | 
 7 | data = {
 8 |     "model": "llama3",
 9 |     "prompt": "Why is the sky blue?"
10 | }
11 | 
12 | response = requests.post(url, json=data, stream=True)
13 | 
14 | if response.status_code == 200:
15 |     for line in response.iter_lines():
16 |         if line:
17 |             json_data = json.loads(line.decode('utf-8'))
18 |             print(json_data) 
19 | else:
20 |     print("Failed to get valid response, status code:", response.status_code)
21 | 


--------------------------------------------------------------------------------
/microservice/RAG/templates/chat.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en">
 3 | <head>
 4 |     <meta charset="UTF-8">
 5 |     <meta name="viewport" content="width=device-width, initial-scale=1.0">
 6 |     <title>ChatGPT Interface</title>
 7 |     <link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">
 8 | </head>
 9 | <body>
10 |     <div class="chat_container">
11 |         <header>RAG with Ollama and LlamaIndex</header>
12 |         <main>
13 | 
14 |         </main>
15 |         <div class="chatbox">
16 |             <input type="text" placeholder="How can I help you today?" autofocus>
17 |         </div>
18 |         <button>Send</button>
19 |     </div>
20 | 
21 | </body>
22 | </html>
23 | 


--------------------------------------------------------------------------------
/microservice/RAG/static/style.css:
--------------------------------------------------------------------------------
 1 | body {
 2 |     font-family: Arial, sans-serif;
 3 |     background-color: #333;
 4 |     color: white;
 5 |     margin: 0;
 6 |     padding: 0;
 7 |     display: flex;
 8 |     justify-content: center;
 9 |     align-items: center;
10 |     height: 100vh;
11 | }
12 | 
13 | .chat_container {
14 |     width: 600px;
15 |     text-align: center;
16 | }
17 | 
18 | .chatbox input {
19 |     width: 80%;
20 |     padding: 10px;
21 |     margin: 20px 0;
22 |     border: none;
23 |     border-radius: 4px;
24 | }
25 | 
26 | button {
27 |     padding: 10px 20px;
28 |     border: none;
29 |     background-color: #4CAF50;
30 |     color: white;
31 |     border-radius: 4px;
32 |     cursor: pointer;
33 | }
34 | 


--------------------------------------------------------------------------------
/microservice/LLM/README.md:
--------------------------------------------------------------------------------
 1 | 
 2 | 
 3 | 启动本地大模型服务
 4 | ```sh
 5 | jetson-containers run --name ollama $(autotag ollama)
 6 | ```
 7 | 
 8 | 
 9 | reComputer J4012 with Jetpack 5.1.2
10 | ```sh
11 | docker run --runtime nvidia -it --rm --network host --volume /tmp/argus_socket:/tmp/argus_socket --volume /etc/enctune.conf:/etc/enctune.conf --volume /etc/nv_tegra_release:/etc/nv_tegra_release --volume /tmp/nv_jetson_model:/tmp/nv_jetson_model --volume /var/run/dbus:/var/run/dbus --volume /var/run/avahi-daemon/socket:/var/run/avahi-daemon/socket --volume /var/run/docker.sock:/var/run/docker.sock --volume /home/seeed/aa/microservice/LLM/jetson-containers/data:/data --device /dev/snd --device /dev/bus/usb --name ollama dustynv/ollama:r35.4.1
12 | ```
13 | 
14 | 
15 | 
16 | ```sh
17 | curl http://localhost:11434/api/generate -d '{
18 |   "model": "llama2",
19 |   "prompt": "Why is the sky blue?"
20 | }'
21 | ```
22 | 
23 | 


--------------------------------------------------------------------------------
/microservice/RAG/app.py:
--------------------------------------------------------------------------------
 1 | from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, Settings
 2 | from llama_index.core.embeddings import resolve_embed_model
 3 | from llama_index.llms.ollama import Ollama
 4 | from flask import Flask, jsonify, request, render_template
 5 | 
 6 | 
 7 | class RAG:
 8 |     def __init__(self):
 9 |         self.ragapp = Flask(__name__)
10 |         self.port=7774
11 |         self.setup_routes()
12 | 
13 |         self.input_dir_doc = '/home/seeed/aa/raw_meeting_transcript'
14 |         self.documents = self.read_directory(input_dir=self.input_dir_doc)
15 | 
16 |         Settings.embed_model = resolve_embed_model("local:BAAI/bge-small-en-v1.5")
17 |         Settings.llm = Ollama(model="llama3", request_timeout=120.0)
18 | 
19 |         self.index = VectorStoreIndex.from_documents(self.documents)
20 |         self.query_engine = self.index.as_query_engine()
21 | 
22 |     def setup_routes(self):
23 |         @self.ragapp.route('/')
24 |         def index():
25 |             return render_template('chat.html')
26 |     
27 |         @self.ragapp.route('/api/update-doc')
28 |         def update_doc():
29 |             self.documents = self.read_directory(input_dir=self.input_dir_doc)
30 |             self.index = VectorStoreIndex.from_documents(self.documents,)
31 |             self.query_engine = self.index.as_query_engine()
32 |             return jsonify({"message": "documents update!"})
33 |         
34 |         @self.ragapp.route('/api/rag-query')
35 |         def rag_query():
36 |             usr_prompt = request.args.get('prompt', 'Hello!')
37 |             print(usr_prompt)
38 |             response = self.query_engine.query(usr_prompt)
39 |             print(response)
40 |             return jsonify({"message": str(response)})
41 |         
42 |     def run(self):
43 |         self.ragapp.run(port=self.port)
44 |     
45 |     def read_directory(self, input_dir):
46 |         reader = SimpleDirectoryReader(input_dir=input_dir)
47 |         documents = reader.load_data()
48 |         print(f"Loaded {len(documents)} docs from {input_dir}")
49 |         return documents
50 |         
51 | 
52 | if __name__ == '__main__':
53 |     re = RAG()
54 |     re.run()
55 | 


--------------------------------------------------------------------------------
/app.py:
--------------------------------------------------------------------------------
  1 | import time
  2 | import os
  3 | import datetime
  4 | import webbrowser
  5 | import glob
  6 | import json
  7 | from flask import render_template, Flask, jsonify, request
  8 | from flask_socketio import SocketIO
  9 | from threading import Thread
 10 | import requests
 11 | 
 12 | 
 13 | class WebUI:
 14 |     def __init__(self):
 15 |         # WebUI
 16 |         self.webapp = Flask(__name__)
 17 |         # self.socketio = SocketIO(self.webapp)
 18 |         self.messages = ["hello, this is a test string", "another line"]  # a list to cache recorded sentence
 19 |         
 20 |         # 创建相应的文件夹
 21 |         self.recorder_path = '/home/seeed/aa/raw_meeting_transcript'
 22 |         self.summary_path = '/home/seeed/aa/meeting_summary'
 23 |         if not os.path.exists(self.recorder_path):
 24 |             os.makedirs(self.recorder_path)
 25 |             print(f"Folder '{self.recorder_path}' was created.")
 26 |         else:
 27 |             print(f"Folder '{self.recorder_path}' already exists.")
 28 |         if not os.path.exists(self.summary_path):
 29 |             os.makedirs(self.summary_path)
 30 |             print(f"Folder '{self.summary_path}' was created.")
 31 |         else:
 32 |             print(f"Folder '{self.summary_path}' already exists.")
 33 | 
 34 |         # ASR
 35 |         self.asr_server_url = 'http://127.0.0.1:7771'
 36 |         self.asr_flag = 0
 37 | 
 38 |         # LLM
 39 |         self.llm_url = 'http://localhost:11434/api/generate'
 40 | 
 41 |         self.save_record_name = None
 42 |         self.webapp_thread = None
 43 |         self.setup_routes()
 44 | 
 45 |     def setup_routes(self):
 46 |         @self.webapp.route('/')
 47 |         def index():
 48 |             return render_template('index.html')
 49 |         
 50 |         @self.webapp.route('/chat-rag', methods=['POST'])
 51 |         def chat_rag():
 52 |             data = request.get_json()
 53 | 
 54 |             url = 'http://127.0.0.1:7774/api/rag-query'
 55 |             params = {'prompt': data['text']}
 56 | 
 57 |             response = requests.get(url, params=params)
 58 | 
 59 |             if response.status_code == 200:
 60 |                 rst = response.json()
 61 |                 print(rst)
 62 |             else:
 63 |                 print('Failed to retrieve data:', response.status_code)
 64 | 
 65 |             return jsonify(response=rst['message'])
 66 |         
 67 |         @self.webapp.route('/get_recorder_messages', methods=['GET'])
 68 |         def get_messages():
 69 |             response = requests.get('http://127.0.0.1:7771/api/get-text')
 70 |             if response.status_code == 200:
 71 |                 data = response.json()
 72 |                 if self.asr_flag == 1:
 73 |                     if data['content'] != self.messages[-1] and data['content'] not in self.messages:
 74 |                         self.messages[-1] = data['content']
 75 | 
 76 |                     if data['final'] is True and self.messages[-1] != ' ':
 77 |                         self.messages.append(' ')
 78 |             else:
 79 |                 print('Failed to retrieve data:', response.status_code)
 80 |             return jsonify(self.messages)
 81 | 
 82 |         @self.webapp.route('/btn-start-record', methods=['POST'])
 83 |         def btn_start_record():
 84 |             self.messages.clear()
 85 |             self.messages.append('')
 86 |             self.asr_flag = 1
 87 |             return '', 200
 88 | 
 89 |         @self.webapp.route('/btn-stop-record', methods=['POST'])
 90 |         def btn_stop_record():
 91 |             self.asr_flag = 0
 92 |             return '', 200
 93 |         
 94 |         @self.webapp.route('/save-record', methods=['POST'])
 95 |         def save_record():
 96 |             now = datetime.datetime.now()
 97 |             formatted_time = now.strftime('%Y-%m-%d_%H-%M-%S')
 98 |             self.save_record_name = f'record_{formatted_time}.txt'
 99 |             print(f'{self.recorder_path}/{self.save_record_name}')
100 |             with open(f'{self.recorder_path}/{self.save_record_name}', 'w') as file:
101 |                 for item in self.messages:
102 |                     file.write(item + '\n')
103 |             return '', 200
104 |         
105 |         @self.webapp.route('/gen-summary', methods=['POST'])
106 |         def gen_summary():
107 |             self.asr_flag = 0
108 | 
109 |             file_path = self.get_latest_txt_file(self.recorder_path)
110 |             print(file_path)
111 |             with open(file_path, 'r', encoding='utf-8') as file:
112 |                 content = file.read() 
113 |             
114 |             data = {
115 |                 "model": "llama3",
116 |                 "prompt": f"Please summarise this meeting transcript:{content}"
117 |             }
118 |             response = requests.post(self.llm_url, json=data, stream=True)
119 |             if response.status_code == 200:
120 |                 self.messages.clear()
121 |                 self.messages.append('')
122 |                 for line in response.iter_lines():
123 |                     if line: 
124 |                         json_data = json.loads(line.decode('utf-8'))
125 |                         self.messages[-1] += json_data['response']
126 |                         print(json_data)
127 |             else:
128 |                 print("Failed to get valid response, status code:", response.status_code)
129 |             return '', 200
130 | 
131 |         @self.webapp.route('/save-summary', methods=['POST'])
132 |         def save_summary():
133 |             self.asr_flag = 0
134 |             record_file_path = self.get_latest_txt_file(self.recorder_path)
135 |             filename = os.path.basename(record_file_path)
136 |             filename = 'summary_' + filename
137 |             save_path = os.path.join(self.summary_path, filename)
138 |             print(save_path)
139 | 
140 |             with open(save_path, 'w') as file:
141 |                 for item in self.messages:
142 |                     file.write(item + '\n')
143 | 
144 |             return '', 200
145 |         
146 |         @self.webapp.route('/get_messages', methods=['GET'])
147 |         def get_messages1():
148 |             return jsonify(self.messages)
149 |         
150 |     def run(self):
151 |         self.webapp.run()
152 | 
153 |     def get_asr_message(self):
154 |         url = self.asr_server_url + '/api/get-text'
155 |         while True:
156 |             response = requests.get(url)
157 |             if response.status_code == 200:
158 |                 data = response.json()
159 |                 if self.asr_flag == 1:
160 |                     if data['content'] != self.messages[-1] and data['content'] not in self.messages:
161 |                         self.messages[-1] = data['content']
162 |                     if data['final'] is True and self.messages[-1] != '':
163 |                         self.messages.append('')
164 |                     # print(self.messages[-1])
165 |             else:
166 |                 print('Failed to retrieve data:', response.status_code)
167 |             time.sleep(1)
168 | 
169 |     @staticmethod
170 |     def get_latest_txt_file(directory):
171 |         txt_files = {}
172 | 
173 |         for filename in os.listdir(directory):
174 |             if filename.endswith('.txt'): 
175 |                 full_path = os.path.join(directory, filename)
176 |                 mod_time = os.path.getmtime(full_path)
177 |                 txt_files[full_path] = mod_time
178 | 
179 |         if txt_files:
180 |             latest_file = max(txt_files, key=txt_files.get)
181 |             return latest_file
182 |         else:
183 |             return None
184 | 
185 | if __name__ == '__main__':
186 |     re = WebUI()
187 |     re.run()
188 | 


--------------------------------------------------------------------------------
/templates/index.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html lang="en">
  3 | <head>
  4 |     <meta charset="UTF-8">
  5 |     <meta name="viewport" content="width=device-width, initial-scale=1.0">
  6 |     <title>Chatbot Interface</title>
  7 | 
  8 |     <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
  9 | 
 10 |     <style>
 11 |         body{
 12 |             margin: 0px;
 13 |             height: 100vh;
 14 |         }
 15 |         .container{
 16 |             margin: 0px auto 0px;
 17 |             display: grid;
 18 |             grid-template-columns: minmax(150px, 25%) 1fr;
 19 |         }
 20 |         .file-menu{
 21 |             padding: 10px;
 22 |             /* border: 1px solid rgb(34, 24, 24); */
 23 |         }
 24 |         .chat-box{
 25 |             display: inline-block;
 26 |             vertical-align: middle;
 27 |         }
 28 |         #chat-container{
 29 | 			height: 92vh;
 30 |             margin: 5px 5px 5px 5px;
 31 |             border: 2px solid rgb(29, 66, 48);
 32 |             border-radius: 5px;
 33 |             overflow-y: scroll;
 34 |             background: #fff;
 35 |         }
 36 |         .message {
 37 |             margin: 5px;
 38 |             padding: 10px;
 39 |             background: #e0e0e0;
 40 |             border-radius: 5px;
 41 |             width: fit-content;
 42 |         }
 43 |         .user {
 44 |             align-self: flex-end;
 45 |             background-color: #007bff;
 46 |             color: white;
 47 |         }
 48 |         #input-container{
 49 |             margin: 2px auto 2px;
 50 |             display: flex;
 51 |             align-items: center;
 52 |             width: 100%;
 53 |         }
 54 |         #user-input {
 55 |             flex-grow: 1;
 56 |             margin: 5px 10px 5px 5px;
 57 |             margin-right: 10px;
 58 |             padding: 10px;
 59 |             width: 70%;
 60 |             border: 2px solid #ccc;
 61 |             border-radius: 5px;
 62 |         }
 63 |         #send-button {
 64 |             padding: 10px 20px;
 65 |             margin-right: 10px;
 66 |             margin-left: 10px;
 67 |             background-color: #007bff;
 68 |             color: white;
 69 |             border: none;
 70 |             border-radius: 5px;
 71 |             cursor: pointer;
 72 |         }
 73 |     </style>
 74 | 
 75 |     
 76 | </head>
 77 | <body>
 78 |     <div class="container">
 79 |         <div class="file-menu">
 80 |             <button class="button" id="start-record">启动记录</button>
 81 |             <button class="button" id="stop-record">结束记录</button>
 82 |             <button class="button" id="save-record">保存记录</button>
 83 |             <button class="button" id="generate-summary">生成摘要</button>
 84 |             <button class="button" id="save-summary">保存摘要</button>
 85 |         </div>
 86 | 
 87 |         <div class="chat-box">
 88 |             <div id="chat-container"></div>
 89 |             <div id="input-container">
 90 |                 <input type="text" id="user-input" placeholder="Type a message..." autofocus>
 91 |                 <button id="send-button">Send</button>
 92 |             </div>
 93 |         </div>
 94 |     </div>
 95 | 
 96 |     <script>
 97 |         const startButton = document.getElementById('start-record');
 98 |         const stopButton = document.getElementById('stop-record');
 99 |         const saveButton = document.getElementById('save-record');
100 |         const genButton = document.getElementById('generate-summary');
101 |         const saveSumButton = document.getElementById('save-summary');
102 | 
103 |         const chatContainer = document.getElementById('chat-container');
104 |         const userInput = document.getElementById('user-input');
105 |         const sendButton = document.getElementById('send-button');
106 |         
107 |         let shouldFetchMessages = 0;
108 | 
109 |         function appendMessage(text, user) {
110 |             const messageDiv = document.createElement('div');
111 |             messageDiv.classList.add('message');
112 |             if (user) {
113 |                 messageDiv.classList.add('user');
114 |             }
115 |             messageDiv.textContent = text;
116 |             chatContainer.appendChild(messageDiv);
117 |             chatContainer.scrollTop = chatContainer.scrollHeight;
118 |         }
119 | 
120 |         sendButton.addEventListener('click', () => {
121 |             const userInputText = userInput.value.trim();
122 |             if (userInputText) {
123 |                 appendMessage(userInputText, true);
124 |                 userInput.value = '';
125 |                 
126 |                 fetch('/chat-rag', {
127 |                     method: 'POST',
128 |                     headers: {
129 |                         'Content-Type': 'application/json'
130 |                     },
131 |                     body: JSON.stringify({ text: userInputText}) 
132 |                 })
133 |                 .then(response => response.json()) 
134 |                 .then(data => {
135 |                     appendMessage("Response: " + data.response, false); 
136 |                 })
137 |                 .catch(error => {
138 |                     console.error('Error:', error);
139 |                     appendMessage("Error: Unable to get response", false);
140 |                 });
141 |             }
142 |         });
143 | 
144 |         userInput.addEventListener('keypress', (event) => {
145 |             if (event.key === 'Enter') {
146 |                 sendButton.click();
147 |             }
148 |         });
149 | 
150 |         startButton.addEventListener('click', function() {
151 |             $.post('/btn-start-record', {}, function() {
152 |                 shouldFetchMessages = 1
153 |                 alert('Start Record!');
154 |             });
155 |         });
156 | 
157 |         stopButton.addEventListener('click', function() {
158 |             $.post('/btn-stop-record', {}, function() {
159 |                 shouldFetchMessages = 0
160 |                 alert('Stop Record!');
161 |             });
162 |         });
163 | 
164 |         saveButton.addEventListener('click', function() {
165 |             shouldFetchMessages = 0
166 |             $.post('/save-record', {}, function() {
167 |                 alert('Save Record!');
168 |             });
169 |         });
170 | 
171 |         genButton.addEventListener('click', function() {
172 |             shouldFetchMessages = 2
173 |             $.post('/gen-summary', {}, function() {
174 |                 alert('Generate Summary!');
175 |             });
176 |         });
177 | 
178 |         saveSumButton.addEventListener('click', function() {
179 |             shouldFetchMessages = 0
180 |             $.post('/save-summary', {}, function() {
181 |                 alert('Save Summary!');
182 |             });
183 |         });        
184 | 
185 |         function fetchMessages() {
186 |             fetch('/get_recorder_messages')
187 |                 .then(response => response.json())
188 |                 .then(data => {
189 |                     chatContainer.innerHTML = '';
190 |                     data.forEach(msg => {
191 |                         appendMessage(msg, true);
192 |                     });
193 |                 })
194 |                 .catch(error => console.error('Error fetching messages:', error));
195 |         }
196 | 
197 |         setInterval(() => {
198 |             if (shouldFetchMessages === 1) {
199 |                 fetchMessages();
200 |             }
201 |             if (shouldFetchMessages === 2) {
202 |                 console.log("Hello world");
203 |                 fetch('/get_messages')
204 |                 .then(response => response.json())
205 |                 .then(data => {
206 |                     chatContainer.innerHTML = '';
207 |                     data.forEach(msg => {
208 |                         appendMessage(msg, true);
209 |                     });
210 |                 })
211 |                 .catch(error => console.error('Error fetching messages:', error));
212 |             }
213 |         }, 1000);
214 | 
215 |     </script>
216 | </body>
217 | </html>
218 | 


--------------------------------------------------------------------------------
/microservice/ASR/app.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | import sys
  3 | import time
  4 | import threading
  5 | import pyaudio
  6 | from flask import Flask, jsonify, request, Response, stream_with_context
  7 | import riva.client
  8 | import riva.client.audio_io
  9 | 
 10 | 
 11 | class RivaBase(threading.Thread):
 12 | 
 13 |     def __init__(self, auth=None, input_device=None, sample_rate_hz=16000, audio_chunk=1600, audio_channels=1,
 14 |                  automatic_punctuation=True, verbatim_transcripts=True, profanity_filter=False,
 15 |                  language_code='en-US', boosted_lm_words=None, boosted_lm_score=4.0, callback=None):
 16 |         super(RivaBase, self).__init__()
 17 | 
 18 |         assert auth is not None, f"Invalid parameter： {auth}"
 19 |         self.asr_service = riva.client.ASRService(auth)
 20 | 
 21 |         self.input_device = input_device
 22 |         self.sample_rate_hz = sample_rate_hz
 23 |         self.audio_chunk = audio_chunk
 24 |         self.callback = callback
 25 |         if self.callback is None:
 26 |             self.callback = self.callback_example
 27 | 
 28 |         self.asr_config = riva.client.StreamingRecognitionConfig(
 29 |             config=riva.client.RecognitionConfig(
 30 |                 encoding=riva.client.AudioEncoding.LINEAR_PCM,
 31 |                 language_code=language_code,
 32 |                 max_alternatives=1,
 33 |                 profanity_filter=profanity_filter,
 34 |                 enable_automatic_punctuation=automatic_punctuation,
 35 |                 verbatim_transcripts=verbatim_transcripts,
 36 |                 sample_rate_hertz=sample_rate_hz,
 37 |                 audio_channel_count=audio_channels,
 38 |             ),
 39 |             interim_results=True,
 40 |         )
 41 | 
 42 |         riva.client.add_word_boosting_to_config(self.asr_config, boosted_lm_words, boosted_lm_score)
 43 |         self.mute_flag = False
 44 |         self.stop_flag = False
 45 | 
 46 |     def run(self):
 47 |         try:
 48 |             with riva.client.audio_io.MicrophoneStream(
 49 |                     self.sample_rate_hz,
 50 |                     self.audio_chunk,
 51 |                     device=self.input_device,
 52 |             ) as audio_chunk_iterator:
 53 |                 responses = self.asr_service.streaming_response_generator(
 54 |                     audio_chunks=audio_chunk_iterator, streaming_config=self.asr_config
 55 |                 )
 56 |                 for response in responses:
 57 |                     if self.stop_flag:
 58 |                         audio_chunk_iterator.close()
 59 |                     if self.get_mute_state():
 60 |                         continue
 61 |                     else:
 62 |                         if not response.results:
 63 |                             continue
 64 |                         self.callback(response)
 65 |             print('ASR Server Stop!')
 66 |         except:
 67 |             sys.exit(0)
 68 | 
 69 |     @staticmethod
 70 |     def callback_example(response):
 71 |         try:
 72 |             for result in response.results:
 73 |                 if not result.alternatives:
 74 |                     continue
 75 |                 transcript = result.alternatives[0].transcript
 76 |                 if result.is_final:
 77 |                     print("## " + transcript)
 78 |                     print(f"Confidence:{result.alternatives[0].confidence:9.4f}" + "\n")
 79 |                     return True
 80 |                 else:
 81 |                     print(">> " + transcript)
 82 |                     print(f"Stability：{result.stability:9.4f}" + "\n")
 83 |         finally:
 84 |             pass
 85 | 
 86 |     def get_mute_state(self):
 87 |         return self.mute_flag
 88 | 
 89 |     @staticmethod
 90 |     def list_devices():
 91 |         riva.client.audio_io.list_input_devices()
 92 | 
 93 | 
 94 | class ASR_Riva:
 95 |     def __init__(self, input_device=None, sample_rate_hz=None):
 96 |         self.asrapp = Flask(__name__)
 97 |         self.port=7771
 98 |         self.setup_routes()
 99 | 
100 |         self.riva_server = "127.0.0.1:50051"
101 | 
102 |         if input_device is None:
103 |             target_device = self.get_asr_devices_list()
104 |             input_device = target_device['index']
105 |             sample_rate_hz = int(target_device['defaultSampleRate'])
106 |             print(f"Microphone parameters : {input_device}, {sample_rate_hz}")
107 | 
108 |         auth = riva.client.Auth(uri=self.riva_server)
109 |         self.asr = RivaBase(auth, input_device, sample_rate_hz, callback=self.asr_callback)
110 |         self.asr.start()
111 | 
112 |         self.text_cache = []
113 |         self.text_stream = {'content':'', 'final': True, 'push': True, 'asr_state': self.asr.stop_flag}
114 | 
115 |     def asr_callback(self, response):
116 |         try:
117 |             
118 |             for result in response.results:
119 |                 if not result.alternatives:
120 |                     continue
121 |                 transcript = result.alternatives[0].transcript
122 | 
123 |                 if result.is_final:
124 |                     print("## " + transcript)
125 |                     print(f"Confidence:{result.alternatives[0].confidence:9.4f}" + "\n")
126 |                     self.text_stream['content'] = transcript
127 |                     self.text_stream['final'] = True
128 |                     self.text_stream['push'] = False
129 |                     self.text_cache.append(self.text_stream)
130 |                     if len(self.text_cache) > 100:
131 |                         del self.text_cache[:50]
132 |                 else:
133 |                     if result.stability > 0.8:
134 |                         print(">> " + transcript)
135 |                         print(f"Stability：{result.stability:9.4f}" + "\n")
136 |                         self.text_stream['content'] = transcript
137 |                         self.text_stream['final'] = False
138 |                         self.text_stream['push'] = False
139 |         finally:
140 |             pass
141 | 
142 |     def setup_routes(self):
143 | 
144 |         @self.asrapp.route('/api/get-text')
145 |         def get_text():
146 |             self.text_stream['push'] = True
147 |             return jsonify(self.text_stream)
148 | 
149 |         @self.asrapp.route('/api/get-cache')
150 |         def get_cache():
151 |             return jsonify(self.text_cache[-1])
152 | 
153 |         @self.asrapp.route('/api/asr-stop')
154 |         def asr_stop():
155 |             self.asr.stop_flag=True
156 |             print('Stop!')
157 |             return jsonify({"response": 'stop'})
158 |         
159 |         @self.asrapp.route('/api/asr-start')
160 |         def update_doc():
161 |             if self.asr.stop_flag == True:
162 |                 self.asr.stop_flag=False
163 |                 self.asr.start()
164 |                 print('Start!')
165 |                 return jsonify({"response": 'start'})
166 |             else:
167 |                 return jsonify({"response": 'running'})
168 |         
169 |         @self.asrapp.route('/stream')
170 |         def stream():
171 |             def generate():
172 |                 while True:
173 |                     if self.text_stream['push'] is False:
174 |                         yield self.text_stream['content'] + '\n'
175 |                     time.sleep(1)
176 |             return Response(stream_with_context(generate()), mimetype='text/plain')
177 | 
178 | 
179 |     def run(self):
180 |         self.asrapp.run(port=self.port)
181 | 
182 |     @staticmethod
183 |     def get_asr_devices_list():
184 |         target_device = None
185 |         p = pyaudio.PyAudio()
186 |         print("Input audio devices:")
187 |         for i in range(p.get_device_count()):
188 |             info = p.get_device_info_by_index(i)
189 |             if info['maxInputChannels'] < 1:
190 |                 continue
191 |             if 'USB' in info["name"]:
192 |                 target_device = info
193 |             print(f"{info['index']}: {info['name']}")
194 |         p.terminate()
195 |         if target_device is None:
196 |             print('No available input device found, please manually config an device.')
197 |             sys.exit(0)
198 |         else:
199 |             return target_device
200 | 
201 | if __name__ == '__main__':
202 |     re = ASR_Riva()
203 |     re.run()
204 | 


--------------------------------------------------------------------------------