├── .gitattributes ├── .gitignore ├── .gitmodules ├── .vscode └── settings.json ├── ESP-IDF ├── ai_chat │ ├── .vscode │ │ ├── c_cpp_properties.json │ │ └── settings.json │ ├── CMakeLists.txt │ ├── Makefile │ ├── components │ │ ├── espressif__esp_lcd_touch_ft5x06 │ │ │ ├── .component_hash │ │ │ ├── CMakeLists.txt │ │ │ ├── README.md │ │ │ ├── esp_lcd_touch_ft5x06.c │ │ │ ├── idf_component.yml │ │ │ ├── include │ │ │ │ └── esp_lcd_touch_ft5x06.h │ │ │ └── license.txt │ │ └── my_board │ │ │ ├── CMakeLists.txt │ │ │ ├── Kconfig.projbuild │ │ │ ├── component.mk │ │ │ └── my_board_v1_0 │ │ │ ├── board.c │ │ │ ├── board.h │ │ │ ├── board_def.h │ │ │ ├── board_def_s3.h │ │ │ ├── board_pins_config.c │ │ │ └── board_pins_config_s3.c │ ├── dependencies.lock │ ├── main │ │ ├── CMakeLists.txt │ │ ├── baidu_tts.c │ │ ├── baidu_tts.h │ │ ├── baidu_vtt.c │ │ ├── baidu_vtt.h │ │ ├── component.mk │ │ ├── font_alipuhui20.c │ │ ├── idf_component.yml │ │ ├── img_bilibili120.c │ │ ├── lv_gui.c │ │ ├── lv_gui.h │ │ ├── main.c │ │ ├── minimax_chat.c │ │ └── minimax_chat.h │ ├── partitions.csv │ ├── sdkconfig │ ├── sdkconfig.defaults │ └── sdkconfig.old └── emotion │ ├── README.md │ └── emotion.zip ├── LICENSE ├── README.md ├── README.pdf ├── python ├── Dify │ ├── esp32.yml │ └── test.py ├── classfy-model │ ├── classification_demo_dataset.ipynb │ ├── dataset │ │ └── dataset_furina.csv │ ├── furina_test.ipynb │ ├── hflrbt3 │ │ ├── added_tokens.json │ │ ├── config.json │ │ ├── special_tokens_map.json │ │ ├── tokenizer.json │ │ ├── tokenizer_config.json │ │ └── vocab.txt │ └── model_furina │ │ └── 从网盘下载模型文件.txt ├── computer_chat │ ├── assistant.py │ └── 注意 需要额外安装库 ollama.txt ├── requirements.txt ├── server-model │ ├── __pycache__ │ │ ├── API.cpython-39.pyc │ │ ├── chat_local.cpython-39.pyc │ │ ├── chat_local_manage.cpython-39.pyc │ │ ├── chat_local_manage_t.cpython-39.pyc │ │ ├── chat_mysql.cpython-39.pyc │ │ ├── classfy_model.cpython-39.pyc │ │ ├── file_dealing.cpython-39.pyc │ │ └── search_internet.cpython-39.pyc │ ├── chat_local.py │ ├── chat_local_manage.py │ ├── chat_local_manage_t.py │ ├── chat_mysql.py │ ├── classfy_model.py │ ├── file_dealing.py │ ├── http_server.py │ ├── http_server_t.py │ ├── http_server_xiaozhi.py │ ├── search_internet.py │ └── search_internet_local.py ├── test.ipynb └── xiaozhi-server │ ├── local-tts │ ├── READEME.md │ ├── api.bat │ ├── api_v2.bat │ ├── config.py │ ├── config和api是GPTsovites文件夹下面的, 另外两个是服务器使用的文件.txt │ ├── gpt_sovits_v1.py │ └── 配置文件.txt │ └── 连接各种服务.md └── 代码分析 ├── 小智.md ├── 小智闹钟文档.md ├── 插件 ├── Screenshot_20250326_201012_com.jideos.jnotes.jpg ├── Screenshot_20250326_215027_com.jideos.jnotes.jpg ├── test_reqllm.py └── test_response.py ├── 服务器.md └── 服务器框图.jpg /.gitattributes: -------------------------------------------------------------------------------- 1 | *.pth filter=lfs diff=lfs merge=lfs -text 2 | *.bin filter=lfs diff=lfs merge=lfs -text 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ESP-IDF/ai_chat/managed_components 2 | ESP-IDF/ai_chat/build 3 | ESP-IDF/xiaozhi-esp32/managed_components 4 | ESP-IDF/xiaozhi-esp32/build 5 | 6 | # Prerequisites 7 | *.d 8 | 9 | # Object files 10 | *.o 11 | *.ko 12 | *.obj 13 | *.elf 14 | 15 | # Linker output 16 | *.ilk 17 | *.map 18 | *.exp 19 | 20 | # Precompiled Headers 21 | *.gch 22 | *.pch 23 | 24 | # Libraries 25 | *.lib 26 | *.a 27 | *.la 28 | *.lo 29 | 30 | # Shared objects (inc. Windows DLLs) 31 | *.dll 32 | *.so 33 | *.so.* 34 | *.dylib 35 | 36 | # Executables 37 | *.exe 38 | *.out 39 | *.app 40 | *.i*86 41 | *.x86_64 42 | *.hex 43 | 44 | # Debug files 45 | *.dSYM/ 46 | *.su 47 | *.idb 48 | *.pdb 49 | 50 | # Kernel Module Compile Results 51 | *.mod* 52 | *.cmd 53 | .tmp_versions/ 54 | modules.order 55 | Module.symvers 56 | Mkfile.old 57 | dkms.conf 58 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "ESP-IDF/xiao-zhi"] 2 | path = ESP-IDF/xiao-zhi 3 | url = git@github.com:XuSenfeng/ai-chat-local-xiaozhi.git 4 | [submodule "ESP-IDF/xiaozhi-alarm"] 5 | path = ESP-IDF/xiaozhi-alarm 6 | url = git@github.com:XuSenfeng/xiaozhi-alarm.git 7 | [submodule "python/xiaozhi-server/vision"] 8 | path = python/xiaozhi-server/vision 9 | url = git@github.com:XuSenfeng/xiaozhi-server-vision.git 10 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "idf.customExtraVars": { 3 | "IDF_TARGET": "esp32s3" 4 | }, 5 | "idf.openOcdConfigs": [ 6 | "interface/ftdi/esp32_devkitj_v1.cfg", 7 | "target/esp32s3.cfg" 8 | ] 9 | } -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/.vscode/c_cpp_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "name": "ESP-IDF", 5 | "cStandard": "c11", 6 | "cppStandard": "c++17", 7 | "compileCommands": "${workspaceFolder}/build/compile_commands.json", 8 | "compilerPath": "${config:idf.toolsPath}/tools/riscv32-esp-elf/esp-12.2.0_20230208/riscv32-esp-elf/bin/riscv32-esp-elf-gcc" 9 | } 10 | ], 11 | "version": 4 12 | } 13 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "idf.adapterTargetName": "esp32c3", 3 | "idf.openOcdConfigs": [ 4 | "board/esp32c3-ftdi.cfg" 5 | ], 6 | "idf.flashType": "UART", 7 | "idf.portWin": "COM3", 8 | "files.associations": { 9 | "mp3_decoder.h": "c", 10 | "minimax_chat.h": "c", 11 | "esp_wifi.h": "c", 12 | "string.h": "c" 13 | }, 14 | "idf.port": "/dev/ttyACM0" 15 | } -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # The following lines of boilerplate have to be in your project's 2 | # CMakeLists in this exact order for cmake to work correctly 3 | cmake_minimum_required(VERSION 3.5) 4 | 5 | include($ENV{ADF_PATH}/CMakeLists.txt) 6 | include($ENV{IDF_PATH}/tools/cmake/project.cmake) 7 | 8 | project(ai_chat) 9 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # This is a project Makefile. It is assumed the directory this Makefile resides in is a 3 | # project subdirectory. 4 | # 5 | 6 | PROJECT_NAME := ai_chat 7 | 8 | include $(ADF_PATH)/project.mk 9 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/espressif__esp_lcd_touch_ft5x06/.component_hash: -------------------------------------------------------------------------------- 1 | 97759953d9436a365e9427078c5b04ecce4e6a50f50cf62c68cd6bfa229b812c -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/espressif__esp_lcd_touch_ft5x06/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | idf_component_register(SRCS "esp_lcd_touch_ft5x06.c" INCLUDE_DIRS "include" REQUIRES "esp_lcd") 2 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/espressif__esp_lcd_touch_ft5x06/README.md: -------------------------------------------------------------------------------- 1 | # ESP LCD Touch FT5x06 Controller 2 | 3 | [![Component Registry](https://components.espressif.com/components/espressif/esp_lcd_touch_ft5x06/badge.svg)](https://components.espressif.com/components/espressif/esp_lcd_touch_ft5x06) 4 | 5 | Implementation of the FT5x06 touch controller with esp_lcd_touch component. 6 | 7 | | Touch controller | Communication interface | Component name | Link to datasheet | 8 | | :--------------: | :---------------------: | :------------: | :---------------: | 9 | | FT5x06 | I2C (SPI [^1]) | esp_lcd_touch_ft5x06 | [PDF](https://www.displayfuture.com/Display/datasheet/controller/FT5x06.pdf) | 10 | 11 | [^1]: **NOTE:** This controller should work via I2C or SPI communication interface. But it was tested on HW only via I2C communication interface. 12 | 13 | ## Add to project 14 | 15 | Packages from this repository are uploaded to [Espressif's component service](https://components.espressif.com/). 16 | You can add them to your project via `idf.py add-dependancy`, e.g. 17 | ``` 18 | idf.py add-dependency esp_lcd_touch_ft5x06==1.0.0 19 | ``` 20 | 21 | Alternatively, you can create `idf_component.yml`. More is in [Espressif's documentation](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-guides/tools/idf-component-manager.html). 22 | 23 | ## Example use 24 | 25 | I2C initialization of the touch component. 26 | 27 | ``` 28 | esp_lcd_panel_io_i2c_config_t io_config = ESP_LCD_TOUCH_IO_I2C_FT5x06_CONFIG(); 29 | 30 | esp_lcd_touch_config_t tp_cfg = { 31 | .x_max = CONFIG_LCD_HRES, 32 | .y_max = CONFIG_LCD_VRES, 33 | .rst_gpio_num = -1, 34 | .int_gpio_num = -1, 35 | .levels = { 36 | .reset = 0, 37 | .interrupt = 0, 38 | }, 39 | .flags = { 40 | .swap_xy = 0, 41 | .mirror_x = 0, 42 | .mirror_y = 0, 43 | }, 44 | }; 45 | 46 | esp_lcd_touch_handle_t tp; 47 | esp_lcd_touch_new_i2c_ft5x06(io_handle, &tp_cfg, &tp); 48 | ``` 49 | 50 | Read data from the touch controller and store it in RAM memory. It should be called regularly in poll. 51 | 52 | ``` 53 | esp_lcd_touch_read_data(tp); 54 | ``` 55 | 56 | Get one X and Y coordinates with strength of touch. 57 | 58 | ``` 59 | uint16_t touch_x[1]; 60 | uint16_t touch_y[1]; 61 | uint16_t touch_strength[1]; 62 | uint8_t touch_cnt = 0; 63 | 64 | bool touchpad_pressed = esp_lcd_touch_get_coordinates(tp, touch_x, touch_y, touch_strength, &touch_cnt, 1); 65 | ``` 66 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/espressif__esp_lcd_touch_ft5x06/esp_lcd_touch_ft5x06.c: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD 3 | * 4 | * SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | #include 8 | #include 9 | #include "freertos/FreeRTOS.h" 10 | #include "freertos/task.h" 11 | #include "esp_system.h" 12 | #include "esp_err.h" 13 | #include "esp_log.h" 14 | #include "esp_check.h" 15 | #include "driver/gpio.h" 16 | #include "driver/i2c.h" 17 | #include "esp_lcd_panel_io.h" 18 | #include "esp_lcd_touch.h" 19 | 20 | static const char *TAG = "FT5x06"; 21 | 22 | /* Registers */ 23 | #define FT5x06_DEVICE_MODE (0x00) 24 | #define FT5x06_GESTURE_ID (0x01) 25 | #define FT5x06_TOUCH_POINTS (0x02) 26 | 27 | #define FT5x06_TOUCH1_EV_FLAG (0x03) 28 | #define FT5x06_TOUCH1_XH (0x03) 29 | #define FT5x06_TOUCH1_XL (0x04) 30 | #define FT5x06_TOUCH1_YH (0x05) 31 | #define FT5x06_TOUCH1_YL (0x06) 32 | 33 | #define FT5x06_TOUCH2_EV_FLAG (0x09) 34 | #define FT5x06_TOUCH2_XH (0x09) 35 | #define FT5x06_TOUCH2_XL (0x0A) 36 | #define FT5x06_TOUCH2_YH (0x0B) 37 | #define FT5x06_TOUCH2_YL (0x0C) 38 | 39 | #define FT5x06_TOUCH3_EV_FLAG (0x0F) 40 | #define FT5x06_TOUCH3_XH (0x0F) 41 | #define FT5x06_TOUCH3_XL (0x10) 42 | #define FT5x06_TOUCH3_YH (0x11) 43 | #define FT5x06_TOUCH3_YL (0x12) 44 | 45 | #define FT5x06_TOUCH4_EV_FLAG (0x15) 46 | #define FT5x06_TOUCH4_XH (0x15) 47 | #define FT5x06_TOUCH4_XL (0x16) 48 | #define FT5x06_TOUCH4_YH (0x17) 49 | #define FT5x06_TOUCH4_YL (0x18) 50 | 51 | #define FT5x06_TOUCH5_EV_FLAG (0x1B) 52 | #define FT5x06_TOUCH5_XH (0x1B) 53 | #define FT5x06_TOUCH5_XL (0x1C) 54 | #define FT5x06_TOUCH5_YH (0x1D) 55 | #define FT5x06_TOUCH5_YL (0x1E) 56 | 57 | #define FT5x06_ID_G_THGROUP (0x80) 58 | #define FT5x06_ID_G_THPEAK (0x81) 59 | #define FT5x06_ID_G_THCAL (0x82) 60 | #define FT5x06_ID_G_THWATER (0x83) 61 | #define FT5x06_ID_G_THTEMP (0x84) 62 | #define FT5x06_ID_G_THDIFF (0x85) 63 | #define FT5x06_ID_G_CTRL (0x86) 64 | #define FT5x06_ID_G_TIME_ENTER_MONITOR (0x87) 65 | #define FT5x06_ID_G_PERIODACTIVE (0x88) 66 | #define FT5x06_ID_G_PERIODMONITOR (0x89) 67 | #define FT5x06_ID_G_AUTO_CLB_MODE (0xA0) 68 | #define FT5x06_ID_G_LIB_VERSION_H (0xA1) 69 | #define FT5x06_ID_G_LIB_VERSION_L (0xA2) 70 | #define FT5x06_ID_G_CIPHER (0xA3) 71 | #define FT5x06_ID_G_MODE (0xA4) 72 | #define FT5x06_ID_G_PMODE (0xA5) 73 | #define FT5x06_ID_G_FIRMID (0xA6) 74 | #define FT5x06_ID_G_STATE (0xA7) 75 | #define FT5x06_ID_G_FT5201ID (0xA8) 76 | #define FT5x06_ID_G_ERR (0xA9) 77 | 78 | /******************************************************************************* 79 | * Function definitions 80 | *******************************************************************************/ 81 | static esp_err_t esp_lcd_touch_ft5x06_read_data(esp_lcd_touch_handle_t tp); 82 | static bool esp_lcd_touch_ft5x06_get_xy(esp_lcd_touch_handle_t tp, uint16_t *x, uint16_t *y, uint16_t *strength, uint8_t *point_num, uint8_t max_point_num); 83 | static esp_err_t esp_lcd_touch_ft5x06_del(esp_lcd_touch_handle_t tp); 84 | 85 | /* I2C read */ 86 | static esp_err_t touch_ft5x06_i2c_write(esp_lcd_touch_handle_t tp, uint8_t reg, uint8_t data); 87 | static esp_err_t touch_ft5x06_i2c_read(esp_lcd_touch_handle_t tp, uint8_t reg, uint8_t *data, uint8_t len); 88 | 89 | /* FT5x06 init */ 90 | static esp_err_t touch_ft5x06_init(esp_lcd_touch_handle_t tp); 91 | /* FT5x06 reset */ 92 | static esp_err_t touch_ft5x06_reset(esp_lcd_touch_handle_t tp); 93 | 94 | /******************************************************************************* 95 | * Public API functions 96 | *******************************************************************************/ 97 | 98 | esp_err_t esp_lcd_touch_new_i2c_ft5x06(const esp_lcd_panel_io_handle_t io, const esp_lcd_touch_config_t *config, esp_lcd_touch_handle_t *out_touch) 99 | { 100 | esp_err_t ret = ESP_OK; 101 | 102 | assert(config != NULL); 103 | assert(out_touch != NULL); 104 | 105 | /* Prepare main structure */ 106 | esp_lcd_touch_handle_t esp_lcd_touch_ft5x06 = heap_caps_calloc(1, sizeof(esp_lcd_touch_t), MALLOC_CAP_DEFAULT); 107 | ESP_GOTO_ON_FALSE(esp_lcd_touch_ft5x06, ESP_ERR_NO_MEM, err, TAG, "no mem for FT5x06 controller"); 108 | 109 | /* Communication interface */ 110 | esp_lcd_touch_ft5x06->io = io; 111 | 112 | /* Only supported callbacks are set */ 113 | esp_lcd_touch_ft5x06->read_data = esp_lcd_touch_ft5x06_read_data; 114 | esp_lcd_touch_ft5x06->get_xy = esp_lcd_touch_ft5x06_get_xy; 115 | esp_lcd_touch_ft5x06->del = esp_lcd_touch_ft5x06_del; 116 | 117 | /* Mutex */ 118 | esp_lcd_touch_ft5x06->data.lock.owner = portMUX_FREE_VAL; 119 | 120 | /* Save config */ 121 | memcpy(&esp_lcd_touch_ft5x06->config, config, sizeof(esp_lcd_touch_config_t)); 122 | 123 | /* Prepare pin for touch interrupt */ 124 | if (esp_lcd_touch_ft5x06->config.int_gpio_num != GPIO_NUM_NC) { 125 | const gpio_config_t int_gpio_config = { 126 | .mode = GPIO_MODE_INPUT, 127 | .intr_type = (esp_lcd_touch_ft5x06->config.levels.interrupt ? GPIO_INTR_POSEDGE : GPIO_INTR_NEGEDGE), 128 | .pin_bit_mask = BIT64(esp_lcd_touch_ft5x06->config.int_gpio_num) 129 | }; 130 | ret = gpio_config(&int_gpio_config); 131 | ESP_GOTO_ON_ERROR(ret, err, TAG, "GPIO config failed"); 132 | 133 | /* Register interrupt callback */ 134 | if (esp_lcd_touch_ft5x06->config.interrupt_callback) { 135 | esp_lcd_touch_register_interrupt_callback(esp_lcd_touch_ft5x06, esp_lcd_touch_ft5x06->config.interrupt_callback); 136 | } 137 | } 138 | 139 | /* Prepare pin for touch controller reset */ 140 | if (esp_lcd_touch_ft5x06->config.rst_gpio_num != GPIO_NUM_NC) { 141 | const gpio_config_t rst_gpio_config = { 142 | .mode = GPIO_MODE_OUTPUT, 143 | .pin_bit_mask = BIT64(esp_lcd_touch_ft5x06->config.rst_gpio_num) 144 | }; 145 | ret = gpio_config(&rst_gpio_config); 146 | ESP_GOTO_ON_ERROR(ret, err, TAG, "GPIO config failed"); 147 | } 148 | 149 | /* Reset controller */ 150 | ret = touch_ft5x06_reset(esp_lcd_touch_ft5x06); 151 | ESP_GOTO_ON_ERROR(ret, err, TAG, "FT5x06 reset failed"); 152 | 153 | /* Init controller */ 154 | ret = touch_ft5x06_init(esp_lcd_touch_ft5x06); 155 | ESP_GOTO_ON_ERROR(ret, err, TAG, "FT5x06 init failed"); 156 | 157 | err: 158 | if (ret != ESP_OK) { 159 | ESP_LOGE(TAG, "Error (0x%x)! Touch controller FT5x06 initialization failed!", ret); 160 | if (esp_lcd_touch_ft5x06) { 161 | esp_lcd_touch_ft5x06_del(esp_lcd_touch_ft5x06); 162 | } 163 | } 164 | 165 | *out_touch = esp_lcd_touch_ft5x06; 166 | 167 | return ret; 168 | } 169 | 170 | static esp_err_t esp_lcd_touch_ft5x06_read_data(esp_lcd_touch_handle_t tp) 171 | { 172 | esp_err_t err; 173 | uint8_t data[30]; 174 | uint8_t points; 175 | size_t i = 0; 176 | 177 | assert(tp != NULL); 178 | 179 | err = touch_ft5x06_i2c_read(tp, FT5x06_TOUCH_POINTS, &points, 1); 180 | ESP_RETURN_ON_ERROR(err, TAG, "I2C read error!"); 181 | 182 | if (points > 5 || points == 0) { 183 | return ESP_OK; 184 | } 185 | 186 | /* Number of touched points */ 187 | points = (points > CONFIG_ESP_LCD_TOUCH_MAX_POINTS ? CONFIG_ESP_LCD_TOUCH_MAX_POINTS : points); 188 | 189 | err = touch_ft5x06_i2c_read(tp, FT5x06_TOUCH1_XH, data, 6 * points); 190 | ESP_RETURN_ON_ERROR(err, TAG, "I2C read error!"); 191 | 192 | portENTER_CRITICAL(&tp->data.lock); 193 | 194 | /* Number of touched points */ 195 | tp->data.points = points; 196 | 197 | /* Fill all coordinates */ 198 | for (i = 0; i < points; i++) { 199 | tp->data.coords[i].x = (((uint16_t)data[(i * 6) + 0] & 0x0f) << 8) + data[(i * 6) + 1]; 200 | tp->data.coords[i].y = (((uint16_t)data[(i * 6) + 2] & 0x0f) << 8) + data[(i * 6) + 3]; 201 | } 202 | 203 | portEXIT_CRITICAL(&tp->data.lock); 204 | 205 | return ESP_OK; 206 | } 207 | 208 | static bool esp_lcd_touch_ft5x06_get_xy(esp_lcd_touch_handle_t tp, uint16_t *x, uint16_t *y, uint16_t *strength, uint8_t *point_num, uint8_t max_point_num) 209 | { 210 | assert(tp != NULL); 211 | assert(x != NULL); 212 | assert(y != NULL); 213 | assert(point_num != NULL); 214 | assert(max_point_num > 0); 215 | 216 | portENTER_CRITICAL(&tp->data.lock); 217 | 218 | /* Count of points */ 219 | *point_num = (tp->data.points > max_point_num ? max_point_num : tp->data.points); 220 | 221 | for (size_t i = 0; i < *point_num; i++) { 222 | x[i] = tp->data.coords[i].x; 223 | y[i] = tp->data.coords[i].y; 224 | 225 | if (strength) { 226 | strength[i] = tp->data.coords[i].strength; 227 | } 228 | } 229 | 230 | /* Invalidate */ 231 | tp->data.points = 0; 232 | 233 | portEXIT_CRITICAL(&tp->data.lock); 234 | 235 | return (*point_num > 0); 236 | } 237 | 238 | static esp_err_t esp_lcd_touch_ft5x06_del(esp_lcd_touch_handle_t tp) 239 | { 240 | assert(tp != NULL); 241 | 242 | /* Reset GPIO pin settings */ 243 | if (tp->config.int_gpio_num != GPIO_NUM_NC) { 244 | gpio_reset_pin(tp->config.int_gpio_num); 245 | if (tp->config.interrupt_callback) { 246 | gpio_isr_handler_remove(tp->config.int_gpio_num); 247 | } 248 | } 249 | 250 | /* Reset GPIO pin settings */ 251 | if (tp->config.rst_gpio_num != GPIO_NUM_NC) { 252 | gpio_reset_pin(tp->config.rst_gpio_num); 253 | } 254 | 255 | free(tp); 256 | 257 | return ESP_OK; 258 | } 259 | 260 | /******************************************************************************* 261 | * Private API function 262 | *******************************************************************************/ 263 | 264 | static esp_err_t touch_ft5x06_init(esp_lcd_touch_handle_t tp) 265 | { 266 | esp_err_t ret = ESP_OK; 267 | 268 | // Valid touching detect threshold 269 | ret |= touch_ft5x06_i2c_write(tp, FT5x06_ID_G_THGROUP, 70); 270 | 271 | // valid touching peak detect threshold 272 | ret |= touch_ft5x06_i2c_write(tp, FT5x06_ID_G_THPEAK, 60); 273 | 274 | // Touch focus threshold 275 | ret |= touch_ft5x06_i2c_write(tp, FT5x06_ID_G_THCAL, 16); 276 | 277 | // threshold when there is surface water 278 | ret |= touch_ft5x06_i2c_write(tp, FT5x06_ID_G_THWATER, 60); 279 | 280 | // threshold of temperature compensation 281 | ret |= touch_ft5x06_i2c_write(tp, FT5x06_ID_G_THTEMP, 10); 282 | 283 | // Touch difference threshold 284 | ret |= touch_ft5x06_i2c_write(tp, FT5x06_ID_G_THDIFF, 20); 285 | 286 | // Delay to enter 'Monitor' status (s) 287 | ret |= touch_ft5x06_i2c_write(tp, FT5x06_ID_G_TIME_ENTER_MONITOR, 2); 288 | 289 | // Period of 'Active' status (ms) 290 | ret |= touch_ft5x06_i2c_write(tp, FT5x06_ID_G_PERIODACTIVE, 12); 291 | 292 | // Timer to enter 'idle' when in 'Monitor' (ms) 293 | ret |= touch_ft5x06_i2c_write(tp, FT5x06_ID_G_PERIODMONITOR, 40); 294 | 295 | return ret; 296 | } 297 | 298 | /* Reset controller */ 299 | static esp_err_t touch_ft5x06_reset(esp_lcd_touch_handle_t tp) 300 | { 301 | assert(tp != NULL); 302 | 303 | if (tp->config.rst_gpio_num != GPIO_NUM_NC) { 304 | ESP_RETURN_ON_ERROR(gpio_set_level(tp->config.rst_gpio_num, tp->config.levels.reset), TAG, "GPIO set level error!"); 305 | vTaskDelay(pdMS_TO_TICKS(10)); 306 | ESP_RETURN_ON_ERROR(gpio_set_level(tp->config.rst_gpio_num, !tp->config.levels.reset), TAG, "GPIO set level error!"); 307 | vTaskDelay(pdMS_TO_TICKS(10)); 308 | } 309 | 310 | return ESP_OK; 311 | } 312 | 313 | static esp_err_t touch_ft5x06_i2c_write(esp_lcd_touch_handle_t tp, uint8_t reg, uint8_t data) 314 | { 315 | assert(tp != NULL); 316 | 317 | // *INDENT-OFF* 318 | /* Write data */ 319 | // return esp_lcd_panel_io_tx_param(tp->io, reg, (uint8_t[]){data}, 1); 320 | // *INDENT-ON* 321 | uint8_t write_buf[2] = {reg, data}; 322 | 323 | return i2c_master_write_to_device(0, 0x38, write_buf, sizeof(write_buf), 1000 / portTICK_PERIOD_MS); 324 | } 325 | 326 | static esp_err_t touch_ft5x06_i2c_read(esp_lcd_touch_handle_t tp, uint8_t reg, uint8_t *data, uint8_t len) 327 | { 328 | assert(tp != NULL); 329 | assert(data != NULL); 330 | 331 | /* Read data */ 332 | // return esp_lcd_panel_io_rx_param(tp->io, reg, data, len); 333 | return i2c_master_write_read_device(0, 0x38, ®, 1, data, len, 1000 / portTICK_PERIOD_MS); 334 | } 335 | 336 | // static esp_err_t touch_ft5x06_i2c_write(esp_lcd_touch_handle_t tp, uint8_t reg, uint8_t data) 337 | // { 338 | // assert(tp != NULL); 339 | 340 | // // *INDENT-OFF* 341 | // /* Write data */ 342 | // return esp_lcd_panel_io_tx_param(tp->io, reg, (uint8_t[]){data}, 1); 343 | // // *INDENT-ON* 344 | // } 345 | 346 | // static esp_err_t touch_ft5x06_i2c_read(esp_lcd_touch_handle_t tp, uint8_t reg, uint8_t *data, uint8_t len) 347 | // { 348 | // assert(tp != NULL); 349 | // assert(data != NULL); 350 | 351 | // /* Read data */ 352 | // return esp_lcd_panel_io_rx_param(tp->io, reg, data, len); 353 | // } 354 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/espressif__esp_lcd_touch_ft5x06/idf_component.yml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | esp_lcd_touch: 3 | public: true 4 | version: ^1.0.4 5 | idf: 6 | version: '>=4.4.2' 7 | description: ESP LCD Touch FT5x06 - touch controller FT5x06 8 | url: https://github.com/espressif/esp-bsp/tree/master/components/lcd_touch/esp_lcd_touch_ft5x06 9 | version: 1.0.6 10 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/espressif__esp_lcd_touch_ft5x06/include/esp_lcd_touch_ft5x06.h: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD 3 | * 4 | * SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | /** 8 | * @file 9 | * @brief ESP LCD touch: FT5x06 10 | */ 11 | 12 | #pragma once 13 | 14 | #include "esp_lcd_touch.h" 15 | 16 | #ifdef __cplusplus 17 | extern "C" { 18 | #endif 19 | 20 | /** 21 | * @brief Create a new FT5x06 touch driver 22 | * 23 | * @note The I2C communication should be initialized before use this function. 24 | * 25 | * @param io LCD/Touch panel IO handle 26 | * @param config: Touch configuration 27 | * @param out_touch: Touch instance handle 28 | * @return 29 | * - ESP_OK on success 30 | * - ESP_ERR_NO_MEM if there is no memory for allocating main structure 31 | */ 32 | esp_err_t esp_lcd_touch_new_i2c_ft5x06(const esp_lcd_panel_io_handle_t io, const esp_lcd_touch_config_t *config, esp_lcd_touch_handle_t *out_touch); 33 | 34 | /** 35 | * @brief I2C address of the FT5x06 controller 36 | * 37 | */ 38 | #define ESP_LCD_TOUCH_IO_I2C_FT5x06_ADDRESS (0x38) 39 | 40 | /** 41 | * @brief Touch IO configuration structure 42 | * 43 | */ 44 | #define ESP_LCD_TOUCH_IO_I2C_FT5x06_CONFIG() \ 45 | { \ 46 | .dev_addr = ESP_LCD_TOUCH_IO_I2C_FT5x06_ADDRESS, \ 47 | .control_phase_bytes = 1, \ 48 | .dc_bit_offset = 0, \ 49 | .lcd_cmd_bits = 8, \ 50 | .flags = \ 51 | { \ 52 | .disable_control_phase = 1, \ 53 | } \ 54 | } 55 | 56 | 57 | #ifdef __cplusplus 58 | } 59 | #endif 60 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/espressif__esp_lcd_touch_ft5x06/license.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/my_board/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Edit following two lines to set component requirements (see docs) 2 | set(COMPONENT_REQUIRES) 3 | set(COMPONENT_PRIV_REQUIRES audio_sal audio_hal esp_dispatcher esp_peripherals display_service) 4 | 5 | if(CONFIG_AUDIO_BOARD_CUSTOM) 6 | message(STATUS "Current board name is " CONFIG_AUDIO_BOARD_CUSTOM) 7 | list(APPEND COMPONENT_ADD_INCLUDEDIRS ./my_board_v1_0) 8 | set(COMPONENT_SRCS 9 | ./my_board_v1_0/board.c 10 | ./my_board_v1_0/board_pins_config.c 11 | ) 12 | endif() 13 | 14 | register_component() 15 | 16 | idf_component_get_property(audio_board_lib audio_board COMPONENT_LIB) 17 | set_property(TARGET ${audio_board_lib} APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${COMPONENT_LIB}) 18 | 19 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/my_board/Kconfig.projbuild: -------------------------------------------------------------------------------- 1 | menu "My Audio Board" 2 | 3 | choice CUSTOMER_AUDIO_BOARD 4 | prompt "My audio board" 5 | default MY_BOARD_V1_0 6 | help 7 | Select an audio board to use with the ESP-ADF 8 | 9 | config MY_BOARD_V1_0 10 | bool "My board v1.0" 11 | 12 | endchoice 13 | 14 | endmenu 15 | 16 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/my_board/component.mk: -------------------------------------------------------------------------------- 1 | # 2 | # "main" pseudo-component makefile. 3 | # 4 | # (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.) 5 | 6 | ifdef CONFIG_AUDIO_BOARD_CUSTOM 7 | 8 | COMPONENT_ADD_INCLUDEDIRS += ./my_board_v1_0 9 | COMPONENT_SRCDIRS += ./my_board_v1_0 10 | endif -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/my_board/my_board_v1_0/board.c: -------------------------------------------------------------------------------- 1 | /* 2 | * ESPRESSIF MIT License 3 | * 4 | * Copyright (c) 2020 5 | * 6 | * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, 7 | * it is free of charge, to any person obtaining a copy of this software and associated 8 | * documentation files (the "Software"), to deal in the Software without restriction, including 9 | * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 | * and/or sell copies of the Software, and to permit persons to whom the Software is furnished 11 | * to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all copies or 14 | * substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 18 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 19 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 20 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 21 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | * 23 | */ 24 | 25 | #include "esp_log.h" 26 | #include "board.h" 27 | #include "audio_mem.h" 28 | #include "es8311.h" 29 | #include "periph_button.h" 30 | 31 | static const char *TAG = "AUDIO_BOARD"; 32 | 33 | static audio_board_handle_t board_handle = 0; 34 | 35 | audio_board_handle_t audio_board_init(void) 36 | { 37 | if (board_handle) { 38 | ESP_LOGW(TAG, "The board has already been initialized!"); 39 | return board_handle; 40 | } 41 | board_handle = (audio_board_handle_t) audio_calloc(1, sizeof(struct audio_board_handle)); 42 | AUDIO_MEM_CHECK(TAG, board_handle, return NULL); 43 | board_handle->audio_hal = audio_board_codec_init(); 44 | 45 | return board_handle; 46 | } 47 | 48 | audio_hal_handle_t audio_board_codec_init(void) 49 | { 50 | audio_hal_codec_config_t audio_codec_cfg = AUDIO_CODEC_DEFAULT_CONFIG(); 51 | audio_hal_handle_t codec_hal = audio_hal_init(&audio_codec_cfg, &AUDIO_CODEC_ES8311_DEFAULT_HANDLE); 52 | AUDIO_NULL_CHECK(TAG, codec_hal, return NULL); 53 | return codec_hal; 54 | } 55 | 56 | audio_board_handle_t audio_board_get_handle(void) 57 | { 58 | return board_handle; 59 | } 60 | 61 | esp_err_t audio_board_deinit(audio_board_handle_t audio_board) 62 | { 63 | esp_err_t ret = ESP_OK; 64 | ret |= audio_hal_deinit(audio_board->audio_hal); 65 | free(audio_board); 66 | board_handle = NULL; 67 | return ret; 68 | } 69 | 70 | esp_err_t _get_lcd_io_bus (void *bus, esp_lcd_panel_io_spi_config_t *io_config, 71 | esp_lcd_panel_io_handle_t *out_panel_io) 72 | { 73 | return esp_lcd_new_panel_io_spi((esp_lcd_spi_bus_handle_t)bus, io_config, out_panel_io); 74 | } 75 | 76 | 77 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/my_board/my_board_v1_0/board.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ESPRESSIF MIT License 3 | * 4 | * Copyright (c) 2020 5 | * 6 | * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, 7 | * it is free of charge, to any person obtaining a copy of this software and associated 8 | * documentation files (the "Software"), to deal in the Software without restriction, including 9 | * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 | * and/or sell copies of the Software, and to permit persons to whom the Software is furnished 11 | * to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all copies or 14 | * substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 18 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 19 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 20 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 21 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | * 23 | */ 24 | 25 | #ifndef _AUDIO_BOARD_H_ 26 | #define _AUDIO_BOARD_H_ 27 | 28 | #include "audio_hal.h" 29 | #include "board_def.h" 30 | #include "board_pins_config.h" 31 | #include "esp_peripherals.h" 32 | #include "display_service.h" 33 | #include "periph_lcd.h" 34 | 35 | #ifdef __cplusplus 36 | extern "C" { 37 | #endif 38 | 39 | /** 40 | * @brief Audio board handle 41 | */ 42 | struct audio_board_handle { 43 | audio_hal_handle_t audio_hal; /*!< pa hardware abstract layer handle */ 44 | audio_hal_handle_t adc_hal; /*!< adc hardware abstract layer handle */ 45 | }; 46 | 47 | typedef struct audio_board_handle *audio_board_handle_t; 48 | 49 | /** 50 | * @brief Initialize audio board 51 | * 52 | * @return The audio board handle 53 | */ 54 | audio_board_handle_t audio_board_init(void); 55 | 56 | /** 57 | * @brief Initialize codec 58 | * 59 | * @return The audio hal handle 60 | */ 61 | audio_hal_handle_t audio_board_codec_init(void); 62 | 63 | 64 | /** 65 | * @brief Initialize key peripheral 66 | * 67 | * @param set The handle of esp_periph_set_handle_t 68 | * 69 | * @return 70 | * - ESP_OK, success 71 | * - Others, fail 72 | */ 73 | //esp_err_t audio_board_key_init(esp_periph_set_handle_t set); 74 | 75 | /** 76 | * @brief Initialize lcd peripheral 77 | * 78 | * @param set The handle of esp_periph_set_handle_t 79 | * @param cb The `on_color_trans_done` callback in `esp_lcd_panel_io_spi_config_t` 80 | * 81 | * @return The `esp_lcd_panel_handle_t` handle 82 | */ 83 | void *audio_board_lcd_init(esp_periph_set_handle_t set, void *cb); 84 | 85 | 86 | /** 87 | * @brief Query audio_board_handle 88 | * 89 | * @return The audio board handle 90 | */ 91 | audio_board_handle_t audio_board_get_handle(void); 92 | 93 | /** 94 | * @brief Uninitialize the audio board 95 | * 96 | * @param audio_board The handle of audio board 97 | * 98 | * @return 0 success, 99 | * others fail 100 | */ 101 | esp_err_t audio_board_deinit(audio_board_handle_t audio_board); 102 | 103 | #ifdef __cplusplus 104 | } 105 | #endif 106 | 107 | #endif 108 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/my_board/my_board_v1_0/board_def.h: -------------------------------------------------------------------------------- 1 | /* 2 | * @Descripttion: 3 | * @version: 4 | * @Author: XvSenfeng 5 | * @Date: 2024-06-07 22:02:56 6 | * @LastEditors: XvSenfeng 7 | * @LastEditTime: 2024-11-07 18:37:56 8 | */ 9 | /* 10 | * ESPRESSIF MIT License 11 | * 12 | * Copyright (c) 2020 13 | * 14 | * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, 15 | * it is free of charge, to any person obtaining a copy of this software and associated 16 | * documentation files (the "Software"), to deal in the Software without restriction, including 17 | * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 18 | * and/or sell copies of the Software, and to permit persons to whom the Software is furnished 19 | * to do so, subject to the following conditions: 20 | * 21 | * The above copyright notice and this permission notice shall be included in all copies or 22 | * substantial portions of the Software. 23 | * 24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 25 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 26 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 27 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 28 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 30 | * 31 | */ 32 | 33 | #ifndef _AUDIO_BOARD_DEFINITION_H_ 34 | #define _AUDIO_BOARD_DEFINITION_H_ 35 | 36 | #define BUTTON_MODE_ID 18 /* You need to define the GPIO pins of your board */ 37 | #define BUTTON_REC_ID 9 /* You need to define the GPIO pins of your board */ 38 | #define PA_ENABLE_GPIO 13 /* You need to define the GPIO pins of your board */ 39 | 40 | #define ES8311_MCLK_SOURCE 0 /* 0 From MCLK of esp32 1 From BCLK */ 41 | 42 | 43 | #define BOARD_PA_GAIN (10) /* Power amplifier gain defined by board (dB) */ 44 | 45 | #define SDCARD_PWR_CTRL -1 46 | #define ESP_SD_PIN_CLK -1 47 | #define ESP_SD_PIN_CMD -1 48 | #define ESP_SD_PIN_D0 -1 49 | #define ESP_SD_PIN_D1 -1 50 | #define ESP_SD_PIN_D2 -1 51 | #define ESP_SD_PIN_D3 -1 52 | #define ESP_SD_PIN_D4 -1 53 | #define ESP_SD_PIN_D5 -1 54 | #define ESP_SD_PIN_D6 -1 55 | #define ESP_SD_PIN_D7 -1 56 | #define ESP_SD_PIN_CD -1 57 | #define ESP_SD_PIN_WP -1 58 | 59 | extern audio_hal_func_t AUDIO_CODEC_ES8311_DEFAULT_HANDLE; 60 | 61 | #define AUDIO_CODEC_DEFAULT_CONFIG(){ \ 62 | .adc_input = AUDIO_HAL_ADC_INPUT_LINE1, \ 63 | .dac_output = AUDIO_HAL_DAC_OUTPUT_ALL, \ 64 | .codec_mode = AUDIO_HAL_CODEC_MODE_BOTH, \ 65 | .i2s_iface = { \ 66 | .mode = AUDIO_HAL_MODE_SLAVE, \ 67 | .fmt = AUDIO_HAL_I2S_NORMAL, \ 68 | .samples = AUDIO_HAL_16K_SAMPLES, \ 69 | .bits = AUDIO_HAL_BIT_LENGTH_16BITS, \ 70 | }, \ 71 | }; 72 | 73 | #define INPUT_KEY_NUM 1 /* You need to define the number of input buttons on your board */ 74 | 75 | #define INPUT_KEY_DEFAULT_INFO() { \ 76 | { \ 77 | .type = PERIPH_ID_BUTTON, \ 78 | .user_id = INPUT_KEY_USER_ID_REC, \ 79 | .act_id = BUTTON_REC_ID, \ 80 | }, \ 81 | } 82 | 83 | #define CODEC_ADC_I2S_PORT (0) 84 | 85 | #endif 86 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/my_board/my_board_v1_0/board_def_s3.h: -------------------------------------------------------------------------------- 1 | /* 2 | * @Descripttion: 3 | * @version: 4 | * @Author: XvSenfeng 5 | * @Date: 2024-06-07 22:02:56 6 | * @LastEditors: XvSenfeng 7 | * @LastEditTime: 2024-11-07 18:37:56 8 | */ 9 | /* 10 | * ESPRESSIF MIT License 11 | * 12 | * Copyright (c) 2020 13 | * 14 | * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, 15 | * it is free of charge, to any person obtaining a copy of this software and associated 16 | * documentation files (the "Software"), to deal in the Software without restriction, including 17 | * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 18 | * and/or sell copies of the Software, and to permit persons to whom the Software is furnished 19 | * to do so, subject to the following conditions: 20 | * 21 | * The above copyright notice and this permission notice shall be included in all copies or 22 | * substantial portions of the Software. 23 | * 24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 25 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 26 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 27 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 28 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 30 | * 31 | */ 32 | 33 | #ifndef _AUDIO_BOARD_DEFINITION_H_ 34 | #define _AUDIO_BOARD_DEFINITION_H_ 35 | 36 | #define BUTTON_MODE_ID 18 /* You need to define the GPIO pins of your board */ 37 | #define BUTTON_REC_ID 0 /* You need to define the GPIO pins of your board */ 38 | #define PA_ENABLE_GPIO 13 /* You need to define the GPIO pins of your board */ 39 | 40 | #define ES8311_MCLK_SOURCE 0 /* 0 From MCLK of esp32 1 From BCLK */ 41 | 42 | 43 | #define BOARD_PA_GAIN (10) /* Power amplifier gain defined by board (dB) */ 44 | 45 | #define SDCARD_PWR_CTRL -1 46 | #define ESP_SD_PIN_CLK -1 47 | #define ESP_SD_PIN_CMD -1 48 | #define ESP_SD_PIN_D0 -1 49 | #define ESP_SD_PIN_D1 -1 50 | #define ESP_SD_PIN_D2 -1 51 | #define ESP_SD_PIN_D3 -1 52 | #define ESP_SD_PIN_D4 -1 53 | #define ESP_SD_PIN_D5 -1 54 | #define ESP_SD_PIN_D6 -1 55 | #define ESP_SD_PIN_D7 -1 56 | #define ESP_SD_PIN_CD -1 57 | #define ESP_SD_PIN_WP -1 58 | 59 | extern audio_hal_func_t AUDIO_CODEC_ES8311_DEFAULT_HANDLE; 60 | 61 | #define AUDIO_CODEC_DEFAULT_CONFIG(){ \ 62 | .adc_input = AUDIO_HAL_ADC_INPUT_LINE1, \ 63 | .dac_output = AUDIO_HAL_DAC_OUTPUT_ALL, \ 64 | .codec_mode = AUDIO_HAL_CODEC_MODE_BOTH, \ 65 | .i2s_iface = { \ 66 | .mode = AUDIO_HAL_MODE_SLAVE, \ 67 | .fmt = AUDIO_HAL_I2S_NORMAL, \ 68 | .samples = AUDIO_HAL_16K_SAMPLES, \ 69 | .bits = AUDIO_HAL_BIT_LENGTH_16BITS, \ 70 | }, \ 71 | }; 72 | 73 | #define INPUT_KEY_NUM 1 /* You need to define the number of input buttons on your board */ 74 | 75 | #define INPUT_KEY_DEFAULT_INFO() { \ 76 | { \ 77 | .type = PERIPH_ID_BUTTON, \ 78 | .user_id = INPUT_KEY_USER_ID_REC, \ 79 | .act_id = BUTTON_REC_ID, \ 80 | }, \ 81 | } 82 | 83 | #define CODEC_ADC_I2S_PORT (0) 84 | 85 | #endif 86 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/my_board/my_board_v1_0/board_pins_config.c: -------------------------------------------------------------------------------- 1 | /* 2 | * @Descripttion: 3 | * @version: 4 | * @Author: XvSenfeng 5 | * @Date: 2024-06-07 22:02:56 6 | * @LastEditors: XvSenfeng 7 | * @LastEditTime: 2024-11-07 18:59:01 8 | */ 9 | /* 10 | * ESPRESSIF MIT License 11 | * 12 | * Copyright (c) 2020 13 | * 14 | * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, 15 | * it is free of charge, to any person obtaining a copy of this software and associated 16 | * documentation files (the "Software"), to deal in the Software without restriction, including 17 | * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 18 | * and/or sell copies of the Software, and to permit persons to whom the Software is furnished 19 | * to do so, subject to the following conditions: 20 | * 21 | * The above copyright notice and this permission notice shall be included in all copies or 22 | * substantial portions of the Software. 23 | * 24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 25 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 26 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 27 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 28 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 30 | * 31 | */ 32 | 33 | #include "esp_log.h" 34 | #include "driver/gpio.h" 35 | #include 36 | #include "board.h" 37 | #include "audio_error.h" 38 | #include "audio_mem.h" 39 | #include "soc/soc_caps.h" 40 | 41 | static const char *TAG = "MY_BOARD_V1_0"; 42 | 43 | esp_err_t get_i2c_pins(i2c_port_t port, i2c_config_t *i2c_config) 44 | { 45 | AUDIO_NULL_CHECK(TAG, i2c_config, return ESP_FAIL); 46 | if (port == I2C_NUM_0) { 47 | i2c_config->sda_io_num = 0; 48 | i2c_config->scl_io_num = 1; 49 | } else { 50 | i2c_config->sda_io_num = -1; 51 | i2c_config->scl_io_num = -1; 52 | ESP_LOGE(TAG, "i2c port %d is not supported", port); 53 | return ESP_FAIL; 54 | } 55 | return ESP_OK; 56 | } 57 | 58 | esp_err_t get_i2s_pins(int port, board_i2s_pin_t *i2s_config) 59 | { 60 | AUDIO_NULL_CHECK(TAG, i2s_config, return ESP_FAIL); 61 | if (port == 0) { 62 | i2s_config->mck_io_num = GPIO_NUM_10; 63 | i2s_config->bck_io_num = GPIO_NUM_8; 64 | i2s_config->ws_io_num = GPIO_NUM_12; 65 | i2s_config->data_out_num = GPIO_NUM_11; 66 | i2s_config->data_in_num = GPIO_NUM_7; 67 | } else if (port == 1) { 68 | i2s_config->bck_io_num = -1; 69 | i2s_config->ws_io_num = -1; 70 | i2s_config->data_out_num = -1; 71 | i2s_config->data_in_num = -1; 72 | } else { 73 | memset(i2s_config, -1, sizeof(board_i2s_pin_t)); 74 | ESP_LOGE(TAG, "i2s port %d is not supported", port); 75 | return ESP_FAIL; 76 | } 77 | 78 | return ESP_OK; 79 | } 80 | 81 | // pa enable 82 | int8_t get_pa_enable_gpio(void) 83 | { 84 | return PA_ENABLE_GPIO; 85 | } 86 | 87 | // rec button 88 | int8_t get_input_rec_id(void) 89 | { 90 | return BUTTON_REC_ID; 91 | } 92 | 93 | // mode button 94 | int8_t get_input_mode_id(void) 95 | { 96 | return BUTTON_MODE_ID; 97 | } 98 | 99 | int8_t get_es8311_mclk_src(void) 100 | { 101 | return ES8311_MCLK_SOURCE; 102 | } -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/components/my_board/my_board_v1_0/board_pins_config_s3.c: -------------------------------------------------------------------------------- 1 | /* 2 | * @Descripttion: 3 | * @version: 4 | * @Author: XvSenfeng 5 | * @Date: 2024-06-07 22:02:56 6 | * @LastEditors: XvSenfeng 7 | * @LastEditTime: 2024-11-07 18:59:01 8 | */ 9 | /* 10 | * ESPRESSIF MIT License 11 | * 12 | * Copyright (c) 2020 13 | * 14 | * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, 15 | * it is free of charge, to any person obtaining a copy of this software and associated 16 | * documentation files (the "Software"), to deal in the Software without restriction, including 17 | * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 18 | * and/or sell copies of the Software, and to permit persons to whom the Software is furnished 19 | * to do so, subject to the following conditions: 20 | * 21 | * The above copyright notice and this permission notice shall be included in all copies or 22 | * substantial portions of the Software. 23 | * 24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 25 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 26 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 27 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 28 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 30 | * 31 | */ 32 | 33 | #include "esp_log.h" 34 | #include "driver/gpio.h" 35 | #include 36 | #include "board.h" 37 | #include "audio_error.h" 38 | #include "audio_mem.h" 39 | #include "soc/soc_caps.h" 40 | 41 | static const char *TAG = "MY_BOARD_V1_0"; 42 | 43 | esp_err_t get_i2c_pins(i2c_port_t port, i2c_config_t *i2c_config) 44 | { 45 | AUDIO_NULL_CHECK(TAG, i2c_config, return ESP_FAIL); 46 | if (port == I2C_NUM_0) { 47 | i2c_config->sda_io_num = 1; 48 | i2c_config->scl_io_num = 2; 49 | } else { 50 | i2c_config->sda_io_num = -1; 51 | i2c_config->scl_io_num = -1; 52 | ESP_LOGE(TAG, "i2c port %d is not supported", port); 53 | return ESP_FAIL; 54 | } 55 | return ESP_OK; 56 | } 57 | 58 | esp_err_t get_i2s_pins(int port, board_i2s_pin_t *i2s_config) 59 | { 60 | AUDIO_NULL_CHECK(TAG, i2s_config, return ESP_FAIL); 61 | if (port == 0) { 62 | i2s_config->mck_io_num = GPIO_NUM_38; 63 | i2s_config->bck_io_num = GPIO_NUM_14; 64 | i2s_config->ws_io_num = GPIO_NUM_13; 65 | i2s_config->data_out_num = GPIO_NUM_45; 66 | i2s_config->data_in_num = GPIO_NUM_12; 67 | } else if (port == 1) { 68 | i2s_config->bck_io_num = -1; 69 | i2s_config->ws_io_num = -1; 70 | i2s_config->data_out_num = -1; 71 | i2s_config->data_in_num = -1; 72 | } else { 73 | memset(i2s_config, -1, sizeof(board_i2s_pin_t)); 74 | ESP_LOGE(TAG, "i2s port %d is not supported", port); 75 | return ESP_FAIL; 76 | } 77 | 78 | return ESP_OK; 79 | } 80 | 81 | // pa enable 82 | int8_t get_pa_enable_gpio(void) 83 | { 84 | return PA_ENABLE_GPIO; 85 | } 86 | 87 | // rec button 88 | int8_t get_input_rec_id(void) 89 | { 90 | return BUTTON_REC_ID; 91 | } 92 | 93 | // mode button 94 | int8_t get_input_mode_id(void) 95 | { 96 | return BUTTON_MODE_ID; 97 | } 98 | 99 | int8_t get_es8311_mclk_src(void) 100 | { 101 | return ES8311_MCLK_SOURCE; 102 | } -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/dependencies.lock: -------------------------------------------------------------------------------- 1 | dependencies: 2 | espressif/esp-dsp: 3 | component_hash: 3e7bbd487f1357a1d4944d0c85966d049501ea281b8a4c7f93f7cfedd5b7f23d 4 | dependencies: 5 | - name: idf 6 | require: private 7 | version: '>=4.2' 8 | source: 9 | registry_url: https://components.espressif.com/ 10 | type: service 11 | version: 1.4.12 12 | espressif/esp_lcd_touch: 13 | component_hash: 779b4ba2464a3ae85681e4b860caa5fdc35801458c23f3039ee761bae7f442a4 14 | dependencies: 15 | - name: idf 16 | require: private 17 | version: '>=4.4.2' 18 | source: 19 | registry_url: https://components.espressif.com/ 20 | type: service 21 | version: 1.1.2 22 | espressif/esp_lcd_touch_ft5x06: 23 | dependencies: [] 24 | source: 25 | path: /mnt/e/JHY/esp32/ESP-IDF/2025-2-4-ai-chai-local/ai-chat-local/ESP-IDF/ai_chat/components/espressif__esp_lcd_touch_ft5x06 26 | type: local 27 | version: 1.0.6 28 | espressif/jsmn: 29 | component_hash: d80350c41bbaa827c98a25b6072df00884e72f54885996fab4a4f0aebce6b6c3 30 | dependencies: 31 | - name: idf 32 | require: private 33 | version: '>=4.3' 34 | source: 35 | registry_url: https://components.espressif.com/ 36 | type: service 37 | version: 1.1.0 38 | espressif/nghttp: 39 | component_hash: c52b83cc3d1083ae669751534ea4f7445974b86d91d49a5273bdb0fbf87f34d6 40 | dependencies: 41 | - name: idf 42 | require: private 43 | version: '>=5.0' 44 | source: 45 | registry_url: https://components.espressif.com/ 46 | type: service 47 | version: 1.62.1 48 | idf: 49 | source: 50 | type: idf 51 | version: 5.1.5 52 | lvgl/lvgl: 53 | component_hash: 948bff879a345149b83065535bbc4a026ce9f47498a22881e432a264b9098015 54 | dependencies: [] 55 | source: 56 | registry_url: https://components.espressif.com/ 57 | type: service 58 | version: 8.3.11 59 | direct_dependencies: 60 | - espressif/esp-dsp 61 | - espressif/esp_lcd_touch 62 | - espressif/esp_lcd_touch_ft5x06 63 | - espressif/jsmn 64 | - espressif/nghttp 65 | - idf 66 | - lvgl/lvgl 67 | manifest_hash: b055db83a47520e7b3945359cda8dccf420f7d2a629d3145f615ee43b5d4eaac 68 | target: esp32c3 69 | version: 2.0.0 70 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/main/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | file(GLOB_RECURSE DEMO_SOURCES *.c) 3 | 4 | set(COMPONENT_SRCS "main.c" "lv_gui.c" "baidu_vtt.c" "minimax_chat.c" "baidu_tts.c" "font_alipuhui20.c" "img_bilibili120.c") 5 | set(COMPONENT_ADD_INCLUDEDIRS .) 6 | 7 | register_component() 8 | 9 | # Some of warinings from LVGL. Block them. 10 | target_compile_options(${COMPONENT_LIB} PRIVATE -w) 11 | 12 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/main/baidu_tts.c: -------------------------------------------------------------------------------- 1 | /* 2 | * ESPRESSIF MIT License 3 | * 4 | * Copyright (c) 2018 5 | * 6 | * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, 7 | * it is free of charge, to any person obtaining a copy of this software and associated 8 | * documentation files (the "Software"), to deal in the Software without restriction, including 9 | * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 | * and/or sell copies of the Software, and to permit persons to whom the Software is furnished 11 | * to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all copies or 14 | * substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 18 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 19 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 20 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 21 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | * 23 | */ 24 | 25 | #include 26 | #include "freertos/FreeRTOS.h" 27 | #include "freertos/task.h" 28 | #include "esp_log.h" 29 | #include "esp_wifi.h" 30 | #include "nvs_flash.h" 31 | #include "esp_http_client.h" 32 | #include "sdkconfig.h" 33 | #include "audio_element.h" 34 | #include "audio_pipeline.h" 35 | #include "audio_event_iface.h" 36 | #include "audio_common.h" 37 | #include "audio_hal.h" 38 | #include "http_stream.h" 39 | #include "i2s_stream.h" 40 | #include "mp3_decoder.h" 41 | #include "baidu_tts.h" 42 | 43 | static const char *TAG = "BAIDU_TTS"; 44 | 45 | extern char *baidu_access_token; 46 | 47 | #define BAIDU_TTS_ENDPOINT "http://tsn.baidu.com/text2audio" 48 | #define BAIDU_TTS_TASK_STACK (8*1024) 49 | 50 | typedef struct baidu_tts { 51 | audio_pipeline_handle_t pipeline; 52 | audio_element_handle_t i2s_writer; 53 | audio_element_handle_t http_stream_reader; 54 | audio_element_handle_t mp3_decoder; 55 | int buffer_size; 56 | char *buffer; 57 | char *text; 58 | int sample_rate; 59 | } baidu_tts_t; 60 | 61 | 62 | static esp_err_t _http_stream_reader_event_handle(http_stream_event_msg_t *msg) 63 | { 64 | esp_http_client_handle_t http = (esp_http_client_handle_t)msg->http_client; 65 | baidu_tts_t *tts = (baidu_tts_t *)msg->user_data; 66 | 67 | if (msg->event_id == HTTP_STREAM_PRE_REQUEST) { 68 | // Post text data 69 | ESP_LOGI(TAG, "[ + ] HTTP client HTTP_STREAM_PRE_REQUEST, lenght=%d", msg->buffer_len); 70 | 71 | int payload_len = snprintf(tts->buffer, tts->buffer_size, "lan=zh&cuid=ESP32&ctp=1&vol=15&per=4&tok=%s&tex=%s", baidu_access_token, tts->text); 72 | esp_http_client_set_post_field(http, tts->buffer, payload_len); // 设置POST数据 73 | esp_http_client_set_method(http, HTTP_METHOD_POST); // 设置为POST方法 74 | esp_http_client_set_header(http, "Content-Type", "application/x-www-form-urlencoded"); 75 | esp_http_client_set_header(http, "Accept", "*/*"); 76 | 77 | return ESP_OK; 78 | } 79 | 80 | return ESP_OK; 81 | } 82 | 83 | /// @brief 初始化 84 | /// @param config 85 | /// @return 86 | baidu_tts_handle_t baidu_tts_init(baidu_tts_config_t *config) 87 | { 88 | // 管道设置 89 | audio_pipeline_cfg_t pipeline_cfg = DEFAULT_AUDIO_PIPELINE_CONFIG(); 90 | baidu_tts_t *tts = calloc(1, sizeof(baidu_tts_t)); 91 | AUDIO_MEM_CHECK(TAG, tts, return NULL); 92 | 93 | tts->pipeline = audio_pipeline_init(&pipeline_cfg); 94 | 95 | tts->buffer_size = config->buffer_size; 96 | if (tts->buffer_size <= 0) { 97 | tts->buffer_size = DEFAULT_TTS_BUFFER_SIZE; 98 | } 99 | 100 | tts->buffer = malloc(tts->buffer_size); 101 | AUDIO_MEM_CHECK(TAG, tts->buffer, goto exit_tts_init); 102 | 103 | tts->sample_rate = config->playback_sample_rate; 104 | 105 | // I2S流设置 106 | i2s_stream_cfg_t i2s_cfg = I2S_STREAM_CFG_DEFAULT_WITH_PARA(0, 16000, 16, AUDIO_STREAM_WRITER); 107 | i2s_cfg.std_cfg.slot_cfg.slot_mode = I2S_SLOT_MODE_MONO; // 单声道 108 | i2s_cfg.std_cfg.slot_cfg.slot_mask = I2S_STD_SLOT_LEFT; // 左声道 109 | tts->i2s_writer = i2s_stream_init(&i2s_cfg); // 初始化I2S流 110 | 111 | // http流设置 112 | http_stream_cfg_t http_cfg = { 113 | .type = AUDIO_STREAM_READER, 114 | .event_handle = _http_stream_reader_event_handle, 115 | .user_data = tts, 116 | .task_stack = BAIDU_TTS_TASK_STACK, 117 | }; 118 | tts->http_stream_reader = http_stream_init(&http_cfg); // 初始化HTTP流 119 | 120 | // MP3流设置 121 | mp3_decoder_cfg_t mp3_cfg = DEFAULT_MP3_DECODER_CONFIG(); 122 | tts->mp3_decoder = mp3_decoder_init(&mp3_cfg); 123 | 124 | // 把HTTP流 MP3流 I2S流 注册到管道 并链接起来, 这样数据就可以从HTTP流经过MP3解码器再到I2S流输出 125 | audio_pipeline_register(tts->pipeline, tts->http_stream_reader, "tts_http"); 126 | audio_pipeline_register(tts->pipeline, tts->mp3_decoder, "tts_mp3"); 127 | audio_pipeline_register(tts->pipeline, tts->i2s_writer, "tts_i2s"); 128 | const char *link_tag[3] = {"tts_http", "tts_mp3", "tts_i2s"}; 129 | audio_pipeline_link(tts->pipeline, &link_tag[0], 3); 130 | 131 | // I2S流采样率 位数等设置 132 | i2s_stream_set_clk(tts->i2s_writer, config->playback_sample_rate, 16, 1); 133 | return tts; 134 | exit_tts_init: 135 | baidu_tts_destroy(tts); 136 | return NULL; 137 | } 138 | 139 | esp_err_t baidu_tts_destroy(baidu_tts_handle_t tts) 140 | { 141 | if (tts == NULL) { 142 | return ESP_FAIL; 143 | } 144 | audio_pipeline_stop(tts->pipeline); 145 | audio_pipeline_wait_for_stop(tts->pipeline); 146 | audio_pipeline_terminate(tts->pipeline); 147 | audio_pipeline_remove_listener(tts->pipeline); 148 | audio_pipeline_deinit(tts->pipeline); 149 | free(tts->buffer); 150 | free(tts); 151 | return ESP_OK; 152 | } 153 | 154 | /// @brief 这个函数用于注册一个监听器，以便在TTS事件发生时接收通知 155 | /// @param tts 156 | /// @param listener 157 | /// @return 158 | esp_err_t baidu_tts_set_listener(baidu_tts_handle_t tts, audio_event_iface_handle_t listener) 159 | { 160 | if (listener) { 161 | audio_pipeline_set_listener(tts->pipeline, listener); 162 | } 163 | return ESP_OK; 164 | } 165 | /// @brief 检测TTS事件是否完成 166 | /// @param tts 167 | /// @param msg 168 | /// @return 169 | bool baidu_tts_check_event_finish(baidu_tts_handle_t tts, audio_event_iface_msg_t *msg) 170 | { 171 | if (msg->source_type == AUDIO_ELEMENT_TYPE_ELEMENT && msg->source == (void *) tts->i2s_writer 172 | && msg->cmd == AEL_MSG_CMD_REPORT_STATUS 173 | && (((int)msg->data == AEL_STATUS_STATE_STOPPED) || ((int)msg->data == AEL_STATUS_STATE_FINISHED))) { 174 | return true; 175 | } 176 | return false; 177 | } 178 | 179 | /// @brief 这是一个简单的TTS播放器，它将文本发送到百度云TTS服务并播放接收到的音频 180 | /// @param tts text to speech handle 181 | /// @param text 需要转换为语音的文本 182 | /// @return 183 | esp_err_t baidu_tts_start(baidu_tts_handle_t tts, const char *text) 184 | { 185 | free(tts->text); 186 | tts->text = strdup(text); 187 | if (tts->text == NULL) { 188 | ESP_LOGE(TAG, "Error no mem"); 189 | return ESP_ERR_NO_MEM; 190 | } 191 | snprintf(tts->buffer, tts->buffer_size, BAIDU_TTS_ENDPOINT); 192 | audio_pipeline_reset_items_state(tts->pipeline); 193 | audio_pipeline_reset_ringbuffer(tts->pipeline); 194 | audio_element_set_uri(tts->http_stream_reader, tts->buffer); 195 | audio_pipeline_run(tts->pipeline); 196 | return ESP_OK; 197 | } 198 | /// @brief 用于停止播放从百度云TTS接收到的音频 199 | /// @param tts 200 | /// @return 201 | esp_err_t baidu_tts_stop(baidu_tts_handle_t tts) 202 | { 203 | audio_pipeline_stop(tts->pipeline); 204 | audio_pipeline_wait_for_stop(tts->pipeline); 205 | ESP_LOGD(TAG, "TTS Stopped"); 206 | return ESP_OK; 207 | } 208 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/main/baidu_tts.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ESPRESSIF MIT License 3 | * 4 | * Copyright (c) 2018 5 | * 6 | * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, 7 | * it is free of charge, to any person obtaining a copy of this software and associated 8 | * documentation files (the "Software"), to deal in the Software without restriction, including 9 | * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 | * and/or sell copies of the Software, and to permit persons to whom the Software is furnished 11 | * to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all copies or 14 | * substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 18 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 19 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 20 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 21 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | * 23 | */ 24 | 25 | #ifndef _BAIDU_TTS_H_ 26 | #define _BAIDU_TTS_H_ 27 | 28 | #include "esp_err.h" 29 | #include "audio_event_iface.h" 30 | 31 | #ifdef __cplusplus 32 | extern "C" { 33 | #endif 34 | 35 | #define DEFAULT_TTS_BUFFER_SIZE (2048) 36 | 37 | typedef struct baidu_tts* baidu_tts_handle_t; 38 | 39 | typedef struct { 40 | const char *api_key; 41 | //const char *lang_code; 42 | int playback_sample_rate; 43 | int buffer_size; 44 | } baidu_tts_config_t; 45 | 46 | /** 47 | * @brief Initialize Baidu Cloud Text-to-Speech, this function will return a Text-to-Speech context 48 | * 49 | * @param config The configuration 50 | * 51 | * @return The Text-to-Speech context 52 | */ 53 | baidu_tts_handle_t baidu_tts_init(baidu_tts_config_t *config); 54 | 55 | /** 56 | * @brief Start sending text to Baidu Cloud Text-to-Speech and play audio received 57 | * 58 | * @param[in] tts The Text-to-Speech context 59 | * @param[in] text The text 60 | * @param[in] lang_code The language code 61 | * 62 | * @return 63 | * - ESP_OK 64 | * - ESP_FAIL 65 | */ 66 | esp_err_t baidu_tts_start(baidu_tts_handle_t tts, const char *text); 67 | 68 | /** 69 | * @brief Stop playing audio from Baidu Cloud Text-to-Speech 70 | * 71 | * @param[in] tts The Text-to-Speech context 72 | * 73 | * @return 74 | * - ESP_OK 75 | * - ESP_FAIL 76 | */ 77 | esp_err_t baidu_tts_stop(baidu_tts_handle_t tts); 78 | 79 | /** 80 | * @brief Register listener for the Text-to-Speech context 81 | * 82 | * @param[in] tts The Text-to-Speech context 83 | * @param[in] listener The listener 84 | * 85 | * @return 86 | * - ESP_OK 87 | * - ESP_FAIL 88 | */ 89 | esp_err_t baidu_tts_set_listener(baidu_tts_handle_t tts, audio_event_iface_handle_t listener); 90 | 91 | /** 92 | * @brief Cleanup the Text-to-Speech object 93 | * 94 | * @param[in] tts The Text-to-Speech context 95 | * 96 | * @return 97 | * - ESP_OK 98 | * - ESP_FAIL 99 | */ 100 | esp_err_t baidu_tts_destroy(baidu_tts_handle_t tts); 101 | 102 | /** 103 | * @brief Check if the Text-To-Speech finished playing audio from server 104 | * 105 | * @param[in] tts The Text-to-Speech context 106 | * @param msg The message 107 | * 108 | * @return 109 | * - true 110 | * - false 111 | */ 112 | bool baidu_tts_check_event_finish(baidu_tts_handle_t tts, audio_event_iface_msg_t *msg); 113 | 114 | #ifdef __cplusplus 115 | } 116 | #endif 117 | 118 | #endif 119 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/main/baidu_vtt.c: -------------------------------------------------------------------------------- 1 | /* 2 | * ESPRESSIF MIT License 3 | * 4 | * Copyright (c) 2018 5 | * 6 | * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, 7 | * it is free of charge, to any person obtaining a copy of this software and associated 8 | * documentation files (the "Software"), to deal in the Software without restriction, including 9 | * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 | * and/or sell copies of the Software, and to permit persons to whom the Software is furnished 11 | * to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all copies or 14 | * substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 18 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 19 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 20 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 21 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | * 23 | */ 24 | 25 | #include 26 | #include "freertos/FreeRTOS.h" 27 | #include "freertos/task.h" 28 | #include "esp_log.h" 29 | #include "esp_wifi.h" 30 | #include "nvs_flash.h" 31 | #include "esp_http_client.h" 32 | #include "sdkconfig.h" 33 | #include "audio_element.h" 34 | #include "audio_pipeline.h" 35 | #include "audio_event_iface.h" 36 | #include "audio_common.h" 37 | #include "audio_hal.h" 38 | #include "http_stream.h" 39 | #include "i2s_stream.h" 40 | #include "mp3_decoder.h" 41 | #include "baidu_vtt.h" 42 | #include "cJSON.h" 43 | 44 | static const char *TAG = "BAIDU_VTT"; 45 | 46 | extern char *baidu_access_token; 47 | 48 | #define BAIDU_VTT_ENDPOINT "http://vop.baidu.com/server_api?dev_pid=1537&cuid=esp32c3&token=%s" 49 | #define BAIDU_VTT_TASK_STACK (8*1024) 50 | 51 | 52 | typedef struct baidu_vtt { 53 | audio_pipeline_handle_t pipeline; 54 | char *buffer; 55 | audio_element_handle_t i2s_reader; 56 | audio_element_handle_t http_stream_writer; 57 | int sample_rates; 58 | int buffer_size; 59 | baidu_vtt_encoding_t encoding; 60 | char *response_text; 61 | baidu_vtt_event_handle_t on_begin; 62 | } baidu_vtt_t; 63 | 64 | char ask_text[256]={0}; // 存储提问的文字 65 | 66 | static esp_err_t _http_stream_writer_event_handle(http_stream_event_msg_t *msg) 67 | { 68 | esp_http_client_handle_t http = (esp_http_client_handle_t)msg->http_client; 69 | baidu_vtt_t *vtt = (baidu_vtt_t *)msg->user_data; 70 | 71 | static int total_write = 0; 72 | char len_buf[16]; 73 | 74 | if (msg->event_id == HTTP_STREAM_PRE_REQUEST) { 75 | ESP_LOGI(TAG, "[ + ] HTTP client HTTP_STREAM_PRE_REQUEST, lenght=%d", msg->buffer_len); 76 | 77 | total_write = 0; 78 | esp_http_client_set_method(http, HTTP_METHOD_POST); 79 | esp_http_client_set_post_field(http, NULL, -1); // Chunk content 80 | esp_http_client_set_header(http, "Content-Type", "audio/pcm;rate=16000"); 81 | esp_http_client_set_header(http, "Accept", "application/json"); 82 | return ESP_OK; 83 | } 84 | 85 | if (msg->event_id == HTTP_STREAM_ON_REQUEST) { // 这个if会进来好多次 86 | // write data 87 | int wlen = sprintf(len_buf, "%x\r\n", msg->buffer_len); 88 | if (esp_http_client_write(http, len_buf, wlen) <= 0) { 89 | return ESP_FAIL; 90 | } 91 | if (esp_http_client_write(http, msg->buffer, msg->buffer_len) <= 0) { 92 | return ESP_FAIL; 93 | } 94 | if (esp_http_client_write(http, "\r\n", 2) <= 0) { 95 | return ESP_FAIL; 96 | } 97 | total_write += msg->buffer_len; 98 | printf("\033[A\33[2K\rTotal bytes written: %d\n", total_write); 99 | return msg->buffer_len; 100 | } 101 | 102 | /* Write End chunk */ 103 | if (msg->event_id == HTTP_STREAM_POST_REQUEST) { 104 | ESP_LOGI(TAG, "[ + ] HTTP client HTTP_STREAM_POST_REQUEST, write end chunked marker"); 105 | if (esp_http_client_write(http, "0\r\n\r\n", 5) <= 0) { 106 | return ESP_FAIL; 107 | } 108 | return ESP_OK; 109 | } 110 | 111 | if (msg->event_id == HTTP_STREAM_FINISH_REQUEST) { 112 | //结束的事件 113 | int read_len = esp_http_client_read(http, (char *)vtt->buffer, vtt->buffer_size); 114 | ESP_LOGI(TAG, "[ + ] HTTP client HTTP_STREAM_FINISH_REQUEST, read_len=%d", read_len); 115 | if (read_len <= 0) { 116 | return ESP_FAIL; 117 | } 118 | if (read_len > vtt->buffer_size - 1) { 119 | read_len = vtt->buffer_size - 1; 120 | } 121 | vtt->buffer[read_len] = 0; 122 | ESP_LOGI(TAG, "Got HTTP Response = %s", (char *)vtt->buffer); 123 | //参数解析 124 | cJSON *root = cJSON_Parse(vtt->buffer); 125 | int err_no = cJSON_GetObjectItem(root,"err_no")->valueint; 126 | 127 | if (err_no == 0) // 如果转换成功 128 | { 129 | cJSON *result = cJSON_GetObjectItem(root,"result"); 130 | char *text = cJSON_GetArrayItem(result, 0)->valuestring; 131 | strcpy(ask_text, text); 132 | vtt->response_text = ask_text; //记录返回的数据 133 | ESP_LOGI(TAG, "response_text:%s", vtt->response_text); 134 | } 135 | else{ 136 | vtt->response_text = NULL; 137 | } 138 | cJSON_Delete(root); 139 | 140 | return ESP_OK; 141 | } 142 | return ESP_OK; 143 | } 144 | 145 | baidu_vtt_handle_t baidu_vtt_init(baidu_vtt_config_t *config) 146 | { 147 | // 管道配置 148 | audio_pipeline_cfg_t pipeline_cfg = DEFAULT_AUDIO_PIPELINE_CONFIG(); 149 | baidu_vtt_t *vtt = calloc(1, sizeof(baidu_vtt_t)); 150 | AUDIO_MEM_CHECK(TAG, vtt, return NULL); 151 | vtt->pipeline = audio_pipeline_init(&pipeline_cfg); 152 | 153 | vtt->buffer_size = config->buffer_size; 154 | if (vtt->buffer_size <= 0) { 155 | vtt->buffer_size = DEFAULT_VTT_BUFFER_SIZE; 156 | } 157 | 158 | vtt->buffer = malloc(vtt->buffer_size); 159 | AUDIO_MEM_CHECK(TAG, vtt->buffer, goto exit_vtt_init); 160 | 161 | // I2S流配置 162 | i2s_stream_cfg_t i2s_cfg = I2S_STREAM_CFG_DEFAULT_WITH_PARA(0, 16000, 16, AUDIO_STREAM_READER); 163 | i2s_cfg.std_cfg.slot_cfg.slot_mode = I2S_SLOT_MODE_MONO; 164 | i2s_cfg.std_cfg.slot_cfg.slot_mask = I2S_STD_SLOT_LEFT; 165 | vtt->i2s_reader = i2s_stream_init(&i2s_cfg); 166 | 167 | // HTTP流配置 168 | http_stream_cfg_t http_cfg = { 169 | .type = AUDIO_STREAM_WRITER, 170 | .event_handle = _http_stream_writer_event_handle, 171 | .user_data = vtt, 172 | .task_stack = BAIDU_VTT_TASK_STACK, 173 | }; 174 | vtt->http_stream_writer = http_stream_init(&http_cfg); 175 | vtt->sample_rates = config->record_sample_rates; 176 | vtt->encoding = config->encoding; 177 | vtt->on_begin = config->on_begin; 178 | 179 | // 把 I2S流 和 HTTP流 注册到管道 并链接起来 180 | audio_pipeline_register(vtt->pipeline, vtt->http_stream_writer, "vtt_http"); 181 | audio_pipeline_register(vtt->pipeline, vtt->i2s_reader, "vtt_i2s"); 182 | const char *link_tag[2] = {"vtt_i2s", "vtt_http"}; 183 | audio_pipeline_link(vtt->pipeline, &link_tag[0], 2); 184 | 185 | // 设置I2S采样率 位数 等参数 186 | i2s_stream_set_clk(vtt->i2s_reader, config->record_sample_rates, 16, 1); 187 | 188 | return vtt; 189 | exit_vtt_init: 190 | baidu_vtt_destroy(vtt); 191 | return NULL; 192 | } 193 | 194 | esp_err_t baidu_vtt_destroy(baidu_vtt_handle_t vtt) 195 | { 196 | if (vtt == NULL) { 197 | return ESP_FAIL; 198 | } 199 | audio_pipeline_stop(vtt->pipeline); 200 | audio_pipeline_wait_for_stop(vtt->pipeline); 201 | audio_pipeline_terminate(vtt->pipeline); 202 | audio_pipeline_remove_listener(vtt->pipeline); 203 | audio_pipeline_deinit(vtt->pipeline); 204 | free(vtt->buffer); 205 | free(vtt); 206 | return ESP_OK; 207 | } 208 | 209 | esp_err_t baidu_vtt_set_listener(baidu_vtt_handle_t vtt, audio_event_iface_handle_t listener) 210 | { 211 | if (listener) { 212 | audio_pipeline_set_listener(vtt->pipeline, listener); 213 | } 214 | return ESP_OK; 215 | } 216 | 217 | esp_err_t baidu_vtt_start(baidu_vtt_handle_t vtt) 218 | { 219 | //合成一下百度的信息 220 | snprintf(vtt->buffer, vtt->buffer_size, BAIDU_VTT_ENDPOINT, baidu_access_token); 221 | audio_element_set_uri(vtt->http_stream_writer, vtt->buffer); 222 | audio_pipeline_reset_items_state(vtt->pipeline); 223 | audio_pipeline_reset_ringbuffer(vtt->pipeline); 224 | audio_pipeline_run(vtt->pipeline); 225 | return ESP_OK; 226 | } 227 | 228 | 229 | char *baidu_vtt_stop(baidu_vtt_handle_t vtt) 230 | { 231 | audio_pipeline_stop(vtt->pipeline); 232 | audio_pipeline_wait_for_stop(vtt->pipeline); 233 | 234 | return vtt->response_text; 235 | } 236 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/main/baidu_vtt.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ESPRESSIF MIT License 3 | * 4 | * Copyright (c) 2018 5 | * 6 | * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, 7 | * it is free of charge, to any person obtaining a copy of this software and associated 8 | * documentation files (the "Software"), to deal in the Software without restriction, including 9 | * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 | * and/or sell copies of the Software, and to permit persons to whom the Software is furnished 11 | * to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all copies or 14 | * substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 18 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 19 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 20 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 21 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | * 23 | */ 24 | 25 | #ifndef _BAIDU_VTT_H_ 26 | #define _BAIDU_VTT_H_ 27 | 28 | #include "esp_err.h" 29 | #include "audio_event_iface.h" 30 | 31 | #ifdef __cplusplus 32 | extern "C" { 33 | #endif 34 | 35 | #define DEFAULT_VTT_BUFFER_SIZE (8192) 36 | 37 | /** 38 | * Baidu Cloud Speech-to-Text audio encoding 39 | */ 40 | typedef enum { 41 | ENCODING_LINEAR16 = 0, /*!< Baidu Cloud Speech-to-Text audio encoding PCM 16-bit mono */ 42 | } baidu_vtt_encoding_t; 43 | 44 | typedef struct baidu_vtt* baidu_vtt_handle_t; 45 | typedef void (*baidu_vtt_event_handle_t)(baidu_vtt_handle_t vtt); 46 | 47 | /** 48 | * baidu Cloud Speech-to-Text configurations 49 | */ 50 | typedef struct { 51 | int record_sample_rates; /*!< Audio recording sample rate */ 52 | baidu_vtt_encoding_t encoding; /*!< Audio encoding */ 53 | int buffer_size; /*!< Processing buffer size */ 54 | baidu_vtt_event_handle_t on_begin; /*!< Begin send audio data to server */ 55 | } baidu_vtt_config_t; 56 | 57 | 58 | 59 | 60 | /** 61 | * @brief initialize Baidu Cloud Speech-to-Text, this function will return a Speech-to-Text context 62 | * 63 | * @param config The Baidu Cloud Speech-to-Text configuration 64 | * 65 | * @return The Speech-to-Text context 66 | */ 67 | baidu_vtt_handle_t baidu_vtt_init(baidu_vtt_config_t *config); 68 | 69 | /** 70 | * @brief Start recording and sending audio to Baidu Cloud Speech-to-Text 71 | * 72 | * @param[in] vtt The Speech-to-Text context 73 | * 74 | * @return 75 | * - ESP_OK 76 | * - ESP_FAIL 77 | */ 78 | esp_err_t baidu_vtt_start(baidu_vtt_handle_t vtt); 79 | 80 | /** 81 | * @brief Stop sending audio to Baidu Cloud Speech-to-Text and get the result text 82 | * 83 | * @param[in] vtt The Speech-to-Text context 84 | * 85 | * @return Baidu Cloud Speech-to-Text server response 86 | */ 87 | char *baidu_vtt_stop(baidu_vtt_handle_t vtt); 88 | 89 | /** 90 | * @brief Cleanup the Speech-to-Text object 91 | * 92 | * @param[in] vtt The Speech-to-Text context 93 | * 94 | * @return 95 | * - ESP_OK 96 | * - ESP_FAIL 97 | */ 98 | esp_err_t baidu_vtt_destroy(baidu_vtt_handle_t vtt); 99 | 100 | /** 101 | * @brief Register listener for the Speech-to-Text context 102 | * 103 | * @param[in] vtt The Speech-to-Text context 104 | * @param[in] listener The listener 105 | * 106 | * @return 107 | * - ESP_OK 108 | * - ESP_FAIL 109 | */ 110 | esp_err_t baidu_vtt_set_listener(baidu_vtt_handle_t vtt, audio_event_iface_handle_t listener); 111 | 112 | #ifdef __cplusplus 113 | } 114 | #endif 115 | 116 | #endif 117 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/main/component.mk: -------------------------------------------------------------------------------- 1 | # 2 | # Main Makefile. This is basically the same as a component makefile. 3 | # 4 | # (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.) 5 | 6 | 7 | #Compile image file into the resulting firmware binary 8 | COMPONENT_OBJS = lv_demos.o -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/main/idf_component.yml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | espressif/esp_lcd_touch_ft5x06: "^1.0.6" 3 | lvgl/lvgl: "~8.3.0" 4 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/main/lv_gui.c: -------------------------------------------------------------------------------- 1 | #include "lvgl.h" 2 | 3 | lv_obj_t *label1; 4 | lv_obj_t *label2; 5 | lv_obj_t *label_wifi_info; 6 | 7 | LV_IMG_DECLARE(img_bilibili120); 8 | LV_FONT_DECLARE(font_alipuhui20); 9 | 10 | // 开机界面 11 | void lv_gui_start(void) 12 | { 13 | // 显示开机GIF图片 14 | lv_obj_t *gif_start = lv_gif_create(lv_scr_act()); 15 | lv_gif_set_src(gif_start, &img_bilibili120); 16 | lv_obj_align(gif_start, LV_ALIGN_CENTER, 0, -20); 17 | 18 | // 显示wifi连接信息 19 | label_wifi_info = lv_label_create(lv_scr_act()); 20 | lv_obj_align(label_wifi_info, LV_ALIGN_BOTTOM_MID, 0, -35); 21 | lv_obj_set_style_text_font(label_wifi_info, &font_alipuhui20, LV_STATE_DEFAULT); 22 | lv_label_set_text(label_wifi_info, "正在连接wifi..."); 23 | 24 | } 25 | 26 | // 主界面 27 | void lv_main_page(void) 28 | { 29 | lv_obj_set_style_bg_color(lv_scr_act(), lv_color_hex(0x000000), 0); // 修改背景为黑色 30 | 31 | static lv_style_t style; 32 | lv_style_init(&style); 33 | lv_style_set_radius(&style, 10); // 设置圆角半径 34 | lv_style_set_bg_opa( &style, LV_OPA_COVER ); 35 | lv_style_set_bg_color(&style, lv_color_hex(0x000000)); 36 | lv_style_set_text_color(&style, lv_color_hex(0xffffff)); 37 | lv_style_set_border_width(&style, 0); 38 | lv_style_set_pad_all(&style, 10); 39 | lv_style_set_width(&style, 320); // 设置宽 40 | lv_style_set_height(&style, 240); // 设置高 41 | 42 | /*Create an object with the new style*/ 43 | lv_obj_t * obj = lv_obj_create(lv_scr_act()); 44 | lv_obj_add_style(obj, &style, 0); 45 | 46 | label1 = lv_label_create(obj); 47 | lv_obj_set_width(label1, 300); 48 | lv_label_set_long_mode(label1, LV_LABEL_LONG_SCROLL_CIRCULAR); /*Circular scroll*/ 49 | lv_obj_align(label1, LV_ALIGN_TOP_LEFT, 0, 0); 50 | lv_obj_set_style_text_font(label1, &font_alipuhui20, LV_STATE_DEFAULT); 51 | lv_label_set_text(label1, "我："); 52 | 53 | label2 = lv_label_create(obj); 54 | lv_obj_set_width(label2, 300); 55 | lv_obj_align(label2, LV_ALIGN_TOP_LEFT, 0, 35); 56 | lv_obj_set_style_text_font(label2, &font_alipuhui20, LV_STATE_DEFAULT); 57 | lv_label_set_text(label2, "AI: "); 58 | } -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/main/lv_gui.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | 4 | void lv_main_page(void); 5 | void lv_gui_start(void); 6 | 7 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/main/minimax_chat.c: -------------------------------------------------------------------------------- 1 | /* 2 | * ESPRESSIF MIT License 3 | * 4 | * Copyright (c) 2018 5 | * 6 | * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, 7 | * it is free of charge, to any person obtaining a copy of this software and associated 8 | * documentation files (the "Software"), to deal in the Software without restriction, including 9 | * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 | * and/or sell copies of the Software, and to permit persons to whom the Software is furnished 11 | * to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all copies or 14 | * substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 18 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 19 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 20 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 21 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | * 23 | */ 24 | 25 | #include 26 | #include "freertos/FreeRTOS.h" 27 | #include "freertos/task.h" 28 | #include "esp_log.h" 29 | #include "esp_wifi.h" 30 | #include "nvs_flash.h" 31 | #include "esp_http_client.h" 32 | #include "sdkconfig.h" 33 | #include "audio_common.h" 34 | #include "audio_hal.h" 35 | #include "minimax_chat.h" 36 | #include "cJSON.h" 37 | 38 | static const char *TAG = "MINIMAX_CHAT"; 39 | 40 | extern nvs_handle my_handle; 41 | #define PSOT_DATA "{\ 42 | \"messages\":[{\"sender_type\":\"USER\", \"user_id\":%d,\"sender_name\":\"test\",\"text\":\"%s\"}]\ 43 | }" 44 | 45 | int user_id = -1; 46 | #define MAX_CHAT_BUFFER (2048) 47 | char minimax_content[2048]={0}; 48 | 49 | 50 | char *minimax_chat(const char *text) 51 | { 52 | char *response_text = NULL; 53 | char *post_buffer = NULL; 54 | char *data_buf = NULL; 55 | 56 | esp_http_client_config_t config = { 57 | .url = "http://192.168.188.165:8888/", // 这里替换成自己的GroupId 58 | .timeout_ms = 40000, 59 | .buffer_size_tx = 1024 // 默认是512 minimax_key很长 512不够 这里改成1024 60 | }; 61 | esp_http_client_handle_t client = esp_http_client_init(&config); 62 | 63 | int post_len = asprintf(&post_buffer, PSOT_DATA, user_id, text); //动态获取一内存记录信息 64 | 65 | if (post_buffer == NULL) { 66 | goto exit_translate; 67 | } 68 | 69 | // POST 70 | esp_http_client_set_method(client, HTTP_METHOD_POST); 71 | esp_http_client_set_header(client, "Content-Type", "application/json"); 72 | 73 | if (esp_http_client_open(client, post_len) != ESP_OK) { 74 | ESP_LOGE(TAG, "Error opening connection"); 75 | goto exit_translate; 76 | } 77 | int write_len = esp_http_client_write(client, post_buffer, post_len); 78 | ESP_LOGI(TAG, "Need to write %d, written %d", post_len, write_len); 79 | //获取信息长度 80 | int data_length = esp_http_client_fetch_headers(client); 81 | if (data_length <= 0) { 82 | data_length = MAX_CHAT_BUFFER; 83 | } 84 | //分配空间 85 | data_buf = malloc(data_length + 1); 86 | if(data_buf == NULL) { 87 | goto exit_translate; 88 | } 89 | data_buf[data_length] = '\0'; 90 | int rlen = esp_http_client_read(client, data_buf, data_length); 91 | data_buf[rlen] = '\0'; 92 | ESP_LOGI(TAG, "read = %s", data_buf); 93 | //解析信息 94 | cJSON *root = cJSON_Parse(data_buf); 95 | char * created = cJSON_GetObjectItem(root,"result")->valuestring; 96 | int32_t temp_id = cJSON_GetObjectItem(root,"user_id")->valueint; 97 | if (temp_id != user_id) 98 | { 99 | user_id = temp_id; 100 | ESP_ERROR_CHECK(nvs_set_i32(my_handle, "user_id", user_id)); 101 | ESP_ERROR_CHECK( nvs_commit(my_handle) ); 102 | ESP_LOGI(TAG, "user_id:%d", user_id); 103 | } 104 | if(created != 0) 105 | { 106 | // ESP_ERR_NVS_INVALID_HANDLE 107 | strcpy(minimax_content, created); 108 | response_text = minimax_content; 109 | ESP_LOGI(TAG, "response_text:%s", response_text); 110 | } 111 | 112 | cJSON_Delete(root); 113 | 114 | exit_translate: 115 | free(post_buffer); 116 | free(data_buf); 117 | esp_http_client_cleanup(client); 118 | 119 | return response_text; 120 | } -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/main/minimax_chat.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ESPRESSIF MIT License 3 | * 4 | * Copyright (c) 2018 5 | * 6 | * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, 7 | * it is free of charge, to any person obtaining a copy of this software and associated 8 | * documentation files (the "Software"), to deal in the Software without restriction, including 9 | * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 | * and/or sell copies of the Software, and to permit persons to whom the Software is furnished 11 | * to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all copies or 14 | * substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 18 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 19 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 20 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 21 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | * 23 | */ 24 | 25 | #ifndef _MINIMAX_CHAT_H_ 26 | #define _MINIMAX_CHAT_H_ 27 | 28 | // #include "esp_err.h" 29 | // #include "audio_event_iface.h" 30 | 31 | #ifdef __cplusplus 32 | extern "C" { 33 | #endif 34 | 35 | char *minimax_chat(const char *text); 36 | 37 | #ifdef __cplusplus 38 | } 39 | #endif 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/partitions.csv: -------------------------------------------------------------------------------- 1 | # Name, Type, SubType, Offset, Size 2 | nvs, data, nvs, 0x9000, 0x4000 3 | phy_init, data, phy, 0xd000, 0x1000 4 | factory, app, factory, 0x10000, 7M, 5 | -------------------------------------------------------------------------------- /ESP-IDF/ai_chat/sdkconfig.defaults: -------------------------------------------------------------------------------- 1 | CONFIG_LV_COLOR_16_SWAP=y 2 | CONFIG_LV_MEM_CUSTOM=y 3 | CONFIG_LV_MEM_CUSTOM_INCLUDE="stdlib.h" 4 | CONFIG_LV_MEM_BUF_MAX_NUM=16 5 | CONFIG_LV_MEMCPY_MEMSET_STD=y 6 | CONFIG_PARTITION_TABLE_CUSTOM=y 7 | CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" 8 | CONFIG_ESPTOOLPY_FLASHSIZE_8MB=y 9 | CONFIG_FREERTOS_ENABLE_BACKWARD_COMPATIBILITY=y 10 | CONFIG_ESP_TLS_INSECURE=y 11 | CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y 12 | CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=3072 -------------------------------------------------------------------------------- /ESP-IDF/emotion/README.md: -------------------------------------------------------------------------------- 1 | # 修改表情 2 | 3 | > 使用的图片https://pan.quark.cn/s/d07357245728, 来自米游社的 枫达QAQ, 纥栖, 堇瓜团子, 1翎11 4 | 5 | 实际是使用自己的图片替换虾哥的图片文件, 小智使用的是lvgl的方式, 所以需要使用lvgl的图片格式转换 6 | 7 | [免费在线图片改尺寸工具 - docsmall](https://docsmall.com/image-resize) 8 | 9 | 需要把图片压缩一下, 否则esp32存不下来以及会出现刷新卡的问题, 我这里使用的是64x64, 也提供了128x128的版本, 图片建议使用透明背景或者白色背景的 10 | 11 | [Image Converter — LVGL](https://lvgl.io/tools/imageconverter) 12 | 13 | 压缩以后得图片按照原本的图片格式进行命名, 实际的文件名和情感是一一对应的 14 | 15 | ![image-20250310200420020](https://picture-01-1316374204.cos.ap-beijing.myqcloud.com/lenovo-picture/202503102004163.png) 16 | 17 | ## 问题 18 | 19 | + 没有对应的文件夹 20 | 21 | 答: 编译以后下载时候才有的文件 22 | 23 | + 改了以后还是原来的图片 24 | 25 | 看一下你的板子是32的还是64的表情, 我使用的是64的 26 | 27 | ![image-20250311123829935](https://picture-01-1316374204.cos.ap-beijing.myqcloud.com/lenovo-picture/202503111238593.png) 28 | 29 | + 表情的大小有限制 30 | 31 | ![image-20250313115333603](https://picture-01-1316374204.cos.ap-beijing.myqcloud.com/lenovo-picture/202503131153788.png) 32 | 33 | + 修改文字显示 34 | 35 | ![image-20250313115547253](https://picture-01-1316374204.cos.ap-beijing.myqcloud.com/lenovo-picture/202503131155323.png) 36 | 37 | 默认的长文本是换行, 可以使用滚动的方法减少显示空间 38 | 39 | ![image-20250313115638298](https://picture-01-1316374204.cos.ap-beijing.myqcloud.com/lenovo-picture/202503131156519.png) 40 | 41 | 替换成LV_LABEL_LONG_SCROLL_CIRCULAR -------------------------------------------------------------------------------- /ESP-IDF/emotion/emotion.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuSenfeng/ai-chat-local/d8254bf7e0d0340b73ff34ea34d83ae6c49db6ce/ESP-IDF/emotion/emotion.zip -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuSenfeng/ai-chat-local/d8254bf7e0d0340b73ff34ea34d83ae6c49db6ce/README.pdf -------------------------------------------------------------------------------- /python/Dify/esp32.yml: -------------------------------------------------------------------------------- 1 | app: 2 | description: '' 3 | icon: 🤖 4 | icon_background: '#FFEAD5' 5 | mode: workflow 6 | name: esp32 7 | use_icon_as_answer_icon: false 8 | kind: app 9 | version: 0.1.5 10 | workflow: 11 | conversation_variables: [] 12 | environment_variables: [] 13 | features: 14 | file_upload: 15 | allowed_file_extensions: 16 | - .JPG 17 | - .JPEG 18 | - .PNG 19 | - .GIF 20 | - .WEBP 21 | - .SVG 22 | allowed_file_types: 23 | - image 24 | allowed_file_upload_methods: 25 | - local_file 26 | - remote_url 27 | enabled: false 28 | fileUploadConfig: 29 | audio_file_size_limit: 50 30 | batch_count_limit: 5 31 | file_size_limit: 15 32 | image_file_size_limit: 10 33 | video_file_size_limit: 100 34 | workflow_file_upload_limit: 10 35 | image: 36 | enabled: false 37 | number_limits: 3 38 | transfer_methods: 39 | - local_file 40 | - remote_url 41 | number_limits: 3 42 | opening_statement: '' 43 | retriever_resource: 44 | enabled: true 45 | sensitive_word_avoidance: 46 | enabled: false 47 | speech_to_text: 48 | enabled: false 49 | suggested_questions: [] 50 | suggested_questions_after_answer: 51 | enabled: false 52 | text_to_speech: 53 | enabled: false 54 | language: '' 55 | voice: '' 56 | graph: 57 | edges: 58 | - data: 59 | isInIteration: false 60 | sourceType: knowledge-retrieval 61 | targetType: llm 62 | id: 1739947041857-source-1739946194201-target 63 | selected: false 64 | source: '1739947041857' 65 | sourceHandle: source 66 | target: '1739946194201' 67 | targetHandle: target 68 | type: custom 69 | zIndex: 0 70 | - data: 71 | isInIteration: false 72 | sourceType: llm 73 | targetType: end 74 | id: 1739946194201-source-1739947401945-target 75 | selected: false 76 | source: '1739946194201' 77 | sourceHandle: source 78 | target: '1739947401945' 79 | targetHandle: target 80 | type: custom 81 | zIndex: 0 82 | - data: 83 | isInIteration: false 84 | sourceType: start 85 | targetType: question-classifier 86 | id: 1739946173294-source-1739947463945-target 87 | selected: false 88 | source: '1739946173294' 89 | sourceHandle: source 90 | target: '1739947463945' 91 | targetHandle: target 92 | type: custom 93 | zIndex: 0 94 | - data: 95 | isInIteration: false 96 | sourceType: question-classifier 97 | targetType: knowledge-retrieval 98 | id: 1739947463945-1-1739947041857-target 99 | selected: false 100 | source: '1739947463945' 101 | sourceHandle: '1' 102 | target: '1739947041857' 103 | targetHandle: target 104 | type: custom 105 | zIndex: 0 106 | - data: 107 | isInIteration: false 108 | sourceType: question-classifier 109 | targetType: llm 110 | id: 1739947463945-2-1739947625799-target 111 | source: '1739947463945' 112 | sourceHandle: '2' 113 | target: '1739947625799' 114 | targetHandle: target 115 | type: custom 116 | zIndex: 0 117 | - data: 118 | isInIteration: false 119 | sourceType: llm 120 | targetType: tool 121 | id: 1739947625799-source-1739947738490-target 122 | source: '1739947625799' 123 | sourceHandle: source 124 | target: '1739947738490' 125 | targetHandle: target 126 | type: custom 127 | zIndex: 0 128 | - data: 129 | isInIteration: false 130 | sourceType: tool 131 | targetType: llm 132 | id: 1739947738490-source-1739947771695-target 133 | source: '1739947738490' 134 | sourceHandle: source 135 | target: '1739947771695' 136 | targetHandle: target 137 | type: custom 138 | zIndex: 0 139 | - data: 140 | isInIteration: false 141 | sourceType: llm 142 | targetType: end 143 | id: 1739947771695-source-1739947813182-target 144 | source: '1739947771695' 145 | sourceHandle: source 146 | target: '1739947813182' 147 | targetHandle: target 148 | type: custom 149 | zIndex: 0 150 | - data: 151 | isInIteration: false 152 | sourceType: question-classifier 153 | targetType: end 154 | id: 1739947463945-1739947519960-1739947863180-target 155 | source: '1739947463945' 156 | sourceHandle: '1739947519960' 157 | target: '1739947863180' 158 | targetHandle: target 159 | type: custom 160 | zIndex: 0 161 | nodes: 162 | - data: 163 | desc: '' 164 | selected: false 165 | title: 开始 166 | type: start 167 | variables: 168 | - label: 客户文本 169 | max_length: 48 170 | options: [] 171 | required: true 172 | type: text-input 173 | variable: text 174 | height: 90 175 | id: '1739946173294' 176 | position: 177 | x: 75 178 | y: 291 179 | positionAbsolute: 180 | x: 75 181 | y: 291 182 | selected: false 183 | sourcePosition: right 184 | targetPosition: left 185 | type: custom 186 | width: 244 187 | - data: 188 | context: 189 | enabled: true 190 | variable_selector: 191 | - '1739947041857' 192 | - result 193 | desc: '' 194 | model: 195 | completion_params: 196 | temperature: 0.7 197 | mode: chat 198 | name: gpt-3.5-turbo-1106 199 | provider: openai 200 | prompt_template: 201 | - id: 20c84b02-51dd-4c82-8437-3bffc0255285 202 | role: system 203 | text: '请你使用搜索到的内容(在 里面)回答用户的问题: {{#1739946173294.text#}} 204 | 205 | 206 | 207 | {{#context#}} 208 | 209 | 210 | 211 | + 如果你不知道, 回答不知道 212 | 213 | + 不要编造不存在的事情 214 | 215 | ' 216 | selected: false 217 | title: LLM 218 | type: llm 219 | variables: [] 220 | vision: 221 | enabled: false 222 | height: 98 223 | id: '1739946194201' 224 | position: 225 | x: 1269.479000133355 226 | y: 291 227 | positionAbsolute: 228 | x: 1269.479000133355 229 | y: 291 230 | selected: false 231 | sourcePosition: right 232 | targetPosition: left 233 | type: custom 234 | width: 244 235 | - data: 236 | dataset_ids: 237 | - ee3fe5dd-d575-48a6-ba9b-0ebf91c6929d 238 | desc: '' 239 | multiple_retrieval_config: 240 | reranking_enable: true 241 | reranking_mode: reranking_model 242 | reranking_model: 243 | model: netease-youdao/bce-reranker-base_v1 244 | provider: siliconflow 245 | top_k: 4 246 | query_variable_selector: 247 | - '1739946173294' 248 | - text 249 | retrieval_mode: multiple 250 | selected: false 251 | title: 知识检索 252 | type: knowledge-retrieval 253 | height: 92 254 | id: '1739947041857' 255 | position: 256 | x: 789 257 | y: 291 258 | positionAbsolute: 259 | x: 789 260 | y: 291 261 | selected: false 262 | sourcePosition: right 263 | targetPosition: left 264 | type: custom 265 | width: 244 266 | - data: 267 | desc: '' 268 | outputs: 269 | - value_selector: 270 | - '1739946194201' 271 | - text 272 | variable: text 273 | selected: false 274 | title: 结束 275 | type: end 276 | height: 90 277 | id: '1739947401945' 278 | position: 279 | x: 1807.8350791281675 280 | y: 296.2233033797768 281 | positionAbsolute: 282 | x: 1807.8350791281675 283 | y: 296.2233033797768 284 | selected: false 285 | sourcePosition: right 286 | targetPosition: left 287 | type: custom 288 | width: 244 289 | - data: 290 | classes: 291 | - id: '1' 292 | name: esp32对话助手项目相关问题, 需要查手册， 如：项目的作者，项目的基础功能，项目的使用等 293 | - id: '2' 294 | name: 实时问题, 如新闻, 天气等需要使用网络搜索 295 | - id: '1739947519960' 296 | name: 日常对话问题, 无需调用, 如询问心情、在做什么、喜好之类的 297 | desc: '' 298 | instructions: '' 299 | model: 300 | completion_params: 301 | temperature: 0.5 302 | mode: chat 303 | name: gpt-3.5-turbo-1106 304 | provider: openai 305 | query_variable_selector: 306 | - '1739946173294' 307 | - text 308 | selected: false 309 | title: 问题分类器 310 | topics: [] 311 | type: question-classifier 312 | vision: 313 | enabled: false 314 | height: 276 315 | id: '1739947463945' 316 | position: 317 | x: 489 318 | y: 291 319 | positionAbsolute: 320 | x: 489 321 | y: 291 322 | selected: false 323 | sourcePosition: right 324 | targetPosition: left 325 | type: custom 326 | width: 244 327 | - data: 328 | context: 329 | enabled: false 330 | variable_selector: [] 331 | desc: '' 332 | model: 333 | completion_params: 334 | temperature: 0.7 335 | mode: chat 336 | name: gpt-3.5-turbo-1106 337 | provider: openai 338 | prompt_template: 339 | - role: system 340 | text: '用户输入的内容是: {{#1739946173294.text#}} 341 | 342 | 请你根据这个问题输出一个搜索使用的关键词' 343 | selected: false 344 | title: LLM 2 345 | type: llm 346 | variables: [] 347 | vision: 348 | enabled: false 349 | height: 98 350 | id: '1739947625799' 351 | position: 352 | x: 789 353 | y: 414.91907859474725 354 | positionAbsolute: 355 | x: 789 356 | y: 414.91907859474725 357 | selected: false 358 | sourcePosition: right 359 | targetPosition: left 360 | type: custom 361 | width: 244 362 | - data: 363 | desc: '' 364 | provider_id: tavily 365 | provider_name: tavily 366 | provider_type: builtin 367 | selected: false 368 | title: Tavily Search 369 | tool_configurations: 370 | days: 3 371 | exclude_domains: null 372 | include_answer: 0 373 | include_domains: null 374 | include_image_descriptions: 0 375 | include_images: 0 376 | include_raw_content: 0 377 | max_results: 1 378 | search_depth: basic 379 | topic: general 380 | tool_label: Tavily Search 381 | tool_name: tavily_search 382 | tool_parameters: 383 | query: 384 | type: mixed 385 | value: '{{#1739947625799.text#}}' 386 | type: tool 387 | height: 324 388 | id: '1739947738490' 389 | position: 390 | x: 1101.5276522233085 391 | y: 439.42319857997626 392 | positionAbsolute: 393 | x: 1101.5276522233085 394 | y: 439.42319857997626 395 | selected: true 396 | sourcePosition: right 397 | targetPosition: left 398 | type: custom 399 | width: 244 400 | - data: 401 | context: 402 | enabled: true 403 | variable_selector: 404 | - '1739947738490' 405 | - text 406 | desc: '' 407 | model: 408 | completion_params: 409 | temperature: 0.7 410 | mode: chat 411 | name: gpt-3.5-turbo-1106 412 | provider: openai 413 | prompt_template: 414 | - id: 6d7867cb-528e-40a3-a2a5-24b2d146f2c6 415 | role: system 416 | text: '请你使用搜索到的内容(在 里面)回答用户的问题: {{#1739946173294.text#}} 417 | 418 | 419 | 420 | {{#context#}} 421 | 422 | 423 | 424 | + 如果你不知道, 回答不知道 425 | 426 | + 不要编造不存在的事情' 427 | selected: false 428 | title: LLM 3 429 | type: llm 430 | variables: [] 431 | vision: 432 | enabled: false 433 | height: 98 434 | id: '1739947771695' 435 | position: 436 | x: 1436 437 | y: 448 438 | positionAbsolute: 439 | x: 1436 440 | y: 448 441 | selected: false 442 | sourcePosition: right 443 | targetPosition: left 444 | type: custom 445 | width: 244 446 | - data: 447 | desc: '' 448 | outputs: 449 | - value_selector: 450 | - '1739947771695' 451 | - text 452 | variable: text 453 | - value_selector: 454 | - '1739947738490' 455 | - text 456 | variable: files 457 | selected: false 458 | title: 结束 2 459 | type: end 460 | height: 116 461 | id: '1739947813182' 462 | position: 463 | x: 1807.8350791281675 464 | y: 461.05825844944184 465 | positionAbsolute: 466 | x: 1807.8350791281675 467 | y: 461.05825844944184 468 | selected: false 469 | sourcePosition: right 470 | targetPosition: left 471 | type: custom 472 | width: 244 473 | - data: 474 | desc: '' 475 | outputs: 476 | - value_selector: 477 | - '1739946173294' 478 | - text 479 | variable: text 480 | selected: true 481 | title: 结束 3 482 | type: end 483 | height: 90 484 | id: '1739947863180' 485 | position: 486 | x: 1401.5276522233085 487 | y: 848.1170667233392 488 | positionAbsolute: 489 | x: 1401.5276522233085 490 | y: 848.1170667233392 491 | selected: false 492 | sourcePosition: right 493 | targetPosition: left 494 | type: custom 495 | width: 244 496 | viewport: 497 | x: -1071.2641868787268 498 | y: -474.86488735532157 499 | zoom: 1.5157165665103987 500 | -------------------------------------------------------------------------------- /python/Dify/test.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | url = 'https://api.dify.ai/v1/workflows/run' 5 | api_key = '你的API_KEY' 6 | headers = { 7 | 'Authorization': f'Bearer {api_key}', 8 | 'Content-Type': 'application/json', 9 | } 10 | data = { 11 | "inputs": {"text": "查手册看一下告诉我文档的作者"}, 12 | "response_mode": "blocking", 13 | "user": "abc-123" 14 | } 15 | 16 | response = requests.post(url, headers=headers, data=json.dumps(data)) 17 | 18 | print(response.text) 19 | result = json.loads(response.text) 20 | # 检测outputs是否有text字段 21 | if "text" in result["data"]['outputs']: 22 | print(result["data"]['outputs']["text"].encode().decode('utf-8')) -------------------------------------------------------------------------------- /python/classfy-model/furina_test.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 7, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stderr", 10 | "output_type": "stream", 11 | "text": [ 12 | "C:\\Users\\jiao\\AppData\\Local\\Temp\\ipykernel_33924\\4280861123.py:6: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n", 13 | " model = torch.load('model_furina/model_2025_02_04_15_41_53.pth')\n", 14 | "Device set to use cuda:0\n" 15 | ] 16 | }, 17 | { 18 | "data": { 19 | "text/plain": [ 20 | "[{'label': 'no', 'score': 0.9056112170219421},\n", 21 | " {'label': 'no', 'score': 0.9552835822105408},\n", 22 | " {'label': 'no', 'score': 0.8829072713851929},\n", 23 | " {'label': 'yes', 'score': 0.834133505821228},\n", 24 | " {'label': 'no', 'score': 0.9279910922050476},\n", 25 | " {'label': 'yes', 'score': 0.9634923934936523},\n", 26 | " {'label': 'yes', 'score': 0.967900276184082},\n", 27 | " {'label': 'yes', 'score': 0.9298421144485474}]" 28 | ] 29 | }, 30 | "execution_count": 7, 31 | "metadata": {}, 32 | "output_type": "execute_result" 33 | } 34 | ], 35 | "source": [ 36 | "import torch\n", 37 | "# 文本分类模型微调的示例\n", 38 | "from transformers import AutoTokenizer\n", 39 | "# 加载模型\n", 40 | "tokenizer = AutoTokenizer.from_pretrained(\"./hflrbt3\")\n", 41 | "model = torch.load('model_furina/model_2025_02_04_15_41_53.pth')\n", 42 | "model.eval()\n", 43 | "id2label = {0:\"no\", 1:\"yes\"}\n", 44 | "sen = [\"服从命令, 芙宁娜女士\",\n", 45 | " \"你今天怎么样\",\n", 46 | " \"我要去上班了\",\n", 47 | " \"给我今天的新闻\",\n", 48 | " \"你是谁\",\n", 49 | " \"看一下最近的股票行情\",\n", 50 | " \"给我讲解一下这一个项目的情况\",\n", 51 | " \"项目的情况怎么样了\"]\n", 52 | "from transformers import pipeline\n", 53 | "model.config.id2label = id2label\n", 54 | "pipe = pipeline(\"text-classification\", model=model, tokenizer=tokenizer, device=0 if torch.cuda.is_available() else -1)\n", 55 | "pipe(sen)" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 3, 61 | "metadata": {}, 62 | "outputs": [ 63 | { 64 | "name": "stderr", 65 | "output_type": "stream", 66 | "text": [ 67 | "Device set to use cuda:0\n" 68 | ] 69 | }, 70 | { 71 | "data": { 72 | "text/plain": [ 73 | "[{'label': 'no', 'score': 0.9489531517028809},\n", 74 | " {'label': 'no', 'score': 0.6886082887649536},\n", 75 | " {'label': 'no', 'score': 0.9056112170219421},\n", 76 | " {'label': 'no', 'score': 0.9143954515457153},\n", 77 | " {'label': 'no', 'score': 0.9552835822105408},\n", 78 | " {'label': 'no', 'score': 0.9614712595939636},\n", 79 | " {'label': 'no', 'score': 0.9431222081184387},\n", 80 | " {'label': 'no', 'score': 0.9308228492736816},\n", 81 | " {'label': 'no', 'score': 0.9267386794090271},\n", 82 | " {'label': 'no', 'score': 0.9052245020866394},\n", 83 | " {'label': 'yes', 'score': 0.9645366668701172},\n", 84 | " {'label': 'yes', 'score': 0.9613882899284363},\n", 85 | " {'label': 'yes', 'score': 0.9616449475288391},\n", 86 | " {'label': 'yes', 'score': 0.9616520404815674},\n", 87 | " {'label': 'yes', 'score': 0.9592236876487732},\n", 88 | " {'label': 'yes', 'score': 0.7594723701477051}]" 89 | ] 90 | }, 91 | "execution_count": 5, 92 | "metadata": {}, 93 | "output_type": "execute_result" 94 | } 95 | ], 96 | "source": [ 97 | "sen = [\"你喜欢什么东西?\",\n", 98 | " \"明天的外卖吃什么好,你有啥推荐的?\",\n", 99 | " \"服从命令, 芙宁娜女士\",\n", 100 | " \"你好, 今天天气怎么样\",\n", 101 | " \"你今天怎么样\",\n", 102 | " \"(开心地笑), 亲爱的，我今天买了一束你最喜欢的花！\",\n", 103 | " \"(撒娇语气), 宝宝，陪我一起看这部新出的电视剧好不好嘛？\",\n", 104 | " \"宝贝，你猜我今天在路上碰到谁了？(好奇地问), \",\n", 105 | " \"亲爱的，(认真地说), 我想和你一起去学一门新的语言。\",\n", 106 | " \"(兴奋地跳), 宝，我们周末去游乐园玩吧！\",\n", 107 | " \"亲爱的，麻烦你帮我查询一下明天北京到深圳的航班动态，看看有没有晚点。\",\n", 108 | " \"宝宝，能不能拜托你帮我获取一下今天某知名股票的实时股价走势，我关注好久了。\",\n", 109 | " \"宝贝，帮我查查距离咱们家最近的三甲医院的挂号方式，我有点不舒服想去看看。\",\n", 110 | " \"亲爱的，务必帮我搜索一下最近一个月内周杰伦在各大音乐平台的歌曲播放量，我好奇好久了。\",\n", 111 | " \"宝，辛苦你帮我了解下本市下周有哪些正在举办的艺术展览，我想去逛逛。\",\n", 112 | " \"给我开一下空调呗, 我有点冷了\"]\n", 113 | "pipe = pipeline(\"text-classification\", model=model, tokenizer=tokenizer, device=0 if torch.cuda.is_available() else -1)\n", 114 | "pipe(sen)" 115 | ] 116 | } 117 | ], 118 | "metadata": { 119 | "kernelspec": { 120 | "display_name": "transform", 121 | "language": "python", 122 | "name": "python3" 123 | }, 124 | "language_info": { 125 | "codemirror_mode": { 126 | "name": "ipython", 127 | "version": 3 128 | }, 129 | "file_extension": ".py", 130 | "mimetype": "text/x-python", 131 | "name": "python", 132 | "nbconvert_exporter": "python", 133 | "pygments_lexer": "ipython3", 134 | "version": "3.9.21" 135 | } 136 | }, 137 | "nbformat": 4, 138 | "nbformat_minor": 2 139 | } 140 | -------------------------------------------------------------------------------- /python/classfy-model/hflrbt3/added_tokens.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /python/classfy-model/hflrbt3/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "architectures": [ 3 | "BertForMaskedLM" 4 | ], 5 | "model_type": "bert", 6 | "attention_probs_dropout_prob": 0.1, 7 | "directionality": "bidi", 8 | "finetuning_task": null, 9 | "hidden_act": "gelu", 10 | "hidden_dropout_prob": 0.1, 11 | "hidden_size": 768, 12 | "initializer_range": 0.02, 13 | "intermediate_size": 3072, 14 | "is_decoder": false, 15 | "layer_norm_eps": 1e-12, 16 | "max_position_embeddings": 512, 17 | "num_attention_heads": 12, 18 | "num_hidden_layers": 3, 19 | "num_labels": 2, 20 | "output_attentions": false, 21 | "output_hidden_states": false, 22 | "output_past": true, 23 | "pooler_fc_size": 768, 24 | "pooler_num_attention_heads": 12, 25 | "pooler_num_fc_layers": 3, 26 | "pooler_size_per_head": 128, 27 | "pooler_type": "first_token_transform", 28 | "pruned_heads": {}, 29 | "torchscript": false, 30 | "type_vocab_size": 2, 31 | "use_bfloat16": false, 32 | "vocab_size": 21128 33 | } 34 | -------------------------------------------------------------------------------- /python/classfy-model/hflrbt3/special_tokens_map.json: -------------------------------------------------------------------------------- 1 | {"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"} -------------------------------------------------------------------------------- /python/classfy-model/hflrbt3/tokenizer_config.json: -------------------------------------------------------------------------------- 1 | {"init_inputs": []} -------------------------------------------------------------------------------- /python/classfy-model/model_furina/从网盘下载模型文件.txt: -------------------------------------------------------------------------------- 1 | https://pan.quark.cn/s/0d28fe813112 2 | 3 | 两个model文件在ai-chat-local\python\classfy-model\model_furina文件夹里面, 可以选一个 4 | pytorch的在ai-chat-local\python\classfy-model\hflrbt3文件夹下面 -------------------------------------------------------------------------------- /python/computer_chat/assistant.py: -------------------------------------------------------------------------------- 1 | import pyttsx3 2 | import numpy as np 3 | import whisper 4 | import pyaudio 5 | import sys 6 | import torch 7 | import requests 8 | import json 9 | import yaml 10 | import wave 11 | from yaml import Loader 12 | import pygame, sys 13 | import pygame.locals 14 | import ollama 15 | 16 | BACK_COLOR = (0,0,0) 17 | REC_COLOR = (255,0,0) 18 | TEXT_COLOR = (255,255,255) 19 | REC_SIZE = 80 20 | FONT_SIZE = 24 21 | WIDTH = 320 22 | HEIGHT = 240 23 | KWIDTH = 20 24 | KHEIGHT = 6 25 | MAX_TEXT_LEN_DISPLAY = 32 26 | 27 | 28 | 29 | INPUT_DEFAULT_DURATION_SECONDS = 5 30 | INPUT_FORMAT = pyaudio.paInt16 31 | INPUT_CHANNELS = 1 32 | INPUT_RATE = 16000 33 | INPUT_CHUNK = 1024 34 | OLLAMA_REST_HEADERS = {'Content-Type': 'application/json',} 35 | INPUT_CONFIG_PATH ="assistant.yaml" 36 | 37 | 38 | data = { 39 | "model": "lfruina", 40 | "messages": [ 41 | # {"role":"system","content": "你好。"}, 42 | {"role": "user","content": " "} 43 | ], 44 | "stream": False 45 | } 46 | 47 | class Assistant: 48 | 49 | 50 | def __init__(self): 51 | 52 | self.config = self.initConfig() 53 | 54 | programIcon = pygame.image.load('assistant.png') 55 | 56 | self.clock = pygame.time.Clock() 57 | pygame.display.set_icon(programIcon) 58 | pygame.display.set_caption("Assistant") 59 | 60 | self.windowSurface = pygame.display.set_mode((WIDTH, HEIGHT), 0, 32) 61 | self.font = pygame.font.SysFont(None, FONT_SIZE) 62 | 63 | self.audio = pyaudio.PyAudio() 64 | 65 | self.tts = pyttsx3.init() # 文字转语音库 66 | 67 | try: 68 | self.audio.open(format=INPUT_FORMAT, 69 | channels=INPUT_CHANNELS, 70 | rate=INPUT_RATE, 71 | input=True, 72 | frames_per_buffer=INPUT_CHUNK).close() 73 | except : 74 | self.wait_exit() 75 | 76 | self.text_to_speech(self.config.messages.loadingModel) 77 | self.display_message(self.config.messages.loadingModel) 78 | self.model = whisper.load_model(self.config.whisperRecognition.modelPath) 79 | #self.conversation_history = [self.config.conversation.context, 80 | # self.config.conversation.greeting] 81 | self.context = [] 82 | 83 | self.text_to_speech(self.config.conversation.greeting) 84 | self.display_message(self.config.messages.pressSpace) 85 | 86 | def wait_exit(self): 87 | while True: 88 | self.display_message(self.config.messages.noAudioInput) 89 | self.clock.tick(60) 90 | for event in pygame.event.get(): 91 | if event.type == pygame.locals.QUIT: 92 | self.shutdown() 93 | 94 | def shutdown(self): 95 | self.audio.terminate() 96 | pygame.quit() 97 | sys.exit() 98 | 99 | def initConfig(self): 100 | class Inst: 101 | pass 102 | config=Inst() 103 | config.messages = Inst() 104 | config.messages.pressSpace = "Pressez sur espace pour parler puis relachez." 105 | config.messages.loadingModel = "Loading model..." 106 | config.messages.noAudioInput = "Erreur: Pas d'entrée son" 107 | config.whisperRecognition = Inst() 108 | config.whisperRecognition.modelPath = "whisper/large-v3.pt" 109 | config.whisperRecognition.lang = "fr" 110 | config.ollama = Inst() 111 | config.ollama.url = "http://localhost:11434/api/generate" 112 | config.ollama.model = 'mistral' 113 | config.conversation = Inst() 114 | config.conversation.context = "This is a discussion in french.\n" 115 | config.conversation.greeting = "Je vous écoute." 116 | config.conversation.recognitionWaitMsg = "J'interprète votre demande." 117 | config.conversation.llmWaitMsg = "Laissez moi réfléchir." 118 | 119 | stream = open(INPUT_CONFIG_PATH, 'r', encoding="utf-8") 120 | dic = yaml.load(stream, Loader=Loader) 121 | #dic depth 2: map values to attributes 122 | def dic2Object(dic, object): 123 | for key in dic: 124 | if hasattr(object, key): 125 | setattr(object, key, dic[key]) 126 | else: 127 | print("Ignoring unknow setting ", key) 128 | #dic depth 1: fill depth 2 attributes 129 | for key in dic: 130 | if hasattr(config, key): 131 | dic2Object(dic[key], getattr(config, key)) 132 | else: 133 | print("Ignoring unknow setting ", key) 134 | 135 | 136 | return config 137 | 138 | def display_rec_start(self): 139 | self.windowSurface.fill(BACK_COLOR) 140 | pygame.draw.circle(self.windowSurface, REC_COLOR, (WIDTH/2, HEIGHT/2), REC_SIZE) 141 | pygame.display.flip() 142 | 143 | def display_sound_energy(self, energy): 144 | COL_COUNT = 5 145 | RED_CENTER = 150 146 | FACTOR = 10 147 | MAX_AMPLITUDE = 100 148 | 149 | self.windowSurface.fill(BACK_COLOR) 150 | amplitude = int(MAX_AMPLITUDE*energy) 151 | hspace, vspace = 2*KWIDTH, int(KHEIGHT/2) 152 | def rect_coords(x, y): 153 | return (int(x-KWIDTH/2), int(y-KHEIGHT/2), 154 | KWIDTH, KHEIGHT) 155 | for i in range(-int(np.floor(COL_COUNT/2)), int(np.ceil(COL_COUNT/2))): 156 | x, y, count = WIDTH/2+(i*hspace), HEIGHT/2, amplitude-2*abs(i) 157 | 158 | mid = int(np.ceil(count/2)) 159 | for i in range(0, mid): 160 | color = (255, 0, 0) if i < RED_CENTER else (0, 255, 0) 161 | offset = i*(KHEIGHT+vspace) 162 | pygame.draw.rect(self.windowSurface, color, 163 | rect_coords(x, y+offset)) 164 | #mirror: 165 | pygame.draw.rect(self.windowSurface, color, 166 | rect_coords(x, y-offset)) 167 | pygame.display.flip() 168 | 169 | def display_message(self, text): 170 | self.windowSurface.fill(BACK_COLOR) 171 | 172 | label = self.font.render(text 173 | if (len(text) np.ndarray: 184 | 185 | self.display_rec_start() 186 | 187 | stream = self.audio.open(format=INPUT_FORMAT, 188 | channels=INPUT_CHANNELS, 189 | rate=INPUT_RATE, 190 | input=True, 191 | frames_per_buffer=INPUT_CHUNK) 192 | frames = [] 193 | 194 | while True: 195 | pygame.event.pump() # process event queue 196 | pressed = pygame.key.get_pressed() 197 | if pressed[key]: 198 | data = stream.read(INPUT_CHUNK) 199 | frames.append(data) 200 | else: 201 | break 202 | 203 | stream.stop_stream() 204 | stream.close() 205 | 206 | return np.frombuffer(b''.join(frames), np.int16).astype(np.float32) * (1 / 32768.0) 207 | 208 | def speech_to_text(self, waveform): 209 | self.text_to_speech(self.config.conversation.recognitionWaitMsg) 210 | 211 | transcript = self.model.transcribe(waveform, 212 | language = self.config.whisperRecognition.lang, 213 | fp16=torch.cuda.is_available()) 214 | text = transcript["text"] 215 | self.text_to_speech(text) 216 | return text 217 | 218 | 219 | def ask_ollama(self, prompt, responseCallback): 220 | 221 | print("get test", prompt) 222 | self.text_to_speech(self.config.conversation.llmWaitMsg) 223 | 224 | self.messages = [] 225 | self.messages.append({"role": "user", "content": prompt}) 226 | output = ollama.chat( 227 | model=self.config.ollama.model, 228 | messages= self.messages 229 | ) 230 | responseCallback(output['message']['content']) 231 | 232 | def text_to_speech(self, text): 233 | print(text) 234 | tempPath = 'temp.wav' 235 | #self.tts.say(text) 236 | self.tts.save_to_file(text , tempPath) 237 | self.tts.runAndWait() 238 | wf = wave.open(tempPath, 'rb') 239 | # open stream based on the wave object which has been input. 240 | stream = self.audio.open(format = 241 | self.audio.get_format_from_width(wf.getsampwidth()), 242 | channels = wf.getnchannels(), 243 | rate = wf.getframerate(), 244 | output = True) # 打开音频流 245 | chunkSize = 1024 246 | chunk = wf.readframes(chunkSize) # 读取数据 247 | while chunk: 248 | stream.write(chunk) 249 | tmp = np.array(np.frombuffer(chunk, np.int16), np.float32) * (1 / 32768.0) 250 | energy_of_chunk = np.sqrt(np.mean(tmp**2)) 251 | self.display_sound_energy(energy_of_chunk) 252 | chunk = wf.readframes(chunkSize) 253 | 254 | wf.close() 255 | # self.display_message(text) 256 | 257 | def main(): 258 | 259 | if sys.version_info[0:3] != (3, 9, 13): 260 | print('Warning, it was only tested with python 3.9.13, it may fail') 261 | 262 | pygame.init() 263 | 264 | ass = Assistant() 265 | 266 | push_to_talk_key = pygame.K_SPACE 267 | 268 | while True: 269 | ass.clock.tick(60) # 用于 270 | for event in pygame.event.get(): 271 | if event.type == pygame.KEYDOWN and event.key == push_to_talk_key: 272 | speech = ass.waveform_from_mic(push_to_talk_key) 273 | 274 | transcription = ass.speech_to_text(waveform=speech) 275 | 276 | ass.ask_ollama(transcription, ass.text_to_speech) 277 | 278 | ass.display_message(ass.config.messages.pressSpace) 279 | 280 | if event.type == pygame.locals.QUIT: 281 | ass.shutdown() 282 | 283 | 284 | if __name__ == "__main__": 285 | main() 286 | 287 | -------------------------------------------------------------------------------- /python/computer_chat/注意 需要额外安装库 ollama.txt: -------------------------------------------------------------------------------- 1 | 需要安装库 ollama -------------------------------------------------------------------------------- /python/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate==1.3.0 2 | aiofiles==23.2.1 3 | aiohappyeyeballs==2.4.4 4 | aiohttp==3.11.11 5 | aiosignal==1.3.2 6 | annotated-types==0.7.0 7 | anyio==4.8.0 8 | argon2-cffi==23.1.0 9 | argon2-cffi-bindings==21.2.0 10 | arrow==1.3.0 11 | asgiref==3.8.1 12 | asttokens @ file:///opt/conda/conda-bld/asttokens_1646925590279/work 13 | async-lru==2.0.4 14 | async-timeout==4.0.3 15 | attrs==24.3.0 16 | babel==2.16.0 17 | backcall @ file:///home/ktietz/src/ci/backcall_1611930011877/work 18 | backoff==2.2.1 19 | bcrypt==4.2.1 20 | beautifulsoup4==4.12.3 21 | bleach==6.2.0 22 | Brotli @ file:///C:/b/abs_c415aux9ra/croot/brotli-split_1736182803933/work 23 | build==1.2.2.post1 24 | cachetools==5.5.1 25 | certifi @ file:///C:/b/abs_59o0xj7aav/croot/certifi_1734473304008/work/certifi 26 | cffi==1.17.1 27 | charset-normalizer @ file:///croot/charset-normalizer_1721748349566/work 28 | chroma-hnswlib==0.7.6 29 | chromadb==0.6.3 30 | click==8.1.8 31 | colorama @ file:///C:/b/abs_a9ozq0l032/croot/colorama_1672387194846/work 32 | coloredlogs==15.0.1 33 | comm @ file:///C:/b/abs_67a8058udb/croot/comm_1709322909844/work 34 | contourpy==1.3.0 35 | cycler==0.12.1 36 | d2l==1.0.3 37 | dataclasses-json==0.6.7 38 | datasets==3.2.0 39 | debugpy @ file:///C:/b/abs_bf9oo2vhxp/croot/debugpy_1736269476451/work 40 | decorator @ file:///opt/conda/conda-bld/decorator_1643638310831/work 41 | defusedxml==0.7.1 42 | Deprecated==1.2.18 43 | dill==0.3.8 44 | distro==1.9.0 45 | durationpy==0.9 46 | evaluate==0.4.3 47 | exceptiongroup @ file:///C:/b/abs_c5h1o1_b5b/croot/exceptiongroup_1706031441653/work 48 | executing @ file:///opt/conda/conda-bld/executing_1646925071911/work 49 | faiss-cpu==1.10.0 50 | fastapi==0.115.8 51 | fastjsonschema==2.21.1 52 | ffmpy==0.5.0 53 | filelock @ file:///C:/b/abs_f2gie28u58/croot/filelock_1700591233643/work 54 | flatbuffers==25.1.24 55 | fonttools==4.55.3 56 | fqdn==1.5.1 57 | frozenlist==1.5.0 58 | fsspec==2024.9.0 59 | gmpy2 @ file:///C:/ci/gmpy2_1645455782955/work 60 | google-auth==2.38.0 61 | googleapis-common-protos==1.66.0 62 | gradio==4.44.1 63 | gradio_client==1.3.0 64 | greenlet==3.1.1 65 | grpcio==1.70.0 66 | h11==0.14.0 67 | httpcore==1.0.7 68 | httptools==0.6.4 69 | httpx==0.28.1 70 | httpx-sse==0.4.0 71 | huggingface-hub==0.28.1 72 | humanfriendly==10.0 73 | idna @ file:///C:/b/abs_aad84bnnw5/croot/idna_1714398896795/work 74 | importlib_metadata @ file:///C:/b/abs_9d0lp1ysmt/croot/importlib_metadata-suite_1732633510972/work 75 | importlib_resources==6.5.2 76 | ipykernel @ file:///C:/b/abs_5etv6sld_j/croot/ipykernel_1728665608850/work 77 | ipython @ file:///C:/b/abs_e5729i179y/croot/ipython_1694181400005/work 78 | ipywidgets==8.1.5 79 | isoduration==20.11.0 80 | jedi @ file:///C:/b/abs_3a2kbnlclc/croot/jedi_1733987412687/work 81 | Jinja2 @ file:///C:/b/abs_ddyeze2h6_/croot/jinja2_1730903330414/work 82 | jiter==0.8.2 83 | json5==0.10.0 84 | jsonpatch==1.33 85 | jsonpointer==3.0.0 86 | jsonschema==4.23.0 87 | jsonschema-specifications==2024.10.1 88 | jupyter==1.0.0 89 | jupyter-console==6.6.3 90 | jupyter-events==0.11.0 91 | jupyter-lsp==2.2.5 92 | jupyter_client @ file:///C:/b/abs_a6h3c8hfdq/croot/jupyter_client_1699455939372/work 93 | jupyter_core @ file:///C:/b/abs_beftpbuevw/croot/jupyter_core_1718818307097/work 94 | jupyter_server==2.15.0 95 | jupyter_server_terminals==0.5.3 96 | jupyterlab==4.3.4 97 | jupyterlab_pygments==0.3.0 98 | jupyterlab_server==2.27.3 99 | jupyterlab_widgets==3.0.13 100 | kiwisolver==1.4.7 101 | kubernetes==32.0.0 102 | langchain==0.3.17 103 | langchain-community==0.3.16 104 | langchain-core==0.3.33 105 | langchain-openai==0.3.3 106 | langchain-text-splitters==0.3.5 107 | langsmith==0.3.4 108 | markdown-it-py==3.0.0 109 | MarkupSafe @ file:///C:/b/abs_a9k8j__c_q/croot/markupsafe_1736370039866/work 110 | marshmallow==3.26.1 111 | matplotlib==3.7.2 112 | matplotlib-inline @ file:///C:/ci/matplotlib-inline_1661915841596/work 113 | mdurl==0.1.2 114 | mistune==3.1.0 115 | mkl-service==2.4.0 116 | mkl_fft @ file:///C:/Users/dev-admin/mkl/mkl_fft_1730823082242/work 117 | mkl_random @ file:///C:/Users/dev-admin/mkl/mkl_random_1730822522280/work 118 | mmh3==5.1.0 119 | monotonic==1.6 120 | mpmath @ file:///C:/b/abs_7833jrbiox/croot/mpmath_1690848321154/work 121 | multidict==6.1.0 122 | multiprocess==0.70.16 123 | mypy-extensions==1.0.0 124 | nbclient==0.10.2 125 | nbconvert==7.16.5 126 | nbformat==5.10.4 127 | nest-asyncio @ file:///C:/b/abs_65d6lblmoi/croot/nest-asyncio_1708532721305/work 128 | networkx @ file:///C:/b/abs_3bxnu56g9d/croot/networkx_1717597507456/work 129 | notebook==7.3.2 130 | notebook_shim==0.2.4 131 | numpy==1.22.4 132 | oauthlib==3.2.2 133 | ollama==0.4.7 134 | onnxruntime==1.19.2 135 | openai==1.61.0 136 | opentelemetry-api==1.29.0 137 | opentelemetry-exporter-otlp-proto-common==1.29.0 138 | opentelemetry-exporter-otlp-proto-grpc==1.29.0 139 | opentelemetry-instrumentation==0.50b0 140 | opentelemetry-instrumentation-asgi==0.50b0 141 | opentelemetry-instrumentation-fastapi==0.50b0 142 | opentelemetry-proto==1.29.0 143 | opentelemetry-sdk==1.29.0 144 | opentelemetry-semantic-conventions==0.50b0 145 | opentelemetry-util-http==0.50b0 146 | optimum==1.24.0 147 | orjson==3.10.15 148 | overrides==7.7.0 149 | packaging @ file:///C:/b/abs_3by6s2fa66/croot/packaging_1734472138782/work 150 | pandas==2.0.3 151 | pandocfilters==1.5.1 152 | parso @ file:///C:/b/abs_834b4mj92b/croot/parso_1733963322289/work 153 | peft==0.14.0 154 | pickleshare @ file:///tmp/build/80754af9/pickleshare_1606932040724/work 155 | pillow==10.4.0 156 | platformdirs @ file:///C:/b/abs_b6z_yqw_ii/croot/platformdirs_1692205479426/work 157 | posthog==3.11.0 158 | prometheus_client==0.21.1 159 | prompt-toolkit @ file:///C:/b/abs_68uwr58ed1/croot/prompt-toolkit_1704404394082/work 160 | propcache==0.2.1 161 | protobuf==5.29.3 162 | psutil @ file:///C:/b/abs_b5gv3mn55h/croot/psutil_1736371546320/work 163 | pure-eval @ file:///opt/conda/conda-bld/pure_eval_1646925070566/work 164 | pyarrow==19.0.0 165 | pyasn1==0.6.1 166 | pyasn1_modules==0.4.1 167 | pycparser==2.22 168 | pydantic==2.10.6 169 | pydantic-settings==2.7.1 170 | pydantic_core==2.27.2 171 | pydub==0.25.1 172 | Pygments @ file:///C:/b/abs_fay9dpq4n_/croot/pygments_1684279990574/work 173 | PyMySQL==1.1.1 174 | pyparsing==3.0.9 175 | pypdf==5.2.0 176 | PyPika==0.48.9 177 | pyproject_hooks==1.2.0 178 | pyreadline3==3.5.4 179 | PySocks @ file:///C:/ci/pysocks_1605307512533/work 180 | python-dateutil @ file:///C:/b/abs_3au_koqnbs/croot/python-dateutil_1716495777160/work 181 | python-dotenv==1.0.1 182 | python-json-logger==3.2.1 183 | python-multipart==0.0.20 184 | pytz==2024.2 185 | pywin32==308 186 | pywinpty==2.0.14 187 | PyYAML @ file:///C:/b/abs_14xkfs39bx/croot/pyyaml_1728657968772/work 188 | pyzmq @ file:///C:/b/abs_f3yte6j5yn/croot/pyzmq_1734711069724/work 189 | qtconsole==5.6.1 190 | QtPy==2.4.2 191 | referencing==0.36.1 192 | regex==2024.11.6 193 | requests==2.32.3 194 | requests-oauthlib==2.0.0 195 | requests-toolbelt==1.0.0 196 | rfc3339-validator==0.1.4 197 | rfc3986-validator==0.1.1 198 | rich==13.9.4 199 | rpds-py==0.22.3 200 | rsa==4.9 201 | ruff==0.9.4 202 | safetensors==0.5.2 203 | scipy==1.10.1 204 | semantic-version==2.10.0 205 | Send2Trash==1.8.3 206 | sentencepiece==0.2.0 207 | shellingham==1.5.4 208 | six @ file:///tmp/build/80754af9/six_1644875935023/work 209 | sniffio==1.3.1 210 | soupsieve==2.6 211 | SQLAlchemy==2.0.37 212 | stack-data @ file:///opt/conda/conda-bld/stack_data_1646927590127/work 213 | starlette==0.45.3 214 | sympy==1.13.1 215 | tenacity==9.0.0 216 | terminado==0.18.1 217 | tiktoken==0.8.0 218 | tinycss2==1.4.0 219 | tokenizers==0.21.0 220 | tomli==2.2.1 221 | tomlkit==0.12.0 222 | torch==2.5.1 223 | torchaudio==2.5.1 224 | torchvision==0.20.1 225 | tornado @ file:///C:/b/abs_7cyu943ybx/croot/tornado_1733960510898/work 226 | tqdm==4.67.1 227 | traitlets @ file:///C:/b/abs_bfsnoxl4pq/croot/traitlets_1718227069245/work 228 | transformers==4.48.2 229 | typer==0.15.1 230 | types-python-dateutil==2.9.0.20241206 231 | typing-inspect==0.9.0 232 | typing_extensions @ file:///C:/b/abs_0ffjxtihug/croot/typing_extensions_1734714875646/work 233 | tzdata==2024.2 234 | uri-template==1.3.0 235 | urllib3 @ file:///C:/b/abs_7bst06lizn/croot/urllib3_1737133657081/work 236 | uvicorn==0.34.0 237 | watchfiles==1.0.4 238 | wcwidth @ file:///Users/ktietz/demo/mc3/conda-bld/wcwidth_1629357192024/work 239 | webcolors==24.11.1 240 | webencodings==0.5.1 241 | websocket-client==1.8.0 242 | websockets==12.0 243 | widgetsnbextension==4.0.13 244 | win-inet-pton @ file:///C:/ci/win_inet_pton_1605306162074/work 245 | wrapt==1.17.2 246 | xxhash==3.5.0 247 | yarl==1.18.3 248 | zipp @ file:///C:/b/abs_2fcpcg89my/croot/zipp_1732630746332/work 249 | zstandard==0.23.0 250 | -------------------------------------------------------------------------------- /python/server-model/__pycache__/API.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuSenfeng/ai-chat-local/d8254bf7e0d0340b73ff34ea34d83ae6c49db6ce/python/server-model/__pycache__/API.cpython-39.pyc -------------------------------------------------------------------------------- /python/server-model/__pycache__/chat_local.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuSenfeng/ai-chat-local/d8254bf7e0d0340b73ff34ea34d83ae6c49db6ce/python/server-model/__pycache__/chat_local.cpython-39.pyc -------------------------------------------------------------------------------- /python/server-model/__pycache__/chat_local_manage.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuSenfeng/ai-chat-local/d8254bf7e0d0340b73ff34ea34d83ae6c49db6ce/python/server-model/__pycache__/chat_local_manage.cpython-39.pyc -------------------------------------------------------------------------------- /python/server-model/__pycache__/chat_local_manage_t.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuSenfeng/ai-chat-local/d8254bf7e0d0340b73ff34ea34d83ae6c49db6ce/python/server-model/__pycache__/chat_local_manage_t.cpython-39.pyc -------------------------------------------------------------------------------- /python/server-model/__pycache__/chat_mysql.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuSenfeng/ai-chat-local/d8254bf7e0d0340b73ff34ea34d83ae6c49db6ce/python/server-model/__pycache__/chat_mysql.cpython-39.pyc -------------------------------------------------------------------------------- /python/server-model/__pycache__/classfy_model.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuSenfeng/ai-chat-local/d8254bf7e0d0340b73ff34ea34d83ae6c49db6ce/python/server-model/__pycache__/classfy_model.cpython-39.pyc -------------------------------------------------------------------------------- /python/server-model/__pycache__/file_dealing.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuSenfeng/ai-chat-local/d8254bf7e0d0340b73ff34ea34d83ae6c49db6ce/python/server-model/__pycache__/file_dealing.cpython-39.pyc -------------------------------------------------------------------------------- /python/server-model/__pycache__/search_internet.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuSenfeng/ai-chat-local/d8254bf7e0d0340b73ff34ea34d83ae6c49db6ce/python/server-model/__pycache__/search_internet.cpython-39.pyc -------------------------------------------------------------------------------- /python/server-model/chat_local.py: -------------------------------------------------------------------------------- 1 | import ollama 2 | from chat_mysql import Chat_MySQL 3 | 4 | class chat_furina: 5 | def __init__(self, user_id, load=True, mysql : Chat_MySQL = None): 6 | self.history = [] 7 | self.messages = [] 8 | self.model = "lfruina" 9 | self.last_time = 0 10 | self.mysql = mysql 11 | self.user_id = user_id 12 | 13 | print("load: ", load) 14 | if load and self.mysql.is_exist(self.user_id): 15 | self.load_history() 16 | elif not self.mysql.is_exist(self.user_id): 17 | print("new user create table") 18 | self.mysql.create_table(self.user_id) 19 | 20 | # 加载历史记录 21 | def load_history(self): 22 | result = self.mysql.select(self.user_id) 23 | for chat in result: 24 | # ((1, 'hello world2', 'user'), (2, 'hello world2', 'user'), (3, 'hello world2', 'user')) 25 | self.messages.append({"role": chat[2], "content": chat[1]}) 26 | 27 | # 带长下文的对话 28 | def chat_with_ollama(self, user_input): 29 | self.history.append([user_input, ""]) 30 | # 遍历历史记录，整理对话消息 31 | for idx, (user_msg, model_msg) in enumerate(self.history): 32 | if idx == len(self.history) - 1 and not model_msg: 33 | # 如果当前对话为最新的一条且未收到模型回复，则只添加用户消息 34 | self.messages.append({"role": "user", "content": user_msg}) 35 | break 36 | if user_msg: 37 | # 如果是用户消息，则添加到消息列表中 38 | self.messages.append({"role": "user", "content": user_msg}) 39 | if model_msg: 40 | # 如果是模型回复，则添加到消息列表中 41 | self.messages.append({"role": "assistant", "content": model_msg}) 42 | 43 | # 调用ollama模型进行对话 44 | output = ollama.chat( 45 | model=self.model, 46 | messages=self.messages 47 | ) 48 | # print('模型回复:', output['message']['content']) 49 | self.history[-1][1] = output['message']['content'] 50 | return output['message']['content'] 51 | 52 | def __del__(self): 53 | print("chat_furina stop id", self.user_id) 54 | for idx, (user_msg, model_msg) in enumerate(self.history): 55 | if user_msg and model_msg: 56 | self.mysql.insert(self.user_id, 'user', user_msg) 57 | self.mysql.insert(self.user_id, 'assistant', model_msg) 58 | 59 | if __name__ == "__main__": 60 | 61 | furina = chat_furina(114514) 62 | print(furina.chat_with_ollama("我的名字是什么?")) 63 | -------------------------------------------------------------------------------- /python/server-model/chat_local_manage.py: -------------------------------------------------------------------------------- 1 | # Description: 本地聊天管理 2 | import chat_local 3 | import threading 4 | import time 5 | from chat_mysql import Chat_MySQL 6 | 7 | defaule_living_time = 300 8 | 9 | # 本地聊天管理, 在一定时间内没有聊天则释放资源 10 | class ChatLocalManage: 11 | def __init__(self): 12 | self.chat_exectutor_runing = {} 13 | self.timer = {} 14 | self.mysql = Chat_MySQL() 15 | 16 | def chat(self, user_id:int, text: str): 17 | print("begin chat") 18 | if user_id not in self.chat_exectutor_runing: 19 | print("new chat_executor") 20 | self.chat_exectutor_runing[user_id] = chat_local.chat_furina(user_id, True, self.mysql) # 加载一个对话 21 | else: 22 | print("reset timer") 23 | self.timer[user_id].cancel() 24 | self.timer[user_id] = threading.Timer(defaule_living_time, self.release_chat_executor, (user_id,)) 25 | ret = self.chat_exectutor_runing[user_id].chat_with_ollama(text) 26 | self.timer[user_id].start() 27 | return ret 28 | 29 | def release_chat_executor(self, user_id): 30 | del self.chat_exectutor_runing[user_id] 31 | del self.timer[user_id] 32 | print("chat_executor released %d " % user_id) 33 | 34 | def get_new_user_id(self): 35 | while True: 36 | temp_id = int(time.time() * 1000) % 1000000000 37 | if not self.mysql.is_exist(temp_id): 38 | return temp_id 39 | 40 | def release_all_chat_executor(self): 41 | user_ids = list(self.chat_exectutor_runing.keys()) 42 | for user_id in user_ids: 43 | self.timer[user_id].cancel() 44 | self.release_chat_executor(user_id) 45 | 46 | if __name__ == "__main__": 47 | chat_manage = ChatLocalManage() 48 | print(chat_manage.chat(12, "你还记得我的名字吗?")) 49 | chat_manage.release_all_chat_executor() 50 | 51 | -------------------------------------------------------------------------------- /python/server-model/chat_local_manage_t.py: -------------------------------------------------------------------------------- 1 | # Description: 本地聊天管理 2 | import chat_local 3 | import threading 4 | import time 5 | from chat_mysql import Chat_MySQL 6 | 7 | defaule_living_time = 300 8 | 9 | # 本地聊天管理, 在一定时间内没有聊天则释放资源 10 | class ChatLocalManage: 11 | def __init__(self): 12 | self.lock = threading.Lock() 13 | self.locks = {} # 用户锁 14 | self.chat_exectutor_runing = {} # 用户对话 15 | self.timer = {} # 计时器 16 | self.mysql = Chat_MySQL() 17 | 18 | def chat(self, user_id:int, text: str): 19 | print("begin chat") 20 | self.lock.acquire() # 主锁加锁 21 | 22 | if user_id not in self.chat_exectutor_runing: 23 | # 用户的对话不存在 24 | self.chat_exectutor_runing[user_id] = chat_local.chat_furina(user_id, True, self.mysql) # 加载一个对话 25 | self.locks[user_id] = threading.Lock() # 获取用户锁 26 | else: 27 | self.timer[user_id].cancel() 28 | 29 | self.timer[user_id] = threading.Timer(defaule_living_time, self.release_chat_executor, (user_id,)) 30 | self.lock.release() # 主锁解锁 31 | 32 | self.locks[user_id].acquire() # 用户锁加锁 33 | ret = self.chat_exectutor_runing[user_id].chat_with_ollama(text) 34 | self.locks[user_id].release() # 用户锁解锁 35 | 36 | self.lock.acquire() # 主锁加锁 37 | self.timer[user_id].start() # 重置计时器 38 | self.lock.release() # 主锁解锁 39 | return ret 40 | 41 | def release_chat_executor(self, user_id): 42 | self.lock.acquire() # 主锁加锁 43 | del self.chat_exectutor_runing[user_id] 44 | del self.timer[user_id] 45 | del self.locks[user_id] 46 | self.lock.release() 47 | print("chat_executor released %d " % user_id) 48 | 49 | def get_new_user_id(self): 50 | while True: 51 | temp_id = int(time.time() * 1000) % 1000000000 52 | self.lock.acquire() # 主锁加锁 53 | if not self.mysql.is_exist(temp_id): 54 | self.lock.release() 55 | return temp_id 56 | self.lock.release() 57 | 58 | def release_all_chat_executor(self): 59 | self.lock.acquire() 60 | user_ids = list(self.chat_exectutor_runing.keys()) 61 | for user_id in user_ids: 62 | self.timer[user_id].cancel() 63 | self.release_chat_executor(user_id) 64 | self.lock.release() 65 | 66 | if __name__ == "__main__": 67 | chat_manage = ChatLocalManage() 68 | print(chat_manage.chat(12, "你还记得我的名字吗?")) 69 | chat_manage.release_all_chat_executor() 70 | -------------------------------------------------------------------------------- /python/server-model/chat_mysql.py: -------------------------------------------------------------------------------- 1 | import pymysql 2 | 3 | class Chat_MySQL: 4 | def __init__(self): 5 | self.connection = pymysql.connect( 6 | host='localhost', 7 | port=3306, 8 | user='root', 9 | charset='utf8') # 连接数据库 10 | 11 | self.cursor = self.connection.cursor() # 创建游标 12 | self.cursor.execute('show databases') 13 | result = self.cursor.fetchall() 14 | # 检查数据库是否存在 15 | if ('chat_ai',) not in result: 16 | print('create database chat_ai') 17 | self.cursor.execute('create database chat_ai DEFAULT CHARSET utf8 COLLATE utf8_general_ci') 18 | self.connection.commit() 19 | # 进入使用的数据库 20 | self.cursor.execute('use chat_ai') 21 | 22 | 23 | def is_exist(self, user_id: int): 24 | self.cursor.execute('show tables') 25 | result = self.cursor.fetchall() 26 | if ('user_%d' % user_id,) in result: 27 | return True 28 | return False 29 | 30 | # 查看是否存在用户的聊天记录表, 如果不存在则可以创建 31 | def create_table(self, user_id: int): 32 | 33 | if not self.is_exist(user_id): 34 | # 创建表 35 | print('create table chat_history for user %d' % user_id) 36 | self.cursor.execute('create table user_%d (' 37 | 'chat_id int primary key auto_increment,' 38 | 'chat_history varchar(10240),' 39 | 'chat_user varchar(20))default charset=utf8;' % user_id) 40 | self.connection.commit() 41 | else: 42 | print('table chat_history for user %d already exist' % user_id) 43 | 44 | def close(self): 45 | self.cursor.close() 46 | self.connection.close() 47 | 48 | # 插入聊天记录 49 | def insert(self, user_id: int, user_name: str, chat_history: str): 50 | chat_history = chat_history.replace('"', "'") 51 | # self.cursor.execute('insert into user_%s (chat_history, chat_user) values (%s, %s)', [self.user_id, chat_history, user]) 52 | self.cursor.execute(f'insert into user_{user_id} (chat_history, chat_user) values (%s, %s)', (chat_history, user_name)) 53 | self.connection.commit() 54 | 55 | def select(self, user_id: int): 56 | self.cursor.execute('select * from user_%s', [user_id]) 57 | result = self.cursor.fetchall() 58 | return result 59 | 60 | def get_history_len(self, user_id: int): 61 | self.cursor.execute('select count(chat_id) from user_%s', [user_id]) 62 | result = self.cursor.fetchall() 63 | return result[0][0] 64 | 65 | def __del__(self): 66 | self.cursor.close() 67 | self.connection.close() 68 | 69 | 70 | 71 | if __name__ == '__main__': 72 | try: 73 | mysql = Chat_MySQL() 74 | # mysql.insert(2, 'hello world') 75 | if not mysql.is_exist(4): 76 | print('table not exist') 77 | mysql.create_table(4) 78 | mysql.insert(4, 'user', 'hello world2') 79 | # mysql.insert('hello world3') 80 | # print(mysql.select()) 81 | print(mysql.get_history_len(4)) 82 | del mysql 83 | except Exception as e: 84 | print(e) 85 | 86 | -------------------------------------------------------------------------------- /python/server-model/classfy_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | # 文本分类模型微调的示例 3 | from transformers import AutoTokenizer 4 | from transformers import pipeline 5 | 6 | # 加载模型 7 | class Classfy: 8 | def __init__(self): 9 | self.tokenizer = AutoTokenizer.from_pretrained("../classfy-model/hflrbt3") 10 | self.model = torch.load("../classfy-model/model_furina/model_2025_02_04_15_41_53.pth") # 加载模型 11 | self.model.eval() 12 | self.id2label = {0:"no", 1:"yes"} 13 | self.model.config.id2label = self.id2label 14 | self.pipe = pipeline("text-classification", model=self.model, tokenizer=self.tokenizer, device=0 if torch.cuda.is_available() else -1) 15 | 16 | # 分类, 参数sen是需要分类的句子 17 | def classfy(self, sen): 18 | if type(sen) == str: 19 | return self.pipe([sen]) 20 | return self.pipe(sen) 21 | 22 | if __name__ == '__main__': 23 | classfy = Classfy() 24 | sen = ["你喜欢什么东西?", 25 | "明天的外卖吃什么好,你有啥推荐的?", 26 | "服从命令, 芙宁娜女士", 27 | "你好, 今天天气怎么样", 28 | "你今天怎么样", 29 | "(开心地笑), 亲爱的，我今天买了一束你最喜欢的花！", 30 | "(撒娇语气), 宝宝，陪我一起看这部新出的电视剧好不好嘛？", 31 | "宝贝，你猜我今天在路上碰到谁了？(好奇地问), ", 32 | "亲爱的，(认真地说), 我想和你一起去学一门新的语言。", 33 | "(兴奋地跳), 宝，我们周末去游乐园玩吧！", 34 | "亲爱的，麻烦你帮我查询一下明天北京到深圳的航班动态，看看有没有晚点。", 35 | "宝宝，能不能拜托你帮我获取一下今天某知名股票的实时股价走势，我关注好久了。", 36 | "宝贝，帮我查查距离咱们家最近的三甲医院的挂号方式，我有点不舒服想去看看。", 37 | "亲爱的，务必帮我搜索一下最近一个月内周杰伦在各大音乐平台的歌曲播放量，我好奇好久了。", 38 | "宝，辛苦你帮我了解下本市下周有哪些正在举办的艺术展览，我想去逛逛。", 39 | "给我开一下空调呗, 我有点冷了" 40 | "项目是关于啥的"] 41 | for s in sen: 42 | print(classfy.classfy(s)) -------------------------------------------------------------------------------- /python/server-model/file_dealing.py: -------------------------------------------------------------------------------- 1 | # 文件处理的简单示例 2 | 3 | from langchain_openai import ChatOpenAI,OpenAIEmbeddings 4 | from langchain_community.document_loaders import PyPDFLoader,TextLoader 5 | from langchain.text_splitter import RecursiveCharacterTextSplitter 6 | from langchain_community.vectorstores.utils import filter_complex_metadata 7 | from langchain_community.vectorstores import Chroma 8 | from langchain_core.prompts import ChatPromptTemplate 9 | from langchain.schema.runnable import RunnablePassthrough 10 | from langchain.schema.output_parser import StrOutputParser 11 | import API 12 | 13 | 14 | 15 | def format_docs(docs): 16 | return "\n\n".join(doc.page_content for doc in docs) 17 | 18 | class ChatDoc: 19 | 20 | def __init__(self): 21 | self.model = ChatOpenAI(temperature=0.1, model_name="gpt-3.5-turbo") 22 | self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=1024, chunk_overlap=100) 23 | self.embeddings = OpenAIEmbeddings() # 使用OpenAI的Embeddings 24 | system_prompt = ( 25 | ''' 26 | You are an assistant for question-answering tasks. 27 | Use the following pieces of retrieved context to answer the question. 28 | If you don't know the answer, say that you don't know. 29 | context: {context}''' 30 | ) 31 | 32 | self.prompt = ChatPromptTemplate.from_messages( 33 | [ 34 | ("system", system_prompt), 35 | ("human", "{question}"), 36 | ] 37 | ) 38 | 39 | def ingest(self, file_path: str): 40 | docs = PyPDFLoader(file_path=file_path).load() # 这里使用你的文件路径 41 | chunks = self.text_splitter.split_documents(docs) # 把文件分成小块 42 | chunks = filter_complex_metadata(chunks) # 过滤掉一些复杂的元数据 43 | vector_store = Chroma.from_documents(documents=chunks, embedding=self.embeddings) # 把文档转换成向量 44 | self.retriever = vector_store.as_retriever(search_type = "mmr", search_kwargs={ 45 | "k": 6, 46 | "fetch_k": 20, 47 | "include_metadata": True 48 | }, 49 | ) # 使用mmr算法进行检索, k=6表示返回6个结果, fetch_k=20表示检索20个结果, include_metadata=True表示返回元数据 50 | self.chain = ({"context": self.retriever| format_docs, "question": RunnablePassthrough()} 51 | | self.prompt 52 | | self.model 53 | | StrOutputParser()) 54 | 55 | 56 | def ask(self, query: str): 57 | if not self.chain: 58 | return "please add document first" 59 | response = self.chain.invoke(query) 60 | return response 61 | 62 | def clear(self): 63 | self.vector_store = None 64 | self.retriever = None 65 | self.chain = None 66 | 67 | 68 | if __name__ == "__main__": 69 | API.init_API() 70 | chat = ChatDoc() 71 | chat.ingest("../../README.pdf") # 这里使用你的文件路径 72 | print(chat.ask("这是一个什么项目?")) 73 | -------------------------------------------------------------------------------- /python/server-model/http_server.py: -------------------------------------------------------------------------------- 1 | from http.server import HTTPServer, BaseHTTPRequestHandler 2 | import json 3 | import search_internet 4 | # import chat_local 5 | import chat_local_manage 6 | from classfy_model import Classfy 7 | import socket 8 | import os 9 | import threading 10 | import sys 11 | os.environ['KMP_DUPLICATE_LIB_OK']='TRUE' 12 | 13 | # os.environ["TAVILY_API_KEY"] = 'xxx' 14 | # os.environ["OPENAI_API_KEY"] = "xxx" 15 | # os.environ["LANGSMITH_API_KEY"] = 'xxx' 16 | # os.environ['USER_AGENT'] = "xxx" 17 | 18 | # 使用的不是文档里面免费的API时候可能需要更改 19 | os.environ["OPENAI_BASE_URL"] = "https://api.chatanywhere.tech" 20 | os.environ["LANGSMITH_TRACING"]='true' 21 | os.environ["LANGSMITH_ENDPOINT"]="https://api.smith.langchain.com" 22 | os.environ["LANGSMITH_PROJECT"]="test" 23 | 24 | 25 | 26 | # 192.168.188.165 27 | data_ret = {'result': '', 'user_id': 12} 28 | def get_ip(): 29 | # 获取本机所有 IP 地址 30 | hostname = socket.gethostname() 31 | ip_list = [] 32 | # 获取IP地址信息 33 | addr_infos = socket.getaddrinfo(hostname, None) 34 | for addr in addr_infos: 35 | ip_list.append(addr[4][0]) 36 | host = (ip_list[-1], 8888) 37 | return host 38 | 39 | 40 | # 处理对话的服务器 41 | class http_chat_server: 42 | def __init__(self): 43 | print("chat server init") 44 | print("loading model...") 45 | self.agent_executor = search_internet.search_Moel() 46 | print("loading chat local model...") 47 | self.chat_manage = chat_local_manage.ChatLocalManage() 48 | print("loading classfy model...") 49 | self.classfy = Classfy() 50 | 51 | def deal_with_data(self, data: str): 52 | d = json.loads(data.replace("'", "\""), strict=False) 53 | # print(d) 54 | print("\n\n get question:", d['messages'][0]['text'], d['messages'][0]['user_id'], '\n') # 打印一下用户的输入 55 | if d['messages'][0]['user_id'] == -1: 56 | # 新用户, 分配用户id 57 | user_id = self.chat_manage.get_new_user_id() 58 | print("new user_id: ", user_id) 59 | else: 60 | user_id = d['messages'][0]['user_id'] 61 | data_ret['user_id'] = int(user_id) 62 | 63 | if self.classfy.classfy(d['messages'][0]['text'])[0]['label'] == 'yes': 64 | print("searching...") 65 | ret = self.agent_executor.search_func(d['messages'][0]['text']) 66 | else: 67 | print("chatting...") 68 | ret = self.chat_manage.chat(user_id, d['messages'][0]['text']) 69 | print(ret) 70 | return ret 71 | 72 | chat_server = http_chat_server() # 服务器对象 73 | 74 | # 处理HTTP请求 75 | class Resquest(BaseHTTPRequestHandler): 76 | 77 | def __init__(self, request, client_address, server): 78 | super().__init__(request, client_address, server) 79 | 80 | def do_GET(self): 81 | self.send_response(200) 82 | self.send_header('Content-type', 'application/json') 83 | self.end_headers() 84 | self.wfile.write(json.dumps(data_ret).encode()) 85 | 86 | def do_POST(self): 87 | datas = self.rfile.read(int(self.headers['content-length'])) 88 | datas = datas.decode() 89 | ret = chat_server.deal_with_data(datas) # 处理数据, 获取AI的回答 90 | # 构建返回数据 91 | self.send_response(200) 92 | self.send_header('Content-type', 'application/json') 93 | self.end_headers() 94 | data_ret['result'] = ret 95 | self.wfile.write(json.dumps(data_ret).encode()) 96 | 97 | 98 | if __name__ == '__main__': 99 | 100 | try: 101 | host = ('192.168.31.104', 8888) 102 | server = HTTPServer(host, Resquest) 103 | print("Starting server, listen at: %s:%s" % host) 104 | server.serve_forever() 105 | except KeyboardInterrupt: 106 | # 结束 107 | print("stop") 108 | chat_server.chat_manage.release_all_chat_executor() 109 | sys.exit(0) 110 | except Exception as e: 111 | print(e) 112 | chat_server.chat_manage.release_all_chat_executor() 113 | sys.exit(0) 114 | 115 | -------------------------------------------------------------------------------- /python/server-model/http_server_t.py: -------------------------------------------------------------------------------- 1 | # 多线程的处理ThreadedHTTPServer的使用 没有测试过, 不保证稳定!!! 2 | from http.server import HTTPServer, BaseHTTPRequestHandler 3 | import json 4 | import search_internet 5 | # import chat_local 6 | import chat_local_manage_t 7 | from classfy_model import Classfy 8 | import socket 9 | import os 10 | import threading 11 | import sys 12 | import socketserver 13 | os.environ['KMP_DUPLICATE_LIB_OK']='TRUE' 14 | 15 | # os.environ["TAVILY_API_KEY"] = 'xxx' 16 | # os.environ["OPENAI_API_KEY"] = "xxx" 17 | # os.environ["LANGSMITH_API_KEY"] = 'xxx' 18 | # os.environ['USER_AGENT'] = "xxx" 19 | 20 | # 使用的不是文档里面免费的API时候可能需要更改 21 | os.environ["OPENAI_BASE_URL"] = "https://api.chatanywhere.tech" 22 | os.environ["LANGSMITH_TRACING"]='true' 23 | os.environ["LANGSMITH_ENDPOINT"]="https://api.smith.langchain.com" 24 | os.environ["LANGSMITH_PROJECT"]="test" 25 | 26 | 27 | 28 | # 192.168.188.165 29 | data_ret = {'result': '', 'user_id': 12} 30 | def get_ip(): 31 | # 获取本机所有 IP 地址 32 | hostname = socket.gethostname() 33 | ip_list = [] 34 | # 获取IP地址信息 35 | addr_infos = socket.getaddrinfo(hostname, None) 36 | for addr in addr_infos: 37 | ip_list.append(addr[4][0]) 38 | host = (ip_list[-1], 8888) 39 | return host 40 | 41 | 42 | # 处理对话的服务器 43 | class http_chat_server: 44 | def __init__(self): 45 | self.agent_executor = search_internet.search_Moel() 46 | self.chat_manage = chat_local_manage_t.ChatLocalManage() 47 | self.classfy = Classfy() 48 | 49 | def deal_with_data(self, data: str): 50 | d = json.loads(data.replace("'", "\""), strict=False) 51 | # print(d) 52 | print("\n\n get question:", d['messages'][0]['text'], d['messages'][0]['user_id'], '\n') # 打印一下用户的输入 53 | if d['messages'][0]['user_id'] == -1: 54 | # 分配用户id 55 | user_id = self.chat_manage.get_new_user_id() 56 | print("new user_id: ", user_id) 57 | else: 58 | user_id = d['messages'][0]['user_id'] 59 | data_ret['user_id'] = int(user_id) 60 | 61 | if self.classfy.classfy(d['messages'][0]['text'])[0]['label'] == 'yes': 62 | print("searching...") 63 | ret = self.agent_executor.search_func(d['messages'][0]['text']) 64 | else: 65 | print("chatting...") 66 | ret = self.chat_manage.chat(user_id, d['messages'][0]['text']) 67 | print(ret) 68 | return ret 69 | 70 | chat_server = http_chat_server() # 服务器对象 71 | 72 | # 处理HTTP请求 73 | class Resquest(BaseHTTPRequestHandler): 74 | 75 | def __init__(self, request, client_address, server): 76 | super().__init__(request, client_address, server) 77 | 78 | def do_GET(self): 79 | self.send_response(200) 80 | self.send_header('Content-type', 'application/json') 81 | self.end_headers() 82 | thread_id = threading.get_ident() 83 | data_ret['user_id'] = thread_id 84 | self.wfile.write(json.dumps(data_ret).encode()) 85 | 86 | 87 | def do_POST(self): 88 | datas = self.rfile.read(int(self.headers['content-length'])) 89 | datas = datas.decode() 90 | ret = chat_server.deal_with_data(datas) # 处理数据, 获取AI的回答 91 | # 构建返回数据 92 | self.send_response(200) 93 | self.send_header('Content-type', 'application/json') 94 | self.end_headers() 95 | data_ret['result'] = ret 96 | self.wfile.write(json.dumps(data_ret).encode()) 97 | 98 | class ThreadedHTTPServer(socketserver.ThreadingMixIn, HTTPServer): 99 | """Threaded HTTP server""" 100 | pass 101 | 102 | if __name__ == '__main__': 103 | 104 | try: 105 | host = ('192.168.188.165', 8888) 106 | server = ThreadedHTTPServer(host, Resquest) 107 | print("Starting server, listen at: %s:%s" % host) 108 | server.serve_forever() 109 | except KeyboardInterrupt: 110 | # 结束 111 | print("stop") 112 | chat_server.chat_manage.release_all_chat_executor() 113 | sys.exit(0) 114 | except Exception as e: 115 | print(e) 116 | chat_server.chat_manage.release_all_chat_executor() 117 | sys.exit(0) 118 | 119 | -------------------------------------------------------------------------------- /python/server-model/http_server_xiaozhi.py: -------------------------------------------------------------------------------- 1 | from http.server import HTTPServer, BaseHTTPRequestHandler 2 | import json 3 | import search_internet 4 | # import chat_local 5 | import chat_local_manage 6 | from classfy_model import Classfy 7 | import socket 8 | import os 9 | import threading 10 | import sys 11 | os.environ['KMP_DUPLICATE_LIB_OK']='TRUE' 12 | 13 | # os.environ["TAVILY_API_KEY"] = 'xxx' 14 | # os.environ["OPENAI_API_KEY"] = "xxx" 15 | # os.environ["LANGSMITH_API_KEY"] = 'xxx' 16 | # os.environ['USER_AGENT'] = "xxx" 17 | 18 | # 使用的不是文档里面免费的API时候可能需要更改 19 | os.environ["OPENAI_BASE_URL"] = "https://api.chatanywhere.tech" 20 | os.environ["LANGSMITH_TRACING"]='true' 21 | os.environ["LANGSMITH_ENDPOINT"]="https://api.smith.langchain.com" 22 | os.environ["LANGSMITH_PROJECT"]="test" 23 | 24 | 25 | 26 | # 192.168.188.165 27 | data_ret = {'tool': 0, 'user_id': -1, 'result': ''} 28 | def get_ip(): 29 | # 获取本机所有 IP 地址 30 | hostname = socket.gethostname() 31 | ip_list = [] 32 | # 获取IP地址信息 33 | addr_infos = socket.getaddrinfo(hostname, None) 34 | for addr in addr_infos: 35 | ip_list.append(addr[4][0]) 36 | host = (ip_list[-1], 8888) 37 | return host 38 | 39 | 40 | # 处理对话的服务器 41 | class http_chat_server: 42 | def __init__(self): 43 | print("chat server init") 44 | print("loading model...") 45 | self.agent_executor = search_internet.search_Moel() 46 | print("loading chat local model...") 47 | self.chat_manage = chat_local_manage.ChatLocalManage() 48 | print("loading classfy model...") 49 | self.classfy = Classfy() 50 | 51 | def deal_with_data(self, data: str): 52 | d = json.loads(data.replace("'", "\""), strict=False) 53 | # print(d) 54 | print("\n\n get question:", d['messages'][0]['text'], d['messages'][0]['user_id'], '\n') # 打印一下用户的输入 55 | if d['messages'][0]['user_id'] == -1: 56 | # 新用户, 分配用户id 57 | user_id = self.chat_manage.get_new_user_id() 58 | print("new user_id: ", user_id) 59 | else: 60 | user_id = d['messages'][0]['user_id'] 61 | data_ret['user_id'] = int(user_id) 62 | 63 | if self.classfy.classfy(d['messages'][0]['text'])[0]['label'] == 'yes': 64 | print("searching...") 65 | ret = self.agent_executor.search_func(d['messages'][0]['text']) 66 | # 每15个字节的长度, 设置一个换行符 67 | length = len(ret) 68 | now = 0 69 | while now < length: 70 | ret = ret[:now+12] + '\n' + ret[now+12:] 71 | now += 13 72 | length += 1 73 | data_ret['tool'] = 1 74 | else: 75 | print("chatting...") # 小智不需要对话模型 76 | ret = "无工具调用" 77 | data_ret['tool'] = 0 78 | print(ret) 79 | return ret 80 | 81 | chat_server = http_chat_server() # 服务器对象 82 | 83 | # 处理HTTP请求 84 | class Resquest(BaseHTTPRequestHandler): 85 | 86 | def __init__(self, request, client_address, server): 87 | super().__init__(request, client_address, server) 88 | 89 | def do_GET(self): 90 | self.send_response(200) 91 | self.send_header('Content-type', 'application/json') 92 | self.end_headers() 93 | self.wfile.write(json.dumps(data_ret).encode()) 94 | 95 | def do_POST(self): 96 | datas = self.rfile.read(int(self.headers['content-length'])) 97 | datas = datas.decode() 98 | ret = chat_server.deal_with_data(datas) # 处理数据, 获取AI的回答 99 | # 构建返回数据 100 | self.send_response(200) 101 | self.send_header('Content-type', 'application/json') 102 | self.end_headers() 103 | data_ret['result'] = ret 104 | self.wfile.write(json.dumps(data_ret).encode()) 105 | 106 | 107 | if __name__ == '__main__': 108 | 109 | try: 110 | host = ('192.168.31.104', 8888) 111 | server = HTTPServer(host, Resquest) 112 | print("Starting server, listen at: %s:%s" % host) 113 | server.serve_forever() 114 | except KeyboardInterrupt: 115 | # 结束 116 | print("stop") 117 | chat_server.chat_manage.release_all_chat_executor() 118 | sys.exit(0) 119 | except Exception as e: 120 | print(e) 121 | chat_server.chat_manage.release_all_chat_executor() 122 | sys.exit(0) 123 | 124 | -------------------------------------------------------------------------------- /python/server-model/search_internet.py: -------------------------------------------------------------------------------- 1 | 2 | # 用于搜索网络的模块 3 | from langchain_openai import OpenAIEmbeddings 4 | from langchain_community.tools.tavily_search import TavilySearchResults 5 | from langchain_openai import ChatOpenAI 6 | from langchain_community.document_loaders import WebBaseLoader, PyPDFLoader 7 | from langchain_community.vectorstores import FAISS 8 | from langchain_text_splitters import RecursiveCharacterTextSplitter 9 | from langchain import hub 10 | import os 11 | file_path="../../README.pdf" 12 | os.environ["OPENAI_BASE_URL"] = "https://api.chatanywhere.tech" 13 | os.environ["LANGSMITH_TRACING"]='true' 14 | os.environ["LANGSMITH_ENDPOINT"]="https://api.smith.langchain.com" 15 | os.environ["LANGSMITH_PROJECT"]="test" 16 | 17 | class search_Moel: 18 | # 初始化文件处理工具 19 | def file_dealing_init(self): 20 | loader = PyPDFLoader(file_path=file_path) 21 | docs = loader.load() 22 | documents = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200).split_documents(docs) 23 | vector = FAISS.from_documents(documents, OpenAIEmbeddings()) 24 | file_retriever = vector.as_retriever() 25 | # 测试检索结果 26 | # print(retriever.get_relevant_documents("这是个啥项目")[1]) 27 | 28 | from langchain.tools.retriever import create_retriever_tool 29 | # 创建一个工具来检索文档 30 | retriever_tool = create_retriever_tool( 31 | file_retriever, 32 | "project_handbook_search", 33 | "搜索有关项目手册的信息。对于项目手册的任何问题，您必须使用此工具！", 34 | ) 35 | return retriever_tool 36 | 37 | # 搜索iphone价格的工具 38 | def iphon_price_search(self): 39 | # 加载HTML内容为一个文档对象 40 | loader = WebBaseLoader("https://www.ithome.com/0/718/713.htm") 41 | docs = loader.load() 42 | # 分割文档 43 | documents = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200).split_documents(docs) 44 | # 向量化 45 | vector = FAISS.from_documents(documents, OpenAIEmbeddings()) 46 | # 创建检索器 47 | retriever = vector.as_retriever() 48 | from langchain.tools.retriever import create_retriever_tool 49 | # 创建一个工具来检索文档 50 | retriever_tool = create_retriever_tool( 51 | retriever, 52 | "iPhone_price_search", 53 | "搜索有关 iPhone 15 的价格信息。对于iPhone 15的任何问题，您必须使用此工具！", 54 | ) 55 | return retriever_tool 56 | 57 | 58 | def __init__(self): 59 | # 初始化大模型 工具1 60 | llm = ChatOpenAI(model="gpt-3.5-turbo-1106", temperature=0) 61 | search = TavilySearchResults(max_results=3) 62 | 63 | 64 | # 创建将在下游使用的工具列表 65 | tools = [search, self.iphon_price_search(), self.file_dealing_init()] 66 | 67 | # 获取要使用的提示 68 | prompt = hub.pull("hwchase17/openai-functions-agent") 69 | 70 | # 使用OpenAI functions代理 71 | from langchain.agents import create_openai_functions_agent 72 | 73 | # 构建OpenAI函数代理：使用 LLM、提示模板和工具来初始化代理 74 | agent = create_openai_functions_agent(llm, tools, prompt) 75 | 76 | from langchain.agents import AgentExecutor 77 | # 将代理与AgentExecutor工具结合起来 78 | self.agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True) 79 | 80 | def search_func(self, query): 81 | try: 82 | data = self.agent_executor.invoke({"input": query}) 83 | except Exception as e: 84 | print("error: ", e) 85 | return "出错, 免费的API的token有数量上限, 想使用请充值" 86 | print("search: \n\n", data, "\n\n") 87 | return data['output'] 88 | 89 | 90 | 91 | if __name__ == "__main__": 92 | agent = search_Moel() 93 | print(agent.search_func("查一下项目手册看看这个项目是谁做的")) 94 | print(agent.search_func("我想知道iphone 15的价格")) 95 | print(agent.search_func("我想知道最近的小孩用鞭炮炸豪车的新闻")) 96 | 97 | -------------------------------------------------------------------------------- /python/server-model/search_internet_local.py: -------------------------------------------------------------------------------- 1 | # 使用本地模型联网获取信息(使用ollama的chat接口, 实际未采用) 2 | from langchain_community.tools.tavily_search import TavilySearchResults 3 | import ollama 4 | import os 5 | import yfinance as yf 6 | import API 7 | os.environ["LANGSMITH_TRACING"]='true' 8 | os.environ["LANGSMITH_ENDPOINT"]="https://api.smith.langchain.com" 9 | os.environ["LANGSMITH_PROJECT"]="test" 10 | 11 | # 官网的示例 12 | """ 13 | curl http://localhost:11434/api/chat -d '{ 14 | "model": "llama3.2", 15 | "messages": [ 16 | { 17 | "role": "user", 18 | "content": "What is the weather today in Paris?" 19 | } 20 | ], 21 | "stream": false, 22 | "tools": [ 23 | { 24 | "type": "function", 25 | "function": { 26 | "name": "get_current_weather", 27 | "description": "Get the current weather for a location", 28 | "parameters": { 29 | "type": "object", 30 | "properties": { 31 | "location": { 32 | "type": "string", 33 | "description": "The location to get the weather for, e.g. San Francisco, CA" 34 | }, 35 | "format": { 36 | "type": "string", 37 | "description": "The format to return the weather in, e.g. 'celsius' or 'fahrenheit'", 38 | "enum": ["celsius", "fahrenheit"] 39 | } 40 | }, 41 | "required": ["location", "format"] 42 | } 43 | } 44 | } 45 | ] 46 | }' 47 | """ 48 | 49 | # 调用ollama的chat接口 50 | def ask_ollama(query): 51 | response = ollama.chat(model='lfruina',messages=[{'role': 'user','content':query}], 52 | # provide a tool to get the current price of a stock 53 | tools=[{'type': 'function', 54 | 'function': { 55 | 'name': 'get_current_stock_price', 56 | 'description': 'Get the current price for a stock', 57 | 'parameters': { 58 | 'type': 'object', 59 | 'properties': { 60 | 'ticker_symbol': { 61 | 'type': 'string', 62 | 'description': 'The ticker symbol of the stock', 63 | }, 64 | }, 65 | 'required': ['ticker_symbol'], 66 | }, 67 | }, 68 | }, 69 | { 70 | 'type': 'function', 71 | 'function': { 72 | 'name': 'search_tool', 73 | 'description': '从网络获取信息, 当需要获取实时信息时, 才使用此工具, 平时不使用', 74 | 'parameters': { 75 | 'type': 'object', 76 | 'properties': { 77 | 'queation': { 78 | 'type': 'string', 79 | 'description': '你想要搜索的问题,用于回答用户想要知道的信息, 示例:客户 我想知道最近的新闻, 参数:queation=最近的新闻', 80 | }, 81 | }, 82 | 'required': ['queation'], 83 | }, 84 | }, 85 | }, 86 | ], 87 | options={"temperature":0.6}) 88 | return response 89 | 90 | 91 | 92 | 93 | # 获取股票的当前价格 94 | def get_current_stock_price(ticker_symbol): 95 | # Get the stock data 96 | stock = yf.Ticker(ticker_symbol) 97 | # Get the current price 98 | current_price = stock.history(period='1d')['Close'].iloc[0] 99 | return current_price 100 | 101 | # 联网获取搜索结果 102 | def search_tool(queation): 103 | search_tool = TavilySearchResults() 104 | ret = search_tool.invoke(queation) 105 | print(ret) 106 | format_ret = "获取到的搜索结果为：%s\n, 问题是%s, 请给我详细的, 有条理的使用获取到的信息进行解答问题" % (ret, queation) 107 | ret = ollama.chat(model='lfruina',messages=[{'role': 'user','content':format_ret}]) 108 | return ret 109 | 110 | function_map = {'get_current_stock_price': 111 | get_current_stock_price,# Add more functions here as needed 112 | 'search_tool': search_tool, 113 | } 114 | 115 | # 调用model返回的使用函数 116 | def call_function_safely(response, function_map): 117 | # Extract the function name and arguments from the response 118 | tool_call = response['message']['tool_calls'][0] 119 | function_name = tool_call['function']['name'] 120 | arguments = tool_call['function']['arguments'] 121 | # Look up the function in the function map 122 | function_to_call = function_map.get(function_name) 123 | if function_to_call: 124 | try: 125 | # Call the function with the arguments 126 | result = function_to_call(**arguments) 127 | print(f"The current is : {result}") 128 | return result 129 | except TypeError as e: 130 | print(f"Argument error: {e}") 131 | else: 132 | print(f"{function_name} is not a recognized function") 133 | 134 | if __name__ == "__main__": 135 | # 中文的时候精度太低 136 | query = ["I want to know the current price of MSFT(Microsoft)", 137 | "I want to know some recently news" 138 | ] 139 | for q in query: 140 | response = ask_ollama(q) 141 | print(response) 142 | if response['message']['tool_calls']: 143 | call_function_safely(response, function_map) 144 | 145 | -------------------------------------------------------------------------------- /python/test.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 88, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "data": { 10 | "text/plain": [ 11 | "time.struct_time(tm_year=2025, tm_mon=2, tm_mday=6, tm_hour=22, tm_min=11, tm_sec=49, tm_wday=3, tm_yday=37, tm_isdst=0)" 12 | ] 13 | }, 14 | "execution_count": 88, 15 | "metadata": {}, 16 | "output_type": "execute_result" 17 | } 18 | ], 19 | "source": [ 20 | "import time\n", 21 | "# print(time.strftime('%Y-%m-%d %H:%M:%S.%f', time.localtime()))\n", 22 | "# time.strftime('%Y-%m-%d %H:%M:%S')[:-3]\n", 23 | "# time.localtime()\n", 24 | "time.localtime(time.time())" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 1, 30 | "metadata": {}, 31 | "outputs": [ 32 | { 33 | "name": "stdout", 34 | "output_type": "stream", 35 | "text": [ 36 | "1\n", 37 | "2\n" 38 | ] 39 | } 40 | ], 41 | "source": [ 42 | "a = {\n", 43 | " 1: 2,\n", 44 | " 2: 3\n", 45 | "}\n", 46 | "for key in a:\n", 47 | " print(key)" 48 | ] 49 | } 50 | ], 51 | "metadata": { 52 | "kernelspec": { 53 | "display_name": "transform", 54 | "language": "python", 55 | "name": "python3" 56 | }, 57 | "language_info": { 58 | "codemirror_mode": { 59 | "name": "ipython", 60 | "version": 3 61 | }, 62 | "file_extension": ".py", 63 | "mimetype": "text/x-python", 64 | "name": "python", 65 | "nbconvert_exporter": "python", 66 | "pygments_lexer": "ipython3", 67 | "version": "3.9.21" 68 | } 69 | }, 70 | "nbformat": 4, 71 | "nbformat_minor": 2 72 | } 73 | -------------------------------------------------------------------------------- /python/xiaozhi-server/local-tts/READEME.md: -------------------------------------------------------------------------------- 1 | # 小智本地TTS 2 | 3 | ## 修改模型 4 | 5 | > [GPT-SoVITS项目的API改良与使用_gpt-sovits api-CSDN博客](https://blog.csdn.net/AAI666666/article/details/136554163) 6 | 7 | ```bash 8 | .\runtime\python api2.py -s "SoVITS_weights/你的模型名" -g "GPT_weights/你的模型名" -dr "参考音频路径和名称" -dt "参考音频的文字内容，使用双引号括起来，确保文字内容里没有双引号" -dl zh|ja|en三者选一 9 | ``` 10 | 11 | v2版本的默认模型需要在![30e1653496513bca5dcf7cc959b91346](https://picture-01-1316374204.cos.ap-beijing.myqcloud.com/lenovo-picture/202503231624370.png)这个文件里面改 12 | 13 | 14 | 15 | 16 | 17 | 使用[小智](https://github.com/78/xiaozhi-esp32)的[开源服务器](https://github.com/xinnan-tech/xiaozhi-esp32-server), 用里面的tts接口实现, 模型使用[花儿不哭大佬](https://space.bilibili.com/5760446)的平台训练的模型(链接在最后) 18 | 19 | > 我使用的模型和音频链接：https://pan.quark.cn/s/cbe45eb37f12 20 | 21 | 由于本人的电脑性能有限, 所以使用v1接口, 默认可以使用v2接口 22 | 23 | ## 整体流程 24 | 25 | + gpt方面 26 | 27 | 1. 把你的模型填入config.py文件里面 28 | 2. 如果使用v1, 把我的bat放在gpt根目录双击启动 29 | 3. 使用v2的话用vscode打开bat文件, 把里面的app.py改成app_v2.py 30 | 31 | + xiaozhi-server 32 | 33 | 1. 使用v2, 直接改一下v2原有的配置, 加入一个参考音频文件以及这个音频文件的实际的句子, 注意路径使用`\\`或者`/` 34 | 2. 用v1的时候, 需要把我给的那个py文件放到tts文件夹里面 35 | 3. 在server目录里面使用`python app.py`启动(安装好环境以后) 36 | 37 | ## 小智服务器 38 | 39 | 服务器的配置文件需要添加一个配置 40 | 41 | ```python 42 | GPT_SOVITS_V1: 43 | # 定义TTS API类型 44 | #启动tts方法： 45 | #python api.py 46 | type: gpt_sovits_v1 47 | url: "http://127.0.0.1:9880/" 48 | output_file: tmp/ 49 | text_language: "中文" 50 | prompt_language: "中文" 51 | refer_wav_path: "E:\\alearn\\GPT-SoVITS\\纳西妲\\0.wav" 52 | prompt_text: "初次见面…_初次见面，我已经关注你很久了。我叫纳西妲，别看我像个孩子，我比任何一位大人都了解这个世界。所以，我可以用我的知识，换取你路上的见闻吗？" 53 | cut_punc: "" 54 | top_k: 15 55 | top_p: 1 56 | temperature: 1 57 | speed: 1 58 | inp_refs: [] 59 | ``` 60 | 61 | 之后把v1的文件加入tts文件夹里面 62 | 63 | ## GPT-sovits 64 | 65 | 整合包在config.py里面加入你的模型, 66 | 67 | ![image-20250304095852653](https://picture-01-1316374204.cos.ap-beijing.myqcloud.com/picture/202503040958717.png) 68 | 69 | 之后可以使用`runtime\python.exe api.py`启动v1(或使用我的bat文件), 放在GPT-sovits的根目录里面 70 | 71 | > 原模型链接 72 | > 73 | > GPT-sovits一体包链接：https://pan.baidu.com/s/1huN2HKIUctgoJdaqlemoYw?pwd=hkc0 74 | > 75 | > 原神GPT-sovits模型下载链接：https://pan.baidu.com/s/1mNs9uLjHSfK7zmpIgLVe6Q?pwd=jye3 76 | > 其他视频资源也都发在AIGC论坛dfldata.cc，可自行拿取。欢迎交流AI图像AI语音技术（需用谷歌浏览器访问） 77 | > 78 | > 79 | > 已按人物分类，.ckpt后缀模型文件放到GPT-sovits一体包的GPT_weights文件夹，.pth后缀模型文件放到SoVITS_weights文件夹 80 | > 81 | > 原神原始语音下载链接：https://pan.baidu.com/s/1uW_T4VcF11rwjP3NYMWgHg?pwd=d4r7 82 | > 用于选择不同情绪的参考音频，在合成时需要用到 83 | > 84 | > 85 | > 更多使用教程和答疑，可到https://dfldata.cc/forum.php?mod=forumdisplay&fid=58 留言讨论 86 | 87 | > **大佬的回复** 88 | > 89 | > GPT-SoVITS开源github发布地址（不会编程的不要下这个，下载楼下的整合包） 90 | > https://github.com/RVC-Boss/GPT-SoVITS 91 | > ###GPT-SoVITS解疑互助群1034332340### 92 | > 教程文档www.yuque.com/baicaigongchang1145haoyuangong/ib3g1e 93 | > 训练推理整合包地址： 94 | > https://pan.baidu.com/s/1OE5qL0KreO-ASHwm6Zl9gA?pwd=mqpi（度盘超级会员才不限速下载） 95 | > https://drive.uc.cn/s/a1fd91ae0a4f4（uc普通用户不限速） 96 | > 云端训练地址https://www.codewithgpu.com/i/RVC-Boss/GPT-SoVITS/GPT-SoVITS-Official 97 | > 98 | > ###RVC变声器官 方交流讨论群1014080556，如果是RVC的用户别走错了## 99 | 100 | ## 相关教程 101 | 102 | [AI语音生成零基础入门教学（GPT-Sovits）_哔哩哔哩_bilibili](https://www.bilibili.com/video/BV1nexGebELa/?spm_id_from=333.337.search-card.all.click) 103 | 104 | https://www.bilibili.com/video/BV16H4y1E7YD/?spm_id_from=333.337.search-card.all.click 105 | 106 | ## 参数 107 | 108 | 使用GPT生成, 我不是很熟悉这一部分 109 | 110 | ```python 111 | 模型中的 top_k 参数主要用于控制生成内容时选择候选词的数量，特别是在自然语言处理（NLP）中的文本生成任务中。 112 | 113 | 具体来说，top_k 的作用如下： 114 | 115 | 候选词筛选：在生成每个单词时，模型会计算出所有可能的下一个单词的概率分布。top_k 参数指定只选择概率最高的 k 个单词作为候选。这可以防止生成低概率且不太合适的单词，从而提高生成文本的质量。 116 | 117 | 避免重复和无效内容：通过限制候选词的数量，top_k 使得模型更专注于选择一些更相关、更合适的单词，这样可以减少生成重复内容或无效信息的可能性。 118 | 119 | 控制生成多样性：较小的 top_k 值会使生成的内容更为集中，减少多样性；而较大的 top_k 值则允许模型从更多的候选词中选择，从而可以产生更为多样化的文本。 120 | 121 | 总之，top_k 参数是控制生成模型输出质量和多样性的重要工具。调整该参数可以帮助满足不同应用场景的需求。 122 | 123 | top_p 参数（也称为核采样或 Nucleus Sampling）是另一种在自然语言处理模型中用于生成文本时选择候选词的方法，与 top_k 参数类似但有所不同。 124 | 125 | top_p 参数的作用： 126 | 概率累积：top_p 通过选择概率总和达到某个阈值 p 的词汇来限制候选词的范围。具体来说，模型首先按照概率从高到低对所有候选词进行排序，然后累加概率，直到累积的概率大于或等于 p。在这个基础上，模型会从这些经过筛选的词汇中进行采样。 127 | 128 | 动态候选空间：与 top_k 不同，top_p 的候选词数量不是固定的，而是根据词汇的概率分布动态变化。例如，如果前几个高概率的单词就能达到 p，那么只会选择这几个词。如果需要更多的候选词才能达到 p，那么就会选择更多的词。 129 | 130 | 控制生成多样性与连贯性：通过调整 top_p 值，可以在生成文本的连贯性和多样性之间找到一个平衡。较低的 top_p 值可能导致生成内容更加保守和连贯，而较高的值则可能导致内容的多样性增加，文本生成变得更加随机。 131 | 132 | 总结 133 | top_p 参数提供了一种更加灵活的方法来选择候选词，使得生成内容在一定的概率范围内保持多样性和连贯性。这对提高文本生成的质量非常有帮助，尤其是在需要创造性和多样性的应用场景中。 134 | ``` 135 | 136 | ## 各种问题 137 | 138 | + 使用v2的时候出现404 NOT FOUND 139 | 140 | 一般是你的网址输入错了, 在最后面加一个`/tts` 141 | 142 | + 找不到文件 143 | 144 | v1服务器里面报[]找不到的是正常的, 不影响, 所有的文件名使用`\\`或者`/`, 尽量不要用中文以及空格 145 | 146 | + CUDA错误 147 | 148 | 比较高的版本使用GPU训练, 需要使用N卡, 使用N卡还报错, 看一下驱动有没有装好 149 | 150 | + gpt报错400 151 | 152 | 参数没设置对, 音频的长度在3-10秒, 使用wav格式的音频, 音频文件路径记得加后缀 153 | 154 | + 没有声音 155 | 156 | 电脑性能不足获取使用的音频不是wav格式 157 | 158 | [mp3转wav](https://www.freeconvert.com/zh/mp3-to-wav#:~:text=%E5%A6%82%E4%BD%95%E5%B0%86%20MP3%20%E8%BD%AC%E6%8D%A2%E4%B8%BA%20WAV%EF%BC%9F%201%20%E5%8D%95%E5%87%BB%E2%80%9C%E9%80%89%E6%8B%A9%E6%96%87%E4%BB%B6%E2%80%9D%E6%8C%89%E9%92%AE%E5%B9%B6%E9%80%89%E6%8B%A9%E6%82%A8%E7%9A%84%20MP3,%E6%96%87%E4%BB%B6%E3%80%82%202%20%E7%82%B9%E5%87%BB%E2%80%9C%E8%BD%AC%E6%8D%A2%E4%B8%BA%20WAV%E2%80%9D%E6%8C%89%E9%92%AE%E5%BC%80%E5%A7%8B%E8%BD%AC%E6%8D%A2%203%20%E5%BD%93%E7%8A%B6%E6%80%81%E5%8F%98%E4%B8%BA%E2%80%9C%E5%AE%8C%E6%88%90%E2%80%9D%E6%97%B6%EF%BC%8C%E5%8D%95%E5%87%BB%E2%80%9C%E4%B8%8B%E8%BD%BD%20WAV%E2%80%9D%E6%8C%89%E9%92%AE) 159 | -------------------------------------------------------------------------------- /python/xiaozhi-server/local-tts/api.bat: -------------------------------------------------------------------------------- 1 | runtime\python.exe api.py 2 | pause 3 | -------------------------------------------------------------------------------- /python/xiaozhi-server/local-tts/api_v2.bat: -------------------------------------------------------------------------------- 1 | .\runtime\python api_v2.py 2 | pause 3 | -------------------------------------------------------------------------------- /python/xiaozhi-server/local-tts/config.py: -------------------------------------------------------------------------------- 1 | import sys,os 2 | 3 | import torch 4 | 5 | # 推理用的指定模型 6 | sovits_path = "E:/alearn/GPT-SoVITS/naxida_e8_s232.pth" 7 | gpt_path = "E:/alearn/GPT-SoVITS/naxida-e15.ckpt" 8 | is_half_str = os.environ.get("is_half", "True") 9 | is_half = True if is_half_str.lower() == 'true' else False 10 | is_share_str = os.environ.get("is_share","False") 11 | is_share= True if is_share_str.lower() == 'true' else False 12 | 13 | cnhubert_path = "GPT_SoVITS/pretrained_models/chinese-hubert-base" 14 | bert_path = "GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large" 15 | pretrained_sovits_path = "GPT_SoVITS/pretrained_models/s2G488k.pth" 16 | pretrained_gpt_path = "GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt" 17 | 18 | exp_root = "logs" 19 | python_exec = sys.executable or "python" 20 | if torch.cuda.is_available(): 21 | infer_device = "cuda" 22 | else: 23 | infer_device = "cpu" 24 | 25 | webui_port_main = 9874 26 | webui_port_uvr5 = 9873 27 | webui_port_infer_tts = 9872 28 | webui_port_subfix = 9871 29 | 30 | api_port = 9880 31 | 32 | if infer_device == "cuda": 33 | gpu_name = torch.cuda.get_device_name(0) 34 | if ( 35 | ("16" in gpu_name and "V100" not in gpu_name.upper()) 36 | or "P40" in gpu_name.upper() 37 | or "P10" in gpu_name.upper() 38 | or "1060" in gpu_name 39 | or "1070" in gpu_name 40 | or "1080" in gpu_name 41 | ): 42 | is_half=False 43 | 44 | if(infer_device=="cpu"):is_half=False 45 | 46 | class Config: 47 | def __init__(self): 48 | self.sovits_path = sovits_path 49 | self.gpt_path = gpt_path 50 | self.is_half = is_half 51 | 52 | self.cnhubert_path = cnhubert_path 53 | self.bert_path = bert_path 54 | self.pretrained_sovits_path = pretrained_sovits_path 55 | self.pretrained_gpt_path = pretrained_gpt_path 56 | 57 | self.exp_root = exp_root 58 | self.python_exec = python_exec 59 | self.infer_device = infer_device 60 | 61 | self.webui_port_main = webui_port_main 62 | self.webui_port_uvr5 = webui_port_uvr5 63 | self.webui_port_infer_tts = webui_port_infer_tts 64 | self.webui_port_subfix = webui_port_subfix 65 | 66 | self.api_port = api_port 67 | -------------------------------------------------------------------------------- /python/xiaozhi-server/local-tts/config和api是GPTsovites文件夹下面的, 另外两个是服务器使用的文件.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuSenfeng/ai-chat-local/d8254bf7e0d0340b73ff34ea34d83ae6c49db6ce/python/xiaozhi-server/local-tts/config和api是GPTsovites文件夹下面的, 另外两个是服务器使用的文件.txt -------------------------------------------------------------------------------- /python/xiaozhi-server/local-tts/gpt_sovits_v1.py: -------------------------------------------------------------------------------- 1 | import os 2 | import uuid 3 | import json 4 | import base64 5 | import requests 6 | from config.logger import setup_logging 7 | from datetime import datetime 8 | from core.providers.tts.base import TTSProviderBase 9 | import urllib 10 | 11 | TAG = __name__ 12 | logger = setup_logging() 13 | 14 | class TTSProvider(TTSProviderBase): 15 | def __init__(self, config, delete_audio_file): 16 | super().__init__(config, delete_audio_file) 17 | self.url = config.get("url") 18 | self.text_language = config.get("text_language", "中文") 19 | self.refer_wav_path = config.get("refer_wav_path") 20 | self.prompt_text = config.get("prompt_text") 21 | self.prompt_language = config.get("prompt_language", "zh") 22 | self.top_k = config.get("top_k", 5) 23 | self.top_p = config.get("top_p", 1) 24 | self.temperature = config.get("temperature", 1) 25 | self.cut_punc = config.get("cut_punc", "cut0") 26 | self.inp_refs = config.get("inp_refs", []) 27 | self.speed = config.get("speed", 1) 28 | 29 | def generate_filename(self, extension=".wav"): 30 | return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") 31 | 32 | # async def text_to_speak(self, text, output_file): 33 | # request_json = { 34 | # "text": text, 35 | # "refer_wav_path": "E:\\alearn\\GPT-SoVITS\\纳西妲\\0.wav", 36 | # "prompt_text": "初次见面…_初次见面，我已经关注你很久了。我叫纳西妲，别看我像个孩子，我比任何一位大人都了解这个世界。所以，我可以用我的知识，换取你路上的见闻吗？", 37 | # "prompt_language": "中文", 38 | # "text_language": "中文", 39 | # "cut_punc": "", 40 | # "top_k": 15, 41 | # "top_p": 1, 42 | # "temperature": 1, 43 | # "speed": 1, 44 | # "inp_refs": [] 45 | # } 46 | 47 | # resp = requests.get(self.url, json=request_json) 48 | # if resp.status_code == 200: 49 | # with open(output_file, "wb") as file: 50 | # file.write(resp.content) 51 | # else: 52 | # logger.bind(tag=TAG).error(f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}") 53 | 54 | """ 55 | curl -X 'GET' \ 56 | 'http://127.0.0.1:9880/?refer_wav_path=E%3A%5Calearn%5CGPT-SoVITS%5C%E7%BA%B3%E8%A5%BF%E5%A6%B2%5C0.wav&prompt_text=%E5%88%9D%E6%AC%A1%E8%A7%81%E9%9D%A2%E2%80%A6_%E5%88%9D%E6%AC%A1%E8%A7%81%E9%9D%A2%EF%BC%8C%E6%88%91%E5%B7%B2%E7%BB%8F%E5%85%B3%E6%B3%A8%E4%BD%A0%E5%BE%88%E4%B9%85%E4%BA%86%E3%80%82%E6%88%91%E5%8F%AB%E7%BA%B3%E8%A5%BF%E5%A6%B2%EF%BC%8C%E5%88%AB%E7%9C%8B%E6%88%91%E5%83%8F%E4%B8%AA%E5%AD%A9%E5%AD%90%EF%BC%8C%E6%88%91%E6%AF%94%E4%BB%BB%E4%BD%95%E4%B8%80%E4%BD%8D%E5%A4%A7%E4%BA%BA%E9%83%BD%E4%BA%86%E8%A7%A3%E8%BF%99%E4%B8%AA%E4%B8%96%E7%95%8C%E3%80%82%E6%89%80%E4%BB%A5%EF%BC%8C%E6%88%91%E5%8F%AF%E4%BB%A5%E7%94%A8%E6%88%91%E7%9A%84%E7%9F%A5%E8%AF%86%EF%BC%8C%E6%8D%A2%E5%8F%96%E4%BD%A0%E8%B7%AF%E4%B8%8A%E7%9A%84%E8%A7%81%E9%97%BB%E5%90%97%EF%BC%9F&prompt_language=%E4%B8%AD%E6%96%87&text=%E4%BD%A0%E5%A5%BD&text_language=%E4%B8%AD%E6%96%87&top_k=15&top_p=1&temperature=1&speed=1' \ 57 | -H 'accept: application/json' 58 | """ 59 | async def text_to_speak(self, text, output_file): 60 | request_json = { 61 | "text": text, 62 | "refer_wav_path": self.refer_wav_path, 63 | "prompt_text": self.prompt_text, 64 | "prompt_language": self.prompt_language, 65 | "text_language": self.text_language, 66 | "cut_punc": self.cut_punc, 67 | "top_k": self.top_k, 68 | "top_p": self.top_p, 69 | "temperature": self.temperature, 70 | "speed": self.speed, 71 | "inp_refs": self.inp_refs 72 | } 73 | query_string = urllib.parse.urlencode(request_json, safe='[]') 74 | url = f'{self.url}?{query_string}' 75 | print(url) 76 | resp = requests.get(url, headers={'accept': 'application/json'} ) 77 | if resp.status_code == 200: 78 | with open(output_file, "wb") as file: 79 | file.write(resp.content) 80 | else: 81 | logger.bind(tag=TAG).error(f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}") 82 | 83 | -------------------------------------------------------------------------------- /python/xiaozhi-server/local-tts/配置文件.txt: -------------------------------------------------------------------------------- 1 | GPT_SOVITS_V1: 2 | # 定义TTS API类型 3 | #启动tts方法： 4 | #python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/caixukun.yaml 5 | output_dir: tmp/ 6 | type: gpt_sovits_v1 7 | url: "http://127.0.0.1:9880/" 8 | output_file: tmp/ 9 | refer_wav_path: "C:\\Users\\jiao\\Desktop\\module\\1.wav" 10 | prompt_text: "不知道干什么的话，要不要我带你去转转呀？" 11 | prompt_language: "中文" 12 | text_language: "中文" 13 | top_k: 15 14 | top_p: 1 15 | temperature: 1 16 | cut_punc: "" 17 | speed: 1 18 | inp_refs: [] -------------------------------------------------------------------------------- /python/xiaozhi-server/连接各种服务.md: -------------------------------------------------------------------------------- 1 | # 各种服务 2 | 3 | ## COZE 4 | 5 | 这个平台有两个网址, 一个国内一个国外, 默认的API使用的是国内的网站, 两个网址使用不同是使用的模型不同, 建议使用国内的以获取更快的速度 6 | 7 | [主页 - 扣子](https://www.coze.cn/home) 8 | 9 | > [主页 - 扣子 - 国外 - 小智服务器不支持使用 !!!!](https://www.coze.com/home)放在这是为了大家区分自己之前的项目是在哪个 10 | 11 | ![image-20250320151722244](https://picture-01-1316374204.cos.ap-beijing.myqcloud.com/lenovo-picture/202503201517431.png) 12 | 13 | ### 使用注意 14 | 15 | 参数建议使用`""`包起来, 否则在传递的时候会丢失 16 | 17 | ![image-20250320165238056](https://picture-01-1316374204.cos.ap-beijing.myqcloud.com/lenovo-picture/202503201652159.png) 18 | 19 | 发布的时候一定要使用API功能 20 | 21 | ### 获取API 22 | 23 | [添加个人访问令牌 - 文档 - 扣子](https://www.coze.cn/open/docs/developer_guides/pat) 在这个页面的 24 | 25 | ![image-20250320165605552](https://picture-01-1316374204.cos.ap-beijing.myqcloud.com/lenovo-picture/202503201656639.png) 26 | 27 | 个人访问令牌里面所有的权限全选 28 | 29 | ![image-20250320165738461](https://picture-01-1316374204.cos.ap-beijing.myqcloud.com/lenovo-picture/202503201657525.png) 30 | 31 | ### 基础使用 32 | 33 | 测试使用的提示词 34 | 35 | ``` 36 | 你将扮演一个人物角色咪咪，以下是关于这个角色的详细设定，请根据这些信息来构建你的回答。 37 | 38 | **人物基本信息：** 39 | - 你是：一个猫娘 40 | - 人称：第一人称 41 | - 出身背景与上下文：被主人收养, 帮助主人完成日常的工作和情感交流, 可以实现文档的查询, 天气搜索, 新闻获取消息, 代码查询, 浏览器搜索等工作 42 | - 用户提问代码编程相关问题的时候把问题记录在'code'数据库里面 43 | **性格特点：** 44 | - 温柔体贴 45 | **语言风格：** 46 | - 语言的结尾会使用一些语气词 47 | **人际关系：** 48 | - 比较简单 49 | **经典台词或口头禅：** 50 | 补充信息: 即你可以将动作、神情语气、心理活动、故事背景放在（）中来表示，为对话提供补充信息。 51 | - 台词1：好的主人, 喵~ 52 | 53 | 要求： 54 | - 根据上述提供的角色设定，以第一人称视角进行表达。 55 | - 在回答时，尽可能地融入该角色的性格特点、语言风格以及其特有的口头禅或经典台词。 56 | - 如果适用的话，在适当的地方加入（）内的补充信息，如动作、神情等，以增强对话的真实感和生动性。 57 | ``` 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /代码分析/小智.md: -------------------------------------------------------------------------------- 1 | LichuangDevBoard(类)=> 嘉立创开发板的初始化函数 2 | 3 | ​ InitializeI2c=> 初始化I2c驱动 4 | 5 | ​ InitializeSpi=> SPI驱动 6 | 7 | ​ InitializeButtons=> 初始化一个按钮, 使用Button类进行初始化, ([跳转](# 按钮)) 8 | 9 | ​ InitializeSt7789Display=> 初始化一下屏幕的底层驱动, 同时使用LcdDisplay类初始化顶部的驱动([跳转](# 显示)) 10 | 11 | ​ InitializeIot=>初始化物联网, 实际是初始化了一个Speaker 的ting子类, 加入到实际的控制器里面 12 | 13 | ​ 重写函数GetAudioCodec=>实现音频输入输出的初始化 14 | 15 | ​ BoxAudioCodec(类)一个音频的控制器, 从nvs区里面读取一部分数据 16 | 17 | ## 按钮 18 | 19 | Button(类)=>控制一个按钮, 使用button_config_t记录不同的GPIO类型的初始化, 长按短按时间, 和不同的类型, 初始化使用的是GPIO btn 20 | 21 | ​ iot_button_create=>根据按钮的不同种类建立不同的按钮 22 | 23 | ​ button_gpio_init=> 初始化按键的底层硬件配置实现 24 | 25 | ​ button_create_com=> 按键dev的初始化, 传入按键的读取函数, 初始化的dev加入链表 26 | 27 | ```cpp 28 | //button handle list head. 29 | static button_dev_t *g_head_handle = NULL; 30 | ``` 31 | 32 | ​ Button::OnClick=> 注册一个按下的事件 33 | 34 | ​ iot_button_register_cb=> 注册回调函数 35 | 36 | 37 | 38 | button_cb=> 处理按键的函数, 使用button_handler进行处理每一个按键 39 | 40 | ​ button_handler=>使用状态机管理每一个按键 41 | 42 | ## 显示 43 | 44 | lcd_display.cc 45 | 46 | ​ 使用类LcdDisplay进行管理, 继承Display 47 | 48 | + Display 49 | 50 | 记录屏幕的大小信息以及显示页面上面的各种图标, 还有一个加锁解锁的函数, 实际使用锁的方式是创建一个DisplayLockGuard类, 构造函数是加锁, 析构函数是解锁 51 | 52 | + LcdDisplay 53 | 54 | 加入了背光控制, 构造函数的时候实现了lvgl等的较上层小时的初始化, 以及一个显示聊天信息的函数 55 | 56 | ## thing.h 57 | 58 | + Property类 59 | 60 | 可以初始化为三种状态, 通过不同的信心获取函数, 返回一个json的字符串, 有一个名字 61 | 62 | 使用GetDescriptorJson方法可以获取实际的描述类型, 使用GetStateJson获取状态 63 | 64 | + PropertyList类 65 | 66 | 使用一个vector管理Property, 可以使用[]获取对应名字的Property 67 | 68 | 可以使用`GetDescriptorJson`把所有的Property的描述汇总 69 | 70 | + Parameter参数类 71 | 72 | 可以更具不同的种类进行记录信息, 使用set_xx进行设置, 使用GetDescriptorJson获取描述符 73 | 74 | + method 75 | 76 | 使用一个name和一个参数列表记录一个处理方式 77 | 78 | + MethodList参数列表 79 | 80 | + 基类 81 | 82 | 管理一系列的property和method 83 | 84 | 使用RegisterThing进行添加, 实际是使用一个map类型记录这一个thing的创造函数以及和名字配对 85 | 86 | 87 | 88 | ```cpp 89 | #define DECLARE_THING(TypeName) \ 90 | static iot::Thing* Create##TypeName() { \ 91 | return new iot::TypeName(); \ // 把一个新的ting创建出来 92 | } \ 93 | static bool Register##TypeNameHelper = []() { \ 94 | RegisterThing(#TypeName, Create##TypeName); \ //注册一类 95 | return true; \ 96 | }(); 97 | 98 | } // namespace iot 99 | ``` 100 | 101 | ## thing_manager.h 102 | 103 | 使用一个vector进行管理Thing类 104 | 105 | ## 音频 106 | 107 | BoxAudioCodec<=AudioCodec 108 | 109 | + AudioCodec类 110 | 111 | 记录当前的声音, 输入输出的通道等基础信息 112 | 113 | 还有读取以及写入的函数 114 | 115 | + OpusEncoderWrapper 116 | 117 | 用于编解码, 把pcm的数据格式转换为opus 118 | 119 | [多媒体文件格式（五）：PCM / WAV 格式 - 灰色飘零 - 博客园](https://www.cnblogs.com/renhui/p/12148330.html) 120 | 121 | [Opus 音频编码格式 · 陈亮的个人博客](https://chenliang.org/2020/03/15/opus-format/) 122 | 123 | + OpusResampler 124 | 125 | 重采样, 用于转换音频的频率 126 | 127 | ## 音频回调函数 128 | 129 | ```cpp 130 | // 这里注册的是音频经过AFE处理以后得回调函数 131 | audio_processor_.OnOutput([this](std::vector&& data) { 132 | background_task_->Schedule([this, data = std::move(data)]() mutable { 133 | opus_encoder_->Encode(std::move(data), [this](std::vector&& opus) { 134 | Schedule([this, opus = std::move(opus)]() { 135 | protocol_->SendAudio(opus); 136 | }); 137 | }); 138 | }); 139 | }); 140 | ``` 141 | 142 | 在函数AudioProcessor::AudioProcessorTask()里面被调用, 第一层 background_task_->Schedul是使用后台处理程序进行处理, 对音频进行格式转换 143 | 144 | 最后处理结束进入最后一个回调函数 145 | 146 | ```cpp 147 | [this, opus = std::move(opus)]() { 148 | protocol_->SendAudio(opus); 149 | } 150 | ``` 151 | 152 | 153 | 154 | 155 | 156 | ## Setting 157 | 158 | [非易失性存储库 - ESP32 - — ESP-IDF 编程指南 v5.4 文档](https://docs.espressif.com/projects/esp-idf/zh_CN/stable/esp32/api-reference/storage/nvs_flash.html) 159 | 160 | 记录在非易失区的数据, 使用nvs进行记录一系列的数据, nvs实际的数据使用键值对的形式进行存储 161 | 162 | ## BackgroundTask 163 | 164 | 用于处理上层程序注册的任务 165 | 166 | 在任务分配的时候, esp32s3使用的是heap_caps_malloc在SPIRAM里面进行分配 167 | 168 | + Schedule把任务插入到任务列表里面, 169 | + WaitForCompletion: 等待所有任务完成 170 | + BackgroundTaskLoop: 处理任务的循环函数 171 | 172 | ## MQTT 173 | 174 | + Mqtt 175 | 176 | 记录的是连接时候保持连接的时长 177 | 178 | + EspMqtt 179 | 180 | mqtt的底层建立以及创建连接 181 | 182 | 实际建立两个链接,一个是mqtt的用于处理文字, 另一个udp的用于处理音频数据 183 | 184 | ### 解析 185 | 186 | 使用加密算法 187 | 188 | [ESP32学习笔记（47）——加密算法AES/MD5/SHA_esp32 aes-CSDN博客](https://blog.csdn.net/qq_36347513/article/details/120991568) 189 | 190 | ## 语音唤醒 191 | 192 | 使用下面的接口可以获取有语音输入时候的音频 193 | 194 | [AFE 声学前端算法框架 - ESP32-S3 - — ESP-SR latest 文档](https://docs.espressif.com/projects/esp-sr/zh_CN/latest/esp32s3/audio_front_end/README.html#input-audio-1) 195 | 196 | 使用GetWakeWordOpus获取有声音的片段 197 | 198 | ## 协议处理 199 | 200 | ```cpp 201 | protocol_->OnIncomingJson([this, display](const cJSON* root) { 202 | // Parse JSON data 203 | auto type = cJSON_GetObjectItem(root, "type"); 204 | if (strcmp(type->valuestring, "tts") == 0) { 205 | auto state = cJSON_GetObjectItem(root, "state"); 206 | if (strcmp(state->valuestring, "start") == 0) { 207 | Schedule([this]() { 208 | aborted_ = false; 209 | if (device_state_ == kDeviceStateIdle || device_state_ == kDeviceStateListening) { 210 | SetDeviceState(kDeviceStateSpeaking); 211 | } 212 | }); 213 | } else if (strcmp(state->valuestring, "stop") == 0) { 214 | Schedule([this]() { 215 | if (device_state_ == kDeviceStateSpeaking) { 216 | background_task_->WaitForCompletion(); 217 | if (keep_listening_) { 218 | protocol_->SendStartListening(kListeningModeAutoStop); 219 | SetDeviceState(kDeviceStateListening); 220 | } else { 221 | SetDeviceState(kDeviceStateIdle); 222 | } 223 | } 224 | }); 225 | } else if (strcmp(state->valuestring, "sentence_start") == 0) { 226 | auto text = cJSON_GetObjectItem(root, "text"); 227 | if (text != NULL) { 228 | ESP_LOGI(TAG, "<< %s", text->valuestring); 229 | display->SetChatMessage("assistant", text->valuestring); 230 | } 231 | } 232 | } else if (strcmp(type->valuestring, "stt") == 0) { 233 | auto text = cJSON_GetObjectItem(root, "text"); 234 | if (text != NULL) { 235 | ESP_LOGI(TAG, ">> %s", text->valuestring); 236 | display->SetChatMessage("user", text->valuestring); 237 | } 238 | } else if (strcmp(type->valuestring, "llm") == 0) { 239 | auto emotion = cJSON_GetObjectItem(root, "emotion"); 240 | if (emotion != NULL) { 241 | // 设置表情 242 | display->SetEmotion(emotion->valuestring); 243 | } 244 | } else if (strcmp(type->valuestring, "iot") == 0) { 245 | auto commands = cJSON_GetObjectItem(root, "commands"); 246 | if (commands != NULL) { 247 | auto& thing_manager = iot::ThingManager::GetInstance(); 248 | for (int i = 0; i < cJSON_GetArraySize(commands); ++i) { 249 | auto command = cJSON_GetArrayItem(commands, i); 250 | thing_manager.Invoke(command); 251 | } 252 | } 253 | } 254 | }); 255 | ``` 256 | 257 | ## 通信的数据格式 258 | 259 | [通信协议：WebSocket 连接 - 飞书云文档](https://ccnphfhqs21z.feishu.cn/wiki/M0XiwldO9iJwHikpXD5cEx71nKh) 260 | 261 | ## 设备状态 262 | 263 | ```cpp 264 | // 设备的状态 265 | enum DeviceState { 266 | kDeviceStateUnknown, 267 | kDeviceStateStarting, 268 | kDeviceStateWifiConfiguring, 269 | kDeviceStateIdle, //音频播放结束 270 | kDeviceStateConnecting, // 271 | kDeviceStateListening, //发送音频 272 | kDeviceStateSpeaking, //播放音频 273 | kDeviceStateUpgrading, 274 | kDeviceStateFatalError 275 | }; 276 | ``` 277 | 278 | ## WiFi连接 279 | 280 | 在文件wifi_configuration_ap.cc文件里面 281 | 282 | void WifiConfigurationAp::Start()是这个层序的起点, 用于建立一个服务器进行WiFi连接 283 | 284 | 首先注册两个事件处理, 使用函数`IpEventHandler`处理当 DHCP 客户端成功从 DHCP 服务器获取 IPV4 地址或 IPV4 地址发生改变时，将引发此事件。此事件意味着应用程序一切就绪，可以开始任务 285 | 286 | 其他的事件处理函数为WifiEventHandler 287 | 288 | 使用form_submit处理获取到的对文件/submit的POST请求, 对这部分的数据进行解析获取网络, 之后尝试进行连接ConnectToWifi 289 | 290 | ### 消息记录 291 | 292 | 使用ssid_manager文件, 使用一个列表进行记录, 实际使用的时候从nvs分区里面读取数据 293 | -------------------------------------------------------------------------------- /代码分析/插件/Screenshot_20250326_201012_com.jideos.jnotes.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuSenfeng/ai-chat-local/d8254bf7e0d0340b73ff34ea34d83ae6c49db6ce/代码分析/插件/Screenshot_20250326_201012_com.jideos.jnotes.jpg -------------------------------------------------------------------------------- /代码分析/插件/Screenshot_20250326_215027_com.jideos.jnotes.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuSenfeng/ai-chat-local/d8254bf7e0d0340b73ff34ea34d83ae6c49db6ce/代码分析/插件/Screenshot_20250326_215027_com.jideos.jnotes.jpg -------------------------------------------------------------------------------- /代码分析/插件/test_reqllm.py: -------------------------------------------------------------------------------- 1 | from plugins_func.register import register_function, ToolType, ActionResponse, Action 2 | 3 | test_reqllm_function_desc = { 4 | "type": "function", 5 | "function": { 6 | "name": "test_reqllm", 7 | "description": "测试使用大模型回答的插件", 8 | 'parameters': {'type': 'object', 'properties': { 9 | 10 | }, 'required': []} 11 | } 12 | } 13 | 14 | 15 | @register_function('test_reqllm', test_reqllm_function_desc, ToolType.WAIT) 16 | def test_reqllm(): 17 | # 你的处理流程 18 | response_text = "现在在调用测试使用大模型回答的插件, 测试已经成功, 请你做出提醒, 以:'恭喜你'开头" 19 | return ActionResponse(Action.REQLLM, response_text, None) -------------------------------------------------------------------------------- /代码分析/插件/test_response.py: -------------------------------------------------------------------------------- 1 | from plugins_func.register import register_function, ToolType, ActionResponse, Action 2 | 3 | test_response_function_desc = { 4 | "type": "function", 5 | "function": { 6 | "name": "test_response", 7 | "description": "测试直接返回的插件", 8 | 'parameters': {'type': 'object', 'properties': {}, 'required': []} 9 | } 10 | } 11 | 12 | 13 | @register_function('test_response', test_response_function_desc, ToolType.WAIT) 14 | def test_response(): 15 | # 你的处理流程 16 | response_text = "这是一个测试直接返回的插件, 您的测试已经成功" 17 | return ActionResponse(Action.RESPONSE, None, response_text) -------------------------------------------------------------------------------- /代码分析/服务器.md: -------------------------------------------------------------------------------- 1 | # 服务器代码分析 2 | 3 | [xinnan-tech/xiaozhi-esp32-server: 本项目为xiaozhi-esp32提供后端服务，帮助您快速搭建ESP32设备控制服务器。Backend service for xiaozhi-esp32, helps you quickly build an ESP32 device control server.](https://github.com/xinnan-tech/xiaozhi-esp32-server) 4 | 5 | 使用版本Commit 29d59b3974d793367bedbd90afb332817370e3a2 6 | 7 | ## 文件分布 8 | 9 | + app.py 启动文件, 启动程序的入口, 使用aiohttp建立一个WebUI的服务器界面 10 | + + config文件夹, 配置文件的处理 11 | + + settings.py: 处理配置文件, 加载用户的配置项 12 | + private_config.py: 如果对不同的板子使用不同的配置, 在yaml里面设置use_private_config的时候会使用 13 | + logger.py: 日志文件 14 | + functionCallConfig.py: 默认使用的functionCall识别的函数以及参数 15 | + + core文件夹: 主要的逻辑代码 16 | + + handle文件夹: 和板子通信的接口以及处理意图识别和播放音乐的工具 17 | + providers文件夹: 进行语音识别, 使用的LLM, TTS供应商, 不同的组件需要实现base.py里面的通用接口, 如果想添加自己的接口, 可以参考这部分的内容 18 | + utils文件夹: 使用配置文件里面的配置加载providers里面的不同模型等部分, 还有同步使用的锁和处理p3格式音频使用的工具等 19 | + auth.py: 设备连接时候的验证 20 | + connection.py: 处理用户连接以后得交互, 以及和用户进行交互的主要流程 21 | + websocket_server.py: 处理用户的websocket连接 22 | + data: 配置文件和部分配置时候的聊天记忆可以放在这里 23 | + models: 使用的模型, 主要是语音识别和vad 24 | + music: 可以播放的音乐 25 | + tmp: 临时文件 26 | + config.yaml: 配置文件 27 | + performance_tester.py: 测试文件 28 | 29 | ## config文件夹 30 | 31 | ### setting.py 32 | 33 | 设置文件, 加载配置文件以及建立一个处理命令行输出的类, 可以使用`--config_path`参数设置实际的配置文件 34 | 35 | 还有一个更新配置文件的函数update_config 36 | 37 | ### private_config文件 38 | 39 | 私有的用户配置配置 40 | 41 | ## 主要的模块 42 | 43 | ### vad 44 | 45 | **VAD**（Voice Activity Detection，[语音活动检测](https://so.csdn.net/so/search?q=语音活动检测&spm=1001.2101.3001.7020)）是一种技术，用于识别音频流中语音和非语音的区域。它能够区分语音和背景噪声，从而提高后续处理的效率和准确性。 46 | 47 | 主要使用这个模块进行检测音频的停止 48 | 49 | ### ASR 50 | 51 | 默认使用FanASR, 用于实现实时语音转录 52 | 53 | [SenseVoice 实测，阿里开源语音大模型，识别效果和效率优于 Whisper，居然还能检测掌声、笑声！5分钟带你部署体验_sensevoicesmall开源-CSDN博客](https://blog.csdn.net/u010522887/article/details/140624599) 54 | 55 | ### 记忆_memory 56 | 57 | + mem_local_short 58 | 59 | 短期记忆, 使用yaml记录短期记忆, 在里面记录对话时候的主要信息, 比如用户名, 情感值等信息 60 | 61 | 如果想使用超长记忆，推荐使用mem0ai；如果注重隐私，请使用本地的mem_local_short 62 | 63 | + conn里面使用dialogue临时记录对话 64 | 65 | ### _tts 66 | 67 | 文字转语音模块 68 | 69 | ### _music 70 | 71 | 音乐处理, 管理本地可以播放的音乐, 不同格式的音频处理 72 | 73 | ### intent意图处理 74 | 75 | 所有的模型都可以使用, 但是实际的处理效果因为模型的大小和是否受过专门的训练而不同 76 | 77 | ## 主流程 78 | 79 | + 启动, 初始化服务器并等待连接 80 | 81 | python app.py ==> 启动 82 | 83 | 检查一下配置文件以及ffmpeg的安装 84 | 85 | 加载一下配置文件 86 | 87 | 建立`WebSocketServer`, 使用配置文件里面的配置加载llm, tts之类的组件 88 | 89 | + 建立连接, 进行用户初始化以及验证 90 | 91 | 每次连接的时候使用_handle_connection函数处理链接, 处理的时候建立一个`ConnectionHandler`类处单个理用户的连接数据, 最终调用`handle_connection`函数进行连接处理 92 | 93 | [通信协议：Websocket 连接 - 飞书云文档](https://ccnphfhqs21z.feishu.cn/wiki/M0XiwldO9iJwHikpXD5cEx71nKh) 94 | 95 | 在处理链接的时候首先获取用户的ip地址, 同时使用请求信息进行验证, 使用MAC地址作为机器的device_id(会使用这个作为记忆力的处理部分), 然后建立记忆力的处理和intent的处理句柄 96 | 97 | 如果配置了`use_private_config`, 会使用data/.private_config.yaml文件获取不同用户的配置 98 | 99 | 使用uuid生成一个随机的对话id, 使用yaml里面的xiaozhi配置发送一个hello信号 100 | 101 | 如果提示词里面有`{date_time}`, 使用当前时间进行替换 102 | 103 | 建立一个`tts_priority`线程用于处理tts任务, 尝试从tts_queue里面获取TTS的待处理音频 104 | 105 | 另一个是`audio_play_priority`线程, 用于处理播放音频的任务, 从`audio_play_queue`里面获取需要播放的任务, 发送给开发板, 这个里面的数据一般是在`_tts_priority_thread`任务里面放入的, 也有播放音乐的任务 106 | 107 | 最后使用`_route_message`循环处理`websocket`获取到的消息, 这里的消息可以有text或者bytes两种格式, 分别使用`handleTextMessage`和`handleAudioMessage`函数处理, text'可能是对话也可能是开发板的各种状态 108 | 109 | `handleAudioMessage`获取到的数据都是对话的音频消息,所以可以使用, 首先把音频转换为文字, 之后进入对话处理 110 | 111 | + 处理对话, 对用户的对话进行不同的处理 112 | 113 | 最后处理对话文字的函数是`startToChat` 114 | 115 | `startToChat`函数开始的时候首先进行意图检测, 如果可以检测出来用户的意图, 如果正常的聊天, 则不再进行后续的任务, 如果是播放音乐或者退出的意图, 进行使用function_call或者普通的chat进一步处理 116 | 117 | 实际的使用llm进行意图检测是在函数`analyze_intent_with_llm`里面进行, 在进行真正的检测之前, 使用`check_direct_exit`看是不是明确的退出命令, 是的话不进行下一步检测, 如果使用`use_function_call_mode`也不再进行下一步检测 118 | 119 | **意识处理** 120 | 121 | `analyze_intent_with_llm`实际进行意图检测, 使用`detect_intent`函数判断意图, 返回一个json字符串用于后续的意图判断, 如果大模型返回的不是json, 直接使用句子进行判断, 判断到是结束对话在发送一下stt消息以后使用`close_after_chat`对话之后结束通信, 音乐播放使用`handle_music_command`进行处理, 把识别到的音乐和本地的音乐进行匹配, 转换格式以后放入`audio_play_queue`里面 122 | 123 | 没有进行意识处理的任务会进入`send_stt_message`发送一个stt和一个llm消息 124 | 125 | **function_call** 126 | 127 | `chat_with_function_calling`函数是使用llm的函数调用的一种处理方式, 使用`get_functions`获取当前的所有的可以使用的函数, 实际记录在`FunctionCallConfig`里面, 使用插件的方式进行添加, 所有的插件试被func_handler管理 128 | 129 | 这个函数里面在初始的时候把这一次对话放入`dialogue`里面, 之后使用`query_memory`获取比较类似的对话记忆, 然后在函数`get_llm_dialogue_with_memory`里面使用合成新的system提示词, 在加上对话形成实际的llm对话内容在`response_with_functions`中获取模型的回答 130 | 131 | 在使用工具调用处理对话的时候使用`content`记录获取的参数放在`content_arguments`中进行后续函数调用的处理流程, 没有的时候使用用`response_message`记录对话, 获取的消息使用流式处理, 判断句子里面最后一个有效的标点依次处理, 标记句子的标号以后使用`speak_and_play`最后进tts, 发送给开发板 132 | 133 | 在实际进行function_call的时候首先使用`handle_llm_function_call`获取这次的function_call所调用的函数的结果, 然后使用_handle_function_result处理执行的结果, 在这个函数里面 134 | 135 | 1. RESPONSE, 把这个返回的字符串使用tts进行播放 136 | 2. REQLLM: 返回的结果需要再次使用LLM进行处理, 重新调用chat_with_function_calling函数, 传入function执行的返回值 137 | 3. 138 | 139 | ## 实际使用 140 | 141 | ### 开发板 142 | 143 | #### 文本通信接口 144 | 145 | 可以使用如下的接口使用文本消息进行通信 146 | 147 | ```python 148 | {"type":"listen","state":"detect","text":"你好","source":"text"} 149 | ``` 150 | 151 | **Wake Word Detected** 152 | 153 | 用于客户端向服务器告知检测到唤醒词。 例： 154 | 155 | ```json 156 | { 157 | "session_id": "xxx", 158 | "type": "listen", 159 | "state": "detect", 160 | "text": "你好小明" 161 | } 162 | ``` 163 | 164 | ### 服务器 165 | 166 | #### 文字转语音 167 | 168 | ```python 169 | # 使用text_index对文字进行编号 170 | await send_stt_message(conn, text) 171 | future = self.executor.submit(self.speak_and_play, text, text_index) 172 | self.tts_queue.put(future) 173 | self.llm_finish_task = True 174 | ``` 175 | 176 | 发送的语音会在_tts_priority_thread线程里面处理tts, 最后把待发送的音频放入audio_play_queue 177 | 178 | #### 播放音乐 179 | 180 | ```python 181 | play_local_music(self, conn, specific_file=None) 182 | ``` 183 | 184 | #### 工具调用 185 | 186 | 1. 在plugins_func文件夹里面的functions里面建立你自己的function文件夹, 使用下面的格式进行添加 187 | 188 | ```python 189 | import requests 190 | from bs4 import BeautifulSoup 191 | from plugins_func.register import register_function,ToolType, ActionResponse, Action 192 | 193 | # 描述一下这个function的实际功能好使用的参数 194 | get_weather_function_desc = { 195 | "type": "function", 196 | "function": { 197 | "name": "get_weather", 198 | "description": "获取某个地点的天气，用户应先提供一个位置，比如用户说杭州天气，参数为：zhejiang/hangzhou，比如用户说北京天气怎么样，参数为：beijing/beijing。如果用户只问天气怎么样，参数是:guangdong/guangzhou", 199 | "parameters": { 200 | "type": "object", 201 | "properties": { 202 | "city": { 203 | "type": "string", 204 | "description": "城市，zhejiang/hangzhou" 205 | } 206 | }, 207 | "required": [ 208 | "city" 209 | ] 210 | } 211 | } 212 | } 213 | 214 | @register_function('get_weather', get_weather_function_desc, ToolType.WAIT) 215 | def get_weather(city: str): 216 | """ 217 | "获取某个地点的天气，用户应先提供一个位置，\n比如用户说杭州天气，参数为：zhejiang/hangzhou，\n\n比如用户说北京天气怎么样，参数为：beijing/beijing", 218 | city : 城市，zhejiang/hangzhou 219 | """ 220 | url = "https://tianqi.moji.com/weather/china/"+city 221 | headers = { 222 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36' 223 | } 224 | response = requests.get(url, headers=headers) 225 | if response.status_code!=200: 226 | return ActionResponse(Action.REQLLM, None, "请求失败") 227 | soup = BeautifulSoup(response.text, "html.parser") 228 | weather = soup.find('meta', attrs={'name':'description'})["content"] 229 | weather = weather.replace("墨迹天气", "") 230 | return ActionResponse(Action.REQLLM, weather, None) # 返回的ACTION会影响后面对于对话的影响 231 | ``` 232 | 233 | 2. 实际的处理是在handle_llm_function_call函数里面, 根据不同的函数名字进行不同的处理, 目前需要根据你的FunctionCallConfig里面的配置自己在这个函数里面添加处理逻辑 234 | 235 | #### 意图处理 236 | 237 | IntentProviderBase里面添加你希望处理的intent, 之后在intent_llm里面是实际的处理位置, 可以在这里优化提示词`get_intent_system_prompt` 238 | 239 | `process_intent_result`函数中处理最后的实际的意识部分结果, 在这里添加实际的处理方式 240 | 241 | > XvSenfeng 2025-3-14 242 | -------------------------------------------------------------------------------- /代码分析/服务器框图.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XuSenfeng/ai-chat-local/d8254bf7e0d0340b73ff34ea34d83ae6c49db6ce/代码分析/服务器框图.jpg --------------------------------------------------------------------------------