├── .gitignore ├── test └── README ├── platformio.ini ├── lib └── README ├── src ├── config.h ├── sound.h ├── sound.cpp └── main.cpp ├── include └── README └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .pio 2 | .vscode/.browse.c_cpp.db* 3 | .vscode/c_cpp_properties.json 4 | .vscode/launch.json 5 | .vscode/ipch 6 | -------------------------------------------------------------------------------- /test/README: -------------------------------------------------------------------------------- 1 | 2 | This directory is intended for PlatformIO Test Runner and project tests. 3 | 4 | Unit Testing is a software testing method by which individual units of 5 | source code, sets of one or more MCU program modules together with associated 6 | control data, usage procedures, and operating procedures, are tested to 7 | determine whether they are fit for use. Unit testing finds problems early 8 | in the development cycle. 9 | 10 | More information about PlatformIO Unit Testing: 11 | - https://docs.platformio.org/en/latest/advanced/unit-testing/index.html 12 | -------------------------------------------------------------------------------- /platformio.ini: -------------------------------------------------------------------------------- 1 | ; PlatformIO Project Configuration File 2 | ; 3 | ; Build options: build flags, source filter 4 | ; Upload options: custom upload port, speed and extra flags 5 | ; Library options: dependencies, extra library storages 6 | ; Advanced options: extra scripting 7 | ; 8 | ; Please visit documentation for the other options and examples 9 | ; https://docs.platformio.org/page/projectconf.html 10 | 11 | [env:m5stack-stamps3] 12 | platform = espressif32 13 | board = m5stack-stamps3 14 | framework = arduino 15 | monitor_speed = 115200 16 | lib_deps = 17 | M5Stack/M5Unified 18 | M5Stack/M5Cardputer 19 | M5Stack/M5Module-LLM#dev 20 | -------------------------------------------------------------------------------- /lib/README: -------------------------------------------------------------------------------- 1 | 2 | This directory is intended for project specific (private) libraries. 3 | PlatformIO will compile them to static libraries and link into executable file. 4 | 5 | The source code of each library should be placed in an own separate directory 6 | ("lib/your_library_name/[here are source files]"). 7 | 8 | For example, see a structure of the following two libraries `Foo` and `Bar`: 9 | 10 | |--lib 11 | | | 12 | | |--Bar 13 | | | |--docs 14 | | | |--examples 15 | | | |--src 16 | | | |- Bar.c 17 | | | |- Bar.h 18 | | | |- library.json (optional, custom build options, etc) https://docs.platformio.org/page/librarymanager/config.html 19 | | | 20 | | |--Foo 21 | | | |- Foo.c 22 | | | |- Foo.h 23 | | | 24 | | |- README --> THIS FILE 25 | | 26 | |- platformio.ini 27 | |--src 28 | |- main.c 29 | 30 | and a contents of `src/main.c`: 31 | ``` 32 | #include 33 | #include 34 | 35 | int main (void) 36 | { 37 | ... 38 | } 39 | 40 | ``` 41 | 42 | PlatformIO Library Dependency Finder will find automatically dependent 43 | libraries scanning project source files. 44 | 45 | More information about PlatformIO Library Dependency Finder 46 | - https://docs.platformio.org/page/librarymanager/ldf.html 47 | -------------------------------------------------------------------------------- /src/config.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file config.h 3 | * @brief M5Cardputer LLM Assistant の設定ファイル 4 | * @version 0.1 5 | * @date 2024-02-08 6 | * 7 | * @details 8 | * アプリケーションの動作に関する各種設定パラメータを定義します。 9 | * フォント、通信設定、音量、使用するLLMモデルなどの設定が含まれます。 10 | */ 11 | 12 | #pragma once 13 | #include // for NULL 14 | 15 | /** 16 | * @brief 表示に使用するフォント 17 | * @details 日本語表示に対応したLGFXJapanGothicフォントを使用 18 | */ 19 | #define FONTNAME fonts::lgfxJapanGothic_24 20 | 21 | /** 22 | * @brief M5Module-LLM との通信用UARTピン設定 23 | * @note M5Cardputer の Serial2 を使用 24 | */ 25 | #define MODULE_LLM_UART_RX 2 ///< UART受信ピン 26 | #define MODULE_LLM_UART_TX 1 ///< UART送信ピン 27 | 28 | /** 29 | * @brief スピーカーの音量設定 30 | * @details 0-255の範囲で指定 (0: 無音, 255: 最大音量) 31 | */ 32 | const int SOUND_VOLUME = 100; 33 | 34 | /** 35 | * @brief 使用するLLMモデルの設定 36 | * @details 37 | * - NULL: デフォルトモデルを使用 38 | * - 以下のモデルが利用可能: 39 | * - qwen2.5-0.5B-prefill-20e: 小規模な高速モデル 40 | * - qwen2.5-1.5b-ax630c: 中規模な汎用モデル 41 | * - deepseek-r1-1.5B-ax630c: DeepSeekベースの高性能モデル 42 | */ 43 | const char *MODEL = NULL; // NULL: Default model 44 | 45 | //const char *MODEL = "qwen2.5-0.5B-prefill-20e"; 46 | //const char *MODEL = "qwen2.5-1.5b-ax630c"; 47 | //const char *MODEL = "deepseek-r1-1.5B-ax630c"; 48 | -------------------------------------------------------------------------------- /include/README: -------------------------------------------------------------------------------- 1 | 2 | This directory is intended for project header files. 3 | 4 | A header file is a file containing C declarations and macro definitions 5 | to be shared between several project source files. You request the use of a 6 | header file in your project source file (C, C++, etc) located in `src` folder 7 | by including it, with the C preprocessing directive `#include'. 8 | 9 | ```src/main.c 10 | 11 | #include "header.h" 12 | 13 | int main (void) 14 | { 15 | ... 16 | } 17 | ``` 18 | 19 | Including a header file produces the same results as copying the header file 20 | into each source file that needs it. Such copying would be time-consuming 21 | and error-prone. With a header file, the related declarations appear 22 | in only one place. If they need to be changed, they can be changed in one 23 | place, and programs that include the header file will automatically use the 24 | new version when next recompiled. The header file eliminates the labor of 25 | finding and changing all the copies as well as the risk that a failure to 26 | find one copy will result in inconsistencies within a program. 27 | 28 | In C, the usual convention is to give header files names that end with `.h'. 29 | It is most portable to use only letters, digits, dashes, and underscores in 30 | header file names, and at most one dot. 31 | 32 | Read more about using header files in official GCC documentation: 33 | 34 | * Include Syntax 35 | * Include Operation 36 | * Once-Only Headers 37 | * Computed Includes 38 | 39 | https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html 40 | -------------------------------------------------------------------------------- /src/sound.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file sound.h 3 | * @brief M5Cardputer用サウンド機能のヘッダーファイル 4 | * @version 0.1 5 | * @date 2024-02-08 6 | * 7 | * @details 8 | * M5Cardputerのスピーカーを使用して音を再生するための機能を提供します。 9 | * 矩形波と三角波の2種類の波形に対応し、効果音(SE)の再生機能も備えています。 10 | * 11 | * @note 12 | * Original code by らびやん 13 | * @see https://gist.github.com/lovyan03/19e8a65195f85fbdd415558d149912f6 14 | */ 15 | 16 | #pragma once 17 | #include 18 | 19 | /** 20 | * @brief 波形の種類を定義する列挙型 21 | */ 22 | enum 23 | { 24 | SOUND_SQUARE = 0, ///< 矩形波 (Duty比50%) 25 | SOUND_TRIANGLE = 1, ///< 三角波 26 | }; 27 | 28 | /** 29 | * @brief 効果音(SE)の種類を定義する列挙型 30 | */ 31 | enum SOUND_SE 32 | { 33 | SOUND_SE_START = 0, ///< 起動時のSE 34 | SOUND_SE_END = 1, ///< 終了時のSE 35 | SOUND_SE_TALK = 2, ///< 会話時のSE 36 | }; 37 | 38 | // 音楽の速さを定義 39 | #define BPM 200 ///< テンポ(1分間の拍数) 40 | #define NOTE_32_MS (60000 / BPM / 8) ///< 32分音符の長さ(ms) 41 | #define NOTE_64_MS (60000 / BPM / 16) ///< 64分音符の長さ(ms) 42 | 43 | /** 44 | * @brief 音符を表す構造体 45 | */ 46 | typedef struct 47 | { 48 | int freq; ///< 周波数 (Hz) (0は休符) 49 | int duration; ///< 音の長さ (ms) 50 | } Note; 51 | 52 | 53 | /** 54 | * @brief 指定した波形、周波数、時間で音を再生する 55 | * 56 | * @param type 波形の種類 (SOUND_SQUARE または SOUND_TRIANGLE) 57 | * @param freq 周波数 (Hz) 58 | * @param duration 再生時間 (ms) 59 | */ 60 | void sound_play(const int type, const float freq, const uint32_t duration); 61 | 62 | /** 63 | * @brief 指定した効果音(SE)を再生する 64 | * 65 | * @param no 効果音の種類 (SOUND_SE_START, SOUND_SE_END, SOUND_SE_TALK) 66 | */ 67 | void sound_play_SE(const SOUND_SE no); 68 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LLMCardputer 2 | 3 | [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/GOROman/LLMCardputer) 4 | 5 | ***AI is in the palm of your hands*** 6 | 7 | image 8 | 9 | 10 | Large Language Models (LLMs) like OpenAI’s ChatGPT typically require a CUDA-capable GPU or an Apple Silicon Mac with substantial memory to run locally. However, in recent years, edge LLM modules designed for embedded microcontrollers have emerged. 11 | 12 | In this article, we’ll explore how to combine M5Stack’s credit card-sized computer with a keyboard, Cardputer, and a ModuleLLM capable of running local LLMs. Together, they enable a "palm-sized device running a local LLM." 13 | 14 | **↓ Demonstration** 15 | 16 | https://x.com/GOROman/status/1883032143884103767 17 | 18 | 19 | # How-To Guide 20 | 21 | ## What You’ll Need 22 | 23 | ### ModuleLLM 24 | 25 | First, get the ModuleLLM (LLM module). It’s currently sold out at the official distributor, Switch Science, and M5Stack’s official store. I managed to snag the last one from Aliexpress. Production might resume after the Lunar New Year, so stock could return around March. Alternatively, you might find someone who bought it impulsively but doesn’t know how to use it and persuade them to sell it. 26 | 27 | - [ModuleLLM](https://docs.m5stack.com/en/module/Module-LLM) 28 | - [ModuleLLM - Switch Science](https://www.switch-science.com/products/10034) 29 | 30 | ### Cardputer 31 | 32 | Next, get a Cardputer. Fortunately, its stock recently replenished. 33 | 34 | - [Cardputer](https://docs.m5stack.com/en/core/Cardputer) 35 | - [Cardputer - Switch Science](https://www.switch-science.com/products/9277) 36 | 37 | ## Step 1: Disassembly 38 | 39 | Remove the ModuleLLM from its frame by unscrewing the four screws with a wrench. Disassemble the Cardputer as well. Be careful when detaching the large battery stuck with double-sided tape. Avoid bending it, as this could cause it to ignite. Remove the black component on top of the Cardputer too. 40 | 41 | ![image.png](https://qiita-image-store.s3.ap-northeast-1.amazonaws.com/0/30621/e9f33e0d-a8a6-6d01-96bc-a66c74095e12.png) 42 | 43 | Modify the Cardputer’s case to fit the ModuleLLM snugly by trimming and sanding as needed. 44 | 45 | ![image.png](https://qiita-image-store.s3.ap-northeast-1.amazonaws.com/0/30621/d656d9cb-c5e5-ce5b-a606-b8dc422c1a6f.png) 46 | 47 | Cut out a section at the back of the case to expose the M.BUS pins. To prevent short circuits, insulate the pins with Kapton or acetate tape. 48 | 49 | #### Fitting Result 50 | 51 | ![image.png](https://qiita-image-store.s3.ap-northeast-1.amazonaws.com/0/30621/f667125d-5818-1972-7307-1683f092a009.png) 52 | 53 | ## Step 2: Wiring 54 | 55 | You can communicate with ModuleLLM via UART or TCP (port 10001). For this guide, we’ll use UART. 56 | 57 | ![image.png](https://qiita-image-store.s3.ap-northeast-1.amazonaws.com/0/30621/27e74244-4d01-b4db-8f7e-0a9f5440c224.png) 58 | 59 | Connect the ModuleLLM to the Cardputer as follows: 60 | 61 | - **Cardputer GROVE terminal (G1, G2, +5V, GND)** 62 | - **ModuleLLM UART pins (+5V, GND)** 63 | 64 | While soldering directly to the board worked for this project, there may be better ways to connect them. 65 | 66 | | Cardputer GROVE | ModuleLLM M.BUS | 67 | |-----------------|-----------------| 68 | | G (Black) | GND | 69 | | 5V (Red) | 5V | 70 | | G1 (Yellow) | UART (Tx) | 71 | | G2 (White) | UART (Rx) | 72 | 73 | ![image.png](https://qiita-image-store.s3.ap-northeast-1.amazonaws.com/0/30621/b6b1f52c-d0f8-157c-7818-986509cff129.png) 74 | 75 | ## Step 3: Coding 76 | 77 | 78 | ## 開発環境 79 | 80 | - [PlatformIO](https://platformio.org/) 81 | - Arduino M5Stack Board Manager v2.0.7 82 | 83 | ## 依存ライブラリ 84 | 85 | - [M5GFX](https://github.com/m5stack/M5GFX) 86 | - [M5Unified](https://github.com/m5stack/M5Unified) 87 | - [M5Module-LLM](https://github.com/m5stack/M5Module-LLM) 88 | 89 | ## セットアップ 90 | 91 | 1. PlatformIOをインストール 92 | 2. プロジェクトをクローン 93 | ```bash 94 | git clone https://github.com/GOROman/LLMCardputer.git 95 | ``` 96 | 3. 依存ライブラリをインストール 97 | ```bash 98 | pio pkg install 99 | ``` 100 | 4. ビルドとアップロード 101 | ```bash 102 | pio run -t upload 103 | ``` 104 | 105 | ## 使用方法 106 | 107 | 1. M5CardputerにM5Module-LLMを接続 108 | 2. 電源を入れると起動アニメーションが表示され、LLMの初期化が行われます 109 | 3. キーボードから質問を入力し、Enterキーで送信 110 | 4. LLMが回答を生成し、テキストで応答します 111 | 112 | ## 設定 113 | 114 | `src/config.h` で以下の設定が可能です: 115 | 116 | ### フォント設定 117 | - `FONTNAME`: 日本語表示に対応したLGFXJapanGothicフォントを使用(デフォルト: fonts::lgfxJapanGothic_24) 118 | 119 | ### 通信設定 120 | - `MODULE_LLM_UART_RX`: UART受信ピン(デフォルト: 2) 121 | - `MODULE_LLM_UART_TX`: UART送信ピン(デフォルト: 1) 122 | - M5Cardputerの Serial2 を使用してM5Module-LLMと通信 123 | 124 | ### サウンド設定 125 | - `SOUND_VOLUME`: スピーカーの音量(0-255の範囲、0: 無音, 255: 最大音量) 126 | デフォルト: 100 127 | 128 | ### LLMモデル設定 129 | - `MODEL`: 使用するLLMモデルを指定(デフォルト: NULL) 130 | - NULL: デフォルトモデルを使用 131 | - 利用可能なモデル: 132 | - qwen2.5-0.5B-prefill-20e: 小規模な高速モデル 133 | - qwen2.5-1.5b-ax630c: 中規模な汎用モデル 134 | - deepseek-r1-1.5B-ax630c: DeepSeekベースの高性能モデル 135 | 136 | ## ライセンス 137 | 138 | MIT License 139 | 140 | ## クレジット 141 | 142 | - サウンド機能: [らびやん](https://gist.github.com/lovyan03/19e8a65195f85fbdd415558d149912f6) 143 | -------------------------------------------------------------------------------- /src/sound.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file sound.cpp 3 | * @brief M5Cardputer用サウンド機能の実装 4 | * @version 0.1 5 | * @date 2024-02-08 6 | * 7 | * @details 8 | * M5Cardputerのスピーカーを使用した音声出力機能を実装します。 9 | * 波形データ、メロディーデータ、および音声出力関数を提供します。 10 | * 11 | * @note 12 | * Original code by らびやん 13 | * @see https://gist.github.com/lovyan03/19e8a65195f85fbdd415558d149912f6 14 | */ 15 | 16 | #include "sound.h" 17 | 18 | /** 19 | * @brief 矩形波のデータ (Duty比50%) 20 | * @note 8ステップで1周期を表現 21 | */ 22 | static constexpr const uint8_t step_square_wave[] = { 23 | 0, 24 | 0, 25 | 0, 26 | 0, 27 | 127, 28 | 127, 29 | 127, 30 | 127, 31 | }; 32 | 33 | /** 34 | * @brief 三角波のデータ 35 | * @note 128ステップで1周期を表現 36 | */ 37 | static constexpr const uint8_t step_triangle_wave[] = { 38 | 136, 39 | 136, 40 | 136, 41 | 136, 42 | 153, 43 | 153, 44 | 153, 45 | 153, 46 | 170, 47 | 170, 48 | 170, 49 | 170, 50 | 187, 51 | 187, 52 | 187, 53 | 187, 54 | 204, 55 | 204, 56 | 204, 57 | 204, 58 | 221, 59 | 221, 60 | 221, 61 | 221, 62 | 238, 63 | 238, 64 | 238, 65 | 238, 66 | 255, 67 | 255, 68 | 255, 69 | 255, 70 | 255, 71 | 255, 72 | 255, 73 | 255, 74 | 238, 75 | 238, 76 | 238, 77 | 238, 78 | 221, 79 | 221, 80 | 221, 81 | 221, 82 | 204, 83 | 204, 84 | 204, 85 | 204, 86 | 187, 87 | 187, 88 | 187, 89 | 187, 90 | 170, 91 | 170, 92 | 170, 93 | 170, 94 | 153, 95 | 153, 96 | 153, 97 | 153, 98 | 136, 99 | 136, 100 | 136, 101 | 136, 102 | 119, 103 | 119, 104 | 119, 105 | 119, 106 | 102, 107 | 102, 108 | 102, 109 | 102, 110 | 85, 111 | 85, 112 | 85, 113 | 85, 114 | 68, 115 | 68, 116 | 68, 117 | 68, 118 | 51, 119 | 51, 120 | 51, 121 | 51, 122 | 34, 123 | 34, 124 | 34, 125 | 34, 126 | 17, 127 | 17, 128 | 17, 129 | 17, 130 | 0, 131 | 0, 132 | 0, 133 | 0, 134 | 0, 135 | 0, 136 | 0, 137 | 0, 138 | 17, 139 | 17, 140 | 17, 141 | 17, 142 | 34, 143 | 34, 144 | 34, 145 | 34, 146 | 51, 147 | 51, 148 | 51, 149 | 51, 150 | 68, 151 | 68, 152 | 68, 153 | 68, 154 | 85, 155 | 85, 156 | 85, 157 | 85, 158 | 102, 159 | 102, 160 | 102, 161 | 102, 162 | 119, 163 | 119, 164 | 119, 165 | 119, 166 | }; 167 | 168 | /** 169 | * @brief 起動時の効果音データ 170 | * @note C4->G4->D4->A4 の順で再生 171 | */ 172 | static Note MELODY_SE_START[] = { 173 | {261, NOTE_32_MS}, // C4 (261.63Hz) 174 | {0, NOTE_32_MS}, // 休符 175 | {392, NOTE_32_MS}, // G4 (392.00Hz) 176 | {0, NOTE_32_MS}, // 休符 177 | {294, NOTE_32_MS}, // D4 (293.66Hz) 178 | {0, NOTE_32_MS}, // 休符 179 | {440, NOTE_32_MS}, // A4 (440.00Hz) 180 | {-1, -1}, // Terminater 181 | }; 182 | 183 | /** 184 | * @brief 終了時の効果音データ 185 | * @note F4->C5->E4->B4->D#4->A4->D4->A4->C#4->G#4->C4->C5->C6 の順で再生 186 | */ 187 | static Note MELODY_SE_END[] = { 188 | {349, NOTE_64_MS}, // F4 189 | {0, NOTE_64_MS}, // r 190 | {523, NOTE_64_MS}, // C5 191 | {0, NOTE_64_MS}, // r 192 | {329, NOTE_64_MS}, // E4 193 | {0, NOTE_64_MS}, // r 194 | {494, NOTE_64_MS}, // B4 195 | {0, NOTE_64_MS}, // r 196 | {622, NOTE_64_MS}, // D#4 197 | {0, NOTE_64_MS}, // r 198 | {440, NOTE_64_MS}, // A4 199 | {0, NOTE_64_MS}, // r 200 | {294, NOTE_64_MS}, // D4 201 | {0, NOTE_64_MS}, // r 202 | {440, NOTE_64_MS}, // A4 203 | {0, NOTE_64_MS}, // r 204 | {554, NOTE_64_MS}, // C#4 205 | {0, NOTE_64_MS}, // r 206 | {415, NOTE_64_MS}, // G#4 207 | {0, NOTE_64_MS}, // r 208 | {261, NOTE_32_MS}, // C4 209 | {0, NOTE_32_MS}, // r 210 | {523, NOTE_32_MS}, // C5 211 | {0, NOTE_32_MS}, // r 212 | {1047, NOTE_32_MS},// C6 213 | {0, NOTE_32_MS}, // r 214 | {-1, -1}, // Terminater 215 | }; 216 | 217 | /** 218 | * @brief 効果音データの配列 219 | * @note SOUND_SE 列挙型のインデックスに対応 220 | */ 221 | static Note *MELODY_SE_LIST[] = { 222 | MELODY_SE_START, 223 | MELODY_SE_END, 224 | }; 225 | 226 | /** 227 | * @brief 指定した波形、周波数、時間で音を再生する 228 | * 229 | * @param type 波形の種類 (SOUND_SQUARE または SOUND_TRIANGLE) 230 | * @param freq 周波数 (Hz) 231 | * @param duration 再生時間 (ms) 232 | */ 233 | void sound_play(const int type, const float freq, const uint32_t duration) 234 | { 235 | switch (type) 236 | { 237 | case SOUND_SQUARE: 238 | M5.Speaker.tone(freq, duration, 0, true, step_square_wave, sizeof(step_square_wave), false); 239 | break; 240 | case SOUND_TRIANGLE: 241 | M5.Speaker.tone(freq, duration, 1, true, step_triangle_wave, sizeof(step_triangle_wave), false); 242 | break; 243 | default: 244 | break; 245 | } 246 | } 247 | 248 | /** 249 | * @brief 指定した効果音(SE)を再生する 250 | * 251 | * @param no 効果音の種類 (SOUND_SE_START, SOUND_SE_END, SOUND_SE_TALK) 252 | * @note 効果音は三角波を使用して再生されます 253 | */ 254 | void sound_play_SE(const SOUND_SE no) 255 | { 256 | if (no < 0 || no >= 3) 257 | { 258 | return; 259 | } 260 | 261 | Note *m = MELODY_SE_LIST[no]; 262 | 263 | while (m->freq != -1) 264 | { 265 | sound_play(SOUND_TRIANGLE, m->freq, m->duration); 266 | delay(m->duration); 267 | m++; 268 | } 269 | } 270 | -------------------------------------------------------------------------------- /src/main.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file main.cpp 3 | * @brief M5Cardputer LLM Assistant 4 | * @version 0.1 5 | * @date 2024-02-08 6 | * 7 | * @details 8 | * M5CardputerとM5Module-LLMを使用したチャットアシスタントアプリケーション。 9 | * キーボード入力による対話、音声フィードバック、マルチタスクによる非同期処理を実装。 10 | * 11 | * @Hardwares: M5Cardputer with M5Module-LLM 12 | * @Platform Version: Arduino M5Stack Board Manager v2.0.7 13 | * @Dependent Library: 14 | * M5GFX: https://github.com/m5stack/M5GFX 15 | * M5Unified: https://github.com/m5stack/M5Unified 16 | * M5Module-LLM: https://github.com/m5stack/M5Module-LLM 17 | */ 18 | 19 | #include 20 | #include 21 | #include 22 | #include "config.h" 23 | #include "sound.h" 24 | 25 | /** @brief LLMモジュールのインスタンス */ 26 | static M5ModuleLLM module_llm; 27 | 28 | /** @brief LLMのワークID */ 29 | static String llm_work_id; 30 | /** @brief ユーザーからの質問文 */ 31 | static String question; 32 | /** @brief LLMからの回答文 */ 33 | static String answer; 34 | /** @brief 入力バッファ */ 35 | static String data; 36 | 37 | /** @brief LLMの初期化完了フラグ */ 38 | static bool task_llm_ready = false; 39 | /** @brief LLMの回答完了フラグ */ 40 | static bool end_flag = false; 41 | 42 | /** @brief 描画用キャンバス */ 43 | static M5Canvas canvas(&M5Cardputer.Display); 44 | 45 | /** @brief ディスプレイ描画用の排他制御ミューテックス */ 46 | static portMUX_TYPE display_mutex = portMUX_INITIALIZER_UNLOCKED; 47 | 48 | /** 49 | * @brief ビープ音を鳴らす 50 | * @details 440Hz(A4)の矩形波を50ms再生 51 | */ 52 | static void beep() 53 | { 54 | sound_play(SOUND_SQUARE, 440, 50); 55 | } 56 | 57 | /** 58 | * @brief エラーメッセージを表示 59 | * @param msg 表示するメッセージ 60 | * @details 赤背景でメッセージを表示し、エラー音を鳴らす 61 | */ 62 | static void error_message(String msg) 63 | { 64 | portENTER_CRITICAL_ISR(&display_mutex); 65 | canvas.setTextColor(WHITE, RED); 66 | canvas.println(msg); 67 | canvas.setTextColor(WHITE, BLACK); 68 | canvas.pushSprite(4, 4); 69 | portEXIT_CRITICAL_ISR(&display_mutex); 70 | 71 | M5Cardputer.Speaker.tone(440, 800); // Error 72 | delay(1000); 73 | } 74 | 75 | /** 76 | * @brief システムメッセージを表示 77 | * @param msg 表示するメッセージ 78 | * @param lf 改行の有無 79 | * @details シアン色でメッセージを表示 80 | */ 81 | static void system_message(String msg, bool lf = true) 82 | { 83 | portENTER_CRITICAL_ISR(&display_mutex); 84 | M5Cardputer.Display.fillRect(0, M5Cardputer.Display.height() - 28, 85 | M5Cardputer.Display.width(), 28, 86 | BLACK); 87 | 88 | M5Cardputer.Display.setTextColor(CYAN); 89 | M5Cardputer.Display.drawString(msg, 4, 90 | M5Cardputer.Display.height() - 24); 91 | 92 | portEXIT_CRITICAL_ISR(&display_mutex); 93 | } 94 | 95 | /** 96 | * @brief LLMの初期化 97 | * @param lang 使用する言語 ("en":英語, "jp":日本語) 98 | * @details LLMのリセット、設定、プロンプト設定を行う 99 | */ 100 | static void LLM_setup(String lang) 101 | { 102 | // Reset LLM 103 | int err = module_llm.sys.reset(); 104 | if (err != MODULE_LLM_OK) 105 | { 106 | error_message("Error: Reset LLM failed"); 107 | return; 108 | } 109 | 110 | // Setup LLM 111 | while (1) 112 | { 113 | system_message("Setup LLM"); 114 | 115 | m5_module_llm::ApiLlmSetupConfig_t llm_config; 116 | if (MODEL) 117 | { 118 | llm_config.model = MODEL; 119 | } 120 | llm_config.max_token_len = 1023; 121 | 122 | if (lang == "en") 123 | { 124 | llm_config.prompt = "Please answer in English"; 125 | } 126 | else if (lang == "jp") 127 | { 128 | llm_config.prompt = "Please answer in Japanese."; 129 | } 130 | 131 | llm_work_id = module_llm.llm.setup(llm_config); 132 | if (llm_work_id == "") 133 | { 134 | error_message("Error: Setup LLM failed"); 135 | delay(500); 136 | continue; 137 | } 138 | system_message("LLM Work ID:" + llm_work_id); 139 | break; 140 | } 141 | 142 | beep(); 143 | } 144 | 145 | /** 146 | * @brief LLM制御タスク 147 | * @param pvParameters タスクパラメータ(未使用) 148 | * @details LLMモジュールの初期化と定期的な状態更新を行う 149 | */ 150 | void task_llm(void *pvParameters) 151 | { 152 | // Init module 153 | module_llm.begin(&Serial2); 154 | 155 | // Make sure module is connected 156 | system_message("ModuleLLM connecting", false); 157 | 158 | while (1) 159 | { 160 | if (module_llm.checkConnection()) 161 | { 162 | break; 163 | } 164 | } 165 | 166 | LLM_setup("jp"); 167 | task_llm_ready = true; 168 | 169 | while (1) 170 | { 171 | module_llm.update(); 172 | vTaskDelay(100); 173 | } 174 | } 175 | 176 | /** 177 | * @brief LLMに質問を送信 178 | * @param question 質問文 179 | * @details LLMに質問を送信し、非同期で回答を受け取る 180 | */ 181 | void talk(String question) 182 | { 183 | // Push question to LLM module and wait inference result 184 | module_llm.llm.inferenceAndWaitResult(llm_work_id, question.c_str(), [](String &result) 185 | { answer += result; }, 2000, "llm_inference"); 186 | end_flag = true; 187 | } 188 | 189 | /** 190 | * @brief 回答表示タスク 191 | * @param pvParameters タスクパラメータ(未使用) 192 | * @details LLMからの回答を1文字ずつ表示し、音声フィードバックを行う 193 | */ 194 | void task_print(void *pvParameters) 195 | { 196 | while (1) 197 | { 198 | int len = answer.length(); 199 | String buffer = answer; 200 | answer = ""; 201 | int count = 0; 202 | 203 | for (int i = 0; i < len; i++) 204 | { 205 | // 1文字づつ表示 206 | String str = buffer.substring(i, i + 1); 207 | 208 | if (str == " " || str == "?") 209 | { 210 | } 211 | else 212 | { 213 | count++; 214 | if (count % 2 == 1) 215 | { 216 | // 喋ってる風のSE 217 | sound_play(SOUND_SQUARE, 888, 33); // 888Hz(A5) 218 | vTaskDelay(33); 219 | } 220 | } 221 | 222 | portENTER_CRITICAL_ISR(&display_mutex); 223 | canvas.setTextColor(GREEN); 224 | canvas.printf("%s", str.c_str()); 225 | canvas.pushSprite(4, 4); 226 | portEXIT_CRITICAL_ISR(&display_mutex); 227 | } 228 | 229 | if (end_flag && (len == 0)) 230 | { 231 | sound_play_SE(SOUND_SE_END); 232 | end_flag = false; 233 | } 234 | 235 | vTaskDelay(50); 236 | } 237 | } 238 | 239 | /** 240 | * @brief 画面をクリア 241 | * @details キャンバスをクリアして再描画 242 | */ 243 | void clear() 244 | { 245 | portENTER_CRITICAL_ISR(&display_mutex); 246 | canvas.setCursor(0, 0); 247 | canvas.clear(); 248 | canvas.pushSprite(4, 4); 249 | portEXIT_CRITICAL_ISR(&display_mutex); 250 | } 251 | 252 | /** 253 | * @brief 起動アニメーションを表示 254 | * @details ランダムなパターンと音を使用した起動時のアニメーション効果を表示 255 | */ 256 | static void startup_animation() 257 | { 258 | const int W = M5Cardputer.Display.width(); 259 | const int H = M5Cardputer.Display.height(); 260 | const int STEP = 20; 261 | const uint16_t COLOR[] = { 262 | BLACK, 263 | M5Cardputer.Display.color565(175, 66, 47), 264 | M5Cardputer.Display.color565(139, 227, 77), 265 | M5Cardputer.Display.color565(19, 17, 169), 266 | }; 267 | while (!task_llm_ready) 268 | { 269 | portENTER_CRITICAL_ISR(&display_mutex); 270 | M5Cardputer.Display.startWrite(); 271 | for (int x = 0; x < W; x += STEP) 272 | { 273 | for (int y = 0; y < H; y += STEP) 274 | { 275 | int r = rand() % 4; 276 | 277 | uint16_t color = COLOR[r]; 278 | canvas.fillRect(x, y, STEP, STEP, color); 279 | } 280 | } 281 | canvas.pushSprite(4, 4); 282 | M5Cardputer.Display.endWrite(); 283 | portEXIT_CRITICAL_ISR(&display_mutex); 284 | 285 | // ランダムに S&H 風の音を鳴らす 286 | int freq = 400 + rand() % 800; 287 | sound_play(SOUND_TRIANGLE, (float)freq, 80); 288 | 289 | vTaskDelay(100); 290 | } 291 | clear(); 292 | } 293 | 294 | /** 295 | * @brief セットアップ 296 | * @details デバイスの初期化、タスクの作成、起動アニメーションの表示を行う 297 | */ 298 | void setup() 299 | { 300 | auto cfg = M5.config(); 301 | M5Cardputer.begin(cfg, true); 302 | 303 | // Module LLMのUART 304 | Serial2.begin(115200, SERIAL_8N1, MODULE_LLM_UART_RX, MODULE_LLM_UART_TX); 305 | 306 | M5Cardputer.Speaker.setVolume(SOUND_VOLUME); 307 | 308 | portENTER_CRITICAL_ISR(&display_mutex); 309 | M5Cardputer.Display.startWrite(); 310 | M5Cardputer.Display.setRotation(1); 311 | M5Cardputer.Display.setTextSize(1.0f); 312 | 313 | M5Cardputer.Display.drawRect(0, 0, M5Cardputer.Display.width(), 314 | M5Cardputer.Display.height() - 28, WHITE); 315 | M5Cardputer.Display.drawRect(1, 1, M5Cardputer.Display.width() - 1, 316 | M5Cardputer.Display.height() - 29, WHITE); 317 | M5Cardputer.Display.setFont(&FONTNAME); 318 | 319 | canvas.setFont(&FONTNAME); 320 | canvas.setTextSize(1.0); 321 | canvas.createSprite(M5Cardputer.Display.width() - 8, 322 | M5Cardputer.Display.height() - 36); 323 | canvas.setTextScroll(true); 324 | canvas.pushSprite(4, 4); 325 | M5Cardputer.Display.drawString(data, 4, M5Cardputer.Display.height() - 24); 326 | M5Cardputer.Display.endWrite(); 327 | portEXIT_CRITICAL_ISR(&display_mutex); 328 | 329 | xTaskCreate( 330 | task_llm, "task_llm", 4096, NULL, 1, NULL); 331 | 332 | clear(); 333 | 334 | xTaskCreate( 335 | task_print, "task_print", 4096, NULL, 1, NULL); 336 | 337 | startup_animation(); 338 | 339 | canvas.setTextColor(GREEN); 340 | canvas.pushSprite(4, 4); 341 | 342 | data = ">"; 343 | M5Cardputer.Display.fillRect(0, M5Cardputer.Display.height() - 28, 344 | M5Cardputer.Display.width(), 28, 345 | BLACK); 346 | M5Cardputer.Display.setTextColor(WHITE); 347 | M5Cardputer.Display.drawString(data, 4, 348 | M5Cardputer.Display.height() - 24); 349 | 350 | // 初期の指示 351 | talk("Please introduce yourself."); 352 | } 353 | 354 | /** 355 | * @brief メインループ 356 | * @details キーボード入力の処理、テキスト表示の更新を行う 357 | */ 358 | void loop() 359 | { 360 | M5Cardputer.update(); 361 | 362 | if (M5Cardputer.Keyboard.isChange()) 363 | { 364 | if (M5Cardputer.Keyboard.isPressed()) 365 | { 366 | Keyboard_Class::KeysState status = M5Cardputer.Keyboard.keysState(); 367 | 368 | for (auto i : status.word) 369 | { 370 | M5.Speaker.tone(880, 100); 371 | data += i; 372 | } 373 | 374 | if (status.del) 375 | { 376 | M5.Speaker.tone(440, 100); 377 | data.remove(data.length() - 1); 378 | } 379 | 380 | if (status.enter) 381 | { 382 | sound_play_SE(SOUND_SE_START); 383 | data.remove(0, 1); 384 | canvas.setTextColor(WHITE); 385 | 386 | M5Cardputer.Display.fillRect(0, M5Cardputer.Display.height() - 28, 387 | M5Cardputer.Display.width(), 28, 388 | BLACK); 389 | { 390 | canvas.println("\n[You]:" + data); 391 | canvas.pushSprite(4, 4); 392 | 393 | canvas.setTextColor(GREEN); 394 | canvas.print("[AI]:"); 395 | question = data; 396 | data = ""; 397 | 398 | talk(question); 399 | } 400 | 401 | data = ">"; 402 | } 403 | 404 | M5Cardputer.Display.fillRect(0, M5Cardputer.Display.height() - 28, 405 | M5Cardputer.Display.width(), 28, 406 | BLACK); 407 | 408 | M5Cardputer.Display.setTextColor(WHITE); 409 | M5Cardputer.Display.drawString(data, 4, 410 | M5Cardputer.Display.height() - 24); 411 | } 412 | } 413 | } 414 | --------------------------------------------------------------------------------