├── .gitattributes ├── ArduinoGFX_Keybaord └── ArduinoGFX_Keybaord.ino └── Code └── Code.ino /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /ArduinoGFX_Keybaord/ArduinoGFX_Keybaord.ino: -------------------------------------------------------------------------------- 1 | /* 2 | ESP32 board package 2.0.10 3 | Arduino GFX Library 1.5.6 4 | bb_captouch 1.3.1 5 | */ 6 | #include 7 | #include 8 | 9 | #define TFT_CS 15 10 | #define TFT_MOSI 13 11 | #define TFT_MISO 12 12 | #define TFT_SCLK 14 13 | #define TFT_DC 21 14 | #define TFT_RES -1 15 | #define TFT_BLK 48 16 | 17 | #define TOUCH_INT 40 18 | #define TOUCH_SDA 39 19 | #define TOUCH_SCL 38 20 | #define TOUCH_RST 1 21 | 22 | #define DISPLAY_WIDTH 240 23 | #define DISPLAY_HEIGHT 320 24 | 25 | Arduino_ESP32SPI *bus = new Arduino_ESP32SPI(TFT_DC, TFT_CS, TFT_SCLK, TFT_MOSI, TFT_MISO, HSPI, true); 26 | Arduino_GFX *gfx = new Arduino_ST7789(bus, TFT_RES, 0, true); 27 | BBCapTouch bbct; 28 | 29 | bool isUpper = true; 30 | bool isSymbol = false; 31 | 32 | const char *alphaUpper[] = { 33 | "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", 34 | "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", 35 | "SPACE", "123", "SHIFT", "<-" 36 | }; 37 | const int alphaUpperCount = sizeof(alphaUpper) / sizeof(alphaUpper[0]); 38 | 39 | const char *alphaLower[] = { 40 | "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", 41 | "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", 42 | "SPACE", "123", "SHIFT", "<-" 43 | }; 44 | const int alphaLowerCount = sizeof(alphaLower) / sizeof(alphaLower[0]); 45 | 46 | const char *symbols[] = { 47 | "1", "2", "3", "4", "5", "6", "7", "8", "9", "0", 48 | "!", "@", "#", "$", "%", "^", "&", "*", "(", ")", 49 | "-", "_", "+", "=", "[", "]", "SPACE", "ABC", ".", "<-" 50 | }; 51 | const int symbolsCount = sizeof(symbols) / sizeof(symbols[0]); 52 | 53 | const char **currentLayout = alphaUpper; 54 | int currentLayoutCount = alphaUpperCount; 55 | 56 | #define COLS 8 57 | #define ROWS 4 58 | #define KEY_W 26 59 | #define KEY_H 30 60 | #define KEY_SPACING 2 61 | #define KEYBOARD_LAYOUT_MAX 32 62 | 63 | struct KeyBounds { 64 | int x, y, w, h; 65 | }; 66 | 67 | KeyBounds keyPositions[KEYBOARD_LAYOUT_MAX]; 68 | KeyBounds sendBtnBounds; 69 | 70 | String inputText = ""; 71 | bool keyTouchActive = false; 72 | 73 | int getKeyFromXY(int x, int y) { 74 | for (int i = 0; i < currentLayoutCount; i++) { 75 | KeyBounds b = keyPositions[i]; 76 | if (x >= b.x && x <= (b.x + b.w) && y >= b.y && y <= (b.y + b.h)) return i; 77 | } 78 | return -1; 79 | } 80 | 81 | bool isTouchOnSendButton(int x, int y) { 82 | return x >= sendBtnBounds.x && x <= (sendBtnBounds.x + sendBtnBounds.w) && y >= sendBtnBounds.y && y <= (sendBtnBounds.y + sendBtnBounds.h); 83 | } 84 | 85 | void drawKey(int index, int x, int y, int w, int h) { 86 | gfx->fillRect(x, y, w, h, DARKGREY); 87 | gfx->drawRect(x, y, w, h, WHITE); 88 | gfx->setTextColor(WHITE); 89 | gfx->setTextSize(1); 90 | int16_t x1, y1; 91 | uint16_t tw, th; 92 | gfx->getTextBounds(currentLayout[index], 0, 0, &x1, &y1, &tw, &th); 93 | gfx->setCursor(x + (w - tw) / 2, y + (h - th) / 2); 94 | gfx->print(currentLayout[index]); 95 | } 96 | 97 | void drawKeyboard() { 98 | gfx->fillScreen(BLACK); 99 | 100 | // Draw input area 101 | gfx->fillRect(0, 0, DISPLAY_WIDTH, 60, NAVY); 102 | gfx->setCursor(10, 5); 103 | gfx->setTextSize(1); 104 | gfx->setTextColor(WHITE); 105 | gfx->print("Text Area:"); 106 | gfx->setCursor(10, 20); 107 | gfx->print(inputText); 108 | 109 | // Draw SEND button 110 | int sendBtnW = 100; 111 | int sendBtnH = 30; 112 | int sendBtnX = (DISPLAY_WIDTH - sendBtnW) / 2; 113 | int sendBtnY = DISPLAY_HEIGHT - (KEY_H + KEY_SPACING) * ROWS - sendBtnH - 6; 114 | gfx->fillRect(sendBtnX, sendBtnY, sendBtnW, sendBtnH, GREEN); 115 | gfx->drawRect(sendBtnX, sendBtnY, sendBtnW, sendBtnH, WHITE); 116 | gfx->setTextColor(BLACK); 117 | gfx->setCursor(sendBtnX + 28, sendBtnY + 8); 118 | gfx->print("SEND"); 119 | sendBtnBounds = { sendBtnX, sendBtnY, sendBtnW, sendBtnH }; 120 | 121 | // Draw keyboard 122 | int keyIndex = 0; 123 | int startY = DISPLAY_HEIGHT - (KEY_H + KEY_SPACING) * ROWS; 124 | for (int row = 0; row < ROWS; row++) { 125 | int y = startY + row * (KEY_H + KEY_SPACING); 126 | int x = 0; 127 | for (int col = 0; col < COLS && keyIndex < currentLayoutCount; col++) { 128 | const char *label = currentLayout[keyIndex]; 129 | int keyW = KEY_W; 130 | if (strcmp(label, "SPACE") == 0) keyW = KEY_W * 3 + KEY_SPACING; 131 | else if (strcmp(label, "SHIFT") == 0 || strcmp(label, "123") == 0 || strcmp(label, "ABC") == 0) 132 | keyW = KEY_W + 8; 133 | 134 | drawKey(keyIndex, x, y, keyW, KEY_H); 135 | keyPositions[keyIndex] = { x, y, keyW, KEY_H }; 136 | x += keyW + KEY_SPACING; 137 | keyIndex++; 138 | } 139 | } 140 | } 141 | 142 | void processKeyboardTouch(int x, int y) { 143 | if (isTouchOnSendButton(x, y)) { 144 | Serial.println("User input: " + inputText); 145 | delay(1000); 146 | drawKeyboard(); 147 | return; 148 | } 149 | 150 | int keyIndex = getKeyFromXY(x, y); 151 | if (keyIndex == -1) return; 152 | 153 | const char *key = currentLayout[keyIndex]; 154 | if (strcmp(key, "SPACE") == 0) inputText += " "; 155 | else if (strcmp(key, "SHIFT") == 0) { 156 | isUpper = !isUpper; 157 | currentLayout = isUpper ? alphaUpper : alphaLower; 158 | currentLayoutCount = isUpper ? alphaUpperCount : alphaLowerCount; 159 | } else if (strcmp(key, "123") == 0) { 160 | currentLayout = symbols; 161 | currentLayoutCount = symbolsCount; 162 | } else if (strcmp(key, "ABC") == 0) { 163 | currentLayout = isUpper ? alphaUpper : alphaLower; 164 | currentLayoutCount = isUpper ? alphaUpperCount : alphaLowerCount; 165 | } else if (strcmp(key, "<-") == 0) { 166 | if (inputText.length() > 0) inputText.remove(inputText.length() - 1); 167 | } else { 168 | inputText += key; 169 | } 170 | 171 | drawKeyboard(); 172 | } 173 | 174 | void setup() { 175 | Serial.begin(115200); 176 | pinMode(TFT_BLK, OUTPUT); 177 | digitalWrite(TFT_BLK, HIGH); 178 | gfx->begin(); 179 | gfx->fillScreen(BLACK); 180 | bbct.init(TOUCH_SDA, TOUCH_SCL, TOUCH_RST, TOUCH_INT); 181 | drawKeyboard(); 182 | } 183 | 184 | void loop() { 185 | TOUCHINFO ti; 186 | if (bbct.getSamples(&ti)) { 187 | if (!keyTouchActive) { 188 | processKeyboardTouch(ti.x[0], ti.y[0]); 189 | keyTouchActive = true; 190 | } 191 | } else { 192 | keyTouchActive = false; 193 | } 194 | } -------------------------------------------------------------------------------- /Code/Code.ino: -------------------------------------------------------------------------------- 1 | /* 2 | Arduino IDE 2.3.2 3 | Esp32 by Espressif board package 2.0.10 4 | GFX Library for Arduino by Moon 1.5.6 5 | bb_captouch by Larry Bank 1.3.1 6 | Arduino_JSON by Arduino 0.2.0 7 | Adafruit GFX Library by Adafruit 1.12.1 8 | 9 | 10 | 11 | AI- Camera V2 Project Flow 12 | Overall Flow: 13 | 1. Power-On Initialization 14 | -> On boot, the device: 15 | - Displays "Connecting to Wi-Fi..." on the screen. 16 | - Connects to the configured Wi-Fi network. 17 | - Once connected, displays the Wi-Fi SSID and IP address. 18 | 19 | 2. SD Card Check 20 | -> After Wi-Fi connects: 21 | - It initializes the SD card. 22 | - Shows SD status on the display. 23 | 24 | 3. Start Live Camera Feed 25 | - Camera begins streaming in RGB565 mode for display. 26 | - The display shows the live camera feed with a bottom bar. 27 | - Bottom bar contains Capture button 28 | 29 | 30 | Image Capture Flow 31 | 4. Capture Button Tapped (When the user taps Capture): 32 | -> Freeze Current Frame: 33 | - The current RGB565 camera frame is paused and stored in RAM. 34 | - This prevents flickering while switching modes. 35 | 36 | -> Switch to JPEG Mode: 37 | - The camera is re-initialized with higher resolution (JPEG mode). 38 | 39 | -> Capture High-Res Image: 40 | - Image is captured and saved to the SD card in JPEG format. 41 | 42 | -> Return to Display Mode: 43 | - The camera preview is not resumed immediately. 44 | - Instead, the system proceeds to the next step. 45 | 46 | 47 | Keyboard & Input Flow 48 | 5. On-Screen Keyboard Displayed 49 | -> After image capture: 50 | - A full-screen keyboard is displayed. 51 | - The user can type their question or prompt. 52 | 53 | -> The keyboard supports: 54 | - Uppercase, lowercase, numbers, symbols 55 | - A SEND button to submit the query 56 | 57 | 6. Sending to OpenAI 58 | -> When SEND is tapped: 59 | - The image (from SD card) is read and Base64 encoded. 60 | - The typed question is packaged into a JSON request. 61 | - The request is sent to the OpenAI API via HTTPS. 62 | 63 | 64 | Displaying the AI Response 65 | 7. Show Captured Image While Waiting 66 | -> While waiting for OpenAI response: 67 | - The previously frozen RGB565 image frame is shown on screen. 68 | - A status "Sending to Chat-gpt..." is shown. 69 | 70 | 8. OpenAI Response Received 71 | -> Once the response arrives: 72 | - The textual response is rendered beneath the image. 73 | - The user can scroll through long responses. 74 | 75 | 76 | Reset to Live View 77 | 9. Close Button (appears on top) 78 | -> When tapped: 79 | - The response and image are cleared. 80 | - The live camera feed is restarted. 81 | - The Capture button is shown again, restarting the cycle. 82 | 83 | 84 | Summary of Flow 85 | 86 | [Power ON] 87 | ↓ 88 | [Wi-Fi Connecting] → [Connected] 89 | ↓ 90 | [SD Card Check] → [OK] 91 | ↓ 92 | [Live Camera Preview + Capture Button] 93 | ↓ 94 | [Capture Pressed] 95 | ↓ 96 | [Freeze Frame in RAM] 97 | ↓ 98 | [Switch to JPEG → Capture Image → Save to SD] 99 | ↓ 100 | [Show On-screen Keyboard] 101 | ↓ 102 | [User types question → Presses SEND] 103 | ↓ 104 | [Send image + prompt to OpenAI] 105 | ↓ 106 | [Show captured image from RAM] 107 | ↓ 108 | [Display OpenAI response (scrollable)] 109 | ↓ 110 | [User presses CLOSE] 111 | ↓ 112 | [Return to live preview + capture button] 113 | */ 114 | 115 | #include // Graphics for ST7789 display 116 | #include // Touch input from bb_captouch controller 117 | #include "esp_camera.h" // Camera support 118 | #include // SD card interface 119 | #include // Filesystem access 120 | #include "base64.h" // Base64 encoding for image uploads 121 | #include // Wi-Fi connection 122 | #include // HTTPS support 123 | #include // For sending HTTP POST to OpenAI 124 | #include // For JSON handling (OpenAI API requests/responses) 125 | #include 126 | #include 127 | 128 | 129 | // Wi-Fi credentials 130 | const char *ssid = "SSID"; 131 | const char *password = "PASSWORD"; 132 | 133 | // OpenAI API credentials 134 | const char *api_endpoint = "https://api.openai.com/v1/chat/completions"; 135 | const char *apiKey = "API KEY"; 136 | 137 | // Display Pins 138 | #define TFT_BLK 48 139 | #define TFT_RES -1 140 | #define TFT_CS 15 141 | #define TFT_MOSI 13 142 | #define TFT_MISO 12 143 | #define TFT_SCLK 14 144 | #define TFT_DC 21 145 | 146 | // Touch Pins 147 | #define TOUCH_INT 40 148 | #define TOUCH_SDA 39 149 | #define TOUCH_SCL 38 150 | #define TOUCH_RST 1 151 | 152 | #define CLOSE_BTN_WIDTH 30 153 | #define CLOSE_BTN_HEIGHT 30 154 | #define CLOSE_BTN_X (DISPLAY_WIDTH - CLOSE_BTN_WIDTH - 5) 155 | #define CLOSE_BTN_Y 5 156 | 157 | // SD Card Pins 158 | #define PIN_SD_CMD 2 159 | #define PIN_SD_CLK 42 160 | #define PIN_SD_D0 41 161 | 162 | // Camera Pins 163 | #define PWDN_GPIO_NUM -1 164 | #define RESET_GPIO_NUM -1 165 | #define XCLK_GPIO_NUM 9 166 | #define SIOD_GPIO_NUM 39 167 | #define SIOC_GPIO_NUM 38 168 | #define Y9_GPIO_NUM 46 169 | #define Y8_GPIO_NUM 3 170 | #define Y7_GPIO_NUM 8 171 | #define Y6_GPIO_NUM 16 172 | #define Y5_GPIO_NUM 6 173 | #define Y4_GPIO_NUM 4 174 | #define Y3_GPIO_NUM 5 175 | #define Y2_GPIO_NUM 7 176 | #define VSYNC_GPIO_NUM 11 177 | #define HREF_GPIO_NUM 10 178 | #define PCLK_GPIO_NUM 17 179 | 180 | // Display Setup via SPI bus 181 | Arduino_ESP32SPI *bus = new Arduino_ESP32SPI(TFT_DC, TFT_CS, TFT_SCLK, TFT_MOSI, TFT_MISO, HSPI, true); 182 | Arduino_GFX *gfx = new Arduino_ST7789(bus, TFT_RES, 0, true); 183 | 184 | // Capacitive Touch Setup 185 | BBCapTouch bbct; 186 | 187 | // Globals 188 | int imageCounter = 0; 189 | bool sdCardMounted = false; 190 | bool liveFeedActive = true; // To track if live feed is active or paused 191 | bool keyboardActive = false; // To track if keyboard is active 192 | camera_fb_t *capturedJpegFb = NULL; // Store the captured JPEG frame 193 | bool showingResponse = false; // Flag to indicate if showing response 194 | uint8_t *displayJpegBuffer = NULL; // For storing RGB565 version of captured image for display 195 | bool touchDetected = false; 196 | camera_fb_t *liveFb = NULL; 197 | int responseScrollOffset = 0; 198 | // Store broken lines once 199 | std::vector responseLines; 200 | String responseString; 201 | const int SCROLLBAR_WIDTH = 4; 202 | const int TEXT_PADDING = 5; // for left and right padding 203 | bool linesPrepared = false; 204 | static bool firstPageDrawn = false; 205 | static bool firstLineHold = true; 206 | 207 | 208 | // Display dimensions 209 | #define DISPLAY_WIDTH 240 210 | #define DISPLAY_HEIGHT 320 211 | 212 | // Response display settings 213 | #define RESPONSE_IMAGE_HEIGHT 120 // Height of image in response view 214 | #define RESPONSE_TEXT_Y 130 // Y position for response text 215 | #define RESPONSE_TEXT_HEIGHT 140 // Height of response text area 216 | 217 | // Button parameters - bottom of screen button bar 218 | #define BUTTON_HEIGHT 40 219 | #define BUTTON_Y 260 220 | #define CAPTURE_BUTTON_X 90 221 | #define CAPTURE_BUTTON_WIDTH 70 222 | 223 | // Keyboard layout 224 | bool isUpper = true; 225 | bool isSymbol = false; 226 | 227 | const char *alphaUpper[] = { 228 | "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", 229 | "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", 230 | "U", "V", "W", "X", "Y", "Z", "SPACE", 231 | "123", "^", "<-" 232 | }; 233 | const int alphaUpperCount = sizeof(alphaUpper) / sizeof(alphaUpper[0]); 234 | 235 | const char *alphaLower[] = { 236 | "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", 237 | "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", 238 | "u", "v", "w", "x", "y", "z", "SPACE", 239 | "123", "^", "<-" 240 | }; 241 | const int alphaLowerCount = sizeof(alphaLower) / sizeof(alphaLower[0]); 242 | 243 | const char *symbols[] = { 244 | "1", "2", "3", "4", "5", "6", "7", "8", "9", "0", 245 | "!", "@", "#", "$", "%", "?", "&", "*", "(", ")", 246 | "-", "_", "+", "=", "[", "]", "SPACE", 247 | "ABC", ".", "<-" 248 | }; 249 | const int symbolsCount = sizeof(symbols) / sizeof(symbols[0]); 250 | 251 | const char **currentLayout = alphaUpper; 252 | int currentLayoutCount = alphaUpperCount; 253 | 254 | #define KEYBOARD_LAYOUT_MAX 32 // Max keys = COLS * ROWS = 8 * 4 255 | 256 | struct KeyBounds { 257 | int x, y, w, h; 258 | }; 259 | 260 | KeyBounds sendBtnBounds; 261 | 262 | KeyBounds keyPositions[KEYBOARD_LAYOUT_MAX]; // Make sure size is enough 263 | 264 | #define COLS 8 265 | #define ROWS 4 266 | #define KEY_W 26 267 | #define KEY_H 30 268 | #define KEY_SPACING 2 269 | 270 | String inputText = ""; 271 | bool keyTouchActive = false; 272 | 273 | int keyboardY = DISPLAY_HEIGHT - (ROWS * (KEY_H + KEY_SPACING)); // Start Y of keyboard 274 | 275 | bool isTouchOnSendButton(int x, int y) { 276 | return x >= sendBtnBounds.x && x <= (sendBtnBounds.x + sendBtnBounds.w) && y >= sendBtnBounds.y && y <= (sendBtnBounds.y + sendBtnBounds.h); 277 | } 278 | 279 | 280 | void setup() { 281 | Serial.begin(115200); 282 | delay(500); 283 | 284 | pinMode(TFT_BLK, OUTPUT); 285 | digitalWrite(TFT_BLK, HIGH); 286 | 287 | gfx->begin(); 288 | gfx->fillScreen(BLACK); 289 | gfx->setTextColor(WHITE); 290 | gfx->setFont(&FreeSans9pt7b); // GFX font in use 291 | 292 | int y = 30; // start Y position 293 | int lineSpacing = 28; // adjust based on font height 294 | 295 | gfx->setCursor(10, y); 296 | gfx->println("AI Camera"); 297 | 298 | 299 | y += lineSpacing; 300 | gfx->setCursor(10, y); 301 | gfx->println("Connecting to:"); 302 | 303 | y += lineSpacing; 304 | gfx->setCursor(10, y); 305 | gfx->println(ssid); 306 | 307 | WiFi.begin(ssid, password); 308 | 309 | y += lineSpacing; 310 | gfx->setCursor(10, y); 311 | gfx->print("Connecting"); 312 | 313 | while (WiFi.status() != WL_CONNECTED) { 314 | delay(500); 315 | gfx->print("."); 316 | } 317 | 318 | gfx->println(" done!"); 319 | Serial.println(WiFi.localIP()); 320 | 321 | y += lineSpacing; 322 | gfx->setCursor(10, y); 323 | gfx->println("Checking SD..."); 324 | 325 | SD_MMC.setPins(PIN_SD_CLK, PIN_SD_CMD, PIN_SD_D0); 326 | y += lineSpacing; 327 | gfx->setCursor(10, y); 328 | if (!SD_MMC.begin("/sdcard", true, true)) { 329 | gfx->println("SD Card Failed"); 330 | y += lineSpacing; 331 | gfx->setCursor(10, y); 332 | gfx->println("Recheck the SD Card"); 333 | Serial.println("SD Card initialization failed. Halting."); 334 | while (true) { 335 | delay(100); // Infinite loop to halt further execution 336 | } 337 | } else { 338 | gfx->println("SD Card OK"); 339 | sdCardMounted = true; 340 | } 341 | 342 | y += lineSpacing; 343 | gfx->setCursor(10, y); 344 | //gfx->println("Init camera..."); 345 | initCamera(FRAMESIZE_240X240, PIXFORMAT_RGB565); 346 | 347 | y += lineSpacing; 348 | gfx->setCursor(10, y); 349 | // gfx->println("Init touch..."); 350 | bbct.init(TOUCH_SDA, TOUCH_SCL, TOUCH_RST, TOUCH_INT); 351 | delay(100); 352 | 353 | // Reset font to default and redraw UI 354 | gfx->setFont(NULL); // Use default font for normal UI 355 | gfx->fillScreen(BLACK); 356 | drawUI(); 357 | } 358 | 359 | void drawUI() { 360 | // Clear the bottom button bar 361 | gfx->fillRect(0, BUTTON_Y, DISPLAY_WIDTH, BUTTON_HEIGHT, BLACK); 362 | 363 | // Set font for measuring 364 | gfx->setFont(&FreeSans9pt7b); 365 | gfx->setTextColor(WHITE); 366 | 367 | const char *label = "CAPTURE"; 368 | 369 | // Measure text size 370 | int16_t x1, y1; 371 | uint16_t textW, textH; 372 | gfx->getTextBounds(label, 0, 0, &x1, &y1, &textW, &textH); 373 | 374 | // Padding around text 375 | int paddingX = 16; 376 | int paddingY = 8; 377 | 378 | // Button size based on text size 379 | int buttonW = textW + 2 * paddingX; 380 | int buttonH = textH + 2 * paddingY; 381 | 382 | // Button position centered horizontally at bottom 383 | int buttonX = (DISPLAY_WIDTH - buttonW) / 2; 384 | int buttonY = BUTTON_Y + (BUTTON_HEIGHT - buttonH) / 2; 385 | 386 | // Draw button background 387 | gfx->fillRoundRect(buttonX, buttonY, buttonW, buttonH, 6, BLUE); 388 | 389 | // Draw text centered inside button 390 | int cursorX = buttonX + (buttonW - textW) / 2; 391 | int cursorY = buttonY + (buttonH + textH) / 2 - 3; // Fine-tuned for baseline alignment 392 | 393 | gfx->setCursor(cursorX, cursorY); 394 | gfx->print(label); 395 | } 396 | 397 | 398 | void initCamera(framesize_t size, pixformat_t format) { 399 | camera_config_t config; 400 | config.ledc_channel = LEDC_CHANNEL_0; 401 | config.ledc_timer = LEDC_TIMER_0; 402 | config.pin_d0 = Y2_GPIO_NUM; 403 | config.pin_d1 = Y3_GPIO_NUM; 404 | config.pin_d2 = Y4_GPIO_NUM; 405 | config.pin_d3 = Y5_GPIO_NUM; 406 | config.pin_d4 = Y6_GPIO_NUM; 407 | config.pin_d5 = Y7_GPIO_NUM; 408 | config.pin_d6 = Y8_GPIO_NUM; 409 | config.pin_d7 = Y9_GPIO_NUM; 410 | config.pin_xclk = XCLK_GPIO_NUM; 411 | config.pin_pclk = PCLK_GPIO_NUM; 412 | config.pin_vsync = VSYNC_GPIO_NUM; 413 | config.pin_href = HREF_GPIO_NUM; 414 | config.pin_sccb_sda = SIOD_GPIO_NUM; 415 | config.pin_sccb_scl = SIOC_GPIO_NUM; 416 | config.pin_pwdn = PWDN_GPIO_NUM; 417 | config.pin_reset = RESET_GPIO_NUM; 418 | config.xclk_freq_hz = 20000000; 419 | config.pixel_format = format; 420 | config.frame_size = size; 421 | config.fb_location = CAMERA_FB_IN_PSRAM; 422 | config.jpeg_quality = 10; 423 | config.grab_mode = CAMERA_GRAB_WHEN_EMPTY; 424 | config.fb_count = 1; 425 | 426 | esp_camera_deinit(); 427 | esp_err_t err = esp_camera_init(&config); 428 | if (err != ESP_OK) { 429 | gfx->println("Camera init failed!"); 430 | return; 431 | } 432 | 433 | sensor_t *s = esp_camera_sensor_get(); 434 | if (s) { 435 | s->set_vflip(s, 0); 436 | s->set_hmirror(s, 1); 437 | } 438 | } 439 | 440 | int checkButtonPress(int x, int y) { 441 | if (showingResponse) { 442 | if (x >= CLOSE_BTN_X && x < CLOSE_BTN_X + CLOSE_BTN_WIDTH && y >= CLOSE_BTN_Y && y < CLOSE_BTN_Y + CLOSE_BTN_HEIGHT) { 443 | return 4; // CLOSE button 444 | } 445 | return 0; 446 | } 447 | 448 | // ===== Updated rectangular CAPTURE button detection ===== 449 | const char *label = "CAPTURE"; 450 | gfx->setFont(&FreeSans9pt7b); 451 | int16_t x1, y1; 452 | uint16_t textW, textH; 453 | gfx->getTextBounds(label, 0, 0, &x1, &y1, &textW, &textH); 454 | 455 | int paddingX = 16; 456 | int paddingY = 8; 457 | int buttonW = textW + 2 * paddingX; 458 | int buttonH = textH + 2 * paddingY; 459 | int buttonX = (DISPLAY_WIDTH - buttonW) / 2; 460 | int buttonY = BUTTON_Y + (BUTTON_HEIGHT - buttonH) / 2; 461 | 462 | if (x >= buttonX && x <= buttonX + buttonW && y >= buttonY && y <= buttonY + buttonH) { 463 | return 2; // CAPTURE 464 | } 465 | 466 | return 0; 467 | } 468 | 469 | 470 | void showStatusMessage(const char *message, uint16_t color) { 471 | // Show status message at top of screen 472 | gfx->fillRect(0, 0, DISPLAY_WIDTH, 30, color); 473 | 474 | gfx->setFont(&FreeSans9pt7b); 475 | gfx->setTextColor(BLACK); 476 | 477 | // Compute proper vertical alignment using text bounds 478 | int16_t x1, y1; 479 | uint16_t w, h; 480 | gfx->getTextBounds(message, 0, 0, &x1, &y1, &w, &h); 481 | 482 | int cursorX = 10; 483 | int cursorY = (30 + h) / 2 - 2; // Adjust for vertical centering 484 | 485 | gfx->setCursor(cursorX, cursorY); 486 | gfx->print(message); 487 | 488 | delay(1000); 489 | } 490 | 491 | 492 | void captureImage() { 493 | showStatusMessage("Capturing...", RED); 494 | delay(500); 495 | showStatusMessage("Please hold", ORANGE); 496 | 497 | 498 | //Capture a copy of the current live 240x240 RGB565 frame for display 499 | if (liveFb && liveFb->format == PIXFORMAT_RGB565 && liveFb->width == 240 && liveFb->height == 240) { 500 | if (displayJpegBuffer != NULL) { 501 | free(displayJpegBuffer); 502 | } 503 | displayJpegBuffer = (uint8_t *)malloc(240 * 240 * 2); // RGB565 buffer 504 | 505 | if (displayJpegBuffer) { 506 | memcpy(displayJpegBuffer, liveFb->buf, 240 * 240 * 2); 507 | Serial.println("[CAM] Copied RGB565 live frame to display buffer."); 508 | } 509 | } 510 | 511 | //Now switch to high-res JPEG capture 512 | initCamera(FRAMESIZE_VGA, PIXFORMAT_JPEG); 513 | delay(100); 514 | 515 | camera_fb_t *jpegFb = esp_camera_fb_get(); 516 | if (!jpegFb || jpegFb->format != PIXFORMAT_JPEG) { 517 | Serial.println("[CAM] JPEG capture failed."); 518 | initCamera(FRAMESIZE_240X240, PIXFORMAT_RGB565); 519 | delay(100); 520 | drawUI(); 521 | return; 522 | } 523 | 524 | //Save to SD 525 | String path; 526 | unsigned long saveStart = millis(); 527 | bool saved = saveJpegToSD(jpegFb, path); 528 | Serial.printf("[CAM] Save time: %lu ms\n", millis() - saveStart); 529 | 530 | if (!saved) { 531 | showStatusMessage("Save failed!", RED); 532 | Serial.println("[CAM] Failed to save image."); 533 | esp_camera_fb_return(jpegFb); 534 | initCamera(FRAMESIZE_240X240, PIXFORMAT_RGB565); 535 | delay(100); 536 | drawUI(); 537 | return; 538 | } 539 | 540 | showStatusMessage("Image saved!", GREEN); 541 | Serial.println("[CAM] Image saved to: " + path); 542 | 543 | //Keep JPEG in memory 544 | if (capturedJpegFb) esp_camera_fb_return(capturedJpegFb); 545 | capturedJpegFb = jpegFb; 546 | 547 | //Reset and re-init touch 548 | digitalWrite(TOUCH_RST, LOW); // At least 5–10ms is usually safe 549 | showStatusMessage("Opening Keyboard...", BLUE); 550 | digitalWrite(TOUCH_RST, HIGH); // Allow touch controller to boot up 551 | 552 | bbct.init(TOUCH_SDA, TOUCH_SCL, TOUCH_RST, TOUCH_INT); // Only once 553 | 554 | 555 | //Launch keyboard 556 | keyboardActive = true; 557 | bbct.init(TOUCH_SDA, TOUCH_SCL, TOUCH_RST, TOUCH_INT); // Only once 558 | 559 | inputText = ""; 560 | drawKeyboard(); 561 | } 562 | 563 | bool saveJpegToSD(camera_fb_t *fb, String &filenameOut) { 564 | if (!sdCardMounted) return false; 565 | 566 | char filename[32]; 567 | sprintf(filename, "/IMG_%04d.jpg", imageCounter++); 568 | filenameOut = String(filename); 569 | 570 | File file = SD_MMC.open(filename, FILE_WRITE); 571 | if (!file) return false; 572 | 573 | if (file.write(fb->buf, fb->len) != fb->len) { 574 | file.close(); 575 | return false; 576 | } 577 | 578 | file.close(); 579 | Serial.printf("Saved image to %s\n", filename); 580 | return true; 581 | } 582 | 583 | // Store the last frame for display when feed is paused 584 | camera_fb_t *lastFrame = NULL; 585 | void storeCopyOfFrame(camera_fb_t *fb) { 586 | if (!fb || fb->format != PIXFORMAT_RGB565) return; 587 | 588 | if (lastFrame != NULL) { 589 | free(lastFrame->buf); 590 | free(lastFrame); 591 | lastFrame = NULL; 592 | } 593 | 594 | lastFrame = (camera_fb_t *)malloc(sizeof(camera_fb_t)); 595 | if (!lastFrame) return; 596 | 597 | lastFrame->len = fb->len; 598 | lastFrame->width = fb->width; 599 | lastFrame->height = fb->height; 600 | lastFrame->format = fb->format; 601 | 602 | lastFrame->buf = (uint8_t *)malloc(fb->len); 603 | if (!lastFrame->buf) { 604 | free(lastFrame); 605 | lastFrame = NULL; 606 | return; 607 | } 608 | 609 | memcpy(lastFrame->buf, fb->buf, fb->len); 610 | } 611 | 612 | void drawKeyboard() { 613 | gfx->fillScreen(BLACK); 614 | gfx->setFont(&FreeSans9pt7b); // Apply custom font 615 | 616 | //Input Area 617 | int inputBoxHeight = 60; 618 | gfx->fillRect(0, 0, DISPLAY_WIDTH, inputBoxHeight, NAVY); 619 | gfx->setTextColor(WHITE); 620 | gfx->setCursor(10, 20); // Y adjusted for new font 621 | gfx->print("Insert Question:"); 622 | 623 | // Multi-line input text rendering 624 | int lineHeight = 19; // Adjusted for larger font 625 | int maxWidth = DISPLAY_WIDTH - 20; 626 | int cursorX = 10; 627 | int cursorY = 35; 628 | String word = ""; 629 | 630 | for (int i = 0; i < inputText.length(); i++) { 631 | char c = inputText[i]; 632 | if (c == ' ') { 633 | // Finish current word first (if any) 634 | if (word.length() > 0) { 635 | int16_t x1, y1; 636 | uint16_t tw, th; 637 | gfx->getTextBounds(word, 0, 0, &x1, &y1, &tw, &th); 638 | if (cursorX + tw > maxWidth) { 639 | cursorX = 10; 640 | cursorY += lineHeight; 641 | } 642 | gfx->setCursor(cursorX, cursorY); 643 | gfx->print(word); 644 | cursorX += tw; 645 | word = ""; 646 | } 647 | 648 | // Manually add visual space (e.g., 6px wide) 649 | int spaceWidth = 6; 650 | if (cursorX + spaceWidth > maxWidth) { 651 | cursorX = 10; 652 | cursorY += lineHeight; 653 | } 654 | cursorX += spaceWidth; 655 | 656 | } else if (i == inputText.length() - 1) { 657 | word += c; 658 | 659 | // Last character, draw it too 660 | int16_t x1, y1; 661 | uint16_t tw, th; 662 | gfx->getTextBounds(word, 0, 0, &x1, &y1, &tw, &th); 663 | if (cursorX + tw > maxWidth) { 664 | cursorX = 10; 665 | cursorY += lineHeight; 666 | } 667 | gfx->setCursor(cursorX, cursorY); 668 | gfx->print(word); 669 | cursorX += tw; 670 | word = ""; 671 | 672 | } else { 673 | word += c; 674 | } 675 | } 676 | 677 | 678 | //Keyboard Y start 679 | int keyboardHeight = (KEY_H + KEY_SPACING) * ROWS; 680 | int keyboardStartY = DISPLAY_HEIGHT - keyboardHeight; 681 | 682 | //SEND Button 683 | int sendBtnW = 100; 684 | int sendBtnH = 30; 685 | int sendBtnX = (DISPLAY_WIDTH - sendBtnW) / 2; 686 | int sendBtnY = keyboardStartY - sendBtnH - 6; 687 | 688 | gfx->fillRect(sendBtnX, sendBtnY, sendBtnW, sendBtnH, GREEN); 689 | gfx->drawRect(sendBtnX, sendBtnY, sendBtnW, sendBtnH, WHITE); 690 | 691 | const char *sendLabel = "SEND"; 692 | int16_t x1, y1; 693 | uint16_t textW, textH; 694 | gfx->setTextColor(BLACK); 695 | gfx->getTextBounds(sendLabel, 0, 0, &x1, &y1, &textW, &textH); 696 | int textX = sendBtnX + (sendBtnW - textW) / 2; 697 | int textY = sendBtnY + (sendBtnH + textH) / 2 - 2; 698 | gfx->setCursor(textX, textY); 699 | gfx->print(sendLabel); 700 | 701 | sendBtnBounds = { sendBtnX, sendBtnY, sendBtnW, sendBtnH }; 702 | 703 | int keyIndex = 0; 704 | for (int row = 0; row < ROWS; row++) { 705 | int y = keyboardStartY + row * (KEY_H + KEY_SPACING); 706 | 707 | // Compute row key count 708 | int rowKeyCount = min(COLS, currentLayoutCount - keyIndex); 709 | int totalRowWidth = 0; 710 | 711 | // 1. Compute total width of keys in this row 712 | for (int i = 0; i < rowKeyCount; i++) { 713 | const char *label = currentLayout[keyIndex + i]; 714 | int keyW = KEY_W; 715 | if (strcmp(label, "SPACE") == 0) keyW = KEY_W * 3 + KEY_SPACING; 716 | else if (strcmp(label, "") == 0 || strcmp(label, "123") == 0 || strcmp(label, "ABC") == 0) 717 | keyW = KEY_W + 8; 718 | totalRowWidth += keyW; 719 | if (i < rowKeyCount - 1) totalRowWidth += KEY_SPACING; 720 | } 721 | 722 | // 2. Compute starting x to center the row 723 | int x = (DISPLAY_WIDTH - totalRowWidth) / 2; 724 | 725 | // 3. Draw keys for this row 726 | for (int i = 0; i < rowKeyCount; i++) { 727 | const char *label = currentLayout[keyIndex]; 728 | int keyW = KEY_W; 729 | if (strcmp(label, "SPACE") == 0) keyW = KEY_W * 3 + KEY_SPACING; 730 | else if (strcmp(label, "") == 0 || strcmp(label, "123") == 0 || strcmp(label, "ABC") == 0) 731 | keyW = KEY_W + 8; 732 | 733 | drawKey(keyIndex, x, y, keyW, KEY_H); 734 | keyPositions[keyIndex] = { x, y, keyW, KEY_H }; 735 | x += keyW + KEY_SPACING; 736 | keyIndex++; 737 | } 738 | } 739 | } 740 | 741 | void drawKey(int index, int x, int y, int w, int h) { 742 | gfx->fillRect(x, y, w, h, DARKGREY); 743 | gfx->drawRect(x, y, w, h, WHITE); 744 | const char *label = currentLayout[index]; 745 | gfx->setFont(&FreeSans9pt7b); 746 | gfx->setTextColor(WHITE); 747 | 748 | int16_t x1, y1; 749 | uint16_t tw, th; 750 | gfx->getTextBounds(label, 0, 0, &x1, &y1, &tw, &th); 751 | int tx = x + (w - tw) / 2; 752 | int ty = y + (h + th) / 2 - 2; // Adjusted for vertical centering 753 | gfx->setCursor(tx, ty); 754 | gfx->print(label); 755 | } 756 | 757 | void processKeyboardTouch(int x, int y) { 758 | // Check if the Send button was tapped 759 | if (isTouchOnSendButton(x, y)) { 760 | Serial.println("User input: " + inputText); 761 | keyboardActive = false; 762 | 763 | gfx->fillScreen(BLACK); 764 | 765 | gfx->setFont(&FreeSans9pt7b); // Set custom font 766 | showStatusMessage("Processing...", BLUE); 767 | 768 | if (capturedJpegFb != NULL) { 769 | drawCapturedImage(); // Make sure this uses the correct font too 770 | String base64Image = base64::encode(capturedJpegFb->buf, capturedJpegFb->len); 771 | uploadToAPI(base64Image, inputText); 772 | } 773 | 774 | return; 775 | } 776 | 777 | // Check if a keyboard key was tapped 778 | int keyIndex = getKeyFromXY(x, y); 779 | if (keyIndex != -1) { 780 | const char *key = currentLayout[keyIndex]; 781 | 782 | if (strcmp(key, "SPACE") == 0) { 783 | inputText += ' '; 784 | } else if (strcmp(key, "^") == 0) { 785 | isUpper = !isUpper; 786 | currentLayout = isUpper ? alphaUpper : alphaLower; 787 | currentLayoutCount = isUpper ? alphaUpperCount : alphaLowerCount; 788 | } else if (strcmp(key, "123") == 0) { 789 | currentLayout = symbols; 790 | currentLayoutCount = symbolsCount; 791 | isSymbol = true; 792 | } else if (strcmp(key, "ABC") == 0) { 793 | currentLayout = isUpper ? alphaUpper : alphaLower; 794 | currentLayoutCount = isUpper ? alphaUpperCount : alphaLowerCount; 795 | isSymbol = false; 796 | } else if (strcmp(key, "<-") == 0 || strcmp(key, "DEL") == 0) { 797 | if (inputText.length() > 0) { 798 | inputText.remove(inputText.length() - 1); 799 | } 800 | } else { 801 | inputText += key; 802 | } 803 | 804 | gfx->setFont(&FreeSans9pt7b); //Ensure font is applied before redraw 805 | drawKeyboard(); // Redraw the updated keyboard UI 806 | delay(200); 807 | } 808 | } 809 | 810 | 811 | void drawCapturedImage() { 812 | if (displayJpegBuffer != NULL) { 813 | // Use big-endian draw function to render correctly colored image 814 | gfx->draw16bitBeRGBBitmap(0, 0, (uint16_t *)displayJpegBuffer, 240, 240); 815 | } else { 816 | gfx->fillRect(0, 0, 240, 240, BLACK); 817 | gfx->setFont(&FreeSans9pt7b); // Use custom font 818 | gfx->setCursor(10, 120); 819 | gfx->setTextColor(RED); 820 | gfx->println("No image available"); 821 | } 822 | } 823 | 824 | 825 | int getKeyFromXY(int x, int y) { 826 | for (int i = 0; i < currentLayoutCount; i++) { 827 | int kx = keyPositions[i].x; 828 | int ky = keyPositions[i].y; 829 | int kw = keyPositions[i].w; 830 | int kh = keyPositions[i].h; 831 | 832 | if (x >= kx && x <= kx + kw && y >= ky && y <= ky + kh) { 833 | return i; 834 | } 835 | } 836 | return -1; 837 | } 838 | 839 | void uploadToAPI(const String &base64Image, const String &prompt) { 840 | Serial.println("[API] uploadToAPI() started"); 841 | 842 | String url = "data:image/jpeg;base64," + base64Image; 843 | Serial.printf("[API] Base64 image length: %d\n", base64Image.length()); 844 | Serial.println("[API] Base64 appears valid"); 845 | 846 | // Use custom prompt if provided, otherwise use default 847 | String userPrompt = prompt.length() > 0 ? prompt : "What's in this image?"; 848 | Serial.printf("[API] Using prompt: %s\n", userPrompt.c_str()); 849 | 850 | // Construct the JSON payload 851 | String payload = String("{") + "\"model\": \"gpt-4o\", " + "\"max_tokens\": 400, " + "\"messages\": [{" + "\"role\": \"user\", \"content\": [" + "{\"type\": \"text\", \"text\": \"" + userPrompt + "\"}, " + "{\"type\": \"image_url\", \"image_url\": {\"url\": \"" + url + "\", \"detail\": \"auto\"}}" + "]}]}"; 852 | 853 | Serial.println("[API] Connecting to OpenAI..."); 854 | Serial.println("[API] Sending request..."); 855 | Serial.println("[API] Payload size: " + String(payload.length())); 856 | 857 | showStatusMessage("Sending to Chat-GPT...", BLUE); 858 | 859 | String result; 860 | if (sendPostRequest(payload, result)) { 861 | Serial.println("[API] Response:"); 862 | Serial.println(result); 863 | 864 | // Parse JSON response 865 | DynamicJsonDocument doc(4096); 866 | DeserializationError error = deserializeJson(doc, result); 867 | 868 | if (error) { 869 | Serial.print("[API] JSON Parse Error: "); 870 | Serial.println(error.c_str()); 871 | showStatusMessage("API Error", RED); 872 | return; 873 | } 874 | 875 | String content = doc["choices"][0]["message"]["content"].as(); 876 | Serial.println("[API] Parsed response: " + content); 877 | 878 | // Store response and display it 879 | responseString = content; 880 | showingResponse = true; 881 | 882 | responseLines.clear(); 883 | linesPrepared = false; 884 | responseScrollOffset = 0; 885 | drawResponseView(); 886 | } 887 | 888 | else { 889 | Serial.println("[API] Failed to get a valid response from OpenAI."); 890 | showStatusMessage("API Failed", RED); 891 | 892 | // Set error message so it's visible in response view 893 | responseString = "Failed to connect to OpenAI."; 894 | linesPrepared = false; // So it triggers text wrapping 895 | showingResponse = true; // Show response view even on error 896 | responseScrollOffset = 0; 897 | 898 | drawResponseView(); 899 | } 900 | } 901 | 902 | bool sendPostRequest(const String &payload, String &result) { 903 | HTTPClient http; 904 | 905 | // This will use the internal default secure client (WiFiClientSecure) 906 | if (!http.begin("https://api.openai.com/v1/chat/completions")) { 907 | Serial.println("[API] Failed to begin HTTPS connection"); 908 | return false; 909 | } 910 | 911 | http.addHeader("Content-Type", "application/json"); 912 | http.addHeader("Authorization", String("Bearer ") + apiKey); 913 | http.setTimeout(30000); 914 | 915 | Serial.println("[API] Sending request..."); 916 | Serial.println("[API] Payload size: " + String(payload.length())); 917 | 918 | int httpCode = http.POST(payload); 919 | 920 | if (httpCode > 0) { 921 | Serial.println("[API] HTTP Code: " + String(httpCode)); 922 | result = http.getString(); 923 | http.end(); 924 | return true; 925 | } else { 926 | Serial.println("[API] POST request failed."); 927 | Serial.println("[API] Error: " + http.errorToString(httpCode)); 928 | result = http.errorToString(httpCode); 929 | http.end(); 930 | return false; 931 | } 932 | } 933 | 934 | 935 | std::vector wrapTextToWidth(const String &text, int maxWidth) { 936 | std::vector wrappedLines; 937 | int start = 0; 938 | int end = 0; 939 | int len = text.length(); 940 | 941 | while (start < len) { 942 | end = start + 1; 943 | String line = ""; 944 | 945 | while (end <= len) { 946 | String testLine = text.substring(start, end); 947 | int16_t x1, y1; 948 | uint16_t w, h; 949 | gfx->getTextBounds(testLine.c_str(), 0, 0, &x1, &y1, &w, &h); 950 | if (w + 4 > maxWidth) { 951 | break; 952 | } 953 | line = testLine; 954 | end++; 955 | } 956 | 957 | // Backtrack to last space to avoid mid-word breaks 958 | int lastSpace = line.lastIndexOf(' '); 959 | if (lastSpace != -1 && end < len) { 960 | line = line.substring(0, lastSpace); 961 | end = start + lastSpace + 1; 962 | } 963 | 964 | wrappedLines.push_back(line); 965 | start = end; 966 | } 967 | 968 | return wrappedLines; 969 | } 970 | 971 | 972 | void drawResponseView() { 973 | gfx->fillScreen(BLACK); 974 | gfx->setFont(&FreeSans9pt7b); //Set custom font 975 | 976 | const int imageY = 0; 977 | const int imageHeight = 240; 978 | const int labelY = 240; 979 | const int textY = labelY + 22; 980 | 981 | // Draw image 982 | if (displayJpegBuffer != NULL) { 983 | gfx->draw16bitBeRGBBitmap(0, imageY, (uint16_t *)displayJpegBuffer, 240, 240); 984 | } else { 985 | gfx->fillRect(0, imageY, DISPLAY_WIDTH, imageHeight, DARKGREY); 986 | gfx->setTextColor(WHITE); 987 | gfx->setCursor(DISPLAY_WIDTH / 2 - 22, imageY + imageHeight / 2); 988 | gfx->print("IMAGE"); 989 | } 990 | 991 | gfx->fillRoundRect(CLOSE_BTN_X, CLOSE_BTN_Y, CLOSE_BTN_WIDTH, CLOSE_BTN_HEIGHT, 5, RED); 992 | 993 | // Center the "X" 994 | int16_t x1, y1; 995 | uint16_t w, h; 996 | gfx->getTextBounds("X", 0, 0, &x1, &y1, &w, &h); 997 | 998 | int xTextX = CLOSE_BTN_X + (CLOSE_BTN_WIDTH - w) / 2 - x1; 999 | int xTextY = CLOSE_BTN_Y + (CLOSE_BTN_HEIGHT - h) / 2 - y1; 1000 | 1001 | gfx->setCursor(xTextX, xTextY); 1002 | gfx->setFont(&FreeSans9pt7b); 1003 | gfx->setTextColor(WHITE); 1004 | gfx->print("X"); 1005 | 1006 | 1007 | // Label 1008 | gfx->setFont(NULL); 1009 | gfx->setCursor(5, labelY + 8); // Slight Y offset to align nicely 1010 | gfx->setTextColor(WHITE); 1011 | gfx->print("AI Response:"); 1012 | gfx->setFont(&FreeSans9pt7b); 1013 | 1014 | 1015 | 1016 | int textWrapWidth = DISPLAY_WIDTH - SCROLLBAR_WIDTH - (2 * TEXT_PADDING); 1017 | responseLines = wrapTextToWidth(responseString, textWrapWidth); 1018 | 1019 | 1020 | drawResponseTextArea(responseLines); 1021 | } 1022 | 1023 | 1024 | void drawResponseTextArea(const std::vector &lines) { 1025 | const int textY = 262; 1026 | const int textHeight = DISPLAY_HEIGHT - textY; 1027 | const int lineHeight = 19; // Increased line height for readability 1028 | 1029 | gfx->setFont(&FreeSans9pt7b); 1030 | gfx->setTextColor(WHITE); 1031 | gfx->setTextWrap(false); // Still off, as each line is pre-wrapped externally 1032 | 1033 | int maxVisibleLines = textHeight / lineHeight; 1034 | 1035 | // Clear the response area 1036 | gfx->fillRect(0, textY, DISPLAY_WIDTH - 4, textHeight, NAVY); 1037 | 1038 | for (int i = 0; i < maxVisibleLines; i++) { 1039 | int lineIndex = responseScrollOffset + i; 1040 | if (lineIndex < lines.size()) { 1041 | gfx->setCursor(TEXT_PADDING, textY + i * lineHeight + 11); // Adjusted baseline 1042 | gfx->print(lines[lineIndex]); 1043 | } 1044 | } 1045 | 1046 | // === Scrollbar === 1047 | int totalLines = lines.size(); 1048 | if (totalLines > maxVisibleLines) { 1049 | int scrollbarX = DISPLAY_WIDTH - 4; 1050 | int scrollbarY = textY; 1051 | int scrollbarHeight = textHeight; 1052 | int handleHeight = max(10, (maxVisibleLines * scrollbarHeight) / totalLines); 1053 | int handleY = scrollbarY + (responseScrollOffset * (scrollbarHeight - handleHeight)) / (totalLines - maxVisibleLines); 1054 | 1055 | gfx->fillRect(scrollbarX, scrollbarY, 4, scrollbarHeight, DARKGREY); // track 1056 | gfx->fillRect(scrollbarX, handleY, 4, handleHeight, WHITE); // handle 1057 | } 1058 | } 1059 | 1060 | 1061 | void handleCloseButton(int x, int y) { 1062 | int buttonPressed = checkButtonPress(x, y); 1063 | if (buttonPressed == 4) { 1064 | Serial.println("Close button pressed"); 1065 | returnToLiveView(); 1066 | } 1067 | } 1068 | 1069 | void returnToLiveView() { 1070 | Serial.println("[UI] Returning to live view..."); 1071 | 1072 | showingResponse = false; 1073 | keyboardActive = false; 1074 | 1075 | gfx->fillScreen(BLACK); 1076 | 1077 | initCamera(FRAMESIZE_240X240, PIXFORMAT_RGB565); 1078 | // HARD RE-INIT TOUCH 1079 | pinMode(TOUCH_RST, OUTPUT); 1080 | digitalWrite(TOUCH_RST, LOW); 1081 | digitalWrite(TOUCH_RST, HIGH); 1082 | bbct.init(TOUCH_SDA, TOUCH_SCL, TOUCH_RST, TOUCH_INT); 1083 | 1084 | bbct.init(TOUCH_SDA, TOUCH_SCL, TOUCH_RST, TOUCH_INT); 1085 | liveFeedActive = true; 1086 | drawUI(); 1087 | 1088 | Serial.println("[UI] Camera live feed resumed and UI reset."); 1089 | } 1090 | 1091 | 1092 | void loop() { 1093 | static bool touchDetected = false; 1094 | static unsigned long lastTouch = 0; 1095 | unsigned long now = millis(); 1096 | 1097 | TOUCHINFO ti; 1098 | bool currentTouch = bbct.getSamples(&ti); 1099 | static unsigned long lastScroll = 0; 1100 | static unsigned long responseShownAt = 0; 1101 | static int lastTouchY = 0; 1102 | 1103 | if (showingResponse) { 1104 | const int textY = 262; 1105 | const int lineHeight = 19; 1106 | const int visibleLines = (DISPLAY_HEIGHT - textY) / lineHeight; 1107 | const int totalLines = responseLines.size(); 1108 | 1109 | if (!linesPrepared) { 1110 | gfx->setFont(&FreeSans9pt7b); 1111 | int textWrapWidth = DISPLAY_WIDTH - SCROLLBAR_WIDTH - (2 * TEXT_PADDING); 1112 | 1113 | responseLines = wrapTextToWidth(responseString.length() > 0 ? responseString : "No response from API. Please try again.", textWrapWidth); 1114 | 1115 | linesPrepared = true; 1116 | firstLineHold = true; 1117 | responseScrollOffset = 0; 1118 | firstPageDrawn = false; 1119 | drawResponseView(); 1120 | lastScroll = millis(); 1121 | responseShownAt = millis(); 1122 | } 1123 | 1124 | if (firstLineHold && !firstPageDrawn) { 1125 | drawResponseTextArea(responseLines); 1126 | firstPageDrawn = true; 1127 | responseShownAt = millis(); 1128 | } 1129 | 1130 | if (millis() - responseShownAt > 2500) { 1131 | firstLineHold = false; 1132 | } 1133 | 1134 | // Auto-scroll 1135 | if (!firstLineHold && millis() - lastScroll > 2500) { 1136 | if (responseScrollOffset < totalLines - visibleLines) { 1137 | responseScrollOffset++; 1138 | drawResponseTextArea(responseLines); 1139 | lastScroll = millis(); 1140 | } 1141 | } 1142 | 1143 | // Manual scroll 1144 | if (currentTouch) { 1145 | int deltaY = ti.y[0] - lastTouchY; 1146 | lastTouchY = ti.y[0]; 1147 | 1148 | const int minSwipeDistance = 20; 1149 | 1150 | if (abs(deltaY) >= minSwipeDistance) { 1151 | if (deltaY > 0 && responseScrollOffset > 0) { 1152 | responseScrollOffset--; 1153 | drawResponseTextArea(responseLines); 1154 | lastScroll = millis(); 1155 | } else if (deltaY < 0 && responseScrollOffset < totalLines - visibleLines) { 1156 | responseScrollOffset++; 1157 | drawResponseTextArea(responseLines); 1158 | lastScroll = millis(); 1159 | } 1160 | } 1161 | 1162 | // Close button check 1163 | if (!touchDetected && now - lastTouch > 500) { 1164 | touchDetected = true; 1165 | lastTouch = now; 1166 | handleCloseButton(ti.x[0], ti.y[0]); 1167 | } 1168 | } else { 1169 | lastTouchY = 0; 1170 | touchDetected = false; 1171 | } 1172 | 1173 | return; 1174 | } 1175 | 1176 | // Handle keyboard 1177 | if (keyboardActive) { 1178 | if (currentTouch && !keyTouchActive) { 1179 | processKeyboardTouch(ti.x[0], ti.y[0]); 1180 | keyTouchActive = true; 1181 | } else if (!currentTouch) { 1182 | keyTouchActive = false; 1183 | } 1184 | return; 1185 | } 1186 | 1187 | // Live feed 1188 | if (liveFeedActive) { 1189 | if (liveFb != NULL) { 1190 | esp_camera_fb_return(liveFb); 1191 | liveFb = NULL; 1192 | } 1193 | 1194 | camera_fb_t *fb = esp_camera_fb_get(); 1195 | if (fb && fb->format == PIXFORMAT_RGB565 && fb->width == 240 && fb->height == 240) { 1196 | liveFb = fb; 1197 | gfx->draw16bitBeRGBBitmap(0, 0, (uint16_t *)fb->buf, fb->width, fb->height); 1198 | drawUI(); 1199 | } else if (fb) { 1200 | esp_camera_fb_return(fb); 1201 | } 1202 | } 1203 | 1204 | if (currentTouch && !touchDetected && now - lastTouch > 500) { 1205 | touchDetected = true; 1206 | lastTouch = now; 1207 | 1208 | int buttonPressed = checkButtonPress(ti.x[0], ti.y[0]); 1209 | switch (buttonPressed) { 1210 | case 1: 1211 | liveFeedActive = !liveFeedActive; 1212 | Serial.println(liveFeedActive ? "Live feed resumed" : "Live feed paused"); 1213 | 1214 | if (!liveFeedActive) { 1215 | if (liveFb) { 1216 | storeCopyOfFrame(liveFb); 1217 | esp_camera_fb_return(liveFb); 1218 | liveFb = NULL; 1219 | } 1220 | if (lastFrame) { 1221 | gfx->draw16bitBeRGBBitmap(0, 0, (uint16_t *)lastFrame->buf, 240, 240); 1222 | } 1223 | } else { 1224 | camera_fb_t *fb = esp_camera_fb_get(); 1225 | if (fb && fb->format == PIXFORMAT_RGB565 && fb->width == 240 && fb->height == 240) { 1226 | liveFb = fb; 1227 | gfx->draw16bitBeRGBBitmap(0, 0, (uint16_t *)fb->buf, fb->width, fb->height); 1228 | } else if (fb) { 1229 | esp_camera_fb_return(fb); 1230 | } 1231 | } 1232 | 1233 | drawUI(); 1234 | break; 1235 | 1236 | case 2: 1237 | if (liveFeedActive) { 1238 | captureImage(); 1239 | } else { 1240 | showStatusMessage("Resume feed first", YELLOW); 1241 | drawUI(); 1242 | } 1243 | break; 1244 | } 1245 | } 1246 | 1247 | if (!currentTouch) { 1248 | touchDetected = false; 1249 | } 1250 | 1251 | delay(30); 1252 | } 1253 | --------------------------------------------------------------------------------