├── README.md ├── LICENSE ├── server └── tcp server.js └── client └── stream_desktop.ino /README.md: -------------------------------------------------------------------------------- 1 | # esp32-stream-desktop 2 | 3 | Project for TTGO T-Display that streams live video of desktop. Capable of 35+ fps @ 240x135 depending on image quality. 4 | Left button toggles FPS counter, right button flips display. Press both to open and close the brightness menu. 5 | 6 | At the top of the client/stream_desktop.ino file set your WIFI_SSID, WIFI_PASSWORD, and HOST_IP definitions. 7 | 8 | Server is very inefficient, I suspect that it may be a bottleneck. If you're getting poor FPS, delayed video, or stutters your PC may be unable to keep up. Increase the time on the server's screenCapIntervalServer variable. 9 | 10 | The couple stutters you get a minute are probably related to the server as well. 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2021, Steve5451 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /server/tcp server.js: -------------------------------------------------------------------------------- 1 | 'use strict'; // this server is a placeholder until it's rewritten in C# due to the lack of good screenshot/recording libraries 2 | 3 | // fps will depend on your system. if your esp32 has large latency you'll have to increase screenCapInterval. doing so will limit your maximum fps. 4 | // designed to have only 1 client 5 | 6 | const net = require("net"), 7 | { performance } = require("perf_hooks"), 8 | screenshot = require("screenshot-desktop"), // this screenshot library is horrible but it's all i could find. an executable is ran each time which takes about 100ms on my system. might benefit from SSD. 9 | sharp = require("sharp"), 10 | footer = Buffer.from([0x55, 0x44, 0x55, 0x11]), // both jpeg footer and incoming frame request from client 11 | port = 5451, 12 | resolution = { x: 240, y: 135 }, 13 | jpegQuality = 50, // 0-100, higher is better. 50 gets 30-40fps. 70 will average 25-35. 14 | screenCapInterval = 25; 15 | 16 | var screenshotBuffer = { time: null, data: null }, 17 | lastSentTime = null, 18 | frameRequestQueue = false, 19 | connection; 20 | 21 | setInterval(captureScreenshot, screenCapInterval); // fetch many consecutive screenshots as a dirty workaround to the massive latency of the screenshot library. 22 | 23 | async function captureScreenshot() { 24 | var startTime = performance.now(); 25 | 26 | var screen = await screenshot({ format: "jpeg" }); 27 | 28 | var width = resolution.x, 29 | height = Math.floor(width * 0.5625); 30 | 31 | var img = await sharp(screen); 32 | await img.resize(240, 135); 33 | await img.jpeg({ 34 | quality: jpegQuality, 35 | optimiseCoding: true, 36 | trellisQuantisation: true, 37 | overshootDeringing: true, 38 | chromaSubsampling: "4:2:0", 39 | progressive: false, 40 | mozjpeg: false, 41 | force: true 42 | }); 43 | 44 | if(startTime > screenshotBuffer.time || !screenshotBuffer.data) { 45 | screenshotBuffer = { time: startTime, data: await img.toBuffer() }; 46 | 47 | if(frameRequestQueue && connection) { 48 | frameRequestQueue = false; 49 | lastSentTime = startTime; 50 | 51 | getImage().then(buffer => connection.write(buffer, undefined, function() { 52 | //console.log("Frame sent") 53 | })); 54 | } 55 | } else { 56 | //console.log("Rejected old frame"); 57 | } 58 | } 59 | 60 | const server = net.createServer(socket => { 61 | socket.setNoDelay(true); 62 | 63 | console.log("Client connected."); 64 | connection = socket; 65 | 66 | socket.on("data", data => { 67 | if(data.length === 4 && data[0] == footer[0] && data[1] == footer[1] && data[2] == footer[2] && data[3] == footer[3]) { // client is requesting new frame 68 | if(screenshotBuffer.time && screenshotBuffer.data && screenshotBuffer.time > lastSentTime) { 69 | lastSentTime = screenshotBuffer.time; 70 | 71 | getImage().then(buffer => socket.write(buffer, undefined, function() { 72 | //console.log("Frame sent") 73 | })); 74 | } else { 75 | frameRequestQueue = true; 76 | //console.log("Frame queued"); 77 | } 78 | } 79 | }); 80 | 81 | if(screenshotBuffer.data) { 82 | getImage().then(buffer => socket.write(buffer, undefined, function() { 83 | //console.log("Frame sent") 84 | })); 85 | } else { 86 | frameRequestQueue = true; 87 | } 88 | 89 | socket.on("end", () => { 90 | connection = null; 91 | console.log("Client disconnected") 92 | }); 93 | 94 | socket.on("error", (err) => { 95 | if(err.code === "ECONNRESET") { 96 | //connection = null; 97 | console.log("Client connection reset"); 98 | } else { 99 | console.error(err); 100 | } 101 | }); 102 | }); 103 | 104 | async function getImage() { 105 | var newBuffer = await Buffer.concat([screenshotBuffer.data, footer]); // append footer to signal end of data. 106 | 107 | return newBuffer; 108 | } 109 | 110 | server.on("error", err => { 111 | throw err; 112 | }); 113 | 114 | server.listen(port, () => { 115 | console.log("Server running"); 116 | }); 117 | -------------------------------------------------------------------------------- /client/stream_desktop.ino: -------------------------------------------------------------------------------- 1 | #pragma GCC push_options 2 | #pragma GCC optimize ("O3") // O3 boosts fps by 20% 3 | 4 | #define WIFI_SSID "" 5 | #define WIFI_PASSWORD "" 6 | #define HOST_IP "" 7 | #define HOST_PORT 5451 8 | 9 | #define BUFFER_SIZE 25000 // size of incoming jpeg buffer. can be smaller as each frame is less than 10kb at 50 jpeg quality @ 240x135 10 | #define SENSOR_POLL_INTERVAL 100 11 | #define SCREEN_WIDTH 240 12 | #define SCREEN_HEIGHT 135 13 | #define L_BUTTON_GPIO 0 14 | #define R_BUTTON_GPIO 35 15 | 16 | #include "JPEGDEC.h" 17 | #include 18 | #include 19 | 20 | TFT_eSPI tft = TFT_eSPI(); 21 | WiFiClient client; 22 | JPEGDEC jpeg; 23 | 24 | uint8_t *buffer; 25 | int bufferLength = 0; 26 | const byte requestMessage[] = {0x55, 0x44, 0x55, 0x11}; // request message should probably be longer to avoid a false positive 27 | unsigned long lastUpdate = 0; 28 | volatile int updates = 0; 29 | volatile int fps = 0; 30 | bool LPressed = false; 31 | bool RPressed = false; 32 | volatile bool showFPS = false; 33 | volatile int rotation = 3; 34 | volatile int setRotation = rotation; 35 | bool brightnessMode = false; 36 | int brightness = 10; 37 | //unsigned long lastRequestSent = 0; 38 | bool ignoreButtonPress = false; 39 | volatile bool readyToDraw = false; 40 | TFT_eSprite fpsSprite = TFT_eSprite(&tft); 41 | void *fpsPtr; 42 | unsigned long lastSensorRead = 0; 43 | 44 | struct JPEGData { 45 | uint16_t pPixels[20000]; // jpeg mcu's won't be this big but ram isn't the main constraint here 46 | int x; 47 | int y; 48 | int iWidth; 49 | int iHeight; 50 | }; 51 | 52 | JPEGData *jpegBlock; 53 | 54 | void setup() { 55 | buffer = (uint8_t*)malloc(BUFFER_SIZE * sizeof(uint8_t)); 56 | jpegBlock = (JPEGData*)malloc(sizeof(JPEGData)); 57 | fpsSprite.createSprite(24, 16); 58 | fpsPtr = fpsSprite.getPointer(); 59 | 60 | Serial.begin(115200); // fps is reported over serial 61 | 62 | client = WiFi.begin(WIFI_SSID, WIFI_PASSWORD); // TODO: allow device to reconnect if connection is lost 63 | 64 | pinMode(L_BUTTON_GPIO, INPUT); 65 | pinMode(R_BUTTON_GPIO, INPUT); 66 | 67 | tft.init(); 68 | tft.setRotation(rotation); 69 | tft.setTextSize(2); 70 | tft.fillScreen(TFT_BLACK); 71 | tft.setTextColor(TFT_GREEN); 72 | tft.println("Connecting to Wi-Fi"); 73 | tft.setTextColor(TFT_WHITE); 74 | tft.println(WIFI_SSID); 75 | 76 | fpsSprite.setTextSize(2); 77 | fpsSprite.setTextColor(TFT_GREEN); 78 | 79 | pinMode(TFT_BL, OUTPUT); // this pwm output controls display brightness 80 | ledcSetup(0, 5000, 8); 81 | ledcAttachPin(TFT_BL, 0); 82 | ledcWrite(0, brightness * 25); 83 | 84 | xTaskCreatePinnedToCore( // start second thread while waiting on wifi 85 | drawPixels, 86 | "drawPixels", 87 | 5000, 88 | NULL, 89 | 0, 90 | NULL, 91 | 0); 92 | 93 | while (WiFi.status() != WL_CONNECTED) { 94 | vTaskDelay(50); 95 | } 96 | 97 | tft.fillScreen(TFT_BLACK); 98 | tft.setTextColor(TFT_GREEN); 99 | tft.setCursor(0, 0); 100 | tft.println("Wi-Fi connected"); 101 | 102 | if(client.connect(HOST_IP, HOST_PORT)) { 103 | tft.println("Connected to server"); 104 | } else { 105 | tft.setTextColor(TFT_RED); 106 | tft.println("FAILED TO CONNECT"); 107 | tft.println("TO SERVER"); 108 | } 109 | 110 | renderFPS(); 111 | 112 | lastUpdate = millis(); 113 | } 114 | 115 | void loop() { 116 | int dataAvailible = client.available(); 117 | 118 | if(dataAvailible > 0) { 119 | client.read((uint8_t*)buffer + bufferLength, dataAvailible); 120 | bufferLength += dataAvailible; 121 | 122 | if(bufferLength > 4 && buffer[bufferLength - 4] == 0x55 && buffer[bufferLength - 3] == 0x44 && buffer[bufferLength - 2] == 0x55 && buffer[bufferLength - 1] == 0x11) { // more elegent solution required. if multiple images end up in the buffer the jpeg will likely fail to decode 123 | int dataLength = bufferLength - 5; 124 | bufferLength = 0; 125 | 126 | client.write(requestMessage, 4); // queue up a new frame before decoding the current one 127 | 128 | if(jpeg.openRAM(buffer, dataLength, copyJpegBlock)) { 129 | jpeg.setPixelType(RGB565_BIG_ENDIAN); 130 | 131 | if(jpeg.decode(0, 0, 1)) { 132 | if(brightnessMode) { 133 | while(readyToDraw) { // prevent drawing brightness menu while jpeg mcu is being drawn to display 134 | taskYIELD(); 135 | } 136 | 137 | tft.fillRect(0, 30, SCREEN_WIDTH, 15, TFT_BLACK); // TODO: handle brightness menu like the fps counter is handled to avoid flicker 138 | tft.setCursor(30, 30); 139 | tft.print("Brightness: "); 140 | tft.print(brightness); 141 | } 142 | 143 | updates += 1; 144 | } else { 145 | Serial.println("Could not decode jpeg"); 146 | } 147 | } else { 148 | Serial.println("Could not open jpeg"); 149 | } 150 | } 151 | } /*else { 152 | if(lastRequestSent > 0 && millis() + 10000 > lastRequestSent) { 153 | client.write(requestMessage, 4); // request new image after timeout 154 | lastRequestSent = millis(); 155 | } 156 | }*/ 157 | 158 | unsigned long time = millis(); 159 | 160 | if(time - lastUpdate >= 1000) { 161 | float overtime = float(time - lastUpdate) / 1000.0; 162 | fps = floor((float)updates / overtime); 163 | 164 | lastUpdate = time; 165 | updates = 0; 166 | 167 | renderFPS(); 168 | 169 | Serial.print("FPS: "); 170 | Serial.println(fps); 171 | } 172 | } 173 | 174 | void renderFPS() { 175 | fpsSprite.fillRect(0, 0, fpsSprite.width(), fpsSprite.height(), TFT_BLACK); 176 | fpsSprite.setCursor(0, 0); 177 | fpsSprite.print(fps); 178 | } 179 | 180 | void drawPixels(void* pvParameters) { 181 | for(;;) { 182 | if(readyToDraw) { 183 | if(showFPS && jpegBlock->x < 24 && jpegBlock->y < 16) { 184 | placeImageData(jpegBlock->pPixels, fpsPtr, 0, 0, (int)fpsSprite.width(), (int)fpsSprite.height(), jpegBlock->iWidth, jpegBlock->iHeight, min(jpegBlock->iWidth, (int)fpsSprite.width())); 185 | } 186 | 187 | tft.pushImage(jpegBlock->x, jpegBlock->y, jpegBlock->iWidth, jpegBlock->iHeight, jpegBlock->pPixels); 188 | 189 | readyToDraw = false; 190 | 191 | if(setRotation != rotation) { 192 | tft.setRotation(rotation); 193 | setRotation = rotation; 194 | } 195 | } 196 | 197 | unsigned long time = millis(); 198 | 199 | if(time - lastSensorRead >= SENSOR_POLL_INTERVAL) { 200 | lastSensorRead = time; 201 | 202 | bool LStatus = digitalRead(L_BUTTON_GPIO) == LOW; 203 | bool RStatus = digitalRead(R_BUTTON_GPIO) == LOW; 204 | 205 | if((LStatus || LPressed) && (RStatus || RPressed) && !ignoreButtonPress) { 206 | ignoreButtonPress = true; 207 | brightnessMode = !brightnessMode; 208 | LPressed = false; 209 | RPressed = false; 210 | 211 | vTaskDelay(200); 212 | continue; 213 | } 214 | 215 | if(LStatus == false && LPressed == true && !ignoreButtonPress) { 216 | if(brightnessMode == true) { 217 | changeBrightness(1); 218 | } else { 219 | showFPS = !showFPS; 220 | } 221 | } 222 | 223 | if(RStatus == false && RPressed == true && !ignoreButtonPress) { 224 | if(brightnessMode == true) { 225 | changeBrightness(-1); 226 | } else { 227 | if(rotation == 3) { 228 | rotation = 1; 229 | } else { 230 | rotation = 3; 231 | } 232 | } 233 | } 234 | 235 | if(ignoreButtonPress && !LStatus && !RStatus && !LPressed && !RPressed) { 236 | ignoreButtonPress = false; 237 | } 238 | 239 | LPressed = LStatus; 240 | RPressed = RStatus; 241 | } 242 | 243 | //taskYIELD(); 244 | } 245 | } 246 | 247 | void placeImageData(void* destination, void* source, int x, int y, int sourceWidth, int sourceHeight, int destinationWidth, int destinationHeight, int copyWidth) { 248 | for(int row = 0; row < sourceHeight; ++row) { 249 | int index = ((y * SCREEN_WIDTH) + ((row * destinationWidth) + x)) * 2; 250 | memcpy(destination + index, source + (row * sourceWidth * 2), copyWidth * 2); 251 | } 252 | } 253 | 254 | int copyJpegBlock(JPEGDRAW *pDraw) { 255 | for(;;) { 256 | while(readyToDraw) { 257 | taskYIELD(); 258 | } 259 | 260 | memcpy(jpegBlock->pPixels, pDraw->pPixels, pDraw->iWidth * pDraw->iHeight * sizeof(uint16_t)); 261 | jpegBlock->x = pDraw->x; 262 | jpegBlock->y = pDraw->y; 263 | jpegBlock->iWidth = pDraw->iWidth; 264 | jpegBlock->iHeight = pDraw->iHeight; 265 | 266 | readyToDraw = true; 267 | 268 | //taskYIELD(); 269 | return 1; 270 | } 271 | } 272 | 273 | void changeBrightness(int amount) { 274 | int newBrightness = brightness; 275 | newBrightness += amount; 276 | 277 | if(newBrightness > 10) { 278 | newBrightness = 10; 279 | } else if(newBrightness < 1) { 280 | newBrightness = 1; 281 | } 282 | 283 | brightness = newBrightness; 284 | ledcSetup(0, 5000, 8); 285 | ledcAttachPin(TFT_BL, 0); 286 | ledcWrite(0, brightness * 25); 287 | } 288 | 289 | #pragma GCC pop_options 290 | --------------------------------------------------------------------------------