├── README.md ├── doc ├── WT32-SC01.pdf ├── demo1.png ├── diagram.pdf ├── diagram.png ├── image1.jpg └── image2.jpg ├── firmware └── micropython_lvgl_esp32.bin └── src ├── codex.py ├── display_driver.py ├── ft6x36.py ├── google.py ├── hal.py ├── logging.py ├── main.py ├── prompt.py ├── prompt.txt ├── record.wav ├── recorder.py ├── secrets.py ├── st7796s.py ├── urequests.py ├── user_interface.py ├── wave.py ├── whisper.py └── wifi.py /README.md: -------------------------------------------------------------------------------- 1 | esp32_ai_assistant 2 | ================== 3 | 4 | https://hackaday.io/project/188303-esp32-ai-assistant 5 | 6 | ![](doc/image1.jpg) 7 | ![](doc/diagram.png) 8 | ![](doc/demo1.png) -------------------------------------------------------------------------------- /doc/WT32-SC01.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jgpeiro/esp32_ai_assistant/32344095f781eda23533fcc08a7f78b31232bb6d/doc/WT32-SC01.pdf -------------------------------------------------------------------------------- /doc/demo1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jgpeiro/esp32_ai_assistant/32344095f781eda23533fcc08a7f78b31232bb6d/doc/demo1.png -------------------------------------------------------------------------------- /doc/diagram.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jgpeiro/esp32_ai_assistant/32344095f781eda23533fcc08a7f78b31232bb6d/doc/diagram.pdf -------------------------------------------------------------------------------- /doc/diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jgpeiro/esp32_ai_assistant/32344095f781eda23533fcc08a7f78b31232bb6d/doc/diagram.png -------------------------------------------------------------------------------- /doc/image1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jgpeiro/esp32_ai_assistant/32344095f781eda23533fcc08a7f78b31232bb6d/doc/image1.jpg -------------------------------------------------------------------------------- /doc/image2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jgpeiro/esp32_ai_assistant/32344095f781eda23533fcc08a7f78b31232bb6d/doc/image2.jpg -------------------------------------------------------------------------------- /firmware/micropython_lvgl_esp32.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jgpeiro/esp32_ai_assistant/32344095f781eda23533fcc08a7f78b31232bb6d/firmware/micropython_lvgl_esp32.bin -------------------------------------------------------------------------------- /src/codex.py: -------------------------------------------------------------------------------- 1 | import urequests as requests 2 | import ujson as json 3 | 4 | def complete( key, prompt, max_tokens, stop ): 5 | url = "https://api.openai.com/v1/completions" 6 | headers = { 7 | "Content-Type": "application/json", 8 | "Authorization" : "Bearer " + key 9 | } 10 | data = { 11 | "model": "code-davinci-002", 12 | "prompt": prompt, 13 | "temperature": 0.5, 14 | "max_tokens": max_tokens, 15 | "top_p": 1, 16 | "frequency_penalty": 0.25, 17 | "presence_penalty": 0.25, 18 | "stop": stop 19 | } 20 | 21 | resp = requests.post( 22 | url, 23 | headers=headers, 24 | data=json.dumps(data) 25 | ) 26 | 27 | if( resp.status_code != 200 ): 28 | print( "resp.status_code != 200", resp.status_code ) 29 | print( "resp.content", resp.content ) 30 | return None, None 31 | 32 | completion = resp.json()["choices"][0]["text"] 33 | status = resp.json()["choices"][0]["finish_reason"] 34 | return status, completion -------------------------------------------------------------------------------- /src/display_driver.py: -------------------------------------------------------------------------------- 1 | import time 2 | import lvgl as lv 3 | 4 | class Display_Driver: 5 | def __init__( self, lcd, tsc ): 6 | self.lcd = lcd 7 | self.tsc = tsc 8 | 9 | self.fb1 = None 10 | self.fb2 = None 11 | 12 | self.disp_draw_buf = None 13 | self.disp_drv = None 14 | self.indev_drv = None 15 | 16 | self.is_fb1 = True 17 | 18 | self.x_bck = 0 19 | self.y_bck = 0 20 | self.p_bck = 0 21 | 22 | self.initialize() 23 | self.last_tick = time.ticks_ms() 24 | 25 | def initialize( self ): 26 | FB_HEIGHT = self.lcd.HEIGHT//8 27 | self.fb1 = bytearray( self.lcd.WIDTH*FB_HEIGHT*lv.color_t.__SIZE__ ) 28 | self.fb2 = bytearray( self.lcd.WIDTH*FB_HEIGHT*lv.color_t.__SIZE__ ) 29 | 30 | self.disp_draw_buf = lv.disp_draw_buf_t() 31 | self.disp_draw_buf.init( self.fb1, self.fb2, len( self.fb1 )//lv.color_t.__SIZE__ ) 32 | 33 | self.disp_drv = lv.disp_drv_t() 34 | self.disp_drv.init() 35 | self.disp_drv.draw_buf = self.disp_draw_buf 36 | self.disp_drv.flush_cb = self.disp_drv_flush_cb 37 | self.disp_drv.hor_res = self.lcd.WIDTH 38 | self.disp_drv.ver_res = self.lcd.HEIGHT 39 | self.disp_drv.color_format = lv.COLOR_FORMAT.NATIVE_REVERSE 40 | self.disp_drv.register() 41 | 42 | self.indev_drv = lv.indev_drv_t() 43 | self.indev_drv.init() 44 | self.indev_drv.type = lv.INDEV_TYPE.POINTER 45 | self.indev_drv.read_cb = self.indev_drv_read_cb 46 | self.indev_drv.register() 47 | 48 | def disp_drv_flush_cb( self, disp_drv, area, color ): 49 | if( self.is_fb1 ): 50 | fb = memoryview( self.fb1 ) 51 | else: 52 | fb = memoryview( self.fb2 ) 53 | self.is_fb1 = not self.is_fb1 54 | x = area.x1 55 | y = area.y1 56 | w = area.x2 - area.x1 + 1 57 | h = area.y2 - area.y1 + 1 58 | self.lcd.draw( x, y, w, h, fb[0:w*h*lv.color_t.__SIZE__] ) 59 | self.disp_drv.flush_ready() 60 | 61 | def indev_drv_read_cb( self, indev_drv, data ): 62 | p, x, y = self.tsc.read() 63 | if( p ): 64 | self.x_bck = x 65 | self.y_bck = y 66 | self.s_bck = p 67 | 68 | data.point.x = int( self.x_bck ) 69 | data.point.y = int( self.y_bck ) 70 | data.state = int( self.s_bck ) 71 | return False 72 | 73 | def process( self ): 74 | tick = time.ticks_ms() 75 | lv.tick_inc( tick - self.last_tick ) 76 | lv.task_handler() 77 | self.last_tick = tick 78 | -------------------------------------------------------------------------------- /src/ft6x36.py: -------------------------------------------------------------------------------- 1 | import time 2 | import machine 3 | 4 | class Ft6x36: 5 | SLAVE_ADDR = 0x38 6 | 7 | STATUS_REG = 0x02 8 | P1_XH_REG = 0x03 9 | 10 | def __init__( self, i2c, ax=1, bx=0, ay=1, by=0, swap_xy=False ): 11 | self.i2c = i2c 12 | self.ax = ax 13 | self.bx = bx 14 | self.ay = ay 15 | self.by = by 16 | self.swap_xy = swap_xy 17 | self.read_buffer1 = bytearray(1) 18 | self.read_buffer4 = bytearray(4) 19 | 20 | def read( self ): 21 | self.i2c.readfrom_mem_into( self.SLAVE_ADDR, self.STATUS_REG, self.read_buffer1 ) 22 | points = self.read_buffer1[0] & 0x0F 23 | if( points == 1 ): 24 | time.sleep_ms(1) 25 | # Read again to avoid glitches 26 | self.i2c.readfrom_mem_into( self.SLAVE_ADDR, self.STATUS_REG, self.read_buffer1 ) 27 | points = self.read_buffer1[0] & 0x0F 28 | if( points == 1 ): 29 | self.i2c.readfrom_mem_into( self.SLAVE_ADDR, self.P1_XH_REG, self.read_buffer4 ) 30 | x = (self.read_buffer4[0] << 8 | self.read_buffer4[1]) & 0x0FFF 31 | y = (self.read_buffer4[2] << 8 | self.read_buffer4[3]) & 0x0FFF 32 | 33 | if( self.swap_xy ): 34 | tmp = x 35 | x = y 36 | y = tmp 37 | 38 | x = self.ax*x + self.bx 39 | y = self.ay*y + self.by 40 | return 1, x, y 41 | else: 42 | return 0, 0, 0 43 | else: 44 | return 0, 0, 0 -------------------------------------------------------------------------------- /src/google.py: -------------------------------------------------------------------------------- 1 | import time 2 | import urequests as requests 3 | import ujson as json 4 | import ubinascii as binascii 5 | 6 | def replace_non_ascii_chars( text ): 7 | text = text.replace( "á", "a" ) 8 | text = text.replace( "é", "e" ) 9 | text = text.replace( "í", "i" ) 10 | text = text.replace( "ó", "o" ) 11 | text = text.replace( "ú", "u" ) 12 | text = text.replace( "ñ", "n" ) 13 | 14 | text = text.replace( "Á", "A" ) 15 | text = text.replace( "É", "E" ) 16 | text = text.replace( "Í", "I" ) 17 | text = text.replace( "Ó", "O" ) 18 | text = text.replace( "Ú", "U" ) 19 | text = text.replace( "Ñ", "N" ) 20 | return text 21 | 22 | def recognize( key, fl_name ): 23 | url_recognize = "https://speech.googleapis.com/v1/speech:recognize?key=" + key 24 | with open( fl_name, "rb" ) as fl: 25 | buf = fl.read() 26 | headers = { "Content-Type": "application/json" } 27 | data = { 28 | "config": { 29 | "encoding": "LINEAR16", 30 | "sampleRateHertz": 8000, 31 | "languageCode": "es-ES" 32 | #"languageCode": "en-US" 33 | }, 34 | "audio": { 35 | "content": binascii.b2a_base64( buf )[:-1].decode( "utf-8" ) 36 | } 37 | } 38 | resp = requests.post( url_recognize, headers=headers, json=data ) 39 | if( resp.status_code != 200 ): 40 | print( "resp.status_code != 200", resp.status_code ) 41 | print( "resp.content", resp.content ) 42 | return None 43 | 44 | results = resp.json()["results"] 45 | transcript = "".join([result["alternatives"][0]["transcript"] for result in results]) 46 | return transcript 47 | 48 | def translate( key, text ): 49 | text = replace_non_ascii_chars( text ) 50 | url = "https://translation.googleapis.com/language/translate/v2?key=" + key 51 | data = { 52 | "q": text, 53 | #"source": "es", 54 | "target": "en", 55 | "format": "text", 56 | "key": key 57 | } 58 | resp = requests.post( url, json=data ) 59 | if( resp.status_code != 200 ): 60 | print( "resp.status_code != 200", resp.status_code ) 61 | print( "resp.content", resp.content ) 62 | return None 63 | 64 | return resp.json()["data"]["translations"][0]["translatedText"] 65 | 66 | -------------------------------------------------------------------------------- /src/hal.py: -------------------------------------------------------------------------------- 1 | import machine 2 | import st7796s 3 | import ft6x36 4 | 5 | # LEDs 6 | led_green = machine.Pin( 27, machine.Pin.OUT, value=0 ) 7 | led_red = machine.Pin( 12, machine.Pin.OUT, value=0 ) 8 | 9 | # Buttons 10 | btn_left = machine.Pin( 2, machine.Pin.IN, machine.Pin.PULL_UP ) 11 | btn_right = machine.Pin( 4, machine.Pin.IN, machine.Pin.PULL_UP ) 12 | 13 | # LCD Init 14 | LCD_SPI = 2 15 | LCD_SPI_BAUDRATE = 20_000_000 16 | 17 | lcd_rst= machine.Pin( 22, machine.Pin.OUT, value=0 ) 18 | lcd_cs = machine.Pin( 15, machine.Pin.OUT, value=1 ) 19 | lcd_dc = machine.Pin( 21, machine.Pin.OUT, value=0 ) 20 | lcd_bl = machine.Pin( 23, machine.Pin.OUT, value=0 ) 21 | lcd_sck = machine.Pin( 14, machine.Pin.OUT ) 22 | lcd_mosi = machine.Pin( 13, machine.Pin.OUT ) 23 | lcd_miso = None 24 | 25 | lcd_spi = machine.SPI( 26 | LCD_SPI, 27 | LCD_SPI_BAUDRATE, 28 | polarity=0, 29 | phase=0, 30 | sck=lcd_sck, 31 | mosi=lcd_mosi, 32 | miso=lcd_miso, 33 | ) 34 | lcd = st7796s.St7796s( lcd_spi, lcd_rst, lcd_cs, lcd_dc, lcd_bl ) 35 | 36 | # TSC Init 37 | TSC_I2C_ID = 1 38 | TSC_I2C_BAUDRATE = 400_000 39 | tsc_sda = machine.Pin( 18 ) 40 | tsc_scl = machine.Pin( 19 ) 41 | tsc_i2c = machine.I2C( 42 | TSC_I2C_ID, 43 | freq=TSC_I2C_BAUDRATE, 44 | sda=tsc_sda, 45 | scl=tsc_scl 46 | ) 47 | 48 | TSC_CALIB_AX, TSC_CALIB_BX =-1.000, 480.0 49 | TSC_CALIB_AY, TSC_CALIB_BY = 0.956, 6.533 50 | tsc = ft6x36.Ft6x36( 51 | tsc_i2c, 52 | ax=TSC_CALIB_AX, 53 | bx=TSC_CALIB_BX, 54 | ay=TSC_CALIB_AY, 55 | by=TSC_CALIB_BY, 56 | swap_xy=True 57 | ) 58 | 59 | 60 | # I2S init 61 | I2S_ID = 0 62 | I2S_BAUDRATE = 8000 63 | I2S_BUF_SIZE = 8192 64 | I2S_BITS = 16 65 | i2s_sck = machine.Pin(0) 66 | i2s_ws = machine.Pin(5) 67 | i2s_sd = machine.Pin(35) 68 | i2s_lr = machine.Pin(25, machine.Pin.OUT, value=0 ) 69 | 70 | mic = machine.I2S( 71 | I2S_ID, 72 | sck=i2s_sck, 73 | ws=i2s_ws, 74 | sd=i2s_sd, 75 | mode=machine.I2S.RX, 76 | bits=I2S_BITS, 77 | format=machine.I2S.MONO, 78 | rate=I2S_BAUDRATE, 79 | ibuf=I2S_BUF_SIZE, 80 | ) 81 | 82 | #print( "hal done" ) -------------------------------------------------------------------------------- /src/logging.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | CRITICAL = 50 4 | ERROR = 40 5 | WARNING = 30 6 | INFO = 20 7 | DEBUG = 10 8 | NOTSET = 0 9 | 10 | _level_dict = { 11 | CRITICAL: "CRIT", 12 | ERROR: "ERROR", 13 | WARNING: "WARN", 14 | INFO: "INFO", 15 | DEBUG: "DEBUG", 16 | } 17 | 18 | _stream = sys.stderr 19 | 20 | 21 | class LogRecord: 22 | def __init__(self): 23 | self.__dict__ = {} 24 | 25 | def __getattr__(self, key): 26 | return self.__dict__[key] 27 | 28 | 29 | class Handler: 30 | def __init__(self): 31 | pass 32 | 33 | def setFormatter(self, fmtr): 34 | pass 35 | 36 | 37 | class Logger: 38 | 39 | level = NOTSET 40 | handlers = [] 41 | record = LogRecord() 42 | 43 | def __init__(self, name): 44 | self.name = name 45 | 46 | def _level_str(self, level): 47 | l = _level_dict.get(level) 48 | if l is not None: 49 | return l 50 | return "LVL%s" % level 51 | 52 | def setLevel(self, level): 53 | self.level = level 54 | 55 | def isEnabledFor(self, level): 56 | return level >= (self.level or _level) 57 | 58 | def log(self, level, msg, *args): 59 | if self.isEnabledFor(level): 60 | levelname = self._level_str(level) 61 | if args: 62 | msg = msg % args 63 | if self.handlers: 64 | d = self.record.__dict__ 65 | d["levelname"] = levelname 66 | d["levelno"] = level 67 | d["message"] = msg 68 | d["name"] = self.name 69 | for h in self.handlers: 70 | h.emit(self.record) 71 | else: 72 | print(levelname, ":", self.name, ":", msg, sep="", file=_stream) 73 | 74 | def debug(self, msg, *args): 75 | self.log(DEBUG, msg, *args) 76 | 77 | def info(self, msg, *args): 78 | self.log(INFO, msg, *args) 79 | 80 | def warning(self, msg, *args): 81 | self.log(WARNING, msg, *args) 82 | 83 | def error(self, msg, *args): 84 | self.log(ERROR, msg, *args) 85 | 86 | def critical(self, msg, *args): 87 | self.log(CRITICAL, msg, *args) 88 | 89 | def exc(self, e, msg, *args): 90 | self.log(ERROR, msg, *args) 91 | sys.print_exception(e, _stream) 92 | 93 | def exception(self, msg, *args): 94 | self.exc(sys.exc_info()[1], msg, *args) 95 | 96 | def addHandler(self, hndlr): 97 | self.handlers.append(hndlr) 98 | 99 | 100 | _level = INFO 101 | _loggers = {} 102 | 103 | 104 | def getLogger(name="root"): 105 | if name in _loggers: 106 | return _loggers[name] 107 | l = Logger(name) 108 | _loggers[name] = l 109 | return l 110 | 111 | 112 | def info(msg, *args): 113 | getLogger().info(msg, *args) 114 | 115 | 116 | def debug(msg, *args): 117 | getLogger().debug(msg, *args) 118 | 119 | 120 | def basicConfig(level=INFO, filename=None, stream=None, format=None): 121 | global _level, _stream 122 | _level = level 123 | if stream: 124 | _stream = stream 125 | if filename is not None: 126 | print("logging.basicConfig: filename arg is not supported") 127 | if format is not None: 128 | print("logging.basicConfig: format arg is not supported") -------------------------------------------------------------------------------- /src/main.py: -------------------------------------------------------------------------------- 1 | import time 2 | import machine 3 | import hal 4 | 5 | import lvgl as lv 6 | import display_driver 7 | import user_interface 8 | import prompt 9 | 10 | import secrets 11 | import wifi 12 | import google 13 | import whisper 14 | import codex 15 | 16 | import wave 17 | import recorder 18 | 19 | 20 | use_whisper = False 21 | 22 | class Application: 23 | FL_WAV_NAME = "record.wav" 24 | FL_PROMPT_NAME = "prompt.txt" 25 | 26 | def __init__( self ): 27 | lv.init() 28 | self.display_driver = display_driver.Display_Driver( hal.lcd, hal.tsc ) 29 | self.ui = user_interface.User_Interface( self.cb_record, self.cb_codex, self.cb_run ) 30 | with open( self.FL_PROMPT_NAME ) as fl: 31 | self.prompt = prompt.Prompt( fl.read() ) 32 | self.ui.set_terminal( self.prompt.colorize(True) ) 33 | self.prompt.run(True) 34 | self.recorder = recorder.Recorder( hal.mic, self.FL_WAV_NAME ) 35 | self.wifi = wifi.Wifi() 36 | 37 | def cb_record( self, evt ): 38 | print("cb_record") 39 | if( self.ui.button_record.get_state() & lv.STATE.CHECKED ): 40 | self.recorder.start() 41 | 42 | self.ui.button_record.clear_state( lv.STATE.DISABLED ) 43 | self.ui.button_codex.add_state( lv.STATE.DISABLED ) 44 | self.ui.button_run.add_state( lv.STATE.DISABLED ) 45 | else: 46 | self.recorder.stop() 47 | 48 | try: 49 | if( use_whisper ): 50 | t0 = time.ticks_ms() 51 | with open( self.FL_WAV_NAME, "rb" ) as fl: 52 | buf = fl.read() 53 | text = whisper.transcribe( buf=buf ) 54 | t1 = time.ticks_ms() 55 | print( "transcribed", t1-t0, text ) 56 | else: 57 | # Option 2. Google AI 58 | t0 = time.ticks_ms() 59 | text = google.recognize( secrets.GOOGLE_KEY, self.FL_WAV_NAME ) 60 | t1 = time.ticks_ms() 61 | print( "recognized", t1-t0, text ) 62 | if( not text ): 63 | return 64 | 65 | t0 = time.ticks_ms() 66 | text = google.translate( secrets.GOOGLE_KEY, text ) 67 | t1 = time.ticks_ms() 68 | print( "translated", t1-t0, text ) 69 | if( not text ): 70 | return 71 | except Exception as e: 72 | print("except", e ) 73 | return 74 | self.prompt.add_human( text ) 75 | self.ui.set_terminal( self.prompt.colorize(True) ) 76 | 77 | self.ui.button_record.clear_state( lv.STATE.DISABLED ) 78 | self.ui.button_codex.clear_state( lv.STATE.DISABLED ) 79 | self.ui.button_run.add_state( lv.STATE.DISABLED ) 80 | 81 | def cb_codex( self, evt ): 82 | print("cb_codex") 83 | # OpenAI Codex AI 84 | t0 = time.ticks_ms() 85 | status, text = codex.complete( secrets.OPENAI_KEY, self.prompt.text, 1000, [self.prompt.HUMAN_TURN] ) 86 | t1 = time.ticks_ms() 87 | print( "codex", t1-t0, text ) 88 | self.prompt.add_codex( text ) 89 | self.ui.set_terminal( self.prompt.colorize(False) ) 90 | 91 | self.ui.button_record.add_state( lv.STATE.DISABLED ) 92 | self.ui.button_codex.clear_state( lv.STATE.DISABLED ) 93 | self.ui.button_run.clear_state( lv.STATE.DISABLED ) 94 | 95 | def cb_run( self, evt ): 96 | print("cb_run") 97 | self.prompt.run() 98 | 99 | self.ui.button_record.clear_state( lv.STATE.DISABLED ) 100 | self.ui.button_codex.add_state( lv.STATE.DISABLED ) 101 | self.ui.button_run.clear_state( lv.STATE.DISABLED ) 102 | 103 | def run( self ): 104 | self.wifi.connect( secrets.WIFI_SSID, secrets.WIFI_PSWD ) 105 | try: 106 | while( True ): 107 | self.display_driver.process() 108 | time.sleep_ms(10) 109 | except Exception as e: 110 | print( "except", e ) 111 | self.wifi.disconnect() 112 | machine.reset() 113 | 114 | app = Application() 115 | app.run() 116 | 117 | print("done") -------------------------------------------------------------------------------- /src/prompt.py: -------------------------------------------------------------------------------- 1 | 2 | class Prompt: 3 | NEW_LINE = "\r\n" # Be sure it matches your text editor 4 | NEW_LINE_LVGL = "\n" 5 | 6 | HUMAN_TURN = "# Human: " # Note all ends with space. This is not mandatory. 7 | CODEX_TURN = "# Esp32: " 8 | REPL_SIGNS = ">>> " 9 | REPL_DOTS = "... " 10 | 11 | START_COLOR_CMD = "$" 12 | END_COLOR_CMD = "$" 13 | 14 | HUMAN_COLOR = "7F0000" # RED" 15 | REPL_COLOR = "444444" # GREY 16 | CODEX_COLOR = "00007F" # BLUE 17 | DEFAULT_COLOR = "000000" # BLACK 18 | 19 | COLON = ":" 20 | 21 | def __init__( self, text ): 22 | self.text = text 23 | 24 | self.text_human = text # Use two temporal prompts allows kind of "retry" operation without duplicate last text. 25 | self.text_codex = text 26 | self.text_codex_last = "" 27 | 28 | def add_human( self, text ): 29 | self.text = self.text_codex 30 | self.text_human = self.text + text + self.NEW_LINE + self.CODEX_TURN 31 | 32 | def add_codex( self, text ): 33 | if( text[-1] == "\n" ): 34 | text = text[:-1] 35 | if( text[-1] == "\r" ): 36 | text = text[:-1] 37 | self.text_codex_last = text + self.NEW_LINE 38 | self.text = self.text_human 39 | self.text_codex = self.text + text + self.NEW_LINE + self.HUMAN_TURN 40 | 41 | def run( self, is_first=False ): 42 | if( is_first ): 43 | text = self.text.split( self.NEW_LINE ) 44 | else: 45 | text = self.text_codex_last.split( self.NEW_LINE ) 46 | try: 47 | lines = [] 48 | for line in text: 49 | line = line.strip() 50 | 51 | if( line.startswith( self.REPL_SIGNS ) ): 52 | if( len( lines ) ): 53 | code = self.NEW_LINE.join( lines ) 54 | exec( code ) 55 | lines = [] 56 | 57 | if( line.endswith( self.COLON ) ): 58 | lines.append( line.replace( self.REPL_SIGNS , "" ) ) 59 | else: 60 | code = line.replace( self.REPL_SIGNS, "" ) 61 | if( code == self.REPL_SIGNS[:-1] ): 62 | continue 63 | exec( code ) 64 | 65 | elif( line.startswith( self.REPL_DOTS ) ): 66 | lines.append( line.replace( self.REPL_DOTS, "" ) ) 67 | 68 | if( len( lines ) ): 69 | code = self.NEW_LINE.join( lines ) 70 | exec( code ) 71 | lines = [] 72 | except Exception as e: 73 | print( "run failed", e ) 74 | 75 | def colorize_line( self, line, color ): 76 | # The space after the color is needed by LVGL 77 | return self.START_COLOR_CMD + color + " " + line + self.END_COLOR_CMD 78 | 79 | def colorize( self, is_human ): 80 | if( is_human ): 81 | text = self.text_human.split( self.NEW_LINE ) 82 | else: 83 | text = self.text_codex.split( self.NEW_LINE ) 84 | 85 | text_color = "" 86 | for line in text: 87 | if line.startswith( self.HUMAN_TURN ): 88 | text_color += self.colorize_line( line, self.HUMAN_COLOR ) 89 | 90 | elif line.startswith( self.REPL_SIGNS ) or line.startswith( self.REPL_DOTS ) : 91 | text_color += self.colorize_line( line, self.REPL_COLOR ) 92 | 93 | elif line.startswith( self.CODEX_TURN ): 94 | text_color += self.colorize_line( line, self.CODEX_COLOR ) 95 | 96 | else: 97 | text_color += self.colorize_line( line, self.DEFAULT_COLOR ) 98 | 99 | text_color += self.NEW_LINE_LVGL 100 | 101 | return text_color 102 | -------------------------------------------------------------------------------- /src/prompt.txt: -------------------------------------------------------------------------------- 1 | # The following is a conversation with an ESP32 AI assistant. 2 | # The assistant is helpful, creative, clever, and very friendly. 3 | # 4 | # Human: Define the HAL. 5 | # Esp32: Ok, here is the code: 6 | >>> import machine 7 | >>> 8 | >>> # LEDs 9 | >>> led_green = machine.Pin( 27, machine.Pin.OUT, value=0 ) 10 | >>> led_red = machine.Pin( 12, machine.Pin.OUT, value=0 ) 11 | >>> 12 | >>> # Buttons 13 | >>> btn_left = machine.Pin( 2, machine.Pin.IN, machine.Pin.PULL_UP ) 14 | >>> btn_right = machine.Pin( 4, machine.Pin.IN, machine.Pin.PULL_UP ) 15 | # Human: Turn on the led green 16 | # Esp32: Ok, here is the code 17 | >>> led_green.value(1) 18 | # Human: -------------------------------------------------------------------------------- /src/record.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jgpeiro/esp32_ai_assistant/32344095f781eda23533fcc08a7f78b31232bb6d/src/record.wav -------------------------------------------------------------------------------- /src/recorder.py: -------------------------------------------------------------------------------- 1 | import time 2 | import array 3 | import wave 4 | 5 | class Recorder: 6 | STATE_IDLE = 0 7 | STATE_START = 1 8 | STATE_RECORD = 2 9 | STATE_STOP = 3 10 | 11 | def __init__( self, i2s, fl_name, buf_len=800 ): 12 | self.i2s = i2s 13 | self.state = self.STATE_IDLE 14 | self.buf = array.array( "h", [0]*buf_len ) 15 | self.buf_mv = memoryview( self.buf ) 16 | self.wave = wave.Wave( fl_name ) 17 | self.t0 = 0 18 | self.t1 = 0 19 | 20 | def start( self ): 21 | self.state = self.STATE_START 22 | self.process( None ) # Start the fsm. This will continue via i2s interrupts 23 | 24 | def stop( self ): 25 | self.state = self.STATE_STOP 26 | while( self.state != self.STATE_IDLE ): 27 | time.sleep_ms( 100 ) 28 | 29 | def process( self, arg ): 30 | if( self.state == self.STATE_IDLE ): 31 | pass 32 | 33 | elif( self.state == self.STATE_START ): 34 | self.wave.open() 35 | self.i2s.irq( self.process ) 36 | self.i2s.readinto( self.buf_mv ) 37 | self.state = self.STATE_RECORD 38 | 39 | elif( self.state == self.STATE_RECORD ): 40 | self.wave.write( self.buf_mv ) 41 | self.i2s.readinto( self.buf_mv ) 42 | 43 | elif( self.state == self.STATE_STOP ): 44 | self.wave.write( self.buf ) 45 | self.i2s.irq( None ) 46 | self.wave.close() 47 | self.state = self.STATE_IDLE -------------------------------------------------------------------------------- /src/secrets.py: -------------------------------------------------------------------------------- 1 | WIFI_SSID = b"" 2 | WIFI_PSWD = b"" 3 | OPENAI_KEY = "" 4 | GOOGLE_KEY = "" -------------------------------------------------------------------------------- /src/st7796s.py: -------------------------------------------------------------------------------- 1 | import time 2 | import machine 3 | 4 | class St7796s: 5 | WIDTH = 480 6 | HEIGHT = 320 7 | 8 | CASET = 0x2A 9 | RASET = 0x2B 10 | RAMWR = 0x2C 11 | 12 | X_OFFSET = 0 13 | Y_OFFSET = 0 14 | 15 | def __init__( self, spi, rst, cs, dc, bl ): 16 | self.spi = spi 17 | self.rst = rst 18 | self.cs = cs 19 | self.dc = dc 20 | self.bl = bl 21 | self.buf1 = bytearray(1) 22 | self.buf4 = bytearray(4) 23 | self.reset() 24 | self.config() 25 | self.clear() 26 | self.bl.value(1) 27 | 28 | def reset( self ): 29 | self.rst.value(0) 30 | self.cs.value(1) 31 | self.dc.value(0) 32 | self.bl.value(0) 33 | time.sleep_ms( 10 ) 34 | 35 | self.rst.value(1) 36 | time.sleep_ms( 100 ) 37 | 38 | def write_reg( self, cmd, buf ): 39 | self.buf1[0] = cmd 40 | 41 | self.dc(0) 42 | self.cs(0) 43 | self.spi.write( self.buf1 ) 44 | if( buf ): 45 | self.dc(1) 46 | self.spi.write( buf ) 47 | self.cs(1) 48 | self.dc(0) 49 | 50 | def config( self ): 51 | self.write_reg( 0x01, b"\x01" ) 52 | time.sleep_ms( 100 ) 53 | self.write_reg( 0x11, b"" ) 54 | time.sleep_ms( 10 ) 55 | self.write_reg( 0x36, b"\xF8" ) # b"\xF0" ) #b"\x70" ) 56 | self.write_reg( 0x3A, b"\x05" ) 57 | self.write_reg( 0xB2, b"\x0C\x0C\x00\x33\x33" ) 58 | self.write_reg( 0xB7, b"\x35" ) 59 | self.write_reg( 0xBB, b"\x19" ) 60 | self.write_reg( 0xC0, b"\x2C" ) 61 | self.write_reg( 0xC2, b"\x01" ) 62 | self.write_reg( 0xC3, b"\x12" ) 63 | self.write_reg( 0xC4, b"\x20" ) 64 | self.write_reg( 0xC6, b"\x0F" ) 65 | self.write_reg( 0xD0, b"\xA4\xA1" ) 66 | self.write_reg( 0xE0, b"\xD0\x04\x0D\x11\x13\x2B\x3F\x54\x4C\x18\x0D\x0B\x1F\x23" ) 67 | self.write_reg( 0xE1, b"\xD0\x04\x0C\x11\x13\x2C\x3F\x44\x51\x2F\x1F\x1F\x20\x23" ) 68 | #self.write_reg( 0x21, b"" ) # Display Inversion On 69 | self.write_reg( 0x11, b"" ) 70 | self.write_reg( 0x29, b"" ) 71 | time.sleep_ms( 100 ) 72 | 73 | def set_window( self, x, y, w, h ): 74 | x0 = x + self.X_OFFSET 75 | y0 = y + self.Y_OFFSET 76 | x1 = x0 + w - 1 77 | y1 = y0 + h - 1 78 | 79 | self.buf4[0] = x0>>8 80 | self.buf4[1] = x0&0xFF 81 | self.buf4[2] = x1>>8 82 | self.buf4[3] = x1&0xFF 83 | self.write_reg( self.CASET, self.buf4 ) 84 | 85 | self.buf4[0] = y0>>8 86 | self.buf4[1] = y0&0xFF 87 | self.buf4[2] = y1>>8 88 | self.buf4[3] = y1&0xFF 89 | self.write_reg( self.RASET, self.buf4 ) 90 | 91 | def draw( self, x, y, w, h, buf ): 92 | self.set_window( x, y, w, h ) 93 | self.write_reg( self.RAMWR, buf ) 94 | 95 | def clear( self ): 96 | buf = bytearray( [0xFF, 0xFF]*self.WIDTH ) 97 | for i in range( self.HEIGHT ): 98 | self.draw( 0, i, self.WIDTH, 1, buf ) 99 | -------------------------------------------------------------------------------- /src/urequests.py: -------------------------------------------------------------------------------- 1 | import usocket 2 | 3 | 4 | class Response: 5 | def __init__(self, f): 6 | self.raw = f 7 | self.encoding = "utf-8" 8 | self._cached = None 9 | 10 | def close(self): 11 | if self.raw: 12 | self.raw.close() 13 | self.raw = None 14 | self._cached = None 15 | 16 | @property 17 | def content(self): 18 | if self._cached is None: 19 | try: 20 | self._cached = self.raw.read() 21 | finally: 22 | self.raw.close() 23 | self.raw = None 24 | return self._cached 25 | 26 | @property 27 | def text(self): 28 | return str(self.content, self.encoding) 29 | 30 | def json(self): 31 | import ujson 32 | 33 | return ujson.loads(self.content) 34 | 35 | 36 | def request( 37 | method, 38 | url, 39 | data=None, 40 | json=None, 41 | headers={}, 42 | stream=None, 43 | auth=None, 44 | timeout=None, 45 | parse_headers=True, 46 | ): 47 | redirect = None # redirection url, None means no redirection 48 | chunked_data = data and getattr(data, "__iter__", None) and not getattr(data, "__len__", None) 49 | 50 | if auth is not None: 51 | import ubinascii 52 | 53 | username, password = auth 54 | formated = b"{}:{}".format(username, password) 55 | formated = str(ubinascii.b2a_base64(formated)[:-1], "ascii") 56 | headers["Authorization"] = "Basic {}".format(formated) 57 | 58 | try: 59 | proto, dummy, host, path = url.split("/", 3) 60 | except ValueError: 61 | proto, dummy, host = url.split("/", 2) 62 | path = "" 63 | if proto == "http:": 64 | port = 80 65 | elif proto == "https:": 66 | import ussl 67 | 68 | port = 443 69 | else: 70 | raise ValueError("Unsupported protocol: " + proto) 71 | 72 | if ":" in host: 73 | host, port = host.split(":", 1) 74 | port = int(port) 75 | 76 | ai = usocket.getaddrinfo(host, port, 0, usocket.SOCK_STREAM) 77 | ai = ai[0] 78 | 79 | resp_d = None 80 | if parse_headers is not False: 81 | resp_d = {} 82 | 83 | s = usocket.socket(ai[0], usocket.SOCK_STREAM, ai[2]) 84 | 85 | if timeout is not None: 86 | # Note: settimeout is not supported on all platforms, will raise 87 | # an AttributeError if not available. 88 | s.settimeout(timeout) 89 | 90 | try: 91 | s.connect(ai[-1]) 92 | if proto == "https:": 93 | s = ussl.wrap_socket(s, server_hostname=host) 94 | s.write(b"%s /%s HTTP/1.0\r\n" % (method, path)) 95 | if not "Host" in headers: 96 | s.write(b"Host: %s\r\n" % host) 97 | # Iterate over keys to avoid tuple alloc 98 | for k in headers: 99 | s.write(k) 100 | s.write(b": ") 101 | s.write(headers[k]) 102 | s.write(b"\r\n") 103 | if json is not None: 104 | assert data is None 105 | import ujson 106 | 107 | data = ujson.dumps(json) 108 | s.write(b"Content-Type: application/json\r\n") 109 | if data: 110 | if chunked_data: 111 | s.write(b"Transfer-Encoding: chunked\r\n") 112 | else: 113 | s.write(b"Content-Length: %d\r\n" % len(data)) 114 | s.write(b"Connection: close\r\n\r\n") 115 | if data: 116 | if chunked_data: 117 | for chunk in data: 118 | s.write(b"%x\r\n" % len(chunk)) 119 | s.write(chunk) 120 | s.write(b"\r\n") 121 | s.write("0\r\n\r\n") 122 | else: 123 | s.write(data) 124 | 125 | l = s.readline() 126 | # print(l) 127 | l = l.split(None, 2) 128 | if len(l) < 2: 129 | # Invalid response 130 | raise ValueError("HTTP error: BadStatusLine:\n%s" % l) 131 | status = int(l[1]) 132 | reason = "" 133 | if len(l) > 2: 134 | reason = l[2].rstrip() 135 | while True: 136 | l = s.readline() 137 | if not l or l == b"\r\n": 138 | break 139 | # print(l) 140 | if l.startswith(b"Transfer-Encoding:"): 141 | if b"chunked" in l: 142 | raise ValueError("Unsupported " + str(l, "utf-8")) 143 | elif l.startswith(b"Location:") and not 200 <= status <= 299: 144 | if status in [301, 302, 303, 307, 308]: 145 | redirect = str(l[10:-2], "utf-8") 146 | else: 147 | raise NotImplementedError("Redirect %d not yet supported" % status) 148 | if parse_headers is False: 149 | pass 150 | elif parse_headers is True: 151 | l = str(l, "utf-8") 152 | k, v = l.split(":", 1) 153 | resp_d[k] = v.strip() 154 | else: 155 | parse_headers(l, resp_d) 156 | except OSError: 157 | s.close() 158 | raise 159 | 160 | if redirect: 161 | s.close() 162 | if status in [301, 302, 303]: 163 | return request("GET", redirect, None, None, headers, stream) 164 | else: 165 | return request(method, redirect, data, json, headers, stream) 166 | else: 167 | resp = Response(s) 168 | resp.status_code = status 169 | resp.reason = reason 170 | if resp_d is not None: 171 | resp.headers = resp_d 172 | return resp 173 | 174 | 175 | def head(url, **kw): 176 | return request("HEAD", url, **kw) 177 | 178 | 179 | def get(url, **kw): 180 | return request("GET", url, **kw) 181 | 182 | 183 | def post(url, **kw): 184 | return request("POST", url, **kw) 185 | 186 | 187 | def put(url, **kw): 188 | return request("PUT", url, **kw) 189 | 190 | 191 | def patch(url, **kw): 192 | return request("PATCH", url, **kw) 193 | 194 | 195 | def delete(url, **kw): 196 | return request("DELETE", url, **kw) 197 | -------------------------------------------------------------------------------- /src/user_interface.py: -------------------------------------------------------------------------------- 1 | import lvgl as lv 2 | 3 | class User_Interface: 4 | def __init__( self, cb_button_record, cb_button_codex, cb_button_run ): 5 | self.cb_button_record = cb_button_record 6 | self.cb_button_codex = cb_button_codex 7 | self.cb_button_run = cb_button_run 8 | 9 | self.label_terminal = None 10 | self.panel_terminal = None 11 | 12 | self.button_whisper = None 13 | self.button_codex = None 14 | self.button_run = None 15 | 16 | self.build_ui() 17 | 18 | def set_terminal( self, text ): 19 | self.label_terminal.set_text( text ) 20 | self.panel_terminal.scroll_to_y( lv.COORD.MAX , lv.ANIM.OFF ) 21 | self.panel_terminal.update_layout() 22 | 23 | def build_ui( self ): 24 | scr = lv.scr_act() 25 | 26 | style = lv.style_t() 27 | style.init() 28 | style.set_pad_all( 2 ) 29 | style.set_pad_gap( 2 ) 30 | style.set_radius( 2 ) 31 | style.set_border_width( 0 ) 32 | style.set_bg_color( lv.palette_lighten( lv.PALETTE.GREY, 3 ) ) 33 | 34 | col = lv.obj( scr ) 35 | col.add_style( style, 0 ) 36 | col.set_flex_flow( lv.FLEX_FLOW.COLUMN ) 37 | col.set_size( lv.pct(100), lv.pct(100) ) 38 | col.clear_flag( lv.obj.FLAG.SCROLLABLE ) 39 | 40 | self.panel_terminal = lv.obj( col ) 41 | self.panel_terminal.set_size( lv.pct(98), lv.pct(80) ) 42 | self.panel_terminal.clear_flag( lv.obj.FLAG.SCROLL_MOMENTUM ) 43 | 44 | self.label_terminal = lv.label( self.panel_terminal ) 45 | self.label_terminal.set_recolor( True) 46 | 47 | row = lv.obj( col ) 48 | row.add_style( style, 0 ) 49 | row.set_flex_flow( lv.FLEX_FLOW.ROW ) 50 | row.set_size(lv.pct(100), lv.pct(100) ) 51 | 52 | self.button_record = lv.btn( row ) 53 | self.button_record.add_flag( lv.obj.FLAG.CHECKABLE ) 54 | self.button_record.set_size( lv.pct(33), lv.pct(18) ) 55 | self.button_record.add_event_cb( self.cb_button_record, lv.EVENT.VALUE_CHANGED, None ) 56 | label = lv.label( self.button_record ) 57 | label.set_text( "Record" ) 58 | label.center() 59 | 60 | self.button_codex = lv.btn( row ) 61 | self.button_codex.set_size( lv.pct(33), lv.pct(18) ) 62 | self.button_codex.add_event_cb( self.cb_button_codex, lv.EVENT.CLICKED, None ) 63 | label = lv.label( self.button_codex ) 64 | label.set_text( "Codex" ) 65 | label.center() 66 | 67 | self.button_run = lv.btn( row ) 68 | self.button_run.set_size( lv.pct(33), lv.pct(18) ) 69 | self.button_run.add_event_cb( self.cb_button_run, lv.EVENT.CLICKED, None ) 70 | label = lv.label( self.button_run ) 71 | label.set_text( "Run" ) 72 | label.center() 73 | 74 | self.button_record.clear_state( lv.STATE.DISABLED ) 75 | self.button_codex.add_state( lv.STATE.DISABLED ) 76 | self.button_run.add_state( lv.STATE.DISABLED ) 77 | 78 | -------------------------------------------------------------------------------- /src/wave.py: -------------------------------------------------------------------------------- 1 | import struct 2 | 3 | class Wave: 4 | HEADER_FMT = "<4sI4s4sIHHIIHH4sI" 5 | FORMAT_PCM = 1 6 | 7 | def __init__( self, fl_name, freq=8000, bits=16, channels=1 ): 8 | self.fl_name = fl_name 9 | self.freq = freq 10 | self.bits = bits 11 | self.channels = channels 12 | 13 | self.fl = None 14 | self.size = 0 15 | 16 | def build_header( self ): 17 | header = struct.pack( self.HEADER_FMT, 18 | b'RIFF', 19 | self.size + struct.calcsize( self.HEADER_FMT ) - 8, 20 | b'WAVE', 21 | b'fmt ', 22 | self.bits, 23 | self.FORMAT_PCM, 24 | self.channels, 25 | self.freq, 26 | (self.freq*self.channels*self.bits)//8, 27 | (self.channels * self.bits)//8, 28 | self.bits, 29 | b'data', 30 | self.size 31 | ) 32 | return header 33 | 34 | def open( self ): 35 | self.fl = open( self.fl_name, "wb" ) 36 | self.fl.write( self.build_header() ) 37 | 38 | def write( self, buf ): 39 | self.size += self.fl.write( buf ) 40 | 41 | def close( self ): 42 | # Update header with the written data 43 | self.fl.seek( 0 ) 44 | self.fl.write( self.build_header() ) 45 | self.fl.close() 46 | 47 | -------------------------------------------------------------------------------- /src/whisper.py: -------------------------------------------------------------------------------- 1 | import urequests as requests 2 | 3 | def transcribe( server="192.168.1.105", port=5000, buf=b"" ): 4 | url = "http://" + server + ":" + str( port ) + "/transcribe" 5 | resp = requests.post( url, data=buf ) 6 | return resp.text 7 | 8 | 9 | """ 10 | # PC server code 11 | # Requirements 12 | # pip install git+https://github.com/openai/whisper.git 13 | # pip install Flask 14 | 15 | import time 16 | import base64 17 | import flask 18 | import whisper 19 | 20 | # available sizes: base, small, medium 21 | # target: cpu, gpu 22 | model = whisper.load_model( "small", "cpu" ) 23 | 24 | app = flask.Flask("Whisper Server") 25 | @app.route( "/transcribe", methods=["POST"] ) 26 | def transcribe(): 27 | buf = flask.request.data 28 | with open( "tmp.wav", "wb" ) as fl: 29 | fl.write( buf ) 30 | t0 = time.time() 31 | result = model.transcribe( "tmp.wav", language="en" ) 32 | text = result["text"] 33 | t1 = time.time() 34 | print( t1-t0, text ) 35 | return text 36 | 37 | app.run( "192.168.1.105", 5000 ) # Get the IP with ipconfig 38 | print( "done" ) 39 | """ -------------------------------------------------------------------------------- /src/wifi.py: -------------------------------------------------------------------------------- 1 | import time 2 | import network 3 | 4 | class Wifi: 5 | def __init__( self ): 6 | self.wlan = None 7 | 8 | def connect( self, ssid, pswd ): 9 | self.wlan = network.WLAN( network.STA_IF ) 10 | self.wlan.active( True ) 11 | self.wlan.connect( ssid, pswd ) 12 | while not self.wlan.isconnected(): 13 | time.sleep( 0.1 ) 14 | print( self.wlan.ifconfig() ) 15 | 16 | def disconnect( self ): 17 | self.wlan.disconnect() 18 | self.wlan.active( False ) 19 | --------------------------------------------------------------------------------