├── .gitignore ├── .idea ├── .gitignore ├── .name ├── compiler.xml ├── deploymentTargetDropDown.xml ├── gradle.xml ├── kotlinc.xml ├── misc.xml └── vcs.xml ├── LICENSE ├── README.md ├── app ├── .gitignore ├── build.gradle ├── proguard-rules.pro └── src │ ├── androidTest │ └── java │ │ └── com │ │ └── teamopensourcesmartglasses │ │ └── chatgpt │ │ └── ExampleInstrumentedTest.kt │ ├── main │ ├── AndroidManifest.xml │ ├── ic_launcher-playstore.png │ ├── java │ │ └── com │ │ │ └── teamopensourcesmartglasses │ │ │ └── chatgpt │ │ │ ├── ChatGptAppMode.kt │ │ │ ├── ChatGptBackend.kt │ │ │ ├── ChatGptService.kt │ │ │ ├── MainActivity.kt │ │ │ ├── events │ │ │ ├── ChatErrorEvent.kt │ │ │ ├── ChatReceivedEvent.kt │ │ │ ├── ChatSummarizedEvent.kt │ │ │ ├── ClearContextRequestEvent.kt │ │ │ ├── IsLoadingEvent.kt │ │ │ ├── QuestionAnswerReceivedEvent.kt │ │ │ └── UserSettingsChangedEvent.kt │ │ │ ├── ui │ │ │ └── ChatGptSetupFragment.kt │ │ │ └── utils │ │ │ ├── Message.kt │ │ │ └── MessageStore.kt │ └── res │ │ ├── drawable │ │ ├── ic_launcher_background.xml │ │ └── ic_launcher_foreground.xml │ │ ├── layout │ │ ├── activity_main.xml │ │ ├── content_main.xml │ │ └── fragment_chatgptsetup.xml │ │ ├── menu │ │ └── menu_main.xml │ │ ├── mipmap-anydpi-v26 │ │ ├── ic_launcher.xml │ │ └── ic_launcher_round.xml │ │ ├── mipmap-hdpi │ │ ├── ic_launcher.png │ │ └── ic_launcher_round.png │ │ ├── mipmap-mdpi │ │ ├── ic_launcher.png │ │ └── ic_launcher_round.png │ │ ├── mipmap-xhdpi │ │ ├── ic_launcher.png │ │ └── ic_launcher_round.png │ │ ├── mipmap-xxhdpi │ │ ├── ic_launcher.png │ │ └── ic_launcher_round.png │ │ ├── mipmap-xxxhdpi │ │ ├── ic_launcher.png │ │ └── ic_launcher_round.png │ │ ├── navigation │ │ └── nav_graph.xml │ │ ├── values-land │ │ └── dimens.xml │ │ ├── values-night │ │ └── themes.xml │ │ ├── values-w1240dp │ │ └── dimens.xml │ │ ├── values-w600dp │ │ └── dimens.xml │ │ ├── values │ │ ├── colors.xml │ │ ├── dimens.xml │ │ ├── strings.xml │ │ └── themes.xml │ │ └── xml │ │ ├── backup_rules.xml │ │ └── data_extraction_rules.xml │ └── test │ └── java │ └── com │ └── teamopensourcesmartglasses │ └── chatgpt │ └── ExampleUnitTest.kt ├── build.gradle ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── settings.gradle /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | .gradle 3 | /local.properties 4 | /.idea/caches 5 | /.idea/libraries 6 | /.idea/modules.xml 7 | /.idea/workspace.xml 8 | /.idea/navEditor.xml 9 | /.idea/assetWizardSettings.xml 10 | .DS_Store 11 | /build 12 | /captures 13 | .externalNativeBuild 14 | .cxx 15 | local.properties 16 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | -------------------------------------------------------------------------------- /.idea/.name: -------------------------------------------------------------------------------- 1 | chatgpt -------------------------------------------------------------------------------- /.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/deploymentTargetDropDown.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /.idea/gradle.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 19 | 20 | -------------------------------------------------------------------------------- /.idea/kotlinc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 10 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 TeamOpenSmartGlasses 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SmartGlassesChatGPT 2 | 3 | The main purpose of this project is to integrate ChatGPT with your smart glasses. 4 | 5 | We can do a ton of things with ChatGPT already, and having it right up our faces is a lot more efficient 😉 6 | 7 | ## Intro video to the project 8 | 9 | [![Glasses Gpt Intro](http://img.youtube.com/vi/jFTYT9buA4k/0.jpg)](https://www.youtube.com/embed/jFTYT9buA4k) 10 | 11 | ## Installation 12 | 13 | - Install the [SmartGlassesManager](https://github.com/TeamOpenSmartGlasses/SmartGlassesManager) on your phone and make sure it's running 14 | - We need the SmartGlassesManager repo right next to this SmartGlassesChatGPT repo (or you can manually change it in gradle settings) 15 | - In the future, if SmartGlassesManager becomes a package, we might be able to set it up just from Gradle 16 | - Build this app in Android Studio and install it on your Android smartphone 17 | 18 | ## Run the Chatbot 19 | 20 | ### Initial Setup 21 | 22 | 1. Open up Android Smart Glasses app on your glasses 23 | 2. Open up Smart Glasses Manager on your phone and connect to your glasses 24 | 3. Launch the Smart Glasses Chat GPT app on your phone 25 | 4. 2 new commands will appear 26 | 27 | ### Listening Mode 28 | 29 | Activate by saying the phrase `Hey Computer, Listen`, allows the app to listen to your conversation and store them for use for future GPT requests 30 | 31 | ### Conversation Mode 32 | 33 | Activate by saying the phrase `Hey Computer, Conversation`, which allows you to continuously talk to ChatGPT 34 | 35 | ### Question Mode 36 | 37 | Activate by saying the phrase `Hey Computer, Question` allows you to ask one-off questions with ChatGPT 38 | 39 | ### Clear Context 40 | 41 | Resets your entire chat 42 | 43 | ### Difference between Conversation Mode and Question Mode 44 | 45 | You get your response in a card format using Question Mode and will be redirected to the home page once it is done 46 | In your history, questions asked will persist; they will be recorded as the user has asked a question 47 | 48 | ### Example usage flows 49 | 50 | - Turn on ```listening mode```, then switch to ```question mode``` whenever you have a question about a previous conversation 51 | - Turn on ```listening mode```, then switch to ```conversation mode``` to talk to GPT about something continuously based on a previous conversation 52 | 53 | > You also need to manually switch back to listening mode once you are done with your question or conversation with ChatGPT 54 | 55 | ### Customization 56 | 57 | - System prompt, this defines the characteristics of the bot, and will never be removed from the context, so customize your own bot like `Imagine if you are Shakespeare` 58 | - Automatically send messages after `7` seconds or manual mode where you say `send message` 59 | 60 | ## Tech Stack 61 | 62 | - Android + Kotlin 63 | 64 | ## Contributing 65 | 66 | If you would like to contribute to this project, please fork the repository and submit a pull request. We welcome contributions of all kinds, including bug fixes, feature requests, and code improvements. 67 | 68 | Before submitting a pull request, please make sure that your code adheres to the project's coding standards and that all tests pass. 69 | 70 | ### App structure 71 | 72 | A general guide on how to make a 3rd party app for the Smart Glasses Manager can be found here: [SGM Wiki](https://github.com/TeamOpenSmartGlasses/SmartGlassesManager/wiki) 73 | 74 | For our app, it is the same; the main thing you might want to look at is the 75 | 76 | - ```ChatGptBackend.kt``` file for handling the integration logic with the OpenAi Service 77 | - ```ChatGptService.kt``` file for handling the sgmLib integration logic 78 | 79 | ## Future roadmap 80 | 81 | - Add in export or save chat features (or just turn the app into a general intelligent assistant using LangChain) 82 | 83 | ## License 84 | 85 | This project is licensed under the MIT License. See the LICENSE file for details. 86 | -------------------------------------------------------------------------------- /app/.gitignore: -------------------------------------------------------------------------------- 1 | /build -------------------------------------------------------------------------------- /app/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id("com.android.application") 3 | id 'org.jetbrains.kotlin.android' 4 | } 5 | 6 | android { 7 | namespace = "com.teamopensourcesmartglasses.chatgpt" 8 | compileSdk = 34 9 | 10 | defaultConfig { 11 | applicationId = "com.teamopensourcesmartglasses.chatgpt" 12 | minSdk = 30 13 | targetSdk = 34 14 | versionCode = 1 15 | versionName = "1.0" 16 | 17 | testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" 18 | } 19 | 20 | buildTypes { 21 | release { 22 | debuggable = false 23 | minifyEnabled = false 24 | proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro") 25 | } 26 | } 27 | 28 | compileOptions { 29 | sourceCompatibility = JavaVersion.VERSION_17 30 | targetCompatibility = JavaVersion.VERSION_17 31 | } 32 | 33 | buildFeatures { 34 | viewBinding = true 35 | } 36 | } 37 | 38 | dependencies { 39 | implementation("androidx.appcompat:appcompat:1.6.1") 40 | implementation("com.google.android.material:material:1.8.0") 41 | implementation("androidx.constraintlayout:constraintlayout:2.1.4") 42 | implementation("androidx.navigation:navigation-fragment:2.5.3") 43 | implementation("androidx.navigation:navigation-ui:2.5.3") 44 | implementation 'androidx.core:core-ktx:+' 45 | implementation 'androidx.lifecycle:lifecycle-viewmodel-ktx:2.2.0' 46 | testImplementation("junit:junit:4.13.2") 47 | androidTestImplementation("androidx.test.ext:junit:1.1.5") 48 | androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1") 49 | implementation("androidx.lifecycle:lifecycle-extensions:2.2.0") 50 | implementation(project(":SGMLib")) 51 | implementation("com.knuddels:jtokkit:0.4.0") 52 | implementation("org.greenrobot:eventbus:3.3.1") 53 | implementation("io.reactivex.rxjava3:rxandroid:3.0.2") 54 | implementation("io.reactivex.rxjava3:rxjava:3.1.5") 55 | implementation("com.theokanning.openai-gpt3-java:service:0.12.0") 56 | } 57 | -------------------------------------------------------------------------------- /app/proguard-rules.pro: -------------------------------------------------------------------------------- 1 | # Add project specific ProGuard rules here. 2 | # You can control the set of applied configuration files using the 3 | # proguardFiles setting in build.gradle. 4 | # 5 | # For more details, see 6 | # http://developer.android.com/guide/developing/tools/proguard.html 7 | 8 | # If your project uses WebView with JS, uncomment the following 9 | # and specify the fully qualified class name to the JavaScript interface 10 | # class: 11 | #-keepclassmembers class fqcn.of.javascript.interface.for.webview { 12 | # public *; 13 | #} 14 | 15 | # Uncomment this to preserve the line number information for 16 | # debugging stack traces. 17 | #-keepattributes SourceFile,LineNumberTable 18 | 19 | # If you keep the line number information, uncomment this to 20 | # hide the original source file name. 21 | #-renamesourcefileattribute SourceFile -------------------------------------------------------------------------------- /app/src/androidTest/java/com/teamopensourcesmartglasses/chatgpt/ExampleInstrumentedTest.kt: -------------------------------------------------------------------------------- 1 | package com.teamopensourcesmartglasses.chatgpt 2 | 3 | import androidx.test.ext.junit.runners.AndroidJUnit4 4 | import androidx.test.platform.app.InstrumentationRegistry 5 | import org.junit.Assert 6 | import org.junit.Test 7 | import org.junit.runner.RunWith 8 | 9 | /** 10 | * Instrumented test, which will execute on an Android device. 11 | * 12 | * @see [Testing documentation](http://d.android.com/tools/testing) 13 | */ 14 | @RunWith(AndroidJUnit4::class) 15 | class ExampleInstrumentedTest { 16 | @Test 17 | fun useAppContext() { 18 | // Context of the app under test. 19 | val appContext = InstrumentationRegistry.getInstrumentation().targetContext 20 | Assert.assertEquals("com.teamopensourcesmartglasses.chatgpt", appContext.packageName) 21 | } 22 | } -------------------------------------------------------------------------------- /app/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 20 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 34 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /app/src/main/ic_launcher-playstore.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AugmentOS-Community/SmartGlassesChatGPT/ca021a3b659ad292e3bef835898c6b362ed5382b/app/src/main/ic_launcher-playstore.png -------------------------------------------------------------------------------- /app/src/main/java/com/teamopensourcesmartglasses/chatgpt/ChatGptAppMode.kt: -------------------------------------------------------------------------------- 1 | package com.teamopensourcesmartglasses.chatgpt 2 | 3 | enum class ChatGptAppMode { 4 | Conversation, Question, Record, Summarize, Inactive 5 | } -------------------------------------------------------------------------------- /app/src/main/java/com/teamopensourcesmartglasses/chatgpt/ChatGptBackend.kt: -------------------------------------------------------------------------------- 1 | package com.teamopensourcesmartglasses.chatgpt 2 | 3 | import android.util.Log 4 | import com.teamopensourcesmartglasses.chatgpt.events.ChatErrorEvent 5 | import com.teamopensourcesmartglasses.chatgpt.events.ChatReceivedEvent 6 | import com.teamopensourcesmartglasses.chatgpt.events.ChatSummarizedEvent 7 | import com.teamopensourcesmartglasses.chatgpt.events.IsLoadingEvent 8 | import com.teamopensourcesmartglasses.chatgpt.events.QuestionAnswerReceivedEvent 9 | import com.teamopensourcesmartglasses.chatgpt.utils.MessageStore 10 | import com.theokanning.openai.completion.chat.ChatCompletionChoice 11 | import com.theokanning.openai.completion.chat.ChatCompletionRequest 12 | import com.theokanning.openai.completion.chat.ChatMessage 13 | import com.theokanning.openai.completion.chat.ChatMessageRole 14 | import com.theokanning.openai.service.OpenAiService 15 | import org.greenrobot.eventbus.EventBus 16 | import java.time.Duration 17 | import java.util.stream.Collectors 18 | 19 | class ChatGptBackend { 20 | val TAG = "SmartGlassesChatGpt_ChatGptBackend" 21 | private var service: OpenAiService? = null 22 | 23 | // private final List messages = new ArrayList<>(); 24 | private val messages: MessageStore 25 | 26 | // private StringBuffer responseMessageBuffer = new StringBuffer(); 27 | private val chatGptMaxTokenSize = 28 | 3500 // let's play safe and use the 3500 out of 4096, we will leave 500 for custom hardcoded prompts 29 | private val messageDefaultWordsChunkSize = 100 30 | private val openAiServiceTimeoutDuration = 110 31 | private var recordingBuffer = StringBuffer() 32 | 33 | init { 34 | messages = MessageStore(chatGptMaxTokenSize) 35 | } 36 | 37 | fun initChatGptService(token: String?, systemMessage: String?) { 38 | // Setup ChatGpt with a token 39 | service = OpenAiService(token, Duration.ofSeconds(openAiServiceTimeoutDuration.toLong())) 40 | messages.setSystemMessage(systemMessage!!) 41 | } 42 | 43 | fun sendChatToMemory(message: String?) { 44 | // Add to messages here if it is just to record 45 | // It should be chunked into a decent block size 46 | Log.d(TAG, "sendChat: In record mode") 47 | recordingBuffer.append(message) 48 | recordingBuffer.append(" ") 49 | Log.d(TAG, "sendChatToMemory: $recordingBuffer") 50 | if (getWordCount(recordingBuffer.toString()) > messageDefaultWordsChunkSize) { 51 | Log.d(TAG, "sendChatToMemory: size is big enough to be a chunk") 52 | messages.addMessage(ChatMessageRole.USER.value(), recordingBuffer.toString()) 53 | recordingBuffer = StringBuffer() 54 | } 55 | } 56 | 57 | fun sendChatToGpt(message: String?, mode: ChatGptAppMode) { 58 | // Don't run if openAI service is not initialized yet 59 | if (service == null) { 60 | EventBus.getDefault() 61 | .post(ChatErrorEvent("OpenAi Key has not been provided yet. Please do so in the app.")) 62 | return 63 | } 64 | chunkRemainingBufferContent() 65 | 66 | // Add the user message and pass the entire message context to chatgpt 67 | messages.addMessage(ChatMessageRole.USER.value(), message!!) 68 | runChatGpt(message, mode) 69 | } 70 | 71 | private fun runChatGpt(message: String?, mode: ChatGptAppMode) { 72 | class DoGptStuff : Runnable { 73 | override fun run() { 74 | 75 | // Build prompt here 76 | val context: ArrayList 77 | if (mode !== ChatGptAppMode.Summarize) { 78 | context = messages.allMessages 79 | } else { 80 | context = messages.allMessagesWithoutSystemPrompt 81 | if (context.isEmpty()) { 82 | EventBus.getDefault() 83 | .post(ChatErrorEvent("No conversation was recorded, unable to summarize.")) 84 | return 85 | } 86 | val startingPrompt = 87 | """The following text below is a transcript of a conversation. I need your help to summarize the text below. The transcript will be really messy, your first task is to replace all parts of the text that do not makes sense with words or phrases that makes the most sense, The transcript will be really messy, but you must not complain about the quality of the transcript, if it is bad, do not bring it up, No matter what, don't ever complain that the transcript is messy or hard to follow, just tell me the summary and not anything else. After you are done replacing the words with ones that makes sense, I want you to summarize it, You don't need to answer in full sentences as well, be short and concise, just tell me the summary for me in bullet form, each point should be no longer than 20 words long. For the output, I don't want to see the transformed text, I just want the overall summary and it must follow this format, don't put everything in one paragraph, I need it in bullet form as I am working with a really tight schema! 88 | Detected that the user was talking about 89 | - 90 | - 91 | - and so on 92 | 93 | The text can be found within the triple dollar signs here: 94 | ${"$"}${"$"}$ 95 | """ 96 | context.add(0, ChatMessage(ChatMessageRole.SYSTEM.value(), startingPrompt)) 97 | val endingPrompt = "\n $$$" 98 | context.add(ChatMessage(ChatMessageRole.SYSTEM.value(), endingPrompt)) 99 | } 100 | Log.d(TAG, "run: messages: ") 101 | for (message in context) { 102 | Log.d(TAG, "run: message: " + message!!.content) 103 | } 104 | 105 | // Todo: Change completions to streams 106 | val chatCompletionRequest = ChatCompletionRequest.builder() 107 | .model("gpt-3.5-turbo") 108 | .messages(context) 109 | .n(1) 110 | .build() 111 | EventBus.getDefault().post(IsLoadingEvent()) 112 | try { 113 | val result = service!!.createChatCompletion(chatCompletionRequest) 114 | val responses = result.choices 115 | .stream() 116 | .map { obj: ChatCompletionChoice -> obj.message } 117 | .collect(Collectors.toList()) 118 | 119 | // Send a chat received response 120 | val response = responses[0] 121 | Log.d(TAG, "run: ChatGpt response: " + response.content) 122 | 123 | // Send back to chat UI and internal history 124 | if (mode === ChatGptAppMode.Conversation) { 125 | EventBus.getDefault().post(ChatReceivedEvent(response.content)) 126 | messages.addMessage(response.role, response.content) 127 | } 128 | 129 | // Send back one off question and answer 130 | if (mode === ChatGptAppMode.Question) { 131 | EventBus.getDefault().post( 132 | QuestionAnswerReceivedEvent( 133 | message!!, response.content 134 | ) 135 | ) 136 | 137 | // Edit the last user message to specify that it was a question 138 | messages.addPrefixToLastAddedMessage("User asked a question: ") 139 | // Specify on the answer side as well 140 | messages.addMessage( 141 | response.role, 142 | "Assistant responded with an answer: " + response.content 143 | ) 144 | } 145 | if (mode === ChatGptAppMode.Summarize) { 146 | Log.d(TAG, "run: Sending a chat summarized event to service") 147 | EventBus.getDefault().post(ChatSummarizedEvent(response.content)) 148 | } 149 | } catch (e: Exception) { 150 | // Log.d(TAG, "run: encountered error: " + e.getMessage()); 151 | EventBus.getDefault() 152 | .post(ChatErrorEvent("Check if you had set your key correctly or view the error below" + e.message)) 153 | } 154 | 155 | // Log.d(TAG, "Streaming chat completion"); 156 | // service.streamChatCompletion(chatCompletionRequest) 157 | // .doOnError(this::onStreamChatGptError) 158 | // .doOnComplete(this::onStreamComplete) 159 | // .blockingForEach(this::onItemReceivedFromStream); 160 | } // private void onStreamChatGptError(Throwable throwable) { 161 | // Log.d(TAG, throwable.getMessage()); 162 | // EventBus.getDefault().post(new ChatReceivedEvent(throwable.getMessage())); 163 | // throwable.printStackTrace(); 164 | // } 165 | // 166 | // public void onItemReceivedFromStream(ChatCompletionChunk chunk) { 167 | // String textChunk = chunk.getChoices().get(0).getMessage().getContent(); 168 | // Log.d(TAG, "Chunk received from stream: " + textChunk); 169 | // EventBus.getDefault().post(new ChatReceivedEvent(textChunk)); 170 | // responseMessageBuffer.append(textChunk); 171 | // responseMessageBuffer.append(" "); 172 | // } 173 | // 174 | // public void onStreamComplete() { 175 | // String responseMessage = responseMessageBuffer.toString(); 176 | // messages.add(new ChatMessage(ChatMessageRole.ASSISTANT.value(), responseMessage)); 177 | // responseMessageBuffer = new StringBuffer(); 178 | // } 179 | } 180 | Thread(DoGptStuff()).start() 181 | } 182 | 183 | private fun chunkRemainingBufferContent() { 184 | // If there is still words from a previous record session, then add them into the messageQueue 185 | if (recordingBuffer.length != 0) { 186 | Log.d( 187 | TAG, 188 | "sendChatToGpt: There are still words from a recording, adding them to chunk" 189 | ) 190 | messages.addMessage(ChatMessageRole.USER.value(), recordingBuffer.toString()) 191 | recordingBuffer = StringBuffer() 192 | } 193 | } 194 | 195 | private fun getWordCount(message: String): Int { 196 | val words = message.split("\\s+".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() 197 | return words.size 198 | } 199 | 200 | private fun clearSomeMessages() { 201 | for (i in 0 until messages.size() / 2) { 202 | messages.removeOldest() 203 | } 204 | } 205 | 206 | fun summarizeContext() { 207 | // Log.d(TAG, "summarizeContext: Called"); 208 | chunkRemainingBufferContent() 209 | runChatGpt(null, ChatGptAppMode.Summarize) 210 | } 211 | 212 | fun clearConversationContext() { 213 | messages.resetMessages() 214 | recordingBuffer = StringBuffer() 215 | } 216 | } -------------------------------------------------------------------------------- /app/src/main/java/com/teamopensourcesmartglasses/chatgpt/ChatGptService.kt: -------------------------------------------------------------------------------- 1 | package com.teamopensourcesmartglasses.chatgpt 2 | 3 | import android.util.Log 4 | import com.google.android.material.R 5 | import com.teamopensmartglasses.sgmlib.DataStreamType 6 | import com.teamopensmartglasses.sgmlib.FocusStates 7 | import com.teamopensmartglasses.sgmlib.SGMCommand 8 | import com.teamopensmartglasses.sgmlib.SGMLib 9 | import com.teamopensmartglasses.sgmlib.SmartGlassesAndroidService 10 | import com.teamopensourcesmartglasses.chatgpt.events.ChatErrorEvent 11 | import com.teamopensourcesmartglasses.chatgpt.events.ChatReceivedEvent 12 | import com.teamopensourcesmartglasses.chatgpt.events.ChatSummarizedEvent 13 | import com.teamopensourcesmartglasses.chatgpt.events.IsLoadingEvent 14 | import com.teamopensourcesmartglasses.chatgpt.events.QuestionAnswerReceivedEvent 15 | import com.teamopensourcesmartglasses.chatgpt.events.UserSettingsChangedEvent 16 | import org.greenrobot.eventbus.EventBus 17 | import org.greenrobot.eventbus.Subscribe 18 | import java.util.Arrays 19 | import java.util.Timer 20 | import java.util.TimerTask 21 | import java.util.UUID 22 | import java.util.concurrent.ExecutorService 23 | import java.util.concurrent.Executors 24 | import java.util.concurrent.Future 25 | import java.util.concurrent.TimeUnit 26 | 27 | 28 | class ChatGptService : SmartGlassesAndroidService( 29 | MainActivity::class.java, 30 | "chatgpt_app", 31 | 1011, 32 | appName, 33 | "ChatGPT for smart glasses", R.drawable.notify_panel_notification_icon_bg 34 | ) { 35 | 36 | val TAG = "SmartGlassesChatGpt_ChatGptService" 37 | 38 | //our instance of the SGM library 39 | var sgmLib: SGMLib? = null 40 | var focusState: FocusStates? = null 41 | var chatGptBackend: ChatGptBackend? = null 42 | var messageBuffer = StringBuffer() 43 | private var userTurnLabelSet = false 44 | private var chatGptLabelSet = false 45 | private val executorService = Executors.newSingleThreadScheduledExecutor() 46 | private var printExecutorService: ExecutorService? = null 47 | private var future: Future<*>? = null 48 | private var openAiKeyProvided = false 49 | private var mode = ChatGptAppMode.Record // Turn on listening by default 50 | private var useAutoSend = false 51 | private var commandWords: ArrayList? = null 52 | private var scrollingTextTitle = "" 53 | private val messageDisplayDurationMs = 4000 54 | private var loadingTimer: Timer? = null 55 | override fun onCreate() { 56 | super.onCreate() 57 | focusState = FocusStates.OUT_FOCUS 58 | 59 | /* Handle SGMLib specific things */ 60 | 61 | // Create SGMLib instance with context: this 62 | sgmLib = SGMLib(this) 63 | 64 | // Define commands 65 | val startChatCommand = SGMCommand( 66 | appName, 67 | UUID.fromString("c3b5bbfd-4416-4006-8b40-12346ac3abcf"), arrayOf("conversation"), 68 | "Start a ChatGPT session for your smart glasses!" 69 | ) 70 | val askGptCommand = SGMCommand( 71 | appName, 72 | UUID.fromString("c367ba2d-4416-8768-8b15-19046ac3a2af"), arrayOf("question"), 73 | "Ask a one shot question to ChatGpt based on your existing context" 74 | ) 75 | val clearContextCommand = SGMCommand( 76 | appName, 77 | UUID.fromString("2b8d1ba0-f114-11ed-a05b-0242ac120003"), arrayOf("clear"), 78 | "Clear your conversation context" 79 | ) 80 | val recordConversationCommand = SGMCommand( 81 | appName, 82 | UUID.fromString("ea89a5ac-6cbd-4867-bd86-1ebce9a27cb3"), arrayOf("listen"), 83 | "Record your conversation so you can ask ChatGpt for questions later on" 84 | ) 85 | val summarizeConversationCommand = SGMCommand( 86 | appName, 87 | UUID.fromString("9ab3f985-e9d1-4ab2-8d28-0d1e6111bcb4"), arrayOf("summarize"), 88 | "Summarize your conversation using ChatGpt" 89 | ) 90 | 91 | // Save all the wake words so we can detect and remove them during transcriptions with just 1 word 92 | commandWords = ArrayList() 93 | commandWords!!.addAll(startChatCommand.phrases) 94 | commandWords!!.addAll(askGptCommand.phrases) 95 | commandWords!!.addAll(clearContextCommand.phrases) 96 | commandWords!!.addAll(recordConversationCommand.phrases) 97 | commandWords!!.addAll(summarizeConversationCommand.phrases) 98 | 99 | // Register the command 100 | sgmLib!!.registerCommand(startChatCommand) { args: String?, commandTriggeredTime: Long -> 101 | startChatCommandCallback( 102 | args, 103 | commandTriggeredTime 104 | ) 105 | } 106 | sgmLib!!.registerCommand(askGptCommand) { args: String?, commandTriggeredTime: Long -> 107 | askGptCommandCallback( 108 | args, 109 | commandTriggeredTime 110 | ) 111 | } 112 | sgmLib!!.registerCommand(recordConversationCommand) { args: String?, commandTriggeredTime: Long -> 113 | recordConversationCommandCallback( 114 | args, 115 | commandTriggeredTime 116 | ) 117 | } 118 | sgmLib!!.registerCommand(clearContextCommand) { args: String?, commandTriggeredTime: Long -> 119 | clearConversationContextCommandCallback( 120 | args, 121 | commandTriggeredTime 122 | ) 123 | } 124 | sgmLib!!.registerCommand(summarizeConversationCommand) { args: String?, commandTriggeredTime: Long -> 125 | summarizeConversationContextCommandCallback( 126 | args, 127 | commandTriggeredTime 128 | ) 129 | } 130 | 131 | //Subscribe to transcription stream 132 | sgmLib!!.subscribe(DataStreamType.TRANSCRIPTION_ENGLISH_STREAM) { transcript: String, timestamp: Long, isFinal: Boolean -> 133 | processTranscriptionCallback( 134 | transcript, 135 | timestamp, 136 | isFinal 137 | ) 138 | } 139 | Log.d(TAG, "onCreate: ChatGPT service started!") 140 | 141 | /* Handle SmartGlassesChatGPT specific things */EventBus.getDefault().register(this) 142 | chatGptBackend = ChatGptBackend() 143 | 144 | // Putting a separate sharedPreferences here instead of through the event bus from mainActivity 145 | // so I don't have to deal with waiting for this service to finish its startup 146 | val sharedPreferences = getSharedPreferences("user.config", MODE_PRIVATE) 147 | val savedKey = sharedPreferences.getString("openAiKey", "") 148 | if (savedKey!!.isEmpty()) { 149 | // do not start the backend if there is no key, wait for onUserSettingsChanged to init 150 | // Log.d(TAG, "ChatGptService: No key exists"); 151 | return 152 | } 153 | var systemPrompt = sharedPreferences.getString("systemPrompt", "") 154 | if (systemPrompt!!.isEmpty()) { 155 | // Log.d(TAG, "onCreate: empty system prompt, setting it to use the default one"); 156 | systemPrompt = 157 | "Act as a the no 1 ambassador for smart glasses and generative pretrained ai! Find ways to bring up these topics as creative as possible!" 158 | } 159 | chatGptBackend!!.initChatGptService(savedKey, systemPrompt) 160 | useAutoSend = sharedPreferences.getBoolean("autoSendMessages", true) 161 | openAiKeyProvided = true 162 | focusState = FocusStates.IN_FOCUS 163 | recordConversationCommandCallback(null, 0) 164 | // Log.d(TAG, "onCreate: Saved openAi key found, token = " + savedKey); 165 | // Log.d(TAG, "onCreate: systemPrompt = " + systemPrompt); 166 | // Log.d(TAG, "onCreate: useAutoSend = " + useAutoSend); 167 | } 168 | 169 | override fun onDestroy() { 170 | Log.d(TAG, "onDestroy: Called") 171 | EventBus.getDefault().unregister(this) 172 | sgmLib!!.deinit() 173 | super.onDestroy() 174 | } 175 | 176 | fun startChatCommandCallback(args: String?, commandTriggeredTime: Long) { 177 | Log.d(TAG, "startChatCommandCallback: Start ChatGPT command callback called") 178 | Log.d(TAG, "startChatCommandCallback: OpenAiApiKeyProvided:$openAiKeyProvided") 179 | scrollingTextTitle = "Conversation" 180 | // request to be the in focus app so we can continue to show transcripts 181 | sgmLib!!.requestFocus { focusState: FocusStates -> focusChangedCallback(focusState) } 182 | mode = ChatGptAppMode.Conversation 183 | Log.d(TAG, "startChatCommandCallback: Set app mode to conversation") 184 | 185 | // we might had been in the middle of a question, so when we switch back to a conversation, 186 | // we reset our messageBuffer 187 | resetUserMessage() 188 | } 189 | 190 | fun askGptCommandCallback(args: String?, commandTriggeredTime: Long) { 191 | Log.d(TAG, "askGptCommandCallback: Ask ChatGPT command callback called") 192 | // Log.d(TAG, "askGptCommandCallback: OpenAiApiKeyProvided:" + openAiKeyProvided); 193 | 194 | // request to be the in focus app so we can continue to show transcripts 195 | scrollingTextTitle = "Question" 196 | sgmLib!!.requestFocus { focusState: FocusStates -> focusChangedCallback(focusState) } 197 | mode = ChatGptAppMode.Question 198 | Log.d(TAG, "askGptCommandCommand: Set app mode to question") 199 | 200 | // we might had been in the middle of a conversation, so when we switch to a question, 201 | // we need to reset our messageBuffer 202 | resetUserMessage() 203 | } 204 | 205 | fun recordConversationCommandCallback(args: String?, commandTriggeredTime: Long) { 206 | Log.d(TAG, "askGptCommandCallback: Record conversation command callback called") 207 | scrollingTextTitle = "Listening" 208 | // request to be the in focus app so we can continue to show transcripts 209 | sgmLib!!.requestFocus { focusState: FocusStates -> focusChangedCallback(focusState) } 210 | mode = ChatGptAppMode.Record 211 | Log.d(TAG, "askGptCommandCommand: Set app mode to record conversation") 212 | 213 | // we might had been in the middle of a conversation, so when we switch to a question, 214 | // we need to reset our messageBuffer 215 | resetUserMessage() 216 | } 217 | 218 | fun clearConversationContextCommandCallback(args: String?, commandTriggeredTime: Long) { 219 | Log.d(TAG, "askGptCommandCallback: Reset conversation context") 220 | if (loadingTimer != null) { 221 | loadingTimer!!.cancel() 222 | loadingTimer = null 223 | } 224 | sgmLib!!.sendReferenceCard("Clear context", "Cleared conversation context") 225 | mode = ChatGptAppMode.Record 226 | chatGptBackend!!.clearConversationContext() 227 | 228 | // we might had been in the middle of a conversation, so when we switch to a question, 229 | // we need to reset our messageBuffer 230 | resetUserMessage() 231 | } 232 | 233 | fun summarizeConversationContextCommandCallback(args: String?, commandTriggeredTime: Long) { 234 | Log.d(TAG, "askGptCommandCallback: Summarize conversation context") 235 | scrollingTextTitle = "Summarize" 236 | sgmLib!!.requestFocus { focusState: FocusStates -> focusChangedCallback(focusState) } 237 | mode = ChatGptAppMode.Summarize 238 | chatGptBackend!!.summarizeContext() 239 | 240 | // we might had been in the middle of a conversation, so when we switch to a question, 241 | // we need to reset our messageBuffer 242 | resetUserMessage() 243 | } 244 | 245 | fun focusChangedCallback(focusState: FocusStates) { 246 | Log.d(TAG, "Focus callback called with state: $focusState") 247 | this.focusState = focusState 248 | sgmLib!!.stopScrollingText() 249 | //StartScrollingText to show our translation 250 | if (focusState == FocusStates.IN_FOCUS) { 251 | sgmLib!!.startScrollingText(scrollingTextTitle) 252 | Log.d(TAG, "startChatCommandCallback: Added a scrolling text view") 253 | messageBuffer = StringBuffer() 254 | } 255 | } 256 | 257 | fun processTranscriptionCallback(transcript: String, timestamp: Long, isFinal: Boolean) { 258 | // Don't execute if we're not in focus or no mode is set 259 | var transcript = transcript 260 | if (focusState != FocusStates.IN_FOCUS || mode === ChatGptAppMode.Inactive) { 261 | return 262 | } 263 | 264 | // We can ignore command phrases so it is more accurate, tested that this works 265 | if (isFinal && commandWords!!.contains(transcript)) { 266 | return 267 | } 268 | 269 | // If its recording we just save it to memory without even the need to finish the sentence 270 | // It will be saved as a ChatMessage 271 | if (isFinal && mode === ChatGptAppMode.Record) { 272 | Log.d(TAG, "processTranscriptionCallback: $transcript") 273 | chatGptBackend!!.sendChatToMemory(transcript) 274 | sgmLib!!.pushScrollingText(transcript) 275 | } 276 | 277 | // We want to send our message in our message buffer when we stop speaking for like 9 seconds 278 | // If the transcript is finalized, then we add it to our buffer, and reset our timer 279 | if (isFinal && mode !== ChatGptAppMode.Record && openAiKeyProvided) { 280 | Log.d(TAG, "processTranscriptionCallback: $transcript") 281 | if (useAutoSend) { 282 | messageBuffer.append(transcript) 283 | messageBuffer.append(" ") 284 | // Cancel the scheduled job if we get a new transcript 285 | if (future != null) { 286 | future!!.cancel(false) 287 | Log.d(TAG, "processTranscriptionCallback: Cancelled scheduled job") 288 | } 289 | future = executorService.schedule({ sendMessageToChatGpt() }, 7, TimeUnit.SECONDS) 290 | } else { 291 | if (transcript == "send message") { 292 | sendMessageToChatGpt() 293 | } else { 294 | messageBuffer.append(transcript) 295 | messageBuffer.append(" ") 296 | } 297 | } 298 | if (!userTurnLabelSet) { 299 | transcript = "User: $transcript" 300 | userTurnLabelSet = true 301 | } 302 | sgmLib!!.pushScrollingText(transcript) 303 | } 304 | } 305 | 306 | private fun sendMessageToChatGpt() { 307 | val message = messageBuffer.toString() 308 | if (!message.isEmpty()) { 309 | chatGptBackend!!.sendChatToGpt(message, mode) 310 | messageBuffer = StringBuffer() 311 | Log.d(TAG, "processTranscriptionCallback: Ran scheduled job and sent message") 312 | } else { 313 | Log.d(TAG, "processTranscriptionCallback: Can't send because message is empty") 314 | } 315 | } 316 | 317 | private fun resetUserMessage() { 318 | // Cancel the scheduled job if we get a new transcript 319 | if (future != null) { 320 | future!!.cancel(false) 321 | Log.d(TAG, "resetUserMessage: Cancelled scheduled job") 322 | } 323 | messageBuffer = StringBuffer() 324 | } 325 | 326 | @Subscribe 327 | fun onChatReceived(event: ChatReceivedEvent) { 328 | if (loadingTimer != null) { 329 | loadingTimer!!.cancel() 330 | loadingTimer = null 331 | } 332 | chunkLongMessagesAndDisplay(event.message) 333 | userTurnLabelSet = false 334 | mode = ChatGptAppMode.Conversation 335 | } 336 | 337 | private fun chunkLongMessagesAndDisplay(message: String) { 338 | val localPrintExecutorService = Executors.newSingleThreadExecutor() 339 | printExecutorService = localPrintExecutorService 340 | localPrintExecutorService.execute(Runnable execute@{ 341 | val words = 342 | message.split("\\s+".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() 343 | val wordCount = words.size 344 | val groupSize = 15 // depends on glasses size 345 | var i = 0 346 | while (i < wordCount) { 347 | 348 | // Check if the background thread has been interrupted 349 | if (Thread.currentThread().isInterrupted) { 350 | return@execute 351 | } 352 | val endIndex = Math.min(i + groupSize, wordCount) 353 | val group = Arrays.copyOfRange(words, i, endIndex) 354 | val groupText = java.lang.String.join(" ", *group) 355 | if (!chatGptLabelSet) { 356 | // Log.d(TAG, "chunkLongMessagesAndDisplay: " + groupText.trim()); 357 | sgmLib!!.pushScrollingText("ChatGpt: " + groupText.trim { it <= ' ' }) 358 | chatGptLabelSet = true 359 | } else { 360 | sgmLib!!.pushScrollingText(groupText.trim { it <= ' ' }) 361 | } 362 | try { 363 | Thread.sleep(messageDisplayDurationMs.toLong()) // Delay of 3 second (1000 milliseconds) 364 | } catch (e: InterruptedException) { 365 | e.printStackTrace() 366 | // Restore interrupted status and return from the thread 367 | Thread.currentThread().interrupt() 368 | return@execute 369 | } 370 | i += groupSize 371 | } 372 | chatGptLabelSet = false 373 | }) 374 | } 375 | 376 | @Subscribe 377 | fun onQuestionAnswerReceived(event: QuestionAnswerReceivedEvent) { 378 | val body = """ 379 | Q: ${event.question} 380 | 381 | A: ${event.answer} 382 | """.trimIndent() 383 | sgmLib!!.sendReferenceCard("AskGpt", body) 384 | mode = ChatGptAppMode.Record 385 | } 386 | 387 | @Subscribe 388 | fun onChatSummaryReceived(event: ChatSummarizedEvent) { 389 | Log.d(TAG, "onChatSummaryReceived: Received a chat summarized event") 390 | if (loadingTimer != null) { 391 | loadingTimer!!.cancel() 392 | loadingTimer = null 393 | } 394 | val points = 395 | event.summary.split("\n".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() 396 | val localExecutorService = Executors.newSingleThreadExecutor() 397 | printExecutorService = localExecutorService 398 | localExecutorService.execute(Runnable execute@{ 399 | for (point in points) { 400 | if (Thread.currentThread().isInterrupted) { 401 | return@execute 402 | } 403 | if (!chatGptLabelSet) { 404 | sgmLib!!.pushScrollingText("ChatGpt: $point") 405 | chatGptLabelSet = true 406 | } else { 407 | sgmLib!!.pushScrollingText(point) 408 | } 409 | try { 410 | Thread.sleep(messageDisplayDurationMs.toLong()) // Delay of 3 second (1000 milliseconds) 411 | } catch (e: InterruptedException) { 412 | e.printStackTrace() 413 | // Restore interrupted status and return from the thread 414 | Thread.currentThread().interrupt() 415 | return@execute 416 | } 417 | } 418 | }) 419 | chatGptLabelSet = false 420 | // chunkLongMessagesAndDisplay(event.getSummary()); 421 | mode = ChatGptAppMode.Record 422 | } 423 | 424 | 425 | @Subscribe 426 | fun onChatError(event: ChatErrorEvent) { 427 | if (loadingTimer != null) { 428 | loadingTimer!!.cancel() 429 | loadingTimer = null 430 | } 431 | sgmLib!!.sendReferenceCard("Something wrong with ChatGpt", event.errorMessage) 432 | mode = ChatGptAppMode.Record 433 | } 434 | 435 | @Subscribe 436 | fun onLoading(event: IsLoadingEvent?) { 437 | // For those features using scrolling text, it might be useful to let the user know that chatgpt is thinking 438 | if (mode === ChatGptAppMode.Summarize || mode === ChatGptAppMode.Conversation) { 439 | if (loadingTimer == null) { 440 | loadingTimer = Timer() 441 | } 442 | loadingTimer!!.scheduleAtFixedRate(object : TimerTask() { 443 | override fun run() { 444 | sgmLib!!.pushScrollingText("ChatGpt is thinking...") 445 | } 446 | }, 0, 5000) 447 | } 448 | } 449 | 450 | @Subscribe 451 | fun onUserSettingsChanged(event: UserSettingsChangedEvent) { 452 | Log.d(TAG, "onUserSettingsChanged: Enabling ChatGpt with new api key = " + event.openAiKey) 453 | Log.d(TAG, "onUserSettingsChanged: System prompt = " + event.systemPrompt) 454 | chatGptBackend!!.initChatGptService(event.openAiKey, event.systemPrompt) 455 | openAiKeyProvided = true 456 | Log.d( 457 | TAG, 458 | "onUserSettingsChanged: Auto send messages after finish speaking = " + event.useAutoSend 459 | ) 460 | useAutoSend = event.useAutoSend 461 | mode = ChatGptAppMode.Record 462 | recordConversationCommandCallback(null, 0) 463 | } 464 | 465 | companion object { 466 | const val ACTION_START_FOREGROUND_SERVICE = "SGMLIB_ACTION_START_FOREGROUND_SERVICE" 467 | const val ACTION_STOP_FOREGROUND_SERVICE = "SGMLIB_ACTION_STOP_FOREGROUND_SERVICE" 468 | const val appName = "SmartGlassesChatGpt" 469 | } 470 | } -------------------------------------------------------------------------------- /app/src/main/java/com/teamopensourcesmartglasses/chatgpt/MainActivity.kt: -------------------------------------------------------------------------------- 1 | package com.teamopensourcesmartglasses.chatgpt 2 | 3 | import android.app.ActivityManager 4 | import android.content.ComponentName 5 | import android.content.Intent 6 | import android.content.ServiceConnection 7 | import android.os.Bundle 8 | import android.os.IBinder 9 | import android.util.Log 10 | import android.view.View 11 | import android.widget.Button 12 | import android.widget.EditText 13 | import android.widget.RadioButton 14 | import android.widget.Toast 15 | import androidx.appcompat.app.AppCompatActivity 16 | import com.teamopensmartglasses.sgmlib.SmartGlassesAndroidService 17 | import com.teamopensourcesmartglasses.chatgpt.databinding.ActivityMainBinding 18 | import com.teamopensourcesmartglasses.chatgpt.events.UserSettingsChangedEvent 19 | import org.greenrobot.eventbus.EventBus 20 | 21 | class MainActivity : AppCompatActivity() { 22 | private val TAG = "SmartGlassesChatGpt_MainActivity" 23 | var mBound = false 24 | var mService: ChatGptService? = null 25 | private var binding: ActivityMainBinding? = null 26 | private var submitButton: Button? = null 27 | private var openAiKeyText: EditText? = null 28 | private var autoSendRadioButton: RadioButton? = null 29 | private var manualSendRadioButton: RadioButton? = null 30 | private var systemPromptInput: EditText? = null 31 | override fun onCreate(savedInstanceState: Bundle?) { 32 | super.onCreate(savedInstanceState) 33 | binding = ActivityMainBinding.inflate( 34 | layoutInflater 35 | ) 36 | setContentView(binding!!.root) 37 | setSupportActionBar(binding!!.toolbar) 38 | startChatGptService() 39 | val sharedPreferences = getSharedPreferences("user.config", MODE_PRIVATE) 40 | 41 | // Display our previously saved settings 42 | openAiKeyText = findViewById(R.id.edittext_openAiKey) 43 | val savedOpenAiKey = sharedPreferences.getString("openAiKey", "") 44 | // Show toasts and populate openAI key text field if we have or don't have a key saved 45 | if (!savedOpenAiKey!!.isEmpty()) { 46 | openAiKeyText?.setText(savedOpenAiKey) 47 | Toast.makeText(this, "OpenAI key and other app settings found", Toast.LENGTH_LONG) 48 | .show() 49 | } else { 50 | Toast.makeText(this, "No valid OpenAI key found, please add one", Toast.LENGTH_LONG) 51 | .show() 52 | } 53 | systemPromptInput = findViewById(R.id.editTextTextMultiLine_systemPrompt) 54 | val defaultSystemPrompt = 55 | "Act as a the no 1 ambassador for smart glasses and generative pretrained ai! Find ways to bring up these topics as creative as possible!" 56 | val savedSystemPrompt = sharedPreferences.getString("systemPrompt", defaultSystemPrompt) 57 | systemPromptInput?.setText(savedSystemPrompt) 58 | autoSendRadioButton = findViewById(R.id.radioButton_autoSend) 59 | manualSendRadioButton = findViewById(R.id.radioButton_manualSend) 60 | val useAutoSendMethod = sharedPreferences.getBoolean("autoSendMessages", true) 61 | autoSendRadioButton?.isChecked = useAutoSendMethod 62 | manualSendRadioButton?.isChecked = !useAutoSendMethod 63 | 64 | // UI handlers 65 | submitButton = findViewById(R.id.submit_button) 66 | submitButton?.setOnClickListener(View.OnClickListener setOnClickListener@{ v: View? -> 67 | // Save to shared preference 68 | val editor = sharedPreferences.edit() 69 | val openAiApiKey = openAiKeyText?.text.toString().trim { it <= ' ' } 70 | if (openAiApiKey.isEmpty()) { 71 | Toast.makeText(this, "OpenAi key cannot be empty", Toast.LENGTH_LONG).show() 72 | return@setOnClickListener 73 | } 74 | editor.putString("openAiKey", openAiApiKey) 75 | val systemPrompt = systemPromptInput?.text.toString().trim { it <= ' ' } 76 | if (systemPrompt.isEmpty()) { 77 | Toast.makeText(this, "System prompt should not be empty", Toast.LENGTH_LONG).show() 78 | } 79 | editor.putString("systemPrompt", systemPrompt) 80 | val useAutoSendMessages = autoSendRadioButton?.isChecked ?: false 81 | editor.putBoolean("autoSendMessages", useAutoSendMessages) 82 | editor.apply() 83 | EventBus.getDefault().post(UserSettingsChangedEvent(openAiApiKey, systemPrompt, useAutoSendMessages)) 84 | 85 | // Toast to inform user that key has been saved 86 | Toast.makeText(this, "Overall settings changed", Toast.LENGTH_LONG).show() 87 | Toast.makeText(this, "OpenAi key saved for future sessions", Toast.LENGTH_LONG).show() 88 | }) 89 | 90 | } 91 | 92 | /* SGMLib */ 93 | override fun onResume() { 94 | super.onResume() 95 | 96 | //bind to foreground service 97 | bindChatGptService() 98 | } 99 | 100 | override fun onPause() { 101 | super.onPause() 102 | 103 | //unbind foreground service 104 | unbindChatGptService() 105 | } 106 | 107 | fun stopChatGptService() { 108 | unbindChatGptService() 109 | if (!isMyServiceRunning(ChatGptService::class.java)) return 110 | val stopIntent = Intent(this, ChatGptService::class.java) 111 | stopIntent.action = ChatGptService.ACTION_STOP_FOREGROUND_SERVICE 112 | startService(stopIntent) 113 | } 114 | 115 | fun sendChatGptServiceMessage(message: String?) { 116 | if (!isMyServiceRunning(ChatGptService::class.java)) return 117 | val messageIntent = Intent(this, ChatGptService::class.java) 118 | messageIntent.action = message 119 | startService(messageIntent) 120 | } 121 | 122 | fun startChatGptService() { 123 | if (isMyServiceRunning(ChatGptService::class.java)) { 124 | Log.d(TAG, "Not starting service.") 125 | return 126 | } 127 | Log.d(TAG, "Starting service.") 128 | val startIntent = Intent(this, ChatGptService::class.java) 129 | startIntent.action = ChatGptService.ACTION_START_FOREGROUND_SERVICE 130 | startService(startIntent) 131 | bindChatGptService() 132 | } 133 | 134 | //check if service is running 135 | private fun isMyServiceRunning(serviceClass: Class<*>): Boolean { 136 | val manager = getSystemService(ACTIVITY_SERVICE) as ActivityManager 137 | for (service in manager.getRunningServices(Int.MAX_VALUE)) { 138 | if (serviceClass.name == service.service.className) { 139 | return true 140 | } 141 | } 142 | return false 143 | } 144 | 145 | fun bindChatGptService() { 146 | if (!mBound) { 147 | val intent = Intent(this, ChatGptService::class.java) 148 | bindService(intent, chatGptAppServiceConnection, BIND_AUTO_CREATE) 149 | } 150 | } 151 | 152 | fun unbindChatGptService() { 153 | if (mBound) { 154 | unbindService(chatGptAppServiceConnection) 155 | mBound = false 156 | } 157 | } 158 | 159 | /** Defines callbacks for service binding, passed to bindService() */ 160 | private val chatGptAppServiceConnection: ServiceConnection = object : ServiceConnection { 161 | override fun onServiceConnected( 162 | className: ComponentName, 163 | service: IBinder 164 | ) { 165 | // We've bound to LocalService, cast the IBinder and get LocalService instance 166 | val sgmLibServiceBinder = service as SmartGlassesAndroidService.LocalBinder 167 | mService = sgmLibServiceBinder.service as ChatGptService 168 | mBound = true 169 | } 170 | 171 | override fun onServiceDisconnected(arg0: ComponentName) { 172 | mBound = false 173 | } 174 | } 175 | } -------------------------------------------------------------------------------- /app/src/main/java/com/teamopensourcesmartglasses/chatgpt/events/ChatErrorEvent.kt: -------------------------------------------------------------------------------- 1 | package com.teamopensourcesmartglasses.chatgpt.events 2 | 3 | class ChatErrorEvent(val errorMessage: String) -------------------------------------------------------------------------------- /app/src/main/java/com/teamopensourcesmartglasses/chatgpt/events/ChatReceivedEvent.kt: -------------------------------------------------------------------------------- 1 | package com.teamopensourcesmartglasses.chatgpt.events 2 | 3 | class ChatReceivedEvent(val message: String) -------------------------------------------------------------------------------- /app/src/main/java/com/teamopensourcesmartglasses/chatgpt/events/ChatSummarizedEvent.kt: -------------------------------------------------------------------------------- 1 | package com.teamopensourcesmartglasses.chatgpt.events 2 | 3 | class ChatSummarizedEvent(val summary: String) -------------------------------------------------------------------------------- /app/src/main/java/com/teamopensourcesmartglasses/chatgpt/events/ClearContextRequestEvent.kt: -------------------------------------------------------------------------------- 1 | package com.teamopensourcesmartglasses.chatgpt.events 2 | 3 | class ClearContextRequestEvent -------------------------------------------------------------------------------- /app/src/main/java/com/teamopensourcesmartglasses/chatgpt/events/IsLoadingEvent.kt: -------------------------------------------------------------------------------- 1 | package com.teamopensourcesmartglasses.chatgpt.events 2 | 3 | class IsLoadingEvent -------------------------------------------------------------------------------- /app/src/main/java/com/teamopensourcesmartglasses/chatgpt/events/QuestionAnswerReceivedEvent.kt: -------------------------------------------------------------------------------- 1 | package com.teamopensourcesmartglasses.chatgpt.events 2 | 3 | class QuestionAnswerReceivedEvent(val question: String, val answer: String) -------------------------------------------------------------------------------- /app/src/main/java/com/teamopensourcesmartglasses/chatgpt/events/UserSettingsChangedEvent.kt: -------------------------------------------------------------------------------- 1 | package com.teamopensourcesmartglasses.chatgpt.events 2 | 3 | class UserSettingsChangedEvent( 4 | val openAiKey: String, 5 | val systemPrompt: String, 6 | val useAutoSend: Boolean 7 | ) -------------------------------------------------------------------------------- /app/src/main/java/com/teamopensourcesmartglasses/chatgpt/ui/ChatGptSetupFragment.kt: -------------------------------------------------------------------------------- 1 | package com.teamopensourcesmartglasses.chatgpt.ui 2 | 3 | import android.os.Bundle 4 | import android.view.LayoutInflater 5 | import android.view.View 6 | import android.view.ViewGroup 7 | import androidx.fragment.app.Fragment 8 | import com.teamopensourcesmartglasses.chatgpt.databinding.FragmentChatgptsetupBinding 9 | 10 | class ChatGptSetupFragment : Fragment() { 11 | private var binding: FragmentChatgptsetupBinding? = null 12 | override fun onCreateView( 13 | inflater: LayoutInflater, container: ViewGroup?, 14 | savedInstanceState: Bundle? 15 | ): View? { 16 | binding = FragmentChatgptsetupBinding.inflate(inflater, container, false) 17 | return binding!!.root 18 | } 19 | 20 | override fun onViewCreated(view: View, savedInstanceState: Bundle?) { 21 | super.onViewCreated(view, savedInstanceState) 22 | } 23 | 24 | override fun onDestroyView() { 25 | super.onDestroyView() 26 | binding = null 27 | } 28 | } -------------------------------------------------------------------------------- /app/src/main/java/com/teamopensourcesmartglasses/chatgpt/utils/Message.kt: -------------------------------------------------------------------------------- 1 | package com.teamopensourcesmartglasses.chatgpt.utils 2 | 3 | import com.theokanning.openai.completion.chat.ChatMessage 4 | import java.time.LocalDateTime 5 | 6 | class Message(var tokenCount: Int, val timestamp: LocalDateTime, val chatMessage: ChatMessage) { 7 | 8 | override fun toString(): String { 9 | return "$tokenCount $chatMessage" 10 | } 11 | } -------------------------------------------------------------------------------- /app/src/main/java/com/teamopensourcesmartglasses/chatgpt/utils/MessageStore.kt: -------------------------------------------------------------------------------- 1 | package com.teamopensourcesmartglasses.chatgpt.utils 2 | 3 | import android.util.Log 4 | import com.knuddels.jtokkit.Encodings 5 | import com.knuddels.jtokkit.api.EncodingType 6 | import com.theokanning.openai.completion.chat.ChatMessage 7 | import com.theokanning.openai.completion.chat.ChatMessageRole 8 | import java.time.LocalDateTime 9 | import java.util.LinkedList 10 | 11 | class MessageStore(private val maxTokenCount: Int) { 12 | private val TAG = "MessageStore" 13 | private var messageQueue = LinkedList() 14 | private var totalTokenCount = 4 // to account for system role extra tokens 15 | private var systemMessage: ChatMessage? = null 16 | 17 | // CL100K_BASE is for GPT3.5-Turbo 18 | private val encoding = 19 | Encodings.newDefaultEncodingRegistry().getEncoding(EncodingType.CL100K_BASE) 20 | 21 | fun setSystemMessage(systemMessage: String) { 22 | if (this.systemMessage != null) { 23 | totalTokenCount -= getTokenCount(this.systemMessage!!.content) 24 | } 25 | this.systemMessage = ChatMessage(ChatMessageRole.SYSTEM.value(), systemMessage) 26 | totalTokenCount += getTokenCount(systemMessage) 27 | } 28 | 29 | fun getSystemMessage(): String { 30 | return systemMessage.toString() 31 | } 32 | 33 | fun addMessage(role: String?, message: String) { 34 | var tokenCount = getTokenCount(message) 35 | if (role == ChatMessageRole.USER.value()) { 36 | tokenCount += 4 // every message follows {role/name}\n{content} 37 | } else if (role == ChatMessageRole.ASSISTANT.value()) { 38 | tokenCount += 6 // every message follows {role/name}\n{content} 39 | // every reply is primed with assistant 40 | } 41 | totalTokenCount += tokenCount 42 | val chatMessage = ChatMessage(role, message) 43 | messageQueue.add(Message(tokenCount, LocalDateTime.now(), chatMessage)) 44 | Log.d(TAG, "addMessage: Added a new message: $message") 45 | Log.d(TAG, "addMessage: New token count: $totalTokenCount") 46 | // if exceeds new total tokens exceeds the limit, this will evict the old messages 47 | ensureTotalTokensWithinLimit() 48 | } 49 | 50 | /** 51 | * Evicts old messages while total tokens are more than the limit 52 | */ 53 | private fun ensureTotalTokensWithinLimit() { 54 | while (totalTokenCount > maxTokenCount) { 55 | val lastMessage = messageQueue.removeFirst() 56 | totalTokenCount -= lastMessage.tokenCount 57 | Log.d( 58 | TAG, 59 | "ensureTotalTokensWithinLimit: Removed a message " + lastMessage.chatMessage.content 60 | ) 61 | Log.d(TAG, "ensureTotalTokensWithinLimit: New token count: $totalTokenCount") 62 | } 63 | } 64 | 65 | /** 66 | * Gets all chat messages including system prompt message in an arraylist 67 | * @return an array of chat messages 68 | */ 69 | val allMessages: ArrayList 70 | get() { 71 | val result = ArrayList() 72 | result.add(systemMessage) 73 | for (message in messageQueue) { 74 | result.add(message.chatMessage) 75 | } 76 | return result 77 | } 78 | 79 | /** 80 | * Getting all messages without system prompt is useful so you can inject your own prompt 81 | * templates for other use-cases 82 | * @return an array of chat messages without system prompt 83 | */ 84 | val allMessagesWithoutSystemPrompt: ArrayList 85 | get() { 86 | val result = ArrayList() 87 | for (message in messageQueue) { 88 | result.add(message.chatMessage) 89 | } 90 | return result 91 | } 92 | 93 | /** 94 | * Gets the messages for the last x minutes 95 | * @param minutes 96 | * @return array of new messages for the last x minutes 97 | */ 98 | fun getMessagesByTime(minutes: Int): ArrayList { 99 | val result = ArrayList() 100 | result.add(systemMessage) 101 | val startTime = LocalDateTime.now().minusMinutes(minutes.toLong()) 102 | for (message in messageQueue) { 103 | if (message.timestamp.isAfter(startTime)) { 104 | result.add(message.chatMessage) 105 | } 106 | } 107 | return result 108 | } 109 | 110 | /** 111 | * Adds a prefix to the last added message 112 | * @param prefix 113 | */ 114 | fun addPrefixToLastAddedMessage(prefix: String) { 115 | // Removes latest message, add a prefix, then add it back 116 | val mostRecentMessage = removeLatest() 117 | val message = mostRecentMessage.chatMessage 118 | addMessage(message!!.role, prefix + " " + message.content) 119 | } 120 | 121 | fun size(): Int { 122 | return messageQueue.size 123 | } 124 | 125 | fun removeOldest(): Message { 126 | val message = messageQueue.removeFirst() 127 | totalTokenCount -= message.tokenCount 128 | return message 129 | } 130 | 131 | fun removeLatest(): Message { 132 | val message = messageQueue.removeLast() 133 | totalTokenCount -= message.tokenCount 134 | return message 135 | } 136 | 137 | fun resetMessages() { 138 | messageQueue = LinkedList() 139 | } 140 | 141 | private fun getTokenCount(message: String): Int { 142 | return encoding.countTokens(message) 143 | } 144 | } -------------------------------------------------------------------------------- /app/src/main/res/drawable/ic_launcher_background.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 11 | 13 | 15 | 17 | 19 | 21 | 23 | 25 | 27 | 29 | 31 | 33 | 35 | 37 | 39 | 41 | 43 | 45 | 47 | 49 | 51 | 53 | 55 | 57 | 59 | 61 | 63 | 65 | 67 | 69 | 71 | 73 | 75 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /app/src/main/res/drawable/ic_launcher_foreground.xml: -------------------------------------------------------------------------------- 1 | 6 | 10 | 13 | 16 | 19 | 22 | 25 | 28 | 31 | 34 | 37 | 40 | 43 | 46 | 49 | 52 | 55 | 58 | 61 | 64 | 67 | 70 | 73 | 76 | 79 | 82 | 85 | 88 | 91 | 94 | 97 | 100 | 103 | 106 | 109 | 112 | 115 | 118 | 121 | 124 | 127 | 130 | 133 | 136 | 139 | 142 | 145 | 148 | 151 | 154 | 157 | 160 | 161 | 162 | -------------------------------------------------------------------------------- /app/src/main/res/layout/activity_main.xml: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | 13 | 14 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /app/src/main/res/layout/content_main.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 19 | -------------------------------------------------------------------------------- /app/src/main/res/layout/fragment_chatgptsetup.xml: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | 13 | 14 | 18 | 19 | 31 | 32 | 43 | 44 | 56 | 57 | 68 | 69 | 86 | 87 | 98 | 99 | 108 | 109 | 115 | 116 | 123 | 124 | 125 | 126 |