├── .github ├── FUNDING.yml └── workflows │ └── release_on_push.yml ├── .gitignore ├── .idea ├── .gitignore ├── .name ├── AndroidProjectSystem.xml ├── deploymentTargetSelector.xml ├── gradle.xml ├── inspectionProfiles │ └── Project_Default.xml ├── migrations.xml ├── misc.xml ├── runConfigurations.xml ├── runConfigurations │ ├── mabl.xml │ └── plugins_demo.xml └── vcs.xml ├── LICENSE ├── README.md ├── build.gradle.kts ├── build.sh ├── docs └── llm │ ├── README.md │ └── implementation_plan.md ├── gradle.properties ├── gradle ├── libs.versions.toml └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── mabl ├── .gitignore ├── build.gradle.kts └── src │ ├── aipin │ └── java │ │ └── com │ │ └── penumbraos │ │ └── mabl │ │ └── ui │ │ └── aliases.kt │ ├── aipinSimulator │ ├── AndroidManifest.xml │ └── java │ │ └── com │ │ └── penumbraos │ │ └── mabl │ │ ├── simulation │ │ ├── ControlPanel.kt │ │ ├── SimulatedPinDisplay.kt │ │ ├── SimulatedTouchpad.kt │ │ └── services │ │ │ └── SimulatorSttService.kt │ │ └── ui │ │ ├── PlatformUI.kt │ │ ├── SimulatorEventRouter.kt │ │ ├── SimulatorInputHandler.kt │ │ └── UIFactory.kt │ ├── aipincore │ └── java │ │ └── com │ │ └── penumbraos │ │ └── mabl │ │ └── aipincore │ │ ├── ConversationRenderer.kt │ │ ├── PlatformCapabilities.kt │ │ ├── PlatformInputHandler.kt │ │ ├── PlatformUI.kt │ │ ├── SettingsStatusBroadcaster.kt │ │ ├── UIFactory.kt │ │ ├── input │ │ ├── ITouchpadGestureDelegate.kt │ │ └── TouchpadGestureManager.kt │ │ ├── server │ │ ├── HttpServer.kt │ │ └── types │ │ │ └── ConversationWithMessages.kt │ │ └── view │ │ ├── TouchInterceptor.kt │ │ ├── model │ │ ├── ConversationsViewModel.kt │ │ ├── NavViewModel.kt │ │ └── PlatformViewModel.kt │ │ ├── nav │ │ ├── ConversationDisplay.kt │ │ ├── Conversations.kt │ │ ├── Home.kt │ │ ├── Menu.kt │ │ ├── Navigation.kt │ │ ├── Settings.kt │ │ └── util │ │ │ └── WithMenuScene.kt │ │ └── util │ │ └── time.kt │ ├── android │ └── java │ │ └── com │ │ └── penumbraos │ │ └── mabl │ │ └── ui │ │ ├── AndroidPlatformInputHandler.kt │ │ ├── ConversationRenderer.kt │ │ ├── PlatformUI.kt │ │ └── UIFactory.kt │ └── main │ ├── AndroidManifest.xml │ ├── assets │ └── .gitignore │ ├── java │ └── com │ │ └── penumbraos │ │ └── mabl │ │ ├── MainActivity.kt │ │ ├── conversation │ │ ├── ConversationManager.kt │ │ ├── StaticQueryManager.kt │ │ └── StaticQueryToolService.kt │ │ ├── data │ │ ├── AppDatabase.kt │ │ ├── ImageFileManager.kt │ │ ├── dao │ │ │ ├── ConversationDao.kt │ │ │ ├── ConversationImageDao.kt │ │ │ └── ConversationMessageDao.kt │ │ ├── repository │ │ │ ├── ConversationImageRepository.kt │ │ │ └── ConversationRepository.kt │ │ └── types │ │ │ ├── Conversation.kt │ │ │ ├── ConversationImage.kt │ │ │ ├── ConversationMessage.kt │ │ │ └── ConversationWithImages.kt │ │ ├── discovery │ │ └── PluginManager.kt │ │ ├── interaction │ │ ├── IInteractionFlowManager.kt │ │ └── InteractionFlowManager.kt │ │ ├── services │ │ ├── AllControllers.kt │ │ ├── CameraRollService.kt │ │ ├── CameraService.kt │ │ ├── LlmController.kt │ │ ├── ServiceController.kt │ │ ├── SttController.kt │ │ ├── SystemServiceRegistry.kt │ │ ├── ToolController.kt │ │ ├── ToolOrchestrator.kt │ │ ├── ToolSimilarityService.kt │ │ └── TtsController.kt │ │ ├── sound │ │ ├── SoundEffectManager.kt │ │ └── TonePlayer.kt │ │ ├── types │ │ ├── Errors.kt │ │ └── ServiceBundle.kt │ │ ├── ui │ │ ├── SettingsScreen.kt │ │ ├── UIComponents.kt │ │ ├── interfaces │ │ │ ├── IConversationRenderer.kt │ │ │ ├── IPlatformCapabilities.kt │ │ │ └── IPlatformInputHandler.kt │ │ └── theme │ │ │ ├── Color.kt │ │ │ ├── Theme.kt │ │ │ └── Type.kt │ │ └── util │ │ └── SentenceEmbedding.kt │ └── res │ ├── drawable │ ├── ic_launcher_background.xml │ ├── ic_launcher_foreground.xml │ └── outline_voice_chat_24.xml │ ├── mipmap-anydpi │ ├── ic_launcher.xml │ └── ic_launcher_round.xml │ ├── mipmap-hdpi │ ├── ic_launcher.webp │ └── ic_launcher_round.webp │ ├── mipmap-mdpi │ ├── ic_launcher.webp │ └── ic_launcher_round.webp │ ├── mipmap-xhdpi │ ├── ic_launcher.webp │ └── ic_launcher_round.webp │ ├── mipmap-xxhdpi │ ├── ic_launcher.webp │ └── ic_launcher_round.webp │ ├── mipmap-xxxhdpi │ ├── ic_launcher.webp │ └── ic_launcher_round.webp │ └── values │ ├── colors.xml │ ├── strings.xml │ └── themes.xml ├── plugins ├── aipinsystem │ ├── .gitignore │ ├── build.gradle.kts │ └── src │ │ └── main │ │ ├── AndroidManifest.xml │ │ └── java │ │ └── com │ │ └── penumbraos │ │ └── plugins │ │ └── aipinsystem │ │ ├── BatteryToolService.kt │ │ └── TimeToolService.kt ├── demo │ ├── .gitignore │ ├── build.gradle.kts │ └── src │ │ └── main │ │ ├── AndroidManifest.xml │ │ └── java │ │ └── com │ │ └── penumbraos │ │ └── plugins │ │ └── demo │ │ ├── DemoSttService.kt │ │ └── DemoTtsService.kt ├── googlesearch │ ├── .gitignore │ ├── build.gradle.kts │ └── src │ │ └── main │ │ ├── AndroidManifest.xml │ │ └── java │ │ └── com │ │ └── penumbraos │ │ └── plugins │ │ └── googlesearch │ │ ├── GoogleSearchClient.kt │ │ ├── GoogleSearchProcessor.kt │ │ └── GoogleSearchService.kt ├── openai │ ├── .gitignore │ ├── build.gradle.kts │ └── src │ │ └── main │ │ ├── AndroidManifest.xml │ │ └── java │ │ └── com │ │ └── penumbraos │ │ └── plugins │ │ └── openai │ │ ├── LlmConfigManager.kt │ │ └── OpenAiLlmService.kt ├── searxng │ ├── .gitignore │ ├── build.gradle.kts │ └── src │ │ └── main │ │ ├── AndroidManifest.xml │ │ └── java │ │ └── com │ │ └── penumbraos │ │ └── plugins │ │ └── searxng │ │ ├── SearchResultProcessor.kt │ │ ├── SearxngClient.kt │ │ ├── SearxngHtmlParser.kt │ │ └── WebSearchService.kt └── system │ ├── .gitignore │ ├── build.gradle.kts │ └── src │ └── main │ ├── AndroidManifest.xml │ └── java │ └── com │ └── penumbraos │ └── plugins │ └── system │ └── tool │ ├── NetworkService.kt │ └── VolumeService.kt ├── portal ├── .gitignore ├── README.md ├── eslint.config.js ├── index.html ├── package-lock.json ├── package.json ├── postcss.config.cjs ├── src │ ├── App.tsx │ ├── components │ │ ├── CameraRoll.module.scss │ │ ├── CameraRoll.tsx │ │ ├── Conversation.module.scss │ │ ├── Conversation.tsx │ │ ├── Conversations.tsx │ │ ├── ImageGallery.module.scss │ │ ├── ImageGallery.tsx │ │ ├── Navigation.tsx │ │ ├── QueryWrapper.module.scss │ │ └── QueryWrapper.tsx │ ├── hooks │ │ └── useImageGallery.ts │ ├── main.tsx │ ├── routeTree.gen.ts │ ├── routes │ │ ├── __root.tsx │ │ ├── camera-roll.tsx │ │ ├── conversation.$conversationId.tsx │ │ └── index.tsx │ ├── state │ │ ├── api.ts │ │ ├── query.ts │ │ └── types.ts │ ├── styles │ │ └── layout.module.scss │ ├── util │ │ └── date.ts │ └── vite-env.d.ts ├── tsconfig.app.json ├── tsconfig.json ├── tsconfig.node.json └── vite.config.ts ├── sdk ├── .gitignore ├── build.gradle.kts └── src │ └── main │ ├── AndroidManifest.xml │ ├── aidl │ └── com │ │ └── penumbraos │ │ └── mabl │ │ └── sdk │ │ ├── BinderConversationMessage.aidl │ │ ├── ILlmCallback.aidl │ │ ├── ILlmService.aidl │ │ ├── ISttCallback.aidl │ │ ├── ISttService.aidl │ │ ├── ISystemServiceRegistry.aidl │ │ ├── IToolCallback.aidl │ │ ├── IToolService.aidl │ │ ├── ITtsCallback.aidl │ │ ├── ITtsService.aidl │ │ ├── LlmResponse.aidl │ │ ├── ToolCall.aidl │ │ ├── ToolDefinition.aidl │ │ └── ToolParameter.aidl │ └── java │ └── com │ └── penumbraos │ └── mabl │ └── sdk │ ├── DeviceUtils.kt │ ├── MablService.kt │ ├── PluginConstants.kt │ └── ToolService.kt └── settings.gradle.kts /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: agg23 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | .gradle 3 | /local.properties 4 | /.idea/caches 5 | /.idea/libraries 6 | /.idea/modules.xml 7 | /.idea/workspace.xml 8 | /.idea/navEditor.xml 9 | /.idea/assetWizardSettings.xml 10 | .DS_Store 11 | /build 12 | /captures 13 | /test_data 14 | .externalNativeBuild 15 | .cxx 16 | local.properties 17 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | -------------------------------------------------------------------------------- /.idea/.name: -------------------------------------------------------------------------------- 1 | MABL -------------------------------------------------------------------------------- /.idea/AndroidProjectSystem.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | -------------------------------------------------------------------------------- /.idea/deploymentTargetSelector.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 16 | 17 | 19 | 20 | 22 | 23 | 25 | 26 | 28 | 29 | 31 | 32 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /.idea/gradle.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 26 | 27 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 61 | -------------------------------------------------------------------------------- /.idea/migrations.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 9 | 10 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 9 | -------------------------------------------------------------------------------- /.idea/runConfigurations.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 17 | -------------------------------------------------------------------------------- /.idea/runConfigurations/mabl.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 76 | -------------------------------------------------------------------------------- /.idea/runConfigurations/plugins_demo.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 74 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2025 Adam Gastineau 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /build.gradle.kts: -------------------------------------------------------------------------------- 1 | // Top-level build file where you can add configuration options common to all sub-projects/modules. 2 | 3 | plugins { 4 | alias(libs.plugins.android.application) apply false 5 | alias(libs.plugins.kotlin.android) apply false 6 | alias(libs.plugins.kotlin.compose) apply false 7 | alias(libs.plugins.android.library) apply false 8 | } 9 | 10 | tasks { 11 | /** 12 | * Gradle task to install the demo plugin's APK onto connected devices. 13 | * Requires 'adb' to be in the system's PATH or specified with a full path 14 | */ 15 | register("installDemoPlugins") { 16 | description = "Installs the demo plugin's APK on connected device(s) using adb" 17 | group = "install" 18 | 19 | val serviceProject = project(":plugins:demo") 20 | 21 | val serviceApkPath = 22 | "${serviceProject.buildDir}/outputs/apk/debug/${serviceProject.name}-debug.apk" 23 | 24 | commandLine("bash", "-c", "adb install -r $serviceApkPath") 25 | 26 | onlyIf { 27 | File(serviceApkPath).exists() 28 | } 29 | 30 | dependsOn(":plugins:demo:assembleDebug") 31 | } 32 | /** 33 | * Gradle task to install the OpenAI plugin's APK onto connected devices. 34 | * Requires 'adb' to be in the system's PATH or specified with a full path 35 | */ 36 | register("installOpenAiPlugin") { 37 | description = "Installs the OpenAI plugin's APK on connected device(s) using adb" 38 | group = "install" 39 | 40 | val serviceProject = project(":plugins:openai") 41 | 42 | val serviceApkPath = 43 | "${serviceProject.buildDir}/outputs/apk/debug/${serviceProject.name}-debug.apk" 44 | 45 | commandLine("bash", "-c", "adb install -r $serviceApkPath") 46 | 47 | onlyIf { 48 | File(serviceApkPath).exists() 49 | } 50 | 51 | dependsOn(":plugins:openai:assembleDebug") 52 | } 53 | } -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | if ! [ -f mabl/src/main/assets/minilm-l6-v2-qint8-arm64.onnx ]; then 2 | curl -L -o mabl/src/main/assets/minilm-l6-v2-qint8-arm64.onnx https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/onnx/model_qint8_arm64.onnx?download=true 3 | fi 4 | if ! [ -f mabl/src/main/assets/minilm-l6-v2-tokenizer.json ]; then 5 | curl -L -o mabl/src/main/assets/minilm-l6-v2-tokenizer.json https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/tokenizer.json?download=true 6 | fi 7 | ./gradlew :plugins:demo:installDebug :plugins:aipinsystem:installDebug :plugins:system:installDebug :plugins:openai:installDebug :plugins:googlesearch:installDebug :mabl:installAipinDebug 8 | adb shell pm grant com.penumbraos.mabl.pin android.permission.CAMERA 9 | adb shell appops set com.penumbraos.mabl.pin MANAGE_EXTERNAL_STORAGE allow 10 | adb shell appops set com.penumbraos.plugins.openai MANAGE_EXTERNAL_STORAGE allow 11 | adb shell pm disable-user --user 0 humane.experience.systemnavigation 12 | sleep 1 13 | adb shell cmd package set-home-activity com.penumbraos.mabl.pin/com.penumbraos.mabl.MainActivity 14 | sleep 1 15 | # I think one of these works 16 | adb shell settings put secure launcher_component com.penumbraos.mabl.pin/com.penumbraos.mabl.MainActivity 17 | adb shell settings put system home_app com.penumbraos.mabl.pin 18 | adb shell settings put global default_launcher com.penumbraos.mabl.pin/com.penumbraos.mabl.MainActivity 19 | echo "Built on $(date '+%Y-%m-%d %H:%M:%S')" -------------------------------------------------------------------------------- /docs/llm/README.md: -------------------------------------------------------------------------------- 1 | This directory contains entirely LLM generated documentation for the purposes of further implementation. This is an experiment on how LLMs will manage with this kind of "fake" documentation. Humans should not rely on this information as accurate. 2 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | # Project-wide Gradle settings. 2 | # IDE (e.g. Android Studio) users: 3 | # Gradle settings configured through the IDE *will override* 4 | # any settings specified in this file. 5 | # For more details on how to configure your build environment visit 6 | # http://www.gradle.org/docs/current/userguide/build_environment.html 7 | # Specifies the JVM arguments used for the daemon process. 8 | # The setting is particularly useful for tweaking memory settings. 9 | org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 10 | # When configured, Gradle will run in incubating parallel mode. 11 | # This option should only be used with decoupled projects. For more details, visit 12 | # https://developer.android.com/r/tools/gradle-multi-project-decoupled-projects 13 | # org.gradle.parallel=true 14 | # AndroidX package structure to make it clearer which packages are bundled with the 15 | # Android operating system, and which are packaged with your app's APK 16 | # https://developer.android.com/topic/libraries/support-library/androidx-rn 17 | android.useAndroidX=true 18 | # Kotlin code style for this project: "official" or "obsolete": 19 | kotlin.code.style=official 20 | # Enables namespacing of each library's R class so that its R class includes only the 21 | # resources declared in the library itself and none from the library's dependencies, 22 | # thereby reducing the size of the R class for that library 23 | android.nonTransitiveRClass=true -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PenumbraOS/mabl/1218669f9bc43f80d8e4a49c3edcb9029f084e2b/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Fri Jul 18 12:05:30 PDT 2025 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip 5 | zipStoreBase=GRADLE_USER_HOME 6 | zipStorePath=wrapper/dists 7 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /mabl/.gitignore: -------------------------------------------------------------------------------- 1 | /build -------------------------------------------------------------------------------- /mabl/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | alias(libs.plugins.android.application) 3 | alias(libs.plugins.kotlin.android) 4 | alias(libs.plugins.kotlin.compose) 5 | alias(libs.plugins.ksp) 6 | alias(libs.plugins.kotlin.serialization) 7 | } 8 | 9 | android { 10 | namespace = "com.penumbraos.mabl" 11 | compileSdk = 36 12 | 13 | defaultConfig { 14 | applicationId = "com.penumbraos.mabl" 15 | minSdk = 32 16 | targetSdk = 36 17 | versionCode = (project.findProperty("versionCode") as String?)?.toIntOrNull() ?: 1 18 | versionName = project.findProperty("versionName") as String? ?: "1.0" 19 | 20 | ndk { 21 | //noinspection ChromeOsAbiSupport 22 | abiFilters += listOf("arm64-v8a") 23 | } 24 | } 25 | 26 | buildTypes { 27 | release { 28 | isMinifyEnabled = false 29 | signingConfig = signingConfigs.getByName("debug") 30 | } 31 | } 32 | compileOptions { 33 | sourceCompatibility = JavaVersion.VERSION_11 34 | targetCompatibility = JavaVersion.VERSION_11 35 | } 36 | kotlinOptions { 37 | jvmTarget = "11" 38 | } 39 | buildFeatures { 40 | compose = true 41 | buildConfig = true 42 | } 43 | 44 | flavorDimensions += "device" 45 | productFlavors { 46 | create("aipin") { 47 | dimension = "device" 48 | applicationIdSuffix = ".pin" 49 | buildConfigField("boolean", "IS_AI_PIN", "true") 50 | buildConfigField("boolean", "IS_SIMULATOR", "false") 51 | } 52 | create("aipinSimulator") { 53 | dimension = "device" 54 | applicationIdSuffix = ".pinsim" 55 | buildConfigField("boolean", "IS_AI_PIN", "true") 56 | buildConfigField("boolean", "IS_SIMULATOR", "true") 57 | } 58 | // create("android") { 59 | // dimension = "device" 60 | // applicationIdSuffix = ".android" 61 | // buildConfigField("boolean", "IS_AI_PIN", "false") 62 | // buildConfigField("boolean", "IS_SIMULATOR", "false") 63 | // } 64 | } 65 | 66 | sourceSets { 67 | getByName("aipin") { 68 | java.srcDirs("src/aipincore/java") 69 | } 70 | getByName("aipinSimulator") { 71 | java.srcDirs("src/aipincore/java") 72 | } 73 | } 74 | } 75 | 76 | dependencies { 77 | implementation(project(":sdk")) 78 | implementation(libs.penumbraos.sdk) 79 | implementation(libs.androidx.navigation3.ui) 80 | implementation(libs.androidx.navigation3.runtime) 81 | "aipinImplementation"(libs.moonlight.ui) 82 | "aipinSimulatorImplementation"(libs.moonlight.ui) 83 | 84 | implementation(libs.androidx.camera.core) 85 | implementation(libs.androidx.camera.lifecycle) 86 | implementation(libs.androidx.camera.camera2) 87 | 88 | implementation(libs.kotlinx.serialization.json) 89 | 90 | implementation(libs.onnx.runtime.android) 91 | implementation(libs.sentence.embeddings) 92 | 93 | implementation(libs.androidx.core.ktx) 94 | implementation(libs.androidx.lifecycle.runtime.compose) 95 | implementation(libs.androidx.lifecycle.runtime.ktx) 96 | implementation(libs.androidx.lifecycle.process) 97 | implementation(libs.androidx.lifecycle.viewmodel.compose) 98 | implementation(libs.androidx.activity.compose) 99 | implementation(platform(libs.androidx.compose.bom)) 100 | implementation(libs.androidx.ui) 101 | implementation(libs.androidx.ui.graphics) 102 | implementation(libs.androidx.ui.tooling.preview) 103 | implementation(libs.androidx.material3) 104 | implementation(libs.androidx.fragment) 105 | implementation(libs.androidx.room.runtime) 106 | implementation(libs.androidx.room.ktx) 107 | debugImplementation(libs.ui.tooling) 108 | ksp(libs.androidx.room.compiler) 109 | } -------------------------------------------------------------------------------- /mabl/src/aipin/java/com/penumbraos/mabl/ui/aliases.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui 2 | 3 | import androidx.compose.runtime.Composable 4 | 5 | typealias ConversationRenderer = com.penumbraos.mabl.aipincore.ConversationRenderer 6 | 7 | @Composable 8 | fun PlatformUI(uiComponents: UIComponents?) = 9 | com.penumbraos.mabl.aipincore.PlatformUI(uiComponents) 10 | typealias UIFactory = com.penumbraos.mabl.aipincore.UIFactory -------------------------------------------------------------------------------- /mabl/src/aipinSimulator/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 11 | 12 | 13 | 14 | 15 | 18 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /mabl/src/aipinSimulator/java/com/penumbraos/mabl/simulation/SimulatedPinDisplay.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.simulation 2 | 3 | import android.annotation.SuppressLint 4 | import androidx.compose.foundation.background 5 | import androidx.compose.foundation.border 6 | import androidx.compose.foundation.layout.Box 7 | import androidx.compose.foundation.layout.BoxWithConstraints 8 | import androidx.compose.foundation.layout.padding 9 | import androidx.compose.foundation.layout.requiredSize 10 | import androidx.compose.foundation.shape.RoundedCornerShape 11 | import androidx.compose.material3.MaterialTheme 12 | import androidx.compose.material3.Text 13 | import androidx.compose.runtime.Composable 14 | import androidx.compose.ui.Alignment 15 | import androidx.compose.ui.Modifier 16 | import androidx.compose.ui.draw.clip 17 | import androidx.compose.ui.graphics.Color 18 | import androidx.compose.ui.graphics.TransformOrigin 19 | import androidx.compose.ui.graphics.graphicsLayer 20 | import androidx.compose.ui.platform.LocalDensity 21 | import androidx.compose.ui.unit.dp 22 | import com.penumbraos.mabl.ui.UIComponents 23 | 24 | @SuppressLint("UnusedBoxWithConstraintsScope") 25 | @Composable 26 | fun SimulatedPinDisplay( 27 | modifier: Modifier = Modifier, 28 | uiComponents: UIComponents? 29 | ) { 30 | val density = LocalDensity.current 31 | 32 | // Calculate 800x720 aspect ratio while fitting within available space 33 | val targetAspectRatio = 800f / 720f 34 | 35 | BoxWithConstraints( 36 | modifier = modifier, 37 | contentAlignment = Alignment.Center 38 | ) { 39 | val availableWidth = maxWidth 40 | val availableHeight = maxHeight 41 | 42 | // Calculate display size maintaining 800x720 aspect ratio 43 | val (displayWidth, displayHeight, scaleFactor) = with(density) { 44 | val availableWidthPx = availableWidth.toPx() * 0.9f 45 | val availableHeightPx = availableHeight.toPx() * 0.9f 46 | 47 | val scaledHeight = availableWidthPx / targetAspectRatio 48 | val scaledWidth = availableHeightPx * targetAspectRatio 49 | 50 | if (scaledHeight <= availableHeightPx) { 51 | val scale = availableWidthPx / (800f * density.density) 52 | Triple(availableWidthPx.toDp(), scaledHeight.toDp(), scale) 53 | } else { 54 | val scale = availableHeightPx / (720f * density.density) 55 | Triple(scaledWidth.toDp(), availableHeightPx.toDp(), scale) 56 | } 57 | } 58 | 59 | // Simulated Pin Display Container 60 | Box( 61 | modifier = Modifier 62 | .requiredSize(displayWidth, displayHeight) 63 | .clip(RoundedCornerShape(12.dp)) 64 | .border(2.dp, Color.Gray, RoundedCornerShape(12.dp)) 65 | .background(Color.Black) 66 | ) { 67 | Box( 68 | modifier = Modifier 69 | .requiredSize(800.dp, 720.dp) 70 | .graphicsLayer( 71 | scaleX = scaleFactor, 72 | scaleY = scaleFactor, 73 | transformOrigin = TransformOrigin.Center 74 | ) 75 | ) { 76 | com.penumbraos.mabl.aipincore.PlatformUI(uiComponents) 77 | } 78 | } 79 | 80 | // Display dimensions indicator 81 | Text( 82 | text = "${displayWidth.value.toInt()}x${displayHeight.value.toInt()} (scaled from 800x720)", 83 | style = MaterialTheme.typography.labelSmall, 84 | color = Color.Gray, 85 | modifier = Modifier 86 | .align(Alignment.BottomCenter) 87 | .padding(top = 8.dp) 88 | ) 89 | } 90 | } -------------------------------------------------------------------------------- /mabl/src/aipinSimulator/java/com/penumbraos/mabl/ui/PlatformUI.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui 2 | 3 | import androidx.compose.foundation.layout.Column 4 | import androidx.compose.foundation.layout.Row 5 | import androidx.compose.foundation.layout.fillMaxHeight 6 | import androidx.compose.foundation.layout.fillMaxSize 7 | import androidx.compose.foundation.layout.fillMaxWidth 8 | import androidx.compose.foundation.layout.padding 9 | import androidx.compose.foundation.rememberScrollState 10 | import androidx.compose.foundation.verticalScroll 11 | import androidx.compose.material3.Card 12 | import androidx.compose.material3.CardDefaults 13 | import androidx.compose.material3.MaterialTheme 14 | import androidx.compose.material3.Text 15 | import androidx.compose.runtime.Composable 16 | import androidx.compose.ui.Modifier 17 | import androidx.compose.ui.unit.dp 18 | import com.penumbraos.mabl.simulation.ControlPanel 19 | import com.penumbraos.mabl.simulation.SimulatedPinDisplay 20 | import com.penumbraos.mabl.simulation.SimulatedTouchpad 21 | 22 | @Composable 23 | fun PlatformUI(uiComponents: UIComponents?) { 24 | // AI Pin Simulator: Three-panel layout for development and testing 25 | Row( 26 | modifier = Modifier.fillMaxSize() 27 | ) { 28 | // Left Panel: Simulated 800x720 Pin Display 29 | Card( 30 | modifier = Modifier 31 | .weight(0.4f) 32 | .fillMaxHeight() 33 | .padding(8.dp), 34 | elevation = CardDefaults.cardElevation(defaultElevation = 4.dp) 35 | ) { 36 | Column( 37 | modifier = Modifier.padding(8.dp) 38 | ) { 39 | Text( 40 | text = "AI Pin Display (800x720)", 41 | style = MaterialTheme.typography.titleSmall, 42 | modifier = Modifier.padding(bottom = 8.dp) 43 | ) 44 | SimulatedPinDisplay( 45 | modifier = Modifier.fillMaxSize(), 46 | uiComponents = uiComponents 47 | ) 48 | } 49 | } 50 | 51 | // Center Panel: Touchpad Simulation 52 | Card( 53 | modifier = Modifier 54 | .weight(0.3f) 55 | .fillMaxHeight() 56 | .padding(8.dp), 57 | elevation = CardDefaults.cardElevation(defaultElevation = 4.dp) 58 | ) { 59 | Column( 60 | modifier = Modifier.padding(8.dp) 61 | ) { 62 | Text( 63 | text = "Touchpad", 64 | style = MaterialTheme.typography.titleSmall, 65 | modifier = Modifier.padding(bottom = 8.dp) 66 | ) 67 | SimulatedTouchpad( 68 | modifier = Modifier.fillMaxSize() 69 | ) 70 | } 71 | } 72 | 73 | // Right Panel: Simulator Controls 74 | Card( 75 | modifier = Modifier 76 | .weight(0.3f) 77 | .fillMaxHeight() 78 | .padding(8.dp), 79 | elevation = CardDefaults.cardElevation(defaultElevation = 4.dp) 80 | ) { 81 | Column( 82 | modifier = Modifier 83 | .padding(8.dp) 84 | .verticalScroll(rememberScrollState()) 85 | ) { 86 | Text( 87 | text = "Simulator Controls", 88 | style = MaterialTheme.typography.titleSmall, 89 | modifier = Modifier.padding(bottom = 8.dp) 90 | ) 91 | ControlPanel( 92 | modifier = Modifier.fillMaxWidth() 93 | ) 94 | } 95 | } 96 | } 97 | } -------------------------------------------------------------------------------- /mabl/src/aipinSimulator/java/com/penumbraos/mabl/ui/SimulatorEventRouter.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui 2 | 3 | import android.view.MotionEvent 4 | import android.view.KeyEvent 5 | 6 | object SimulatorEventRouter { 7 | var instance: TouchpadEventHandler? = null 8 | 9 | interface TouchpadEventHandler { 10 | fun onSimulatorTouchpadEvent(event: MotionEvent) 11 | } 12 | } 13 | 14 | object SimulatorKeyEventRouter { 15 | var instance: KeyEventHandler? = null 16 | 17 | interface KeyEventHandler { 18 | fun onSimulatorKeyEvent(keyCode: Int, event: KeyEvent?) 19 | } 20 | } 21 | 22 | object SimulatorSttRouter { 23 | var instance: SttEventHandler? = null 24 | 25 | interface SttEventHandler { 26 | fun onSimulatorManualInput(text: String) 27 | fun onSimulatorStartListening() 28 | fun onSimulatorStopListening() 29 | } 30 | } -------------------------------------------------------------------------------- /mabl/src/aipinSimulator/java/com/penumbraos/mabl/ui/SimulatorInputHandler.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui 2 | 3 | import android.content.Context 4 | import android.util.Log 5 | import android.view.KeyEvent 6 | import android.view.MotionEvent 7 | import androidx.lifecycle.LifecycleCoroutineScope 8 | import com.penumbraos.mabl.aipincore.SettingsStatusBroadcaster 9 | import com.penumbraos.mabl.aipincore.view.model.PlatformViewModel 10 | import com.penumbraos.mabl.interaction.IInteractionFlowManager 11 | 12 | private const val TAG = "SimulatorInputHandler" 13 | 14 | class SimulatorInputHandler( 15 | statusBroadcaster: SettingsStatusBroadcaster, 16 | private val viewModel: PlatformViewModel, 17 | private val platformCapabilities: com.penumbraos.mabl.ui.interfaces.IPlatformCapabilities 18 | ) : com.penumbraos.mabl.aipincore.PlatformInputHandler(statusBroadcaster, viewModel), 19 | SimulatorEventRouter.TouchpadEventHandler, 20 | SimulatorKeyEventRouter.KeyEventHandler, 21 | SimulatorSttRouter.SttEventHandler { 22 | 23 | private lateinit var interactionFlowManager: IInteractionFlowManager 24 | 25 | override fun setup( 26 | context: Context, 27 | lifecycleScope: LifecycleCoroutineScope, 28 | interactionFlowManager: IInteractionFlowManager 29 | ) { 30 | SimulatorEventRouter.instance = this 31 | SimulatorKeyEventRouter.instance = this 32 | SimulatorSttRouter.instance = this 33 | this.interactionFlowManager = interactionFlowManager 34 | super.setup(context, lifecycleScope, interactionFlowManager) 35 | } 36 | 37 | override fun onSimulatorTouchpadEvent(event: MotionEvent) { 38 | Log.d(TAG, "Simulator touchpad event received") 39 | super.touchpadGestureManager.processTouchpadEvent(event) 40 | } 41 | 42 | override fun onSimulatorKeyEvent(keyCode: Int, event: KeyEvent?) { 43 | Log.d(TAG, "Simulator key event received: keyCode=$keyCode") 44 | when (keyCode) { 45 | 36 -> { 46 | handleClosedHandGesture() 47 | } 48 | 49 | 54 -> { 50 | handleHandToggledMenuLayer() 51 | } 52 | } 53 | } 54 | 55 | override fun onSimulatorManualInput(text: String) { 56 | Log.d(TAG, "Manual input received: $text") 57 | interactionFlowManager.startConversationFromInput(text) 58 | } 59 | 60 | override fun onSimulatorStartListening() { 61 | Log.d(TAG, "Simulator start listening") 62 | interactionFlowManager.startListening() 63 | } 64 | 65 | override fun onSimulatorStopListening() { 66 | Log.d(TAG, "Simulator stop listening") 67 | if (interactionFlowManager.isFlowActive()) { 68 | interactionFlowManager.finishListening() 69 | } 70 | } 71 | } -------------------------------------------------------------------------------- /mabl/src/aipinSimulator/java/com/penumbraos/mabl/ui/UIFactory.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui 2 | 3 | import android.content.Context 4 | import com.penumbraos.mabl.aipincore.SettingsStatusBroadcaster 5 | import com.penumbraos.mabl.services.AllControllers 6 | import com.penumbraos.mabl.ui.interfaces.IPlatformInputHandler 7 | import kotlinx.coroutines.CoroutineScope 8 | 9 | class UIFactory( 10 | coroutineScope: CoroutineScope, 11 | private val context: Context, 12 | private val controllers: AllControllers, 13 | ) : com.penumbraos.mabl.aipincore.UIFactory(coroutineScope, context, controllers) { 14 | private val statusBroadcaster = SettingsStatusBroadcaster(context, coroutineScope) 15 | 16 | override fun createPlatformInputHandler(): IPlatformInputHandler { 17 | return SimulatorInputHandler( 18 | statusBroadcaster, 19 | controllers.viewModel, 20 | createPlatformCapabilities() 21 | ) 22 | } 23 | } -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/ConversationRenderer.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.aipincore 2 | 3 | import android.content.Context 4 | import android.util.Log 5 | import com.penumbraos.mabl.services.AllControllers 6 | import com.penumbraos.mabl.types.Error 7 | import com.penumbraos.mabl.ui.interfaces.IConversationRenderer 8 | 9 | private const val TAG = "AiPinConversationRenderer" 10 | 11 | class ConversationRenderer( 12 | private val context: Context, 13 | private val controllers: AllControllers, 14 | private val statusBroadcaster: SettingsStatusBroadcaster? = null 15 | ) : IConversationRenderer { 16 | 17 | // val penumbraClient = PenumbraClient(context) 18 | 19 | // init { 20 | // CoroutineScope(Dispatchers.Default).launch { 21 | // penumbraClient.waitForBridge() 22 | // penumbraClient.handTracking.acquireHATSLock() 23 | // Log.d(TAG, "Hand tracking stopped") 24 | // penumbraClient.handTracking.releaseHATSLock() 25 | // delay(1000) 26 | // penumbraClient.handTracking.acquireHATSLock() 27 | // Log.d(TAG, "Hand tracking stopped v2") 28 | // } 29 | // } 30 | 31 | override fun showMessage(message: String, isUser: Boolean) { 32 | Log.d(TAG, "Message: $message (isUser: $isUser)") 33 | 34 | if (isUser) { 35 | statusBroadcaster?.sendUserMessageEvent(message) 36 | statusBroadcaster?.sendAIThinkingStatus(message) 37 | } else { 38 | statusBroadcaster?.sendAIResponseEvent(message, false) 39 | statusBroadcaster?.sendIdleStatus(message) 40 | } 41 | } 42 | 43 | override fun showTranscription(text: String) { 44 | Log.d(TAG, "Transcription: $text") 45 | statusBroadcaster?.sendTranscribingStatus(text) 46 | } 47 | 48 | override fun showListening(isListening: Boolean) { 49 | Log.d(TAG, "Listening: $isListening") 50 | if (isListening) { 51 | controllers.soundEffectManager.playStartListeningEffect() 52 | } 53 | } 54 | 55 | override fun showError(error: Error) { 56 | // TODO: Display onscreen 57 | when (error) { 58 | is Error.TtsError -> {} 59 | is Error.SttError -> { 60 | val lastListenDuration = controllers.stt.lastListenDuration() 61 | if (lastListenDuration != null && lastListenDuration < 2000) { 62 | // Assume this wasn't a real query 63 | return 64 | } 65 | controllers.tts.service?.speakImmediately("Sorry, I could not hear you") 66 | statusBroadcaster?.sendSTTErrorEvent(error.message, "conversationRenderer") 67 | } 68 | 69 | is Error.LlmError -> { 70 | controllers.tts.service?.speakImmediately("Failed to talk to LLM") 71 | statusBroadcaster?.sendLLMErrorEvent(error.message) 72 | statusBroadcaster?.sendErrorStatus(error.message) 73 | } 74 | 75 | is Error.FlowError -> { 76 | controllers.tts.service?.speakImmediately("Failed to talk to LLM") 77 | statusBroadcaster?.sendLLMErrorEvent(error.message) 78 | statusBroadcaster?.sendErrorStatus(error.message) 79 | } 80 | } 81 | } 82 | } -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/PlatformCapabilities.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.aipincore 2 | 3 | import com.penumbraos.mabl.aipincore.server.HttpServer 4 | import com.penumbraos.mabl.aipincore.view.model.PlatformViewModel 5 | import com.penumbraos.mabl.services.AllControllers 6 | import com.penumbraos.mabl.ui.interfaces.IPlatformCapabilities 7 | import com.penumbraos.sdk.PenumbraClient 8 | import kotlinx.coroutines.CoroutineScope 9 | 10 | private const val TAG = "AiPinCapabilities" 11 | 12 | class PlatformCapabilities( 13 | private val coroutineScope: CoroutineScope, 14 | private val allControllers: AllControllers, 15 | private val platformViewModel: PlatformViewModel, 16 | private val client: PenumbraClient 17 | ) : IPlatformCapabilities { 18 | 19 | private val httpServer = HttpServer(allControllers, coroutineScope, client) 20 | 21 | override fun getViewModel(): Any { 22 | return platformViewModel 23 | } 24 | } -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/UIFactory.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.aipincore 2 | 3 | import android.content.Context 4 | import com.penumbraos.mabl.services.AllControllers 5 | import com.penumbraos.mabl.ui.UIComponents 6 | import com.penumbraos.mabl.ui.interfaces.IConversationRenderer 7 | import com.penumbraos.mabl.ui.interfaces.IPlatformCapabilities 8 | import com.penumbraos.mabl.ui.interfaces.IPlatformInputHandler 9 | import com.penumbraos.sdk.PenumbraClient 10 | import kotlinx.coroutines.CoroutineScope 11 | 12 | open class UIFactory( 13 | private val coroutineScope: CoroutineScope, 14 | private val context: Context, 15 | private val controllers: AllControllers, 16 | ) { 17 | private val statusBroadcaster = SettingsStatusBroadcaster(context, coroutineScope) 18 | 19 | private val client = PenumbraClient(context) 20 | 21 | fun createConversationRenderer(): IConversationRenderer { 22 | return ConversationRenderer(context, controllers, statusBroadcaster) 23 | } 24 | 25 | open fun createPlatformInputHandler(): IPlatformInputHandler { 26 | return PlatformInputHandler(statusBroadcaster, controllers.viewModel) 27 | } 28 | 29 | open fun createPlatformCapabilities(): IPlatformCapabilities { 30 | return PlatformCapabilities(coroutineScope, controllers, controllers.viewModel, client) 31 | } 32 | 33 | fun createUIComponents(): UIComponents { 34 | return UIComponents( 35 | conversationRenderer = createConversationRenderer(), 36 | platformInputHandler = createPlatformInputHandler(), 37 | platformCapabilities = createPlatformCapabilities() 38 | ) 39 | } 40 | } -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/input/ITouchpadGestureDelegate.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.aipincore.input 2 | 3 | interface ITouchpadGestureDelegate { 4 | fun onGesture(gesture: TouchpadGesture) 5 | } 6 | 7 | data class TouchpadGesture(val kind: TouchpadGestureKind, val duration: Long, val fingerCount: Int) 8 | 9 | enum class TouchpadGestureKind { 10 | SINGLE_TAP, 11 | DOUBLE_TAP, 12 | HOLD_START, 13 | HOLD_END, 14 | } -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/server/types/ConversationWithMessages.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.aipincore.server.types 2 | 3 | import com.penumbraos.mabl.data.types.ConversationImage 4 | import kotlinx.serialization.Serializable 5 | 6 | @Serializable 7 | data class MessageWithImages( 8 | val id: Long, 9 | val conversationId: String, 10 | val type: String, 11 | val content: String, 12 | val toolCalls: String? = null, 13 | val toolCallId: String? = null, 14 | val timestamp: Long, 15 | val images: List = emptyList() 16 | ) 17 | 18 | @Serializable 19 | data class AugmentedConversation( 20 | val id: String, 21 | val title: String, 22 | val createdAt: Long = System.currentTimeMillis(), 23 | val lastActivity: Long = System.currentTimeMillis(), 24 | val isActive: Boolean = true, 25 | val messages: List 26 | ) 27 | -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/view/TouchInterceptor.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.aipincore.view 2 | 3 | import android.annotation.SuppressLint 4 | import android.content.Context 5 | import android.util.AttributeSet 6 | import android.view.MotionEvent 7 | import android.view.View 8 | import com.open.pin.ui.utils.modifiers.SnapCoordinator 9 | 10 | @SuppressLint("ViewConstructor") 11 | class TouchInterceptor( 12 | val snapCoordinator: SnapCoordinator, 13 | context: Context, 14 | attrs: AttributeSet? = null, 15 | defStyleAttr: Int = 0 16 | ) : 17 | View(context, attrs, defStyleAttr) { 18 | 19 | init { 20 | // Configure view to be interactable, but invisible 21 | isClickable = true 22 | isFocusable = false 23 | background = null 24 | } 25 | 26 | @SuppressLint("ClickableViewAccessibility") 27 | override fun onTouchEvent(event: MotionEvent?): Boolean { 28 | if (event != null) { 29 | snapCoordinator.processTouchEvent(event) 30 | } 31 | return false 32 | } 33 | 34 | override fun dispatchGenericMotionEvent(event: MotionEvent?): Boolean { 35 | if (event != null) { 36 | snapCoordinator.processMotionEvent(event) 37 | } 38 | return super.dispatchGenericMotionEvent(event) 39 | } 40 | } -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/view/model/ConversationsViewModel.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.aipincore.view.model 2 | 3 | import androidx.lifecycle.ViewModel 4 | import com.penumbraos.mabl.data.types.Conversation 5 | import kotlinx.coroutines.flow.Flow 6 | import kotlinx.coroutines.flow.map 7 | 8 | class ConversationsViewModel(private val viewModel: PlatformViewModel) : ViewModel() { 9 | val conversationsWithInjectedTitle: Flow> = 10 | viewModel.database.conversationDao() 11 | .getConversationsWithFirstUserMessage() 12 | .map { conversations -> 13 | conversations.map { conversationWithFirstMessage -> 14 | val conversation = conversationWithFirstMessage.conversation 15 | conversation.copy( 16 | title = conversationWithFirstMessage.firstUserMessage ?: conversation.title 17 | ) 18 | } 19 | } 20 | 21 | fun openConversation(id: String) { 22 | viewModel.navViewModel.pushView(ConversationDisplayNav(id)) 23 | } 24 | } -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/view/model/NavViewModel.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.aipincore.view.model 2 | 3 | import androidx.compose.runtime.derivedStateOf 4 | import androidx.compose.runtime.mutableStateListOf 5 | import androidx.lifecycle.ViewModel 6 | 7 | data object HomeNav 8 | data object MenuNav 9 | data object ConversationsNav 10 | data class ConversationDisplayNav(val conversationId: String) 11 | data object SettingsNav 12 | data object DummyNav 13 | 14 | class NavViewModel() : ViewModel() { 15 | val backStack = mutableStateListOf(HomeNav) 16 | 17 | val isHomeScreen = derivedStateOf { 18 | backStack.lastOrNull() == HomeNav 19 | } 20 | 21 | val isMenuOpen = derivedStateOf { 22 | backStack.lastOrNull() == MenuNav 23 | } 24 | 25 | fun pushView(view: Any) { 26 | val last = backStack.lastOrNull() 27 | 28 | if (last == view) { 29 | return 30 | } 31 | 32 | if (last != null && last::class == view::class) { 33 | // Same class, but not same identity, means this has fields that are likely different 34 | // Replace this element 35 | backStack.removeLastOrNull() 36 | } 37 | 38 | backStack.add(view) 39 | } 40 | 41 | fun popView() { 42 | if (backStack.size > 1) { 43 | backStack.removeLastOrNull() 44 | } 45 | } 46 | 47 | fun replaceLastView(view: Any) { 48 | if (backStack.size > 1) { 49 | backStack.removeLastOrNull() 50 | } 51 | backStack.add(view) 52 | } 53 | 54 | fun jumpHome() { 55 | backStack.clear() 56 | backStack.add(HomeNav) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/view/model/PlatformViewModel.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.aipincore.view.model 2 | 3 | import android.content.Context 4 | import android.util.Log 5 | import androidx.lifecycle.ViewModel 6 | import com.penumbraos.mabl.aipincore.SETTING_APP_ID 7 | import com.penumbraos.mabl.aipincore.SETTING_DEBUG_CATEGORY 8 | import com.penumbraos.mabl.aipincore.SETTING_DEBUG_CURSOR 9 | import com.penumbraos.mabl.data.AppDatabase 10 | import com.penumbraos.mabl.data.repository.ConversationRepository 11 | import com.penumbraos.sdk.PenumbraClient 12 | import com.penumbraos.sdk.api.BooleanSettingListener 13 | import kotlinx.coroutines.CoroutineScope 14 | import kotlinx.coroutines.channels.Channel 15 | import kotlinx.coroutines.flow.receiveAsFlow 16 | import kotlinx.coroutines.launch 17 | 18 | class PlatformViewModel( 19 | coroutineScope: CoroutineScope, 20 | context: Context, 21 | val database: AppDatabase 22 | ) : ViewModel() { 23 | val navViewModel = NavViewModel() 24 | 25 | var appIsForeground: Boolean = false 26 | 27 | private val _backGestureChannel = Channel(Channel.RENDEZVOUS) 28 | val backGestureEvent = _backGestureChannel.receiveAsFlow() 29 | 30 | private val _openCurrentConversationChannel = Channel(Channel.RENDEZVOUS) 31 | val openCurrentConversationEvent = _openCurrentConversationChannel.receiveAsFlow() 32 | 33 | private val _debugChannel = Channel() 34 | val debugChannel = _debugChannel.receiveAsFlow() 35 | 36 | val conversationRepository = ConversationRepository( 37 | database.conversationDao(), 38 | database.conversationMessageDao() 39 | ) 40 | 41 | init { 42 | coroutineScope.launch { 43 | val client = PenumbraClient(context) 44 | client.waitForBridge() 45 | client.settings.addBooleanListener( 46 | SETTING_APP_ID, 47 | SETTING_DEBUG_CATEGORY, 48 | SETTING_DEBUG_CURSOR, 49 | object : BooleanSettingListener { 50 | override fun onSettingChanged(value: Boolean) { 51 | Log.d("PlatformViewModel", "Debug cursor setting changed to $value") 52 | _debugChannel.trySend(value) 53 | } 54 | }) 55 | } 56 | } 57 | 58 | fun openCurrentConversation() { 59 | Log.d("PlatformViewModel", "Opening conversation") 60 | _openCurrentConversationChannel.trySend(Unit) 61 | } 62 | 63 | fun backGesture() { 64 | if (!appIsForeground) { 65 | return 66 | } 67 | 68 | Log.d("PlatformViewModel", "Back gesture received") 69 | _backGestureChannel.trySend(Unit) 70 | } 71 | 72 | fun toggleMenuVisible() { 73 | if (!appIsForeground) { 74 | return 75 | } 76 | 77 | if (!closeMenu()) { 78 | Log.d("PlatformViewModel", "Showing menu") 79 | navViewModel.backStack.add(MenuNav) 80 | } 81 | } 82 | 83 | fun closeMenu(): Boolean { 84 | Log.d("PlatformViewModel", "Closing menu") 85 | if (navViewModel.backStack.lastOrNull() == MenuNav) { 86 | navViewModel.backStack.removeLastOrNull() 87 | return true 88 | } 89 | 90 | return false 91 | } 92 | } -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/view/nav/ConversationDisplay.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.aipincore.view.nav 2 | 3 | import androidx.compose.foundation.background 4 | import androidx.compose.foundation.layout.Box 5 | import androidx.compose.foundation.layout.fillMaxSize 6 | import androidx.compose.runtime.Composable 7 | import androidx.compose.runtime.collectAsState 8 | import androidx.compose.ui.Modifier 9 | import androidx.lifecycle.viewmodel.compose.viewModel 10 | import com.open.pin.ui.PinTheme 11 | import com.penumbraos.mabl.aipincore.ConversationList 12 | import com.penumbraos.mabl.aipincore.view.model.PlatformViewModel 13 | 14 | @Composable 15 | fun ConversationDisplay(conversationId: String) { 16 | val viewModel = viewModel() 17 | val messages = viewModel.conversationRepository.getConversationMessagesFlow(conversationId) 18 | .collectAsState(emptyList()) 19 | 20 | Box( 21 | modifier = Modifier 22 | .fillMaxSize() 23 | .background(color = PinTheme.colors.background) 24 | ) { 25 | ConversationList( 26 | messages = messages.value, 27 | modifier = Modifier.fillMaxSize() 28 | ) 29 | } 30 | } -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/view/nav/Home.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.aipincore.view.nav 2 | 3 | import androidx.compose.foundation.layout.Box 4 | import androidx.compose.foundation.layout.fillMaxSize 5 | import androidx.compose.foundation.layout.offset 6 | import androidx.compose.runtime.Composable 7 | import androidx.compose.runtime.LaunchedEffect 8 | import androidx.compose.runtime.getValue 9 | import androidx.compose.runtime.mutableStateOf 10 | import androidx.compose.runtime.remember 11 | import androidx.compose.runtime.setValue 12 | import androidx.compose.ui.Alignment 13 | import androidx.compose.ui.Modifier 14 | import androidx.compose.ui.text.TextStyle 15 | import androidx.compose.ui.text.style.TextAlign 16 | import androidx.compose.ui.unit.dp 17 | import androidx.compose.ui.unit.sp 18 | import androidx.lifecycle.Lifecycle 19 | import androidx.lifecycle.compose.LocalLifecycleOwner 20 | import androidx.lifecycle.repeatOnLifecycle 21 | import com.open.pin.ui.components.text.PinText 22 | import com.open.pin.ui.debug.AiPinPreview 23 | import com.open.pin.ui.theme.PinTypography 24 | import kotlinx.coroutines.delay 25 | import kotlinx.coroutines.isActive 26 | import java.time.Instant 27 | import java.time.LocalDateTime 28 | import java.time.ZoneId 29 | import java.time.format.DateTimeFormatter 30 | 31 | @Composable 32 | fun Home() { 33 | var currentTime by remember { mutableStateOf(LocalDateTime.now()) } 34 | 35 | val lifecycleOwner = LocalLifecycleOwner.current 36 | 37 | LaunchedEffect(lifecycleOwner) { 38 | lifecycleOwner.lifecycle.repeatOnLifecycle(Lifecycle.State.RESUMED) { 39 | val zone = ZoneId.systemDefault() 40 | while (isActive) { 41 | val currentTimeMillis = System.currentTimeMillis() 42 | 43 | currentTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(currentTimeMillis), zone) 44 | // Tick every ~100ms, normalized based on current time 45 | val nextWall = ((currentTimeMillis / 100) + 1) * 100 46 | val delayMs = (nextWall - currentTimeMillis).coerceIn(0, 100) 47 | delay(delayMs) 48 | } 49 | } 50 | } 51 | 52 | val timeFormatter = DateTimeFormatter.ofPattern("HH:mm:ss") 53 | val dateFormatter = DateTimeFormatter.ofPattern("MMM dd, yyyy") 54 | 55 | Box( 56 | modifier = Modifier.fillMaxSize(), 57 | contentAlignment = Alignment.Center 58 | ) { 59 | PinText( 60 | text = currentTime.format(timeFormatter), 61 | style = TextStyle(fontSize = 160.sp), 62 | textAlign = TextAlign.Center 63 | ) 64 | 65 | PinText( 66 | text = currentTime.format(dateFormatter), 67 | style = PinTypography.displayMedium, 68 | textAlign = TextAlign.Center, 69 | modifier = Modifier.offset(y = (-120).dp) 70 | ) 71 | } 72 | } 73 | 74 | @AiPinPreview 75 | @Composable 76 | fun HomePreview() { 77 | Home() 78 | } 79 | -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/view/nav/Menu.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.aipincore.view.nav 2 | 3 | import android.annotation.SuppressLint 4 | import android.util.Log 5 | import androidx.compose.foundation.background 6 | import androidx.compose.foundation.layout.fillMaxSize 7 | import androidx.compose.material.icons.Icons 8 | import androidx.compose.material.icons.filled.Call 9 | import androidx.compose.material.icons.filled.Home 10 | import androidx.compose.material.icons.filled.Notifications 11 | import androidx.compose.material.icons.filled.Settings 12 | import androidx.compose.runtime.Composable 13 | import androidx.compose.ui.Modifier 14 | import androidx.compose.ui.graphics.Color 15 | import androidx.compose.ui.graphics.vector.ImageVector 16 | import androidx.compose.ui.res.vectorResource 17 | import androidx.compose.ui.unit.Dp 18 | import androidx.compose.ui.unit.dp 19 | import androidx.lifecycle.viewmodel.compose.viewModel 20 | import com.open.pin.ui.components.button.PinCircularButton 21 | import com.open.pin.ui.components.views.RadialView 22 | import com.open.pin.ui.components.views.RadialViewParams 23 | import com.open.pin.ui.debug.AiPinPreview 24 | import com.penumbraos.mabl.R 25 | import com.penumbraos.mabl.aipincore.view.model.ConversationsNav 26 | import com.penumbraos.mabl.aipincore.view.model.DummyNav 27 | import com.penumbraos.mabl.aipincore.view.model.HomeNav 28 | import com.penumbraos.mabl.aipincore.view.model.NavViewModel 29 | import com.penumbraos.mabl.aipincore.view.model.PlatformViewModel 30 | import com.penumbraos.mabl.aipincore.view.model.SettingsNav 31 | 32 | data class MenuItem(val icon: ImageVector, val view: Any, val enabled: Boolean = false) 33 | 34 | @Composable 35 | fun Menu(navViewModel: NavViewModel = viewModel(), animatedRadius: Dp) { 36 | val menuItems = listOf( 37 | MenuItem(Icons.Default.Home, HomeNav, enabled = true), 38 | MenuItem( 39 | ImageVector.vectorResource(R.drawable.outline_voice_chat_24), 40 | ConversationsNav, 41 | enabled = true 42 | ), 43 | MenuItem(Icons.Default.Call, DummyNav), 44 | MenuItem(Icons.Default.Notifications, DummyNav), 45 | MenuItem(Icons.Default.Settings, SettingsNav, enabled = true) 46 | ) 47 | 48 | RadialView( 49 | Modifier 50 | .fillMaxSize() 51 | .background(color = Color(0f, 0f, 0f, 0.9f)), 52 | RadialViewParams(radius = animatedRadius), 53 | menuItems 54 | ) { item -> 55 | PinCircularButton({ 56 | Log.d("Menu", "Navigating to ${item.view}") 57 | if (item.view == HomeNav) { 58 | navViewModel.jumpHome() 59 | } else { 60 | navViewModel.replaceLastView(item.view) 61 | } 62 | }, icon = item.icon, enabled = item.enabled) 63 | } 64 | } 65 | 66 | @SuppressLint("ViewModelConstructorInComposable") 67 | @AiPinPreview 68 | @Composable 69 | fun MenuPreview() { 70 | Menu(navViewModel = NavViewModel(), animatedRadius = 150.dp) 71 | } -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/view/nav/Navigation.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.aipincore.view.nav 2 | 3 | import Settings 4 | import androidx.compose.animation.EnterTransition 5 | import androidx.compose.animation.ExitTransition 6 | import androidx.compose.animation.core.TweenSpec 7 | import androidx.compose.animation.core.animateDpAsState 8 | import androidx.compose.animation.core.tween 9 | import androidx.compose.animation.fadeIn 10 | import androidx.compose.animation.fadeOut 11 | import androidx.compose.animation.togetherWith 12 | import androidx.compose.runtime.Composable 13 | import androidx.compose.runtime.LaunchedEffect 14 | import androidx.compose.runtime.getValue 15 | import androidx.compose.runtime.mutableStateOf 16 | import androidx.compose.runtime.remember 17 | import androidx.compose.ui.unit.Dp 18 | import androidx.compose.ui.unit.dp 19 | import androidx.lifecycle.viewmodel.compose.viewModel 20 | import androidx.navigation3.runtime.NavEntry 21 | import androidx.navigation3.ui.NavDisplay 22 | import com.open.pin.ui.components.text.PinText 23 | import com.penumbraos.mabl.aipincore.view.model.ConversationDisplayNav 24 | import com.penumbraos.mabl.aipincore.view.model.ConversationsNav 25 | import com.penumbraos.mabl.aipincore.view.model.HomeNav 26 | import com.penumbraos.mabl.aipincore.view.model.MenuNav 27 | import com.penumbraos.mabl.aipincore.view.model.NavViewModel 28 | import com.penumbraos.mabl.aipincore.view.model.SettingsNav 29 | import com.penumbraos.mabl.aipincore.view.nav.util.WithMenuSceneStrategy 30 | 31 | val animationSpec = tween(durationMillis = 300) 32 | 33 | @Suppress("UNCHECKED_CAST") 34 | @Composable 35 | fun Navigation(navViewModel: NavViewModel = viewModel()) { 36 | NavDisplay( 37 | backStack = navViewModel.backStack, 38 | onBack = { navViewModel.backStack.removeLastOrNull() }, 39 | sceneStrategy = WithMenuSceneStrategy(), 40 | entryProvider = { key -> 41 | when (key) { 42 | HomeNav -> NavEntry(key) { 43 | Home() 44 | } 45 | 46 | MenuNav -> NavEntry(key, metadata = NavDisplay.transitionSpec { 47 | // TODO: This fade in doesn't seem to work right 48 | fadeIn(animationSpec = tween(500)) togetherWith ExitTransition.KeepUntilTransitionsFinished 49 | } + NavDisplay.popTransitionSpec { 50 | EnterTransition.None togetherWith fadeOut(animationSpec = animationSpec as TweenSpec) 51 | }) { 52 | // We can only reliably detect when the view first starts to render 53 | val menuVisible = remember { mutableStateOf(false) } 54 | // We check if we have removed the menu from the stack to start animating out 55 | val isMenuInBackStack = navViewModel.backStack.contains(MenuNav) 56 | 57 | val animatedRadius by animateDpAsState( 58 | targetValue = if (menuVisible.value && isMenuInBackStack) 150.dp else 300.dp, 59 | animationSpec = animationSpec as TweenSpec, 60 | label = "animatedRadius", 61 | ) 62 | 63 | LaunchedEffect(Unit) { 64 | menuVisible.value = true 65 | } 66 | 67 | Menu(animatedRadius = animatedRadius) 68 | } 69 | 70 | ConversationsNav -> NavEntry(key) { 71 | Conversations() 72 | } 73 | 74 | is ConversationDisplayNav -> NavEntry(key) { 75 | ConversationDisplay(key.conversationId) 76 | } 77 | 78 | SettingsNav -> NavEntry(key) { 79 | Settings() 80 | } 81 | 82 | else -> NavEntry(Unit) { 83 | PinText("Unknown route") 84 | } 85 | } 86 | } 87 | ) 88 | } 89 | -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/view/nav/Settings.kt: -------------------------------------------------------------------------------- 1 | import android.content.ComponentName 2 | import android.content.Intent 3 | import android.util.Log 4 | import androidx.compose.runtime.Composable 5 | import androidx.compose.runtime.LaunchedEffect 6 | import androidx.compose.ui.platform.LocalContext 7 | import androidx.lifecycle.viewmodel.compose.viewModel 8 | import com.penumbraos.mabl.aipincore.view.model.NavViewModel 9 | 10 | @Composable 11 | fun Settings(navViewModel: NavViewModel = viewModel()) { 12 | val context = LocalContext.current 13 | 14 | LaunchedEffect(Unit) { 15 | navViewModel.popView() 16 | 17 | val intent = Intent().apply { 18 | component = ComponentName( 19 | "humane.experience.settings", 20 | "humane.experience.settings.SettingsExperience" 21 | ) 22 | addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TASK) 23 | } 24 | 25 | try { 26 | context.startActivity(intent) 27 | } catch (e: Exception) { 28 | Log.e("Settings", "Failed to start settings", e) 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/view/nav/util/WithMenuScene.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.aipincore.view.nav.util 2 | 3 | import androidx.compose.foundation.layout.Box 4 | import androidx.compose.foundation.layout.fillMaxSize 5 | import androidx.compose.runtime.Composable 6 | import androidx.compose.ui.Modifier 7 | import androidx.navigation3.runtime.NavEntry 8 | import androidx.navigation3.ui.Scene 9 | import androidx.navigation3.ui.SceneStrategy 10 | 11 | class WithMenuScene( 12 | override val key: Any, 13 | override val previousEntries: List>, 14 | val baseEntry: NavEntry, 15 | val menuEntry: NavEntry 16 | ) : Scene { 17 | override val entries: List> = listOf(baseEntry, menuEntry) 18 | override val content: @Composable () -> Unit = { 19 | Box(Modifier.fillMaxSize()) { 20 | baseEntry.Content() 21 | menuEntry.Content() 22 | } 23 | } 24 | } 25 | 26 | class WithMenuSceneStrategy : SceneStrategy { 27 | @Composable 28 | override fun calculateScene(entries: List>, onBack: (Int) -> Unit): Scene? { 29 | val lastTwoEntries = entries.takeLast(2) 30 | 31 | return if (lastTwoEntries.size == 2 && lastTwoEntries[1].contentKey == "MenuNav") { 32 | val baseEntry = lastTwoEntries[0] 33 | val menuEntry = lastTwoEntries[1] 34 | 35 | val sceneKey = Pair(baseEntry, menuEntry) 36 | 37 | WithMenuScene(sceneKey, entries.dropLast(1), baseEntry, menuEntry) 38 | } else { 39 | null 40 | } 41 | } 42 | } -------------------------------------------------------------------------------- /mabl/src/aipincore/java/com/penumbraos/mabl/aipincore/view/util/time.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.aipincore.view.util 2 | 3 | import java.text.SimpleDateFormat 4 | import java.util.Date 5 | import java.util.Locale 6 | 7 | fun formatRelativeTimestamp(timestampMillis: Long): String { 8 | val now = System.currentTimeMillis() 9 | val diffMillis = now - timestampMillis 10 | 11 | val diffMinutes = diffMillis / (1000 * 60) 12 | val diffHours = diffMillis / (1000 * 60 * 60) 13 | val diffDays = diffMillis / (1000 * 60 * 60 * 24) 14 | 15 | return when { 16 | diffMinutes < 1 -> "Just now" 17 | 18 | diffMinutes < 60 -> "${diffMinutes} ${if (diffMinutes == 1L) "Minute" else "Minutes"} ago" 19 | 20 | diffHours < 24 -> "${diffHours} ${if (diffHours == 1L) "Hour" else "Hours"} ago" 21 | 22 | diffDays == 1L -> "Yesterday at ${formatTime(timestampMillis)}" 23 | 24 | else -> { 25 | val date = Date(timestampMillis) 26 | val formatter = SimpleDateFormat("M/d/yy", Locale.US) 27 | "${formatter.format(date)} at ${formatTime(timestampMillis)}" 28 | } 29 | } 30 | } 31 | 32 | fun formatTime(timestampMillis: Long): String { 33 | val date = Date(timestampMillis) 34 | val formatter = SimpleDateFormat("h:mm a", Locale.US) 35 | return formatter.format(date) 36 | } -------------------------------------------------------------------------------- /mabl/src/android/java/com/penumbraos/mabl/ui/AndroidPlatformInputHandler.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui 2 | 3 | import android.content.Context 4 | import androidx.lifecycle.LifecycleCoroutineScope 5 | import com.penumbraos.mabl.interaction.IInteractionFlowManager 6 | import com.penumbraos.mabl.ui.interfaces.IPlatformInputHandler 7 | 8 | private const val TAG = "AndroidPlatformInputHandler" 9 | 10 | class AndroidPlatformInputHandler : IPlatformInputHandler { 11 | 12 | override fun setup( 13 | context: Context, 14 | lifecycleScope: LifecycleCoroutineScope, 15 | interactionFlowManager: IInteractionFlowManager 16 | ) { 17 | } 18 | } -------------------------------------------------------------------------------- /mabl/src/android/java/com/penumbraos/mabl/ui/ConversationRenderer.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui 2 | 3 | import android.content.Context 4 | import android.util.Log 5 | import androidx.compose.runtime.MutableState 6 | import androidx.compose.runtime.mutableStateOf 7 | import com.penumbraos.mabl.services.AllControllers 8 | import com.penumbraos.mabl.types.Error 9 | import com.penumbraos.mabl.ui.interfaces.IConversationRenderer 10 | 11 | private const val TAG = "AndroidConversationRenderer" 12 | 13 | class ConversationRenderer( 14 | private val context: Context, 15 | private val controllers: AllControllers 16 | ) : IConversationRenderer { 17 | 18 | // Compose state for UI updates 19 | val conversationState: MutableState = mutableStateOf("") 20 | val transcriptionState: MutableState = mutableStateOf("") 21 | val listeningState: MutableState = mutableStateOf(false) 22 | val errorState: MutableState = mutableStateOf("") 23 | 24 | override fun showMessage(message: String, isUser: Boolean) { 25 | Log.d(TAG, "Message: $message (isUser: $isUser)") 26 | 27 | val prefix = if (isUser) "You: " else "MABL: " 28 | conversationState.value += "$prefix$message\n" 29 | } 30 | 31 | override fun showTranscription(text: String) { 32 | Log.d(TAG, "Transcription: $text") 33 | transcriptionState.value = text 34 | } 35 | 36 | override fun showListening(isListening: Boolean) { 37 | Log.d(TAG, "Listening: $isListening") 38 | listeningState.value = isListening 39 | 40 | if (!isListening) { 41 | transcriptionState.value = "" 42 | } 43 | } 44 | 45 | override fun showError(error: Error) { 46 | errorState.value = error.message 47 | conversationState.value += "Error: ${error.message}\n" 48 | } 49 | } -------------------------------------------------------------------------------- /mabl/src/android/java/com/penumbraos/mabl/ui/UIFactory.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui 2 | 3 | import android.content.Context 4 | import androidx.lifecycle.LifecycleCoroutineScope 5 | import com.penumbraos.mabl.services.AllControllers 6 | import com.penumbraos.mabl.ui.interfaces.IConversationRenderer 7 | import com.penumbraos.mabl.ui.interfaces.IPlatformInputHandler 8 | import com.penumbraos.mabl.ui.interfaces.IPlatformCapabilities 9 | import kotlinx.coroutines.CoroutineScope 10 | 11 | class UIFactory( 12 | private val coroutineScope: CoroutineScope, 13 | private val context: Context, 14 | private val controllers: AllControllers 15 | ) { 16 | 17 | fun createConversationRenderer(): IConversationRenderer { 18 | return ConversationRenderer(context, controllers) 19 | } 20 | 21 | fun createPlatformInputHandler(): IPlatformInputHandler { 22 | return AndroidPlatformInputHandler() 23 | } 24 | 25 | fun createPlatformCapabilities(): IPlatformCapabilities { 26 | return object : IPlatformCapabilities {} 27 | } 28 | 29 | fun createUIComponents(): UIComponents { 30 | return UIComponents( 31 | conversationRenderer = createConversationRenderer(), 32 | platformInputHandler = createPlatformInputHandler(), 33 | platformCapabilities = createPlatformCapabilities() 34 | ) 35 | } 36 | } 37 | 38 | -------------------------------------------------------------------------------- /mabl/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 8 | 9 | 10 | 11 | 12 | 13 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 38 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /mabl/src/main/assets/.gitignore: -------------------------------------------------------------------------------- 1 | *.onnx 2 | *.json -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/conversation/StaticQueryManager.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.conversation 2 | 3 | import android.content.Context 4 | import com.penumbraos.mabl.sdk.IToolCallback 5 | import com.penumbraos.mabl.sdk.ToolCall 6 | import com.penumbraos.mabl.sdk.ToolDefinition 7 | import com.penumbraos.mabl.services.AllControllers 8 | import kotlinx.coroutines.CoroutineScope 9 | import kotlin.coroutines.resume 10 | import kotlin.coroutines.resumeWithException 11 | import kotlin.coroutines.suspendCoroutine 12 | 13 | class StaticQueryManager( 14 | allControllers: AllControllers, 15 | context: Context, 16 | coroutineScope: CoroutineScope 17 | ) { 18 | private val toolService = StaticQueryToolService(allControllers, context, coroutineScope) 19 | 20 | private var toolMap: Map? = null 21 | 22 | suspend fun evaluateStaticQuery(query: String): String? { 23 | buildQueryMap() 24 | 25 | val query = query.lowercase().trim() 26 | 27 | val tool = toolMap!![query] 28 | if (tool != null) { 29 | try { 30 | return suspendCoroutine { continuation -> 31 | toolService.executeTool(ToolCall().apply { 32 | name = tool.name 33 | }, null, object : IToolCallback.Stub() { 34 | override fun onSuccess(result: String?) { 35 | continuation.resume(result) 36 | } 37 | 38 | override fun onError(error: String?) { 39 | continuation.resumeWithException(Error(error)) 40 | } 41 | }) 42 | } 43 | } catch (e: Exception) { 44 | return null 45 | } 46 | } 47 | 48 | return null 49 | } 50 | 51 | private fun buildQueryMap() { 52 | if (toolMap != null) { 53 | return 54 | } 55 | 56 | val tools = toolService.getToolDefinitions() 57 | val map = mutableMapOf() 58 | for (tool in tools) { 59 | for (staticQuery in tool.examples ?: emptyArray()) { 60 | map[staticQuery] = tool 61 | } 62 | } 63 | toolMap = map 64 | } 65 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/conversation/StaticQueryToolService.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.conversation 2 | 3 | import android.content.ComponentName 4 | import android.content.Context 5 | import android.content.Intent 6 | import android.util.Log 7 | import com.penumbraos.mabl.sdk.IToolCallback 8 | import com.penumbraos.mabl.sdk.ToolCall 9 | import com.penumbraos.mabl.sdk.ToolDefinition 10 | import com.penumbraos.mabl.sdk.ToolService 11 | import com.penumbraos.mabl.services.AllControllers 12 | import com.penumbraos.sdk.PenumbraClient 13 | import kotlinx.coroutines.CoroutineScope 14 | import kotlinx.coroutines.launch 15 | import org.json.JSONObject 16 | 17 | private const val NEW_CONVERSATION = "new_conversation" 18 | private const val OPEN_SETTINGS = "open_settings" 19 | private const val REBOOT_NOW = "reboot_now" 20 | 21 | class StaticQueryToolService( 22 | private val allControllers: AllControllers, 23 | private val context: Context, 24 | val coroutineScope: CoroutineScope 25 | ) : ToolService("StaticQueryToolService") { 26 | // TODO: This should work on non-Pin 27 | private val client = PenumbraClient(context) 28 | 29 | override fun executeTool( 30 | call: ToolCall, 31 | params: JSONObject?, 32 | callback: IToolCallback 33 | ) { 34 | when (call.name) { 35 | NEW_CONVERSATION -> { 36 | coroutineScope.launch { 37 | allControllers.conversationManager.startNewConversation() 38 | 39 | callback.onSuccess("Created new conversation") 40 | } 41 | } 42 | 43 | OPEN_SETTINGS -> { 44 | coroutineScope.launch { 45 | // TODO: This should work on non-Pin 46 | val intent = Intent().apply { 47 | component = ComponentName( 48 | "humane.experience.settings", 49 | "humane.experience.settings.SettingsExperience" 50 | ) 51 | addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TASK) 52 | } 53 | 54 | try { 55 | context.startActivity(intent) 56 | callback.onSuccess("Opened settings") 57 | } catch (e: Exception) { 58 | Log.e("Settings", "Failed to start settings", e) 59 | callback.onSuccess("Failed to open settings") 60 | } 61 | } 62 | } 63 | 64 | REBOOT_NOW -> { 65 | coroutineScope.launch { 66 | try { 67 | client.shell.executeCommand("reboot") 68 | 69 | callback.onSuccess("Rebooting") 70 | } catch (e: Exception) { 71 | callback.onError("Failed to reboot: ${e.message}") 72 | } 73 | } 74 | } 75 | 76 | else -> { 77 | callback.onError("Unknown tool: ${call.name}") 78 | } 79 | } 80 | } 81 | 82 | override fun getToolDefinitions(): Array { 83 | return arrayOf( 84 | ToolDefinition().apply { 85 | name = NEW_CONVERSATION 86 | examples = arrayOf( 87 | "new conversation", 88 | "new chat" 89 | ) 90 | }, 91 | ToolDefinition().apply { 92 | name = OPEN_SETTINGS 93 | examples = arrayOf( 94 | "open settings", 95 | "open system settings", 96 | "open human settings", 97 | "launch settings" 98 | ) 99 | }, 100 | ToolDefinition().apply { 101 | name = REBOOT_NOW 102 | examples = arrayOf( 103 | "reboot now", 104 | "emergency reboot" 105 | ) 106 | } 107 | ) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/data/AppDatabase.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.data 2 | 3 | import android.content.Context 4 | import androidx.room.Database 5 | import androidx.room.Room 6 | import androidx.room.RoomDatabase 7 | import androidx.room.migration.Migration 8 | import androidx.sqlite.db.SupportSQLiteDatabase 9 | import com.penumbraos.mabl.data.dao.ConversationDao 10 | import com.penumbraos.mabl.data.dao.ConversationImageDao 11 | import com.penumbraos.mabl.data.dao.ConversationMessageDao 12 | import com.penumbraos.mabl.data.types.Conversation 13 | import com.penumbraos.mabl.data.types.ConversationImage 14 | import com.penumbraos.mabl.data.types.ConversationMessage 15 | 16 | @Database( 17 | entities = [Conversation::class, ConversationMessage::class, ConversationImage::class], 18 | version = 5, 19 | exportSchema = false 20 | ) 21 | abstract class AppDatabase : RoomDatabase() { 22 | abstract fun conversationDao(): ConversationDao 23 | abstract fun conversationMessageDao(): ConversationMessageDao 24 | abstract fun conversationImageDao(): ConversationImageDao 25 | 26 | companion object { 27 | @Volatile 28 | private var INSTANCE: AppDatabase? = null 29 | 30 | private val MIGRATION_1_2 = object : Migration(1, 2) { 31 | override fun migrate(database: SupportSQLiteDatabase) { 32 | database.execSQL("CREATE TABLE IF NOT EXISTS `conversations` (`id` TEXT NOT NULL, `title` TEXT NOT NULL, `createdAt` INTEGER NOT NULL, `lastActivity` INTEGER NOT NULL, `isActive` INTEGER NOT NULL, PRIMARY KEY(`id`))") 33 | database.execSQL("CREATE TABLE IF NOT EXISTS `conversation_messages` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `conversationId` TEXT NOT NULL, `type` TEXT NOT NULL, `content` TEXT NOT NULL, `toolCalls` TEXT, `toolCallId` TEXT, `timestamp` INTEGER NOT NULL, FOREIGN KEY(`conversationId`) REFERENCES `conversations`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )") 34 | } 35 | } 36 | 37 | private val MIGRATION_2_3 = object : Migration(2, 3) { 38 | override fun migrate(database: SupportSQLiteDatabase) { 39 | database.execSQL( 40 | """ 41 | CREATE TABLE IF NOT EXISTS `conversation_images` ( 42 | `id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, 43 | `messageId` INTEGER NOT NULL, 44 | `fileName` TEXT NOT NULL, 45 | `mimeType` TEXT NOT NULL, 46 | `fileSizeBytes` INTEGER NOT NULL, 47 | `width` INTEGER, 48 | `height` INTEGER, 49 | `timestamp` INTEGER NOT NULL, 50 | FOREIGN KEY(`messageId`) REFERENCES `conversation_messages`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE 51 | ) 52 | """ 53 | ) 54 | } 55 | } 56 | 57 | private val MIGRATION_3_4 = object : Migration(3, 4) { 58 | override fun migrate(database: SupportSQLiteDatabase) { 59 | database.execSQL( 60 | """ 61 | DROP TABLE IF EXISTS `messages` 62 | """ 63 | ) 64 | } 65 | } 66 | 67 | private val MIGRATION_4_5 = object : Migration(4, 5) { 68 | override fun migrate(database: SupportSQLiteDatabase) { 69 | database.execSQL("CREATE INDEX IF NOT EXISTS `index_conversation_messages_conversationId` ON `conversation_messages` (`conversationId`)") 70 | database.execSQL("CREATE INDEX IF NOT EXISTS `index_conversation_messages_conversationId_type_timestamp` ON `conversation_messages` (`conversationId`, `type`, `timestamp`)") 71 | database.execSQL("CREATE INDEX IF NOT EXISTS `index_conversation_images_messageId` ON `conversation_images` (`messageId`)") 72 | } 73 | } 74 | 75 | fun getDatabase(context: Context): AppDatabase { 76 | return INSTANCE ?: synchronized(this) { 77 | val instance = Room.databaseBuilder( 78 | context.applicationContext, 79 | AppDatabase::class.java, 80 | "app_database" 81 | ) 82 | .addMigrations(MIGRATION_1_2, MIGRATION_2_3, MIGRATION_3_4, MIGRATION_4_5) 83 | .build() 84 | INSTANCE = instance 85 | instance 86 | } 87 | } 88 | } 89 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/data/dao/ConversationDao.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.data.dao 2 | 3 | import androidx.room.Dao 4 | import androidx.room.Embedded 5 | import androidx.room.Insert 6 | import androidx.room.Query 7 | import androidx.room.Update 8 | import com.penumbraos.mabl.data.types.Conversation 9 | import kotlinx.coroutines.flow.Flow 10 | 11 | data class ConversationWithFirstUserMessage( 12 | @Embedded val conversation: Conversation, 13 | val firstUserMessage: String? 14 | ) 15 | 16 | @Dao 17 | interface ConversationDao { 18 | @Query("SELECT * FROM conversations ORDER BY lastActivity DESC LIMIT :limit") 19 | fun getAllConversations(limit: Int = 50): Flow> 20 | 21 | @Query( 22 | """ 23 | WITH first_user_messages AS ( 24 | SELECT 25 | cm.conversationId, 26 | cm.content 27 | FROM conversation_messages cm 28 | INNER JOIN ( 29 | SELECT 30 | conversationId, 31 | MIN(timestamp) AS firstTimestamp, 32 | MIN(id) AS firstId 33 | FROM conversation_messages 34 | WHERE type = 'user' 35 | GROUP BY conversationId 36 | ) first_user_timestamp ON first_user_timestamp.conversationId = cm.conversationId 37 | WHERE 38 | cm.type = 'user' AND 39 | cm.timestamp = first_user_timestamp.firstTimestamp AND 40 | cm.id = first_user_timestamp.firstId 41 | ) 42 | SELECT 43 | c.id AS id, 44 | c.title AS title, 45 | c.createdAt AS createdAt, 46 | c.lastActivity AS lastActivity, 47 | c.isActive AS isActive, 48 | fum.content AS firstUserMessage 49 | FROM conversations c 50 | LEFT JOIN first_user_messages fum ON fum.conversationId = c.id 51 | ORDER BY c.lastActivity DESC 52 | LIMIT :limit 53 | """ 54 | ) 55 | fun getConversationsWithFirstUserMessage(limit: Int = 50): Flow> 56 | 57 | @Query("SELECT * FROM conversations WHERE id = :id") 58 | suspend fun getConversation(id: String): Conversation? 59 | 60 | @Query("SELECT * FROM conversations ORDER BY lastActivity DESC LIMIT 1") 61 | fun getLastActiveConversation(): Flow 62 | 63 | @Insert 64 | suspend fun insertConversation(conversation: Conversation) 65 | 66 | @Update 67 | suspend fun updateConversation(conversation: Conversation) 68 | 69 | @Query("UPDATE conversations SET lastActivity = :timestamp WHERE id = :id") 70 | suspend fun updateLastActivity(id: String, timestamp: Long = System.currentTimeMillis()) 71 | 72 | @Query("UPDATE conversations SET title = :title WHERE id = :id") 73 | suspend fun updateTitle(id: String, title: String) 74 | 75 | @Query("DELETE FROM conversations WHERE id = :id") 76 | suspend fun deleteConversation(id: String) 77 | 78 | @Query("SELECT COUNT(*) FROM conversation_messages WHERE conversationId = :conversationId") 79 | suspend fun getMessageCount(conversationId: String): Int 80 | 81 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/data/dao/ConversationImageDao.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.data.dao 2 | 3 | import androidx.room.Dao 4 | import androidx.room.Delete 5 | import androidx.room.Insert 6 | import androidx.room.Query 7 | import com.penumbraos.mabl.data.types.ConversationImage 8 | import kotlinx.coroutines.flow.Flow 9 | 10 | @Dao 11 | interface ConversationImageDao { 12 | @Query("SELECT * FROM conversation_images WHERE messageId = :messageId ORDER BY timestamp ASC") 13 | suspend fun getImagesForMessage(messageId: Long): List 14 | 15 | @Query("SELECT * FROM conversation_images WHERE messageId = :messageId ORDER BY timestamp ASC") 16 | fun getImagesForMessageFlow(messageId: Long): Flow> 17 | 18 | @Query("""SELECT ci.* FROM conversation_images ci 19 | INNER JOIN conversation_messages cm ON ci.messageId = cm.id 20 | WHERE cm.conversationId = :conversationId ORDER BY ci.timestamp ASC""") 21 | suspend fun getImagesForConversation(conversationId: String): List 22 | 23 | @Query("""SELECT ci.* FROM conversation_images ci 24 | INNER JOIN conversation_messages cm ON ci.messageId = cm.id 25 | WHERE cm.conversationId = :conversationId ORDER BY ci.timestamp ASC""") 26 | fun getImagesForConversationFlow(conversationId: String): Flow> 27 | 28 | @Query("SELECT * FROM conversation_images WHERE id = :id") 29 | suspend fun getImage(id: Long): ConversationImage? 30 | 31 | @Query("SELECT * FROM conversation_images WHERE fileName = :fileName") 32 | suspend fun getImageByFileName(fileName: String): ConversationImage? 33 | 34 | @Insert 35 | suspend fun insertImage(image: ConversationImage): Long 36 | 37 | @Delete 38 | suspend fun deleteImage(image: ConversationImage) 39 | 40 | @Query("DELETE FROM conversation_images WHERE id = :id") 41 | suspend fun deleteImageById(id: Long) 42 | 43 | @Query("DELETE FROM conversation_images WHERE messageId = :messageId") 44 | suspend fun deleteImagesForMessage(messageId: Long) 45 | 46 | @Query("""DELETE FROM conversation_images WHERE messageId IN 47 | (SELECT id FROM conversation_messages WHERE conversationId = :conversationId)""") 48 | suspend fun deleteImagesForConversation(conversationId: String) 49 | 50 | @Query("""SELECT COUNT(*) FROM conversation_images ci 51 | INNER JOIN conversation_messages cm ON ci.messageId = cm.id 52 | WHERE cm.conversationId = :conversationId""") 53 | suspend fun getImageCountForConversation(conversationId: String): Int 54 | 55 | @Query("""SELECT SUM(fileSizeBytes) FROM conversation_images ci 56 | INNER JOIN conversation_messages cm ON ci.messageId = cm.id 57 | WHERE cm.conversationId = :conversationId""") 58 | suspend fun getTotalImageSizeForConversation(conversationId: String): Long? 59 | 60 | @Query("SELECT COUNT(*) FROM conversation_images WHERE messageId = :messageId") 61 | suspend fun getImageCountForMessage(messageId: Long): Int 62 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/data/dao/ConversationMessageDao.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.data.dao 2 | 3 | import androidx.room.Dao 4 | import androidx.room.Insert 5 | import androidx.room.Query 6 | import com.penumbraos.mabl.data.types.ConversationMessage 7 | import kotlinx.coroutines.flow.Flow 8 | 9 | @Dao 10 | interface ConversationMessageDao { 11 | @Query("SELECT * FROM conversation_messages WHERE conversationId = :conversationId ORDER BY timestamp ASC") 12 | suspend fun getConversationMessages(conversationId: String): List 13 | 14 | @Query("SELECT * FROM conversation_messages WHERE conversationId = :conversationId ORDER BY timestamp ASC") 15 | fun getConversationMessagesFlow(conversationId: String): Flow> 16 | 17 | @Insert 18 | suspend fun insertMessage(message: ConversationMessage): Long 19 | 20 | @Query("DELETE FROM conversation_messages WHERE conversationId = :conversationId") 21 | suspend fun deleteConversationMessages(conversationId: String) 22 | 23 | @Query( 24 | """ 25 | SELECT content FROM conversation_messages 26 | WHERE conversationId = :conversationId AND type IN ('user', 'assistant') 27 | ORDER BY timestamp DESC LIMIT 1 28 | """ 29 | ) 30 | suspend fun getLastMessageContent(conversationId: String): String? 31 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/data/repository/ConversationRepository.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.data.repository 2 | 3 | import com.penumbraos.mabl.data.dao.ConversationDao 4 | import com.penumbraos.mabl.data.dao.ConversationMessageDao 5 | import com.penumbraos.mabl.data.types.Conversation 6 | import com.penumbraos.mabl.data.types.ConversationMessage 7 | import kotlinx.coroutines.flow.Flow 8 | import kotlinx.coroutines.flow.first 9 | 10 | class ConversationRepository( 11 | private val conversationDao: ConversationDao, 12 | private val conversationMessageDao: ConversationMessageDao, 13 | private val conversationImageRepository: ConversationImageRepository? = null 14 | ) { 15 | 16 | fun getAllConversations(limit: Int = 50): Flow> = 17 | conversationDao.getAllConversations(limit) 18 | 19 | suspend fun getConversation(id: String): Conversation? = conversationDao.getConversation(id) 20 | 21 | suspend fun getLastActiveConversation(): Conversation? = 22 | conversationDao.getLastActiveConversation().first() 23 | 24 | fun getLastActiveConversationFlow(): Flow = 25 | conversationDao.getLastActiveConversation() 26 | 27 | suspend fun createNewConversation(title: String = "New Conversation"): Conversation { 28 | val conversation = Conversation(title = title) 29 | conversationDao.insertConversation(conversation) 30 | return conversation 31 | } 32 | 33 | suspend fun updateLastActivity(conversationId: String) { 34 | conversationDao.updateLastActivity(conversationId) 35 | } 36 | 37 | suspend fun updateConversationTitle(conversationId: String, title: String) { 38 | conversationDao.updateTitle(conversationId, title) 39 | } 40 | 41 | suspend fun deleteConversation(conversationId: String) { 42 | conversationImageRepository?.deleteImagesForConversation(conversationId) 43 | conversationDao.deleteConversation(conversationId) 44 | } 45 | 46 | suspend fun getConversationMessages(conversationId: String): List { 47 | return conversationMessageDao.getConversationMessages(conversationId) 48 | } 49 | 50 | fun getConversationMessagesFlow(conversationId: String): Flow> { 51 | return conversationMessageDao.getConversationMessagesFlow(conversationId) 52 | } 53 | 54 | suspend fun addMessage( 55 | conversationId: String, 56 | type: String, 57 | content: String, 58 | toolCalls: String? = null, 59 | toolCallId: String? = null 60 | ): ConversationMessage { 61 | val message = ConversationMessage( 62 | conversationId = conversationId, 63 | type = type, 64 | content = content, 65 | toolCalls = toolCalls, 66 | toolCallId = toolCallId 67 | ) 68 | val id = conversationMessageDao.insertMessage(message) 69 | conversationDao.updateLastActivity(conversationId) 70 | return message.copy(id = id) 71 | } 72 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/data/types/Conversation.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.data.types 2 | 3 | import androidx.room.Embedded 4 | import androidx.room.Entity 5 | import androidx.room.PrimaryKey 6 | import androidx.room.Relation 7 | import kotlinx.serialization.Serializable 8 | import java.util.UUID 9 | 10 | @Serializable 11 | @Entity(tableName = "conversations") 12 | data class Conversation( 13 | @PrimaryKey val id: String = UUID.randomUUID().toString(), 14 | val title: String, 15 | val createdAt: Long = System.currentTimeMillis(), 16 | val lastActivity: Long = System.currentTimeMillis(), 17 | val isActive: Boolean = true 18 | ) 19 | 20 | @Serializable 21 | data class ConversationWithMessages( 22 | @Embedded val conversation: Conversation, 23 | @Relation( 24 | parentColumn = "id", 25 | entityColumn = "conversationId" 26 | ) 27 | val messages: List = emptyList() 28 | ) 29 | -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/data/types/ConversationImage.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.data.types 2 | 3 | import androidx.room.Entity 4 | import androidx.room.ForeignKey 5 | import androidx.room.Index 6 | import androidx.room.PrimaryKey 7 | import kotlinx.serialization.Serializable 8 | 9 | @Serializable 10 | @Entity( 11 | tableName = "conversation_images", 12 | indices = [ 13 | Index(value = ["messageId"]) 14 | ], 15 | foreignKeys = [ForeignKey( 16 | entity = ConversationMessage::class, 17 | parentColumns = ["id"], 18 | childColumns = ["messageId"], 19 | onDelete = ForeignKey.Companion.CASCADE 20 | )] 21 | ) 22 | data class ConversationImage( 23 | @PrimaryKey(autoGenerate = true) val id: Long = 0, 24 | val messageId: Long, 25 | val fileName: String, 26 | val mimeType: String, 27 | val fileSizeBytes: Long, 28 | val width: Int? = null, 29 | val height: Int? = null, 30 | val timestamp: Long = System.currentTimeMillis() 31 | ) -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/data/types/ConversationMessage.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.data.types 2 | 3 | import androidx.room.Entity 4 | import androidx.room.ForeignKey 5 | import androidx.room.Index 6 | import androidx.room.PrimaryKey 7 | import kotlinx.serialization.Serializable 8 | 9 | @Serializable 10 | @Entity( 11 | tableName = "conversation_messages", 12 | indices = [ 13 | Index(value = ["conversationId"]), 14 | Index(value = ["conversationId", "type", "timestamp"]) 15 | ], 16 | foreignKeys = [ForeignKey( 17 | entity = Conversation::class, 18 | parentColumns = ["id"], 19 | childColumns = ["conversationId"], 20 | onDelete = ForeignKey.Companion.CASCADE 21 | )] 22 | ) 23 | data class ConversationMessage( 24 | @PrimaryKey(autoGenerate = true) val id: Long = 0, 25 | val conversationId: String, 26 | /** 27 | * "user", "assistant", "tool" 28 | */ 29 | val type: String, 30 | val content: String, 31 | /** 32 | * JSON serialized tool calls 33 | */ 34 | val toolCalls: String? = null, 35 | val toolCallId: String? = null, 36 | val timestamp: Long = System.currentTimeMillis() 37 | ) -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/data/types/ConversationWithImages.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.data.types 2 | 3 | import androidx.room.Embedded 4 | import androidx.room.Relation 5 | import kotlinx.serialization.Serializable 6 | 7 | @Serializable 8 | data class ConversationMessageWithImages( 9 | @Embedded val message: ConversationMessage, 10 | @Relation( 11 | parentColumn = "id", 12 | entityColumn = "messageId" 13 | ) 14 | val images: List = emptyList() 15 | ) 16 | 17 | @Serializable 18 | data class ConversationWithImagesData( 19 | @Embedded val conversation: Conversation, 20 | @Relation( 21 | entity = ConversationMessage::class, 22 | parentColumn = "id", 23 | entityColumn = "conversationId" 24 | ) 25 | val messagesWithImages: List = emptyList() 26 | ) { 27 | val allImages: List 28 | get() = messagesWithImages.flatMap { it.images } 29 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/discovery/PluginManager.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.discovery 2 | 3 | import android.content.ComponentName 4 | import android.content.Context 5 | import android.content.Intent 6 | import android.content.ServiceConnection 7 | import android.content.pm.PackageManager 8 | import android.os.IBinder 9 | import com.penumbraos.mabl.sdk.PluginConstants 10 | import com.penumbraos.mabl.sdk.PluginType 11 | import kotlin.coroutines.resume 12 | import kotlin.coroutines.suspendCoroutine 13 | 14 | data class PluginService( 15 | val packageName: String, 16 | val className: String, 17 | val type: PluginType, 18 | val displayName: String?, 19 | val description: String? 20 | ) 21 | 22 | class PluginManager(private val context: Context) { 23 | fun discoverPlugins(): List { 24 | return PluginType.entries.flatMap { discoverServiceType(it) } 25 | } 26 | 27 | fun discoverServices(type: PluginType): List { 28 | return discoverServiceType(type) 29 | } 30 | 31 | suspend fun connectToService( 32 | pluginService: PluginService, 33 | serviceCast: (IBinder) -> T 34 | ): T? = suspendCoroutine { continuation -> 35 | val intent = Intent().apply { 36 | component = ComponentName(pluginService.packageName, pluginService.className) 37 | } 38 | 39 | val connection = object : ServiceConnection { 40 | override fun onServiceConnected(name: ComponentName?, service: IBinder?) { 41 | if (service != null) { 42 | val castedService = serviceCast(service) 43 | continuation.resume(castedService) 44 | } else { 45 | continuation.resume(null) 46 | } 47 | } 48 | 49 | override fun onServiceDisconnected(name: ComponentName?) { 50 | // Handle disconnection if needed 51 | } 52 | } 53 | 54 | val bound = context.bindService(intent, connection, Context.BIND_AUTO_CREATE) 55 | if (!bound) { 56 | continuation.resume(null) 57 | } 58 | } 59 | 60 | private fun discoverServiceType( 61 | type: PluginType 62 | ): List { 63 | val pm = context.packageManager 64 | val intent = Intent(type.action) 65 | val services = pm.queryIntentServices(intent, PackageManager.GET_META_DATA) 66 | 67 | return services.mapNotNull { resolveInfo -> 68 | val serviceInfo = resolveInfo.serviceInfo 69 | val metadata = serviceInfo.metaData 70 | 71 | PluginService( 72 | packageName = serviceInfo.packageName, 73 | className = serviceInfo.name, 74 | type = type, 75 | displayName = metadata?.getString(PluginConstants.METADATA_DISPLAY_NAME), 76 | description = metadata?.getString(PluginConstants.METADATA_DESCRIPTION) 77 | ) 78 | } 79 | } 80 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/interaction/IInteractionFlowManager.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.interaction 2 | 3 | import com.penumbraos.mabl.conversation.ConversationManager 4 | import com.penumbraos.mabl.types.Error 5 | 6 | interface IInteractionFlowManager { 7 | fun startListening(requestImage: Boolean = false) 8 | fun startConversationFromInput(userInput: String) 9 | fun finishListening() 10 | fun isFlowActive(): Boolean 11 | fun getCurrentFlowState(): InteractionFlowState 12 | 13 | fun takePicture() 14 | 15 | fun setConversationManager(conversationManager: ConversationManager?) 16 | fun setStateCallback(callback: InteractionStateCallback?) 17 | fun setContentCallback(callback: InteractionContentCallback?) 18 | } 19 | 20 | enum class InteractionFlowState { 21 | IDLE, 22 | LISTENING, 23 | PROCESSING, 24 | SPEAKING, 25 | CANCELLING 26 | } 27 | 28 | interface InteractionStateCallback { 29 | fun onListeningStarted() 30 | fun onListeningStopped() 31 | fun onUserFinished() 32 | fun onProcessingStarted() 33 | fun onProcessingStopped() 34 | fun onSpeakingStarted() 35 | fun onSpeakingStopped() 36 | fun onError(error: Error) 37 | } 38 | 39 | interface InteractionContentCallback { 40 | fun onPartialTranscription(text: String) 41 | fun onFinalTranscription(text: String) 42 | fun onPartialResponse(token: String) 43 | fun onFinalResponse(response: String) 44 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/services/LlmController.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.services 2 | 3 | import android.os.IBinder 4 | import com.penumbraos.mabl.sdk.ILlmService 5 | import com.penumbraos.mabl.sdk.PluginType 6 | 7 | class LlmController(onConnect: () -> Unit) : ServiceController( 8 | PluginType.LLM, 9 | onConnect 10 | ) { 11 | override fun castService(service: IBinder): ILlmService { 12 | return ILlmService.Stub.asInterface(service) 13 | } 14 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/services/ServiceController.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.services 2 | 3 | import android.content.ComponentName 4 | import android.content.Context 5 | import android.content.Context.BIND_AUTO_CREATE 6 | import android.content.Intent 7 | import android.content.ServiceConnection 8 | import android.os.IBinder 9 | import android.util.Log 10 | import com.penumbraos.mabl.sdk.PluginType 11 | 12 | abstract class ServiceController( 13 | private val pluginType: PluginType, 14 | private val onConnect: () -> Unit 15 | ) { 16 | internal var service: T? = null 17 | 18 | val isConnected: Boolean 19 | get() = service != null 20 | 21 | private val connection = object : ServiceConnection { 22 | override fun onServiceConnected(name: ComponentName?, binder: IBinder?) { 23 | Log.d("MainActivity", "onServiceConnected: ${pluginType.name}") 24 | if (binder == null) { 25 | Log.e("MainActivity", "Service binder for ${pluginType.name} is null") 26 | return 27 | } 28 | service = castService(binder) 29 | onServiceConnected(service!!) 30 | } 31 | 32 | override fun onServiceDisconnected(name: ComponentName?) { 33 | service = null 34 | onServiceDisconnected() 35 | } 36 | } 37 | 38 | abstract fun castService(service: IBinder): T 39 | 40 | open fun onServiceConnected(service: T) { 41 | onConnect() 42 | } 43 | 44 | open fun onServiceDisconnected() { 45 | 46 | } 47 | 48 | fun connect(context: Context, packageName: String, className: String? = null) { 49 | val intent = Intent(pluginType.action).apply { 50 | if (className != null) { 51 | setClassName(packageName, className) 52 | } else { 53 | setPackage(packageName) 54 | } 55 | } 56 | 57 | // Force services to be active using foreground service 58 | context.startForegroundService(intent) 59 | 60 | if (!context.bindService( 61 | intent, 62 | connection, 63 | BIND_AUTO_CREATE 64 | ) 65 | ) { 66 | Log.e("MainActivity", "Could not set up binding for ${pluginType.name} service") 67 | } 68 | } 69 | 70 | fun shutdown(context: Context) { 71 | if (service != null) { 72 | context.unbindService(connection) 73 | } 74 | } 75 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/services/SttController.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.services 2 | 3 | import android.os.IBinder 4 | import com.penumbraos.mabl.sdk.ISttCallback 5 | import com.penumbraos.mabl.sdk.ISttService 6 | import com.penumbraos.mabl.sdk.PluginType 7 | 8 | class SttController(onConnect: () -> Unit) : 9 | ServiceController(PluginType.STT, onConnect) { 10 | 11 | var delegate: ISttCallback? = null 12 | private var isListening = false 13 | 14 | var startTime: Long? = null 15 | private set 16 | var endTime: Long? = null 17 | private set 18 | 19 | fun startListening() { 20 | if (service == null) { 21 | throw IllegalStateException("STT service not connected") 22 | } else if (delegate == null) { 23 | throw IllegalStateException("STT delegate not set") 24 | } 25 | service?.startListening(delegate) 26 | startTime = System.currentTimeMillis() 27 | endTime = null 28 | isListening = true 29 | } 30 | 31 | fun stopListening() { 32 | if (service == null) { 33 | throw IllegalStateException("STT service not connected") 34 | } 35 | service?.stopListening() 36 | endTime = System.currentTimeMillis() 37 | isListening = false 38 | } 39 | 40 | fun cancelListening() { 41 | if (isListening) { 42 | stopListening() 43 | } 44 | } 45 | 46 | fun isCurrentlyListening(): Boolean = isListening 47 | 48 | fun lastListenDuration(): Long? { 49 | if (startTime == null || endTime == null) { 50 | return null 51 | } 52 | return endTime!! - startTime!! 53 | } 54 | 55 | override fun castService(service: IBinder): ISttService { 56 | return ISttService.Stub.asInterface(service) 57 | } 58 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/services/SystemServiceRegistry.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.services 2 | 3 | import com.penumbraos.mabl.sdk.ISystemServiceRegistry 4 | import com.penumbraos.mabl.sdk.ITtsService 5 | 6 | class SystemServiceRegistry( 7 | private val allControllers: AllControllers 8 | ) : ISystemServiceRegistry.Stub() { 9 | 10 | override fun getTtsService(): ITtsService? { 11 | return allControllers.tts.service 12 | } 13 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/services/ToolController.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.services 2 | 3 | import android.os.IBinder 4 | import com.penumbraos.mabl.sdk.IToolService 5 | import com.penumbraos.mabl.sdk.PluginType 6 | 7 | class ToolController(onConnect: () -> Unit) : ServiceController( 8 | PluginType.TOOL, 9 | onConnect 10 | ) { 11 | override fun castService(service: IBinder): IToolService { 12 | return IToolService.Stub.asInterface(service) 13 | } 14 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/services/TtsController.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.services 2 | 3 | import android.os.IBinder 4 | import com.penumbraos.mabl.sdk.ITtsCallback 5 | import com.penumbraos.mabl.sdk.ITtsService 6 | import com.penumbraos.mabl.sdk.PluginType 7 | 8 | class TtsController(onConnect: () -> Unit) : 9 | ServiceController(PluginType.TTS, onConnect) { 10 | 11 | var delegate: ITtsCallback? = null 12 | set(delegate) { 13 | if (delegate != null) { 14 | service?.registerCallback(delegate) 15 | } 16 | } 17 | 18 | override fun onServiceConnected(service: ITtsService) { 19 | if (delegate != null) { 20 | service.registerCallback(delegate) 21 | } 22 | super.onServiceConnected(service) 23 | } 24 | 25 | override fun castService(service: IBinder): ITtsService { 26 | return ITtsService.Stub.asInterface(service) 27 | } 28 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/sound/SoundEffectManager.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sound 2 | 3 | import android.annotation.SuppressLint 4 | import android.media.MediaPlayer 5 | import android.util.Log 6 | import java.io.File 7 | 8 | private const val TAG = "SoundEffectManager" 9 | 10 | class SoundEffectManager() { 11 | private val tonePlayer = TonePlayer() 12 | 13 | private val listeningMediaPlayer = MediaPlayer() 14 | private var listeningMediaPlayerReady = false 15 | 16 | @SuppressLint("SdCardPath") 17 | private val listeningSoundEffectFile = File("/sdcard/penumbra/mabl/sounds/listening.mp3") 18 | 19 | init { 20 | try { 21 | if (listeningSoundEffectFile.exists()) { 22 | listeningMediaPlayer.setDataSource(listeningSoundEffectFile.absolutePath) 23 | listeningMediaPlayer.setOnPreparedListener { 24 | Log.d(TAG, "Loaded listening sound effect") 25 | listeningMediaPlayerReady = true 26 | } 27 | listeningMediaPlayer.setOnErrorListener { player, what, extra -> 28 | Log.e(TAG, "Error loading listening sound effect: $what, $extra") 29 | false 30 | } 31 | listeningMediaPlayer.prepareAsync() 32 | } 33 | } catch (e: Exception) { 34 | Log.e(TAG, "Failed to load listening sound effect", e) 35 | } 36 | } 37 | 38 | fun playStartListeningEffect() { 39 | tonePlayer.stop() 40 | stopStartListeningEffect() 41 | 42 | if (listeningSoundEffectFile.exists() && listeningMediaPlayerReady) { 43 | listeningMediaPlayer.start() 44 | } else { 45 | val g4 = TonePlayer.SoundEvent( 46 | doubleArrayOf(391.995), 47 | 200, 48 | attackDurationMs = 50, 49 | releaseDurationMs = 50 50 | ) 51 | 52 | tonePlayer.playJingle( 53 | listOf(g4) 54 | ) 55 | } 56 | } 57 | 58 | fun stopStartListeningEffect() { 59 | // TODO: This might cause clicking 60 | tonePlayer.stop() 61 | if (listeningMediaPlayer.isPlaying) { 62 | listeningMediaPlayer.pause() 63 | listeningMediaPlayer.seekTo(0) 64 | } 65 | } 66 | 67 | fun playWaitingEffect() { 68 | tonePlayer.stop() 69 | 70 | val g4 = TonePlayer.SoundEvent( 71 | doubleArrayOf(391.995), 72 | 200, 73 | attackDurationMs = 50, 74 | releaseDurationMs = 50 75 | ) 76 | 77 | val bFlat4 = TonePlayer.SoundEvent( 78 | doubleArrayOf(466.164), 79 | 200, 80 | attackDurationMs = 50, 81 | releaseDurationMs = 50 82 | ) 83 | 84 | val c5 = TonePlayer.SoundEvent( 85 | doubleArrayOf(523.251), 86 | 500, 87 | attackDurationMs = 50, 88 | releaseDurationMs = 50 89 | ) 90 | 91 | tonePlayer.playJingle( 92 | listOf( 93 | g4, g4, g4, g4, g4, 94 | 95 | TonePlayer.SoundEvent.rest(600), 96 | 97 | bFlat4, bFlat4, bFlat4, bFlat4, bFlat4, 98 | 99 | TonePlayer.SoundEvent.rest(600), 100 | 101 | c5, 102 | 103 | TonePlayer.SoundEvent.rest(800), 104 | ), 105 | loop = true 106 | ) 107 | } 108 | 109 | fun stopWaitingEffect() { 110 | // TODO: This might cause clicking 111 | tonePlayer.stop() 112 | } 113 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/types/Errors.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.types 2 | 3 | sealed class Error(val message: String) { 4 | class TtsError(message: String) : Error(message) 5 | class SttError(message: String) : Error(message) 6 | class LlmError(message: String) : Error(message) 7 | class FlowError(message: String) : Error(message) 8 | } 9 | -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/types/ServiceBundle.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.types 2 | 3 | import com.penumbraos.mabl.sdk.ILlmService 4 | import com.penumbraos.mabl.sdk.ISttService 5 | import com.penumbraos.mabl.sdk.ITtsService 6 | 7 | // TODO: Unused 8 | data class ServiceBundle( 9 | val stt: ISttService, 10 | val tts: ITtsService, 11 | val llm: ILlmService 12 | ) 13 | -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/ui/SettingsScreen.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui 2 | 3 | import androidx.compose.foundation.layout.Column 4 | import androidx.compose.foundation.layout.Spacer 5 | import androidx.compose.foundation.layout.fillMaxSize 6 | import androidx.compose.foundation.layout.height 7 | import androidx.compose.foundation.layout.padding 8 | import androidx.compose.material.icons.Icons 9 | import androidx.compose.material.icons.filled.ArrowBack 10 | import androidx.compose.material3.ExperimentalMaterial3Api 11 | import androidx.compose.material3.Icon 12 | import androidx.compose.material3.IconButton 13 | import androidx.compose.material3.MaterialTheme 14 | import androidx.compose.material3.Scaffold 15 | import androidx.compose.material3.Text 16 | import androidx.compose.material3.TopAppBar 17 | import androidx.compose.runtime.Composable 18 | import androidx.compose.ui.Modifier 19 | import androidx.compose.ui.unit.dp 20 | 21 | @OptIn(ExperimentalMaterial3Api::class) 22 | @Composable 23 | fun SettingsScreen(onBack: () -> Unit) { 24 | Scaffold( 25 | topBar = { 26 | TopAppBar( 27 | title = { Text("Settings") }, 28 | navigationIcon = { 29 | IconButton(onClick = onBack) { 30 | Icon( 31 | imageVector = Icons.Default.ArrowBack, 32 | contentDescription = "Back" 33 | ) 34 | } 35 | } 36 | ) 37 | } 38 | ) { innerPadding -> 39 | Column( 40 | modifier = Modifier 41 | .fillMaxSize() 42 | .padding(innerPadding) 43 | .padding(16.dp) 44 | ) { 45 | Text( 46 | text = "Settings", 47 | style = MaterialTheme.typography.headlineMedium 48 | ) 49 | Spacer(modifier = Modifier.height(16.dp)) 50 | Text( 51 | text = "Settings screen placeholder", 52 | style = MaterialTheme.typography.bodyMedium 53 | ) 54 | } 55 | } 56 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/ui/UIComponents.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui 2 | 3 | import com.penumbraos.mabl.ui.interfaces.IConversationRenderer 4 | import com.penumbraos.mabl.ui.interfaces.IPlatformInputHandler 5 | import com.penumbraos.mabl.ui.interfaces.IPlatformCapabilities 6 | 7 | data class UIComponents( 8 | val conversationRenderer: IConversationRenderer, 9 | val platformInputHandler: IPlatformInputHandler, 10 | val platformCapabilities: IPlatformCapabilities 11 | ) -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/ui/interfaces/IConversationRenderer.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui.interfaces 2 | 3 | import com.penumbraos.mabl.types.Error 4 | 5 | interface IConversationRenderer { 6 | fun showMessage(message: String, isUser: Boolean) 7 | fun showTranscription(text: String) 8 | fun showListening(isListening: Boolean) 9 | fun showError(error: Error) 10 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/ui/interfaces/IPlatformCapabilities.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui.interfaces 2 | 3 | interface IPlatformCapabilities { 4 | /** 5 | * Get platform-specific view model if available. Returns null for platforms without view models. 6 | */ 7 | fun getViewModel(): Any? { 8 | return null 9 | } 10 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/ui/interfaces/IPlatformInputHandler.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui.interfaces 2 | 3 | import android.content.Context 4 | import androidx.lifecycle.LifecycleCoroutineScope 5 | import com.penumbraos.mabl.interaction.IInteractionFlowManager 6 | 7 | /** 8 | * Platform-specific input handling. 9 | */ 10 | interface IPlatformInputHandler { 11 | fun setup( 12 | context: Context, 13 | lifecycleScope: LifecycleCoroutineScope, 14 | interactionFlowManager: IInteractionFlowManager 15 | ) 16 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/ui/theme/Color.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui.theme 2 | 3 | import androidx.compose.ui.graphics.Color 4 | 5 | val Purple80 = Color(0xFFD0BCFF) 6 | val PurpleGrey80 = Color(0xFFCCC2DC) 7 | val Pink80 = Color(0xFFEFB8C8) 8 | 9 | val Purple40 = Color(0xFF6650a4) 10 | val PurpleGrey40 = Color(0xFF625b71) 11 | val Pink40 = Color(0xFF7D5260) -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/ui/theme/Theme.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui.theme 2 | 3 | import android.app.Activity 4 | import android.os.Build 5 | import androidx.compose.foundation.isSystemInDarkTheme 6 | import androidx.compose.material3.MaterialTheme 7 | import androidx.compose.material3.darkColorScheme 8 | import androidx.compose.material3.dynamicDarkColorScheme 9 | import androidx.compose.material3.dynamicLightColorScheme 10 | import androidx.compose.material3.lightColorScheme 11 | import androidx.compose.runtime.Composable 12 | import androidx.compose.ui.platform.LocalContext 13 | 14 | private val DarkColorScheme = darkColorScheme( 15 | primary = Purple80, 16 | secondary = PurpleGrey80, 17 | tertiary = Pink80 18 | ) 19 | 20 | private val LightColorScheme = lightColorScheme( 21 | primary = Purple40, 22 | secondary = PurpleGrey40, 23 | tertiary = Pink40 24 | 25 | /* Other default colors to override 26 | background = Color(0xFFFFFBFE), 27 | surface = Color(0xFFFFFBFE), 28 | onPrimary = Color.White, 29 | onSecondary = Color.White, 30 | onTertiary = Color.White, 31 | onBackground = Color(0xFF1C1B1F), 32 | onSurface = Color(0xFF1C1B1F), 33 | */ 34 | ) 35 | 36 | @Composable 37 | fun MABLTheme( 38 | darkTheme: Boolean = isSystemInDarkTheme(), 39 | // Dynamic color is available on Android 12+ 40 | dynamicColor: Boolean = true, 41 | content: @Composable () -> Unit 42 | ) { 43 | val colorScheme = when { 44 | dynamicColor && Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> { 45 | val context = LocalContext.current 46 | if (darkTheme) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context) 47 | } 48 | 49 | darkTheme -> DarkColorScheme 50 | else -> LightColorScheme 51 | } 52 | 53 | MaterialTheme( 54 | colorScheme = colorScheme, 55 | typography = Typography, 56 | content = content 57 | ) 58 | } -------------------------------------------------------------------------------- /mabl/src/main/java/com/penumbraos/mabl/ui/theme/Type.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.ui.theme 2 | 3 | import androidx.compose.material3.Typography 4 | import androidx.compose.ui.text.TextStyle 5 | import androidx.compose.ui.text.font.FontFamily 6 | import androidx.compose.ui.text.font.FontWeight 7 | import androidx.compose.ui.unit.sp 8 | 9 | // Set of Material typography styles to start with 10 | val Typography = Typography( 11 | bodyLarge = TextStyle( 12 | fontFamily = FontFamily.Default, 13 | fontWeight = FontWeight.Normal, 14 | fontSize = 16.sp, 15 | lineHeight = 24.sp, 16 | letterSpacing = 0.5.sp 17 | ) 18 | /* Other default text styles to override 19 | titleLarge = TextStyle( 20 | fontFamily = FontFamily.Default, 21 | fontWeight = FontWeight.Normal, 22 | fontSize = 22.sp, 23 | lineHeight = 28.sp, 24 | letterSpacing = 0.sp 25 | ), 26 | labelSmall = TextStyle( 27 | fontFamily = FontFamily.Default, 28 | fontWeight = FontWeight.Medium, 29 | fontSize = 11.sp, 30 | lineHeight = 16.sp, 31 | letterSpacing = 0.5.sp 32 | ) 33 | */ 34 | ) -------------------------------------------------------------------------------- /mabl/src/main/res/drawable/ic_launcher_foreground.xml: -------------------------------------------------------------------------------- 1 | 7 | 8 | 9 | 15 | 18 | 21 | 22 | 23 | 24 | 30 | -------------------------------------------------------------------------------- /mabl/src/main/res/drawable/outline_voice_chat_24.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /mabl/src/main/res/mipmap-anydpi/ic_launcher.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /mabl/src/main/res/mipmap-anydpi/ic_launcher_round.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /mabl/src/main/res/mipmap-hdpi/ic_launcher.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PenumbraOS/mabl/1218669f9bc43f80d8e4a49c3edcb9029f084e2b/mabl/src/main/res/mipmap-hdpi/ic_launcher.webp -------------------------------------------------------------------------------- /mabl/src/main/res/mipmap-hdpi/ic_launcher_round.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PenumbraOS/mabl/1218669f9bc43f80d8e4a49c3edcb9029f084e2b/mabl/src/main/res/mipmap-hdpi/ic_launcher_round.webp -------------------------------------------------------------------------------- /mabl/src/main/res/mipmap-mdpi/ic_launcher.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PenumbraOS/mabl/1218669f9bc43f80d8e4a49c3edcb9029f084e2b/mabl/src/main/res/mipmap-mdpi/ic_launcher.webp -------------------------------------------------------------------------------- /mabl/src/main/res/mipmap-mdpi/ic_launcher_round.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PenumbraOS/mabl/1218669f9bc43f80d8e4a49c3edcb9029f084e2b/mabl/src/main/res/mipmap-mdpi/ic_launcher_round.webp -------------------------------------------------------------------------------- /mabl/src/main/res/mipmap-xhdpi/ic_launcher.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PenumbraOS/mabl/1218669f9bc43f80d8e4a49c3edcb9029f084e2b/mabl/src/main/res/mipmap-xhdpi/ic_launcher.webp -------------------------------------------------------------------------------- /mabl/src/main/res/mipmap-xhdpi/ic_launcher_round.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PenumbraOS/mabl/1218669f9bc43f80d8e4a49c3edcb9029f084e2b/mabl/src/main/res/mipmap-xhdpi/ic_launcher_round.webp -------------------------------------------------------------------------------- /mabl/src/main/res/mipmap-xxhdpi/ic_launcher.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PenumbraOS/mabl/1218669f9bc43f80d8e4a49c3edcb9029f084e2b/mabl/src/main/res/mipmap-xxhdpi/ic_launcher.webp -------------------------------------------------------------------------------- /mabl/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PenumbraOS/mabl/1218669f9bc43f80d8e4a49c3edcb9029f084e2b/mabl/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp -------------------------------------------------------------------------------- /mabl/src/main/res/mipmap-xxxhdpi/ic_launcher.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PenumbraOS/mabl/1218669f9bc43f80d8e4a49c3edcb9029f084e2b/mabl/src/main/res/mipmap-xxxhdpi/ic_launcher.webp -------------------------------------------------------------------------------- /mabl/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PenumbraOS/mabl/1218669f9bc43f80d8e4a49c3edcb9029f084e2b/mabl/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp -------------------------------------------------------------------------------- /mabl/src/main/res/values/colors.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | #FFBB86FC 4 | #FF6200EE 5 | #FF3700B3 6 | #FF03DAC5 7 | #FF018786 8 | #FF000000 9 | #FFFFFFFF 10 | -------------------------------------------------------------------------------- /mabl/src/main/res/values/strings.xml: -------------------------------------------------------------------------------- 1 | 2 | MABL 3 | -------------------------------------------------------------------------------- /mabl/src/main/res/values/themes.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 13 | -------------------------------------------------------------------------------- /plugins/aipinsystem/.gitignore: -------------------------------------------------------------------------------- 1 | /build -------------------------------------------------------------------------------- /plugins/aipinsystem/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | alias(libs.plugins.android.application) 3 | alias(libs.plugins.kotlin.android) 4 | } 5 | 6 | android { 7 | namespace = "com.penumbraos.plugins.aipinsystem" 8 | compileSdk = 35 9 | 10 | defaultConfig { 11 | applicationId = "com.penumbraos.plugins.aipinsystem" 12 | minSdk = 32 13 | targetSdk = 35 14 | versionCode = (project.findProperty("versionCode") as String?)?.toIntOrNull() ?: 1 15 | versionName = project.findProperty("versionName") as String? ?: "1.0" 16 | } 17 | 18 | buildTypes { 19 | release { 20 | isMinifyEnabled = false 21 | signingConfig = signingConfigs.getByName("debug") 22 | } 23 | } 24 | compileOptions { 25 | sourceCompatibility = JavaVersion.VERSION_11 26 | targetCompatibility = JavaVersion.VERSION_11 27 | } 28 | kotlinOptions { 29 | jvmTarget = "11" 30 | } 31 | } 32 | 33 | dependencies { 34 | implementation(project(":sdk")) 35 | 36 | implementation(libs.penumbraos.sdk) 37 | 38 | implementation(libs.androidx.core.ktx) 39 | implementation(libs.androidx.appcompat) 40 | implementation(libs.material) 41 | } -------------------------------------------------------------------------------- /plugins/aipinsystem/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 10 | 11 | 12 | 13 | 14 | 17 | 20 | 21 | 22 | 25 | 26 | 27 | 28 | 29 | 32 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /plugins/demo/.gitignore: -------------------------------------------------------------------------------- 1 | /build -------------------------------------------------------------------------------- /plugins/demo/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | alias(libs.plugins.android.application) 3 | alias(libs.plugins.kotlin.android) 4 | } 5 | 6 | android { 7 | namespace = "com.penumbraos.plugins.demo" 8 | compileSdk = 35 9 | 10 | defaultConfig { 11 | applicationId = "com.penumbraos.plugins.demo" 12 | minSdk = 32 13 | targetSdk = 35 14 | versionCode = (project.findProperty("versionCode") as String?)?.toIntOrNull() ?: 1 15 | versionName = project.findProperty("versionName") as String? ?: "1.0" 16 | } 17 | 18 | buildTypes { 19 | release { 20 | isMinifyEnabled = false 21 | signingConfig = signingConfigs.getByName("debug") 22 | } 23 | } 24 | compileOptions { 25 | sourceCompatibility = JavaVersion.VERSION_11 26 | targetCompatibility = JavaVersion.VERSION_11 27 | } 28 | kotlinOptions { 29 | jvmTarget = "11" 30 | } 31 | } 32 | 33 | dependencies { 34 | implementation(project(":sdk")) 35 | 36 | implementation(libs.penumbraos.sdk) 37 | 38 | implementation(libs.androidx.core.ktx) 39 | implementation(libs.androidx.appcompat) 40 | implementation(libs.material) 41 | } -------------------------------------------------------------------------------- /plugins/demo/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 11 | 12 | 13 | 14 | 15 | 18 | 21 | 22 | 23 | 26 | 27 | 28 | 29 | 30 | 33 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /plugins/demo/src/main/java/com/penumbraos/plugins/demo/DemoSttService.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.plugins.demo 2 | 3 | import android.annotation.SuppressLint 4 | import android.content.Intent 5 | import android.os.Bundle 6 | import android.os.IBinder 7 | import android.os.RemoteException 8 | import android.speech.SpeechRecognizer 9 | import android.util.Log 10 | import com.penumbraos.mabl.sdk.ISttCallback 11 | import com.penumbraos.mabl.sdk.ISttService 12 | import com.penumbraos.mabl.sdk.MablService 13 | import com.penumbraos.sdk.PenumbraClient 14 | import com.penumbraos.sdk.api.types.SttRecognitionListener 15 | import kotlinx.coroutines.CoroutineScope 16 | import kotlinx.coroutines.Dispatchers 17 | import kotlinx.coroutines.launch 18 | 19 | class DemoSttService : MablService("DemoSttService") { 20 | private var currentCallback: ISttCallback? = null 21 | private val scope = CoroutineScope(Dispatchers.IO) 22 | 23 | private lateinit var client: PenumbraClient 24 | 25 | private var isListening = false 26 | 27 | @SuppressLint("ForegroundServiceType") 28 | override fun onCreate() { 29 | super.onCreate() 30 | client = PenumbraClient(applicationContext) 31 | 32 | // Hack to start STT service in advance of usage 33 | client.stt.launchListenerProcess(applicationContext) 34 | 35 | scope.launch { 36 | client.waitForBridge() 37 | Log.i("DemoSttService", "Bridge start received, setting up STT") 38 | 39 | client.stt.initialize(object : SttRecognitionListener() { 40 | override fun onError(error: Int) { 41 | try { 42 | currentCallback?.onError("Recognition error: $error") 43 | } catch (e: RemoteException) { 44 | Log.e("DemoSttService", "Callback error", e) 45 | } 46 | } 47 | 48 | override fun onResults(results: Bundle?) { 49 | results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION) 50 | ?.firstOrNull()?.let { 51 | try { 52 | currentCallback?.onFinalTranscription(it) 53 | } catch (e: RemoteException) { 54 | Log.e("DemoSttService", "Callback error", e) 55 | } 56 | } 57 | } 58 | 59 | override fun onPartialResults(partialResults: Bundle?) { 60 | partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION) 61 | ?.firstOrNull()?.let { 62 | try { 63 | currentCallback?.onPartialTranscription(it) 64 | } catch (e: RemoteException) { 65 | Log.e("DemoSttService", "Callback error", e) 66 | } 67 | } 68 | } 69 | 70 | override fun onEndOfSpeech() { 71 | Log.i("DemoSttService", "End of speech. Continuing") 72 | } 73 | }) 74 | } 75 | } 76 | 77 | private val binder = object : ISttService.Stub() { 78 | override fun startListening(callback: ISttCallback) { 79 | currentCallback = callback 80 | this@DemoSttService.startListening() 81 | } 82 | 83 | override fun stopListening() { 84 | this@DemoSttService.stopListening() 85 | } 86 | } 87 | 88 | fun startListening() { 89 | if (isListening) { 90 | Log.w("DemoSttService", "Already listening. Not starting STT") 91 | return 92 | } 93 | 94 | Log.i("DemoSttService", "Starting STT") 95 | isListening = true 96 | client.stt.startListening() 97 | } 98 | 99 | fun stopListening() { 100 | Log.i("DemoSttService", "Stopping STT") 101 | isListening = false 102 | client.stt.stopListening() 103 | } 104 | 105 | override fun onDestroy() { 106 | client.stt.destroy() 107 | super.onDestroy() 108 | } 109 | 110 | override fun onBind(intent: Intent): IBinder { 111 | return binder 112 | } 113 | } -------------------------------------------------------------------------------- /plugins/googlesearch/.gitignore: -------------------------------------------------------------------------------- 1 | /build -------------------------------------------------------------------------------- /plugins/googlesearch/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | alias(libs.plugins.android.application) 3 | alias(libs.plugins.kotlin.android) 4 | alias(libs.plugins.kotlin.serialization) 5 | } 6 | 7 | android { 8 | namespace = "com.penumbraos.plugins.googlesearch" 9 | compileSdk = 35 10 | 11 | defaultConfig { 12 | applicationId = "com.penumbraos.plugins.googlesearch" 13 | minSdk = 32 14 | targetSdk = 35 15 | versionCode = (project.findProperty("versionCode") as String?)?.toIntOrNull() ?: 1 16 | versionName = project.findProperty("versionName") as String? ?: "1.0" 17 | } 18 | 19 | buildTypes { 20 | release { 21 | isMinifyEnabled = false 22 | signingConfig = signingConfigs.getByName("debug") 23 | } 24 | } 25 | compileOptions { 26 | sourceCompatibility = JavaVersion.VERSION_11 27 | targetCompatibility = JavaVersion.VERSION_11 28 | } 29 | kotlinOptions { 30 | jvmTarget = "11" 31 | } 32 | } 33 | 34 | dependencies { 35 | implementation(project(":sdk")) 36 | 37 | implementation(libs.penumbraos.sdk) 38 | 39 | implementation(libs.ktor.client.android) 40 | implementation(libs.ktor.content.negociation) 41 | implementation(libs.ktor.serialization.kotlinx.json) 42 | implementation(libs.kotlinx.serialization.json) 43 | implementation(libs.kotlinx.coroutines.android) 44 | 45 | implementation(libs.androidx.core.ktx) 46 | implementation(libs.androidx.appcompat) 47 | implementation(libs.material) 48 | } -------------------------------------------------------------------------------- /plugins/googlesearch/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 10 | 11 | 12 | 13 | 14 | 17 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /plugins/googlesearch/src/main/java/com/penumbraos/plugins/googlesearch/GoogleSearchProcessor.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.plugins.googlesearch 2 | 3 | import android.util.Log 4 | import kotlinx.serialization.Serializable 5 | import kotlinx.serialization.json.Json 6 | import java.net.URL 7 | 8 | private const val TAG = "GoogleSearchProcessor" 9 | 10 | @Serializable 11 | data class ProcessedSearchResult( 12 | val title: String, 13 | val url: String, 14 | val snippet: String, 15 | val displayLink: String 16 | ) 17 | 18 | @Serializable 19 | data class ProcessedSearchResponse( 20 | val query: String, 21 | val totalResults: String, 22 | val searchTime: String, 23 | val results: List 24 | ) 25 | 26 | class GoogleSearchProcessor { 27 | 28 | private val json = Json { 29 | prettyPrint = true 30 | encodeDefaults = false 31 | } 32 | 33 | fun processResults(response: GoogleSearchResponse, originalQuery: String): String { 34 | Log.d(TAG, "Processing Google Search results") 35 | 36 | val items = response.items ?: emptyList() 37 | 38 | if (items.isEmpty()) { 39 | return "No search results found for query: '$originalQuery'" 40 | } 41 | 42 | val processedResults = items.map { item -> 43 | ProcessedSearchResult( 44 | title = cleanText(item.title), 45 | url = item.link, 46 | snippet = cleanText(item.snippet), 47 | displayLink = item.displayLink 48 | ) 49 | } 50 | 51 | val searchInfo = response.searchInformation 52 | val processedResponse = ProcessedSearchResponse( 53 | query = originalQuery, 54 | totalResults = searchInfo?.formattedTotalResults ?: searchInfo?.totalResults ?: "Unknown", 55 | searchTime = searchInfo?.formattedSearchTime ?: "${searchInfo?.searchTime ?: 0.0} seconds", 56 | results = processedResults 57 | ) 58 | 59 | return json.encodeToString(ProcessedSearchResponse.serializer(), processedResponse) 60 | } 61 | 62 | private fun cleanText(text: String): String { 63 | if (text.isBlank()) return "" 64 | 65 | return text 66 | // Remove HTML tags 67 | .replace(Regex("<[^>]*>"), "") 68 | // Remove excessive whitespace 69 | .replace(Regex("\\s+"), " ") 70 | // Remove common artifacts 71 | .replace("...", "") 72 | .replace(" ...", "") 73 | .replace("... ", "") 74 | // Fix HTML entities 75 | .replace("&", "and") 76 | .replace("<", "<") 77 | .replace(">", ">") 78 | .replace(""", "\"") 79 | .replace("'", "'") 80 | .replace(" ", " ") 81 | .replace("—", "—") 82 | .replace("–", "–") 83 | // Clean up dates and formatting 84 | .replace(Regex("\\s*-\\s*$"), "") // Remove trailing dash 85 | .replace(Regex("^\\s*-\\s*"), "") // Remove leading dash 86 | .trim() 87 | } 88 | 89 | fun hasValidResults(response: GoogleSearchResponse): Boolean { 90 | return !response.items.isNullOrEmpty() 91 | } 92 | } -------------------------------------------------------------------------------- /plugins/openai/.gitignore: -------------------------------------------------------------------------------- 1 | /build -------------------------------------------------------------------------------- /plugins/openai/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | alias(libs.plugins.android.application) 3 | alias(libs.plugins.kotlin.android) 4 | alias(libs.plugins.kotlin.serialization) 5 | } 6 | 7 | android { 8 | namespace = "com.penumbraos.plugins.openai" 9 | compileSdk = 35 10 | 11 | defaultConfig { 12 | applicationId = "com.penumbraos.plugins.openai" 13 | minSdk = 32 14 | targetSdk = 35 15 | versionCode = (project.findProperty("versionCode") as String?)?.toIntOrNull() ?: 1 16 | versionName = project.findProperty("versionName") as String? ?: "1.0" 17 | } 18 | 19 | buildTypes { 20 | release { 21 | isMinifyEnabled = false 22 | signingConfig = signingConfigs.getByName("debug") 23 | } 24 | } 25 | compileOptions { 26 | sourceCompatibility = JavaVersion.VERSION_11 27 | targetCompatibility = JavaVersion.VERSION_11 28 | } 29 | kotlinOptions { 30 | jvmTarget = "11" 31 | } 32 | } 33 | 34 | dependencies { 35 | implementation(project(":sdk")) 36 | 37 | implementation(libs.penumbraos.sdk) 38 | 39 | implementation(libs.openai.client) 40 | implementation(libs.ktor.client.android) 41 | 42 | implementation(libs.kotlinx.serialization.json) 43 | 44 | implementation(libs.kotlinx.coroutines.android) 45 | 46 | implementation(libs.androidx.core.ktx) 47 | implementation(libs.androidx.appcompat) 48 | implementation(libs.material) 49 | } -------------------------------------------------------------------------------- /plugins/openai/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 11 | 12 | 13 | 17 | 18 | 19 | 20 | 21 | 24 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /plugins/openai/src/main/java/com/penumbraos/plugins/openai/LlmConfigManager.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.plugins.openai 2 | 3 | import android.annotation.SuppressLint 4 | import android.util.Log 5 | import kotlinx.serialization.Serializable 6 | import kotlinx.serialization.json.Json 7 | import java.io.File 8 | 9 | private const val TAG = "LlmConfigService" 10 | 11 | @Serializable 12 | data class LlmConfiguration( 13 | val name: String, 14 | val apiKey: String, 15 | val model: String, 16 | val maxTokens: Int = 1000, 17 | val temperature: Double = 0.7, 18 | val systemPrompt: String? = null, 19 | val baseUrl: String 20 | ) 21 | 22 | @Serializable 23 | data class LlmConfigFile( 24 | val configs: List 25 | ) 26 | 27 | class LlmConfigManager { 28 | 29 | private var configs: List? = null 30 | private val json = Json { ignoreUnknownKeys = true } 31 | 32 | @SuppressLint("SdCardPath") 33 | private val mablDir = File("/sdcard/penumbra/etc/mabl/") 34 | private val configFile = File(mablDir, "llm_configs.json") 35 | 36 | fun getAvailableConfigs(): List { 37 | Log.d(TAG, "Getting available LLM configurations") 38 | 39 | if (configs == null || configs!!.isEmpty()) { 40 | configs = loadConfigsFromFile() 41 | } 42 | 43 | return configs ?: listOf() 44 | } 45 | 46 | private fun loadConfigsFromFile(): List { 47 | return try { 48 | if (configFile.exists()) { 49 | Log.d(TAG, "Attempting to load configs") 50 | val jsonString = configFile.readText() 51 | val configFile = json.decodeFromString(jsonString) 52 | val logMap = configFile.configs.map { config -> 53 | """ 54 | Name: ${config.name} 55 | Model: ${config.model} 56 | Base URL: ${config.baseUrl} 57 | Max Tokens: ${config.maxTokens} 58 | Temperature: ${config.temperature} 59 | """.trimIndent() 60 | } 61 | Log.d(TAG, "Loaded configs from file: ${logMap.joinToString("\n\n")}") 62 | configFile.configs 63 | } else { 64 | Log.e(TAG, "Config file does not exist. Returning empty configs") 65 | listOf() 66 | } 67 | } catch (e: Exception) { 68 | Log.e(TAG, "Error loading configs from file. Returning empty configs", e) 69 | listOf() 70 | } 71 | } 72 | } -------------------------------------------------------------------------------- /plugins/searxng/.gitignore: -------------------------------------------------------------------------------- 1 | /build -------------------------------------------------------------------------------- /plugins/searxng/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | alias(libs.plugins.android.application) 3 | alias(libs.plugins.kotlin.android) 4 | alias(libs.plugins.kotlin.serialization) 5 | } 6 | 7 | android { 8 | namespace = "com.penumbraos.plugins.searxng" 9 | compileSdk = 35 10 | 11 | defaultConfig { 12 | applicationId = "com.penumbraos.plugins.searxng" 13 | minSdk = 32 14 | targetSdk = 35 15 | versionCode = (project.findProperty("versionCode") as String?)?.toIntOrNull() ?: 1 16 | versionName = project.findProperty("versionName") as String? ?: "1.0" 17 | } 18 | 19 | buildTypes { 20 | release { 21 | isMinifyEnabled = false 22 | signingConfig = signingConfigs.getByName("debug") 23 | } 24 | } 25 | compileOptions { 26 | sourceCompatibility = JavaVersion.VERSION_11 27 | targetCompatibility = JavaVersion.VERSION_11 28 | } 29 | kotlinOptions { 30 | jvmTarget = "11" 31 | } 32 | } 33 | 34 | dependencies { 35 | implementation(project(":sdk")) 36 | 37 | implementation(libs.penumbraos.sdk) 38 | 39 | implementation(libs.ktor.client.android) 40 | implementation(libs.ktor.content.negociation) 41 | implementation(libs.ktor.serialization.kotlinx.json) 42 | implementation(libs.kotlinx.serialization.json) 43 | implementation(libs.kotlinx.coroutines.android) 44 | implementation(libs.jsoup) 45 | 46 | implementation(libs.androidx.core.ktx) 47 | implementation(libs.androidx.appcompat) 48 | implementation(libs.material) 49 | } -------------------------------------------------------------------------------- /plugins/searxng/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 10 | 11 | 12 | 13 | 14 | 17 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /plugins/searxng/src/main/java/com/penumbraos/plugins/searxng/SearchResultProcessor.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.plugins.searxng 2 | 3 | import android.util.Log 4 | import kotlinx.serialization.Serializable 5 | import kotlinx.serialization.json.Json 6 | 7 | private const val TAG = "SearchResultProcessor" 8 | 9 | @Serializable 10 | data class CleanedSearchResult( 11 | val title: String, 12 | val url: String, 13 | val content: String, 14 | val source: String 15 | ) 16 | 17 | @Serializable 18 | data class SearchResponse( 19 | val query: String, 20 | val results: List 21 | ) 22 | 23 | class SearchResultProcessor { 24 | 25 | private val json = Json { 26 | prettyPrint = true 27 | encodeDefaults = false 28 | } 29 | 30 | fun processResults(response: SearxngResponse): String { 31 | Log.d(TAG, "Processing ${response.results.size} search results") 32 | 33 | if (response.results.isEmpty()) { 34 | return "No search results found for query: '${response.query}'" 35 | } 36 | 37 | val cleanedResults = response.results.map { result -> 38 | CleanedSearchResult( 39 | title = cleanText(result.title), 40 | url = result.url, 41 | content = cleanText(result.content), 42 | source = extractDomain(result.url) 43 | ) 44 | } 45 | 46 | val searchResponse = SearchResponse( 47 | query = response.query, 48 | results = cleanedResults 49 | ) 50 | 51 | return json.encodeToString(SearchResponse.serializer(), searchResponse) 52 | } 53 | 54 | private fun cleanText(text: String): String { 55 | if (text.isBlank()) return "" 56 | 57 | return text 58 | // Remove HTML tags 59 | .replace(Regex("<[^>]*>"), "") 60 | // Remove excessive whitespace 61 | .replace(Regex("\\s+"), " ") 62 | // Remove common web artifacts 63 | .replace(Regex("\\[\\d+\\]"), "") // Wikipedia reference numbers 64 | .replace("...", "") 65 | .replace("Read more", "") 66 | .replace("Continue reading", "") 67 | // Fix common HTML entities 68 | .replace("&", "and") 69 | .replace("<", "<") 70 | .replace(">", ">") 71 | .replace(""", "\"") 72 | .replace("'", "'") 73 | .replace(" ", " ") 74 | // Remove extra punctuation artifacts 75 | .replace(Regex("\\.{2,}"), ".") 76 | .replace(Regex("\\s*[|•]\\s*"), " - ") 77 | .trim() 78 | } 79 | 80 | private fun extractDomain(url: String): String { 81 | return try { 82 | if (url.isBlank()) return "unknown" 83 | 84 | val cleanUrl = if (!url.startsWith("http")) "https://$url" else url 85 | val domain = java.net.URL(cleanUrl).host.lowercase() 86 | 87 | // Return clean domain name 88 | domain.removePrefix("www.") 89 | } catch (e: Exception) { 90 | Log.w(TAG, "Failed to extract domain from: $url", e) 91 | "unknown" 92 | } 93 | } 94 | } -------------------------------------------------------------------------------- /plugins/system/.gitignore: -------------------------------------------------------------------------------- 1 | /build -------------------------------------------------------------------------------- /plugins/system/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | alias(libs.plugins.android.application) 3 | alias(libs.plugins.kotlin.android) 4 | } 5 | 6 | android { 7 | namespace = "com.penumbraos.plugins.system" 8 | compileSdk = 35 9 | 10 | defaultConfig { 11 | applicationId = "com.penumbraos.plugins.system" 12 | minSdk = 32 13 | targetSdk = 35 14 | versionCode = (project.findProperty("versionCode") as String?)?.toIntOrNull() ?: 1 15 | versionName = project.findProperty("versionName") as String? ?: "1.0" 16 | } 17 | 18 | buildTypes { 19 | release { 20 | isMinifyEnabled = false 21 | signingConfig = signingConfigs.getByName("debug") 22 | } 23 | } 24 | compileOptions { 25 | sourceCompatibility = JavaVersion.VERSION_11 26 | targetCompatibility = JavaVersion.VERSION_11 27 | } 28 | kotlinOptions { 29 | jvmTarget = "11" 30 | } 31 | } 32 | 33 | dependencies { 34 | implementation(project(":sdk")) 35 | 36 | implementation(libs.androidx.core.ktx) 37 | implementation(libs.androidx.appcompat) 38 | implementation(libs.material) 39 | } -------------------------------------------------------------------------------- /plugins/system/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 11 | 12 | 13 | 14 | 15 | 18 | 21 | 22 | 25 | 26 | 27 | 28 | 29 | 32 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /plugins/system/src/main/java/com/penumbraos/plugins/system/tool/NetworkService.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.plugins.system.tool 2 | 3 | import android.Manifest 4 | import android.net.ConnectivityManager 5 | import android.util.Log 6 | import androidx.annotation.RequiresPermission 7 | import com.penumbraos.mabl.sdk.IToolCallback 8 | import com.penumbraos.mabl.sdk.ToolCall 9 | import com.penumbraos.mabl.sdk.ToolDefinition 10 | import com.penumbraos.mabl.sdk.ToolService 11 | import org.json.JSONObject 12 | 13 | 14 | private const val GET_IP = "get_ip" 15 | 16 | private val IPv4_REGEX = """(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(/\d{1,2})?""".toRegex() 17 | 18 | class NetworkService : ToolService("NetworkService") { 19 | @RequiresPermission(Manifest.permission.ACCESS_NETWORK_STATE) 20 | override fun executeTool( 21 | call: ToolCall, 22 | params: JSONObject?, 23 | callback: IToolCallback 24 | ) { 25 | when (call.name) { 26 | GET_IP -> { 27 | val connectivityManager = 28 | getSystemService(CONNECTIVITY_SERVICE) as ConnectivityManager? 29 | 30 | if (connectivityManager == null) { 31 | callback.onError("Failed to get network status") 32 | return 33 | } 34 | 35 | val linkProperties = 36 | connectivityManager.getLinkProperties(connectivityManager.activeNetwork) 37 | 38 | if (linkProperties == null) { 39 | callback.onError("Failed to get network status") 40 | return 41 | } 42 | 43 | Log.d( 44 | "NetworkService", 45 | "Link properties: ${linkProperties.linkAddresses.map { it.toString() }}" 46 | ) 47 | 48 | val address = 49 | linkProperties.linkAddresses.map { 50 | val result = IPv4_REGEX.matchEntire(it.toString()) 51 | result?.groups[1]?.value 52 | }.firstOrNull() 53 | 54 | if (address == null) { 55 | callback.onError("Could not identify IP address") 56 | return 57 | } 58 | 59 | callback.onSuccess("My IP address is $address") 60 | } 61 | } 62 | } 63 | 64 | override fun getToolDefinitions(): Array { 65 | return arrayOf( 66 | ToolDefinition().apply { 67 | name = GET_IP 68 | description = "Get the IP address of the device" 69 | examples = arrayOf( 70 | "what is your IP address", 71 | "what is your address", 72 | "IP address", 73 | "internet address", 74 | "what is the IP" 75 | ) 76 | } 77 | ) 78 | } 79 | } -------------------------------------------------------------------------------- /portal/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /portal/README.md: -------------------------------------------------------------------------------- 1 | # React + TypeScript + Vite 2 | 3 | This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules. 4 | 5 | Currently, two official plugins are available: 6 | 7 | - [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Babel](https://babeljs.io/) for Fast Refresh 8 | - [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh 9 | 10 | ## Expanding the ESLint configuration 11 | 12 | If you are developing a production application, we recommend updating the configuration to enable type-aware lint rules: 13 | 14 | ```js 15 | export default tseslint.config([ 16 | globalIgnores(['dist']), 17 | { 18 | files: ['**/*.{ts,tsx}'], 19 | extends: [ 20 | // Other configs... 21 | 22 | // Remove tseslint.configs.recommended and replace with this 23 | ...tseslint.configs.recommendedTypeChecked, 24 | // Alternatively, use this for stricter rules 25 | ...tseslint.configs.strictTypeChecked, 26 | // Optionally, add this for stylistic rules 27 | ...tseslint.configs.stylisticTypeChecked, 28 | 29 | // Other configs... 30 | ], 31 | languageOptions: { 32 | parserOptions: { 33 | project: ['./tsconfig.node.json', './tsconfig.app.json'], 34 | tsconfigRootDir: import.meta.dirname, 35 | }, 36 | // other options... 37 | }, 38 | }, 39 | ]) 40 | ``` 41 | 42 | You can also install [eslint-plugin-react-x](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-x) and [eslint-plugin-react-dom](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-dom) for React-specific lint rules: 43 | 44 | ```js 45 | // eslint.config.js 46 | import reactX from 'eslint-plugin-react-x' 47 | import reactDom from 'eslint-plugin-react-dom' 48 | 49 | export default tseslint.config([ 50 | globalIgnores(['dist']), 51 | { 52 | files: ['**/*.{ts,tsx}'], 53 | extends: [ 54 | // Other configs... 55 | // Enable lint rules for React 56 | reactX.configs['recommended-typescript'], 57 | // Enable lint rules for React DOM 58 | reactDom.configs.recommended, 59 | ], 60 | languageOptions: { 61 | parserOptions: { 62 | project: ['./tsconfig.node.json', './tsconfig.app.json'], 63 | tsconfigRootDir: import.meta.dirname, 64 | }, 65 | // other options... 66 | }, 67 | }, 68 | ]) 69 | ``` 70 | -------------------------------------------------------------------------------- /portal/eslint.config.js: -------------------------------------------------------------------------------- 1 | import js from '@eslint/js' 2 | import globals from 'globals' 3 | import reactHooks from 'eslint-plugin-react-hooks' 4 | import reactRefresh from 'eslint-plugin-react-refresh' 5 | import tseslint from 'typescript-eslint' 6 | import { globalIgnores } from 'eslint/config' 7 | 8 | export default tseslint.config([ 9 | globalIgnores(['dist']), 10 | { 11 | files: ['**/*.{ts,tsx}'], 12 | extends: [ 13 | js.configs.recommended, 14 | tseslint.configs.recommended, 15 | reactHooks.configs['recommended-latest'], 16 | reactRefresh.configs.vite, 17 | ], 18 | languageOptions: { 19 | ecmaVersion: 2020, 20 | globals: globals.browser, 21 | }, 22 | }, 23 | ]) 24 | -------------------------------------------------------------------------------- /portal/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Vite + React + TS 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /portal/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "portal", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "VITE_HOSTNAME=192.168.1.192:8080 vite", 8 | "build": "tsc -b && vite build", 9 | "lint": "eslint .", 10 | "preview": "vite preview" 11 | }, 12 | "dependencies": { 13 | "@mantine/core": "^8.2.7", 14 | "@mantine/hooks": "^8.2.7", 15 | "@tabler/icons-react": "^3.34.1", 16 | "@tanstack/react-query": "^5.85.6", 17 | "@tanstack/react-router": "^1.131.35", 18 | "@tanstack/react-router-devtools": "^1.131.35", 19 | "react": "^19.1.1", 20 | "react-dom": "^19.1.1" 21 | }, 22 | "devDependencies": { 23 | "@eslint/js": "^9.33.0", 24 | "@tanstack/eslint-plugin-query": "^5.83.1", 25 | "@tanstack/router-plugin": "^1.131.35", 26 | "@types/react": "^19.1.10", 27 | "@types/react-dom": "^19.1.7", 28 | "@vitejs/plugin-react": "^5.0.0", 29 | "eslint": "^9.33.0", 30 | "eslint-plugin-react-hooks": "^5.2.0", 31 | "eslint-plugin-react-refresh": "^0.4.20", 32 | "globals": "^16.3.0", 33 | "postcss": "^8.5.6", 34 | "postcss-preset-mantine": "^1.18.0", 35 | "postcss-simple-vars": "^7.0.1", 36 | "sass": "^1.92.1", 37 | "typescript": "~5.8.3", 38 | "typescript-eslint": "^8.39.1", 39 | "vite": "^7.1.2" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /portal/postcss.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | 'postcss-preset-mantine': {}, 4 | 'postcss-simple-vars': { 5 | variables: { 6 | 'mantine-breakpoint-xs': '36em', 7 | 'mantine-breakpoint-sm': '48em', 8 | 'mantine-breakpoint-md': '62em', 9 | 'mantine-breakpoint-lg': '75em', 10 | 'mantine-breakpoint-xl': '88em', 11 | }, 12 | }, 13 | }, 14 | }; -------------------------------------------------------------------------------- /portal/src/App.tsx: -------------------------------------------------------------------------------- 1 | import { createRouter, RouterProvider } from "@tanstack/react-router"; 2 | import { routeTree } from "./routeTree.gen"; 3 | 4 | const router = createRouter({ routeTree }); 5 | 6 | export const App = () => { 7 | return ; 8 | }; 9 | -------------------------------------------------------------------------------- /portal/src/components/CameraRoll.module.scss: -------------------------------------------------------------------------------- 1 | .imageCard { 2 | cursor: pointer; 3 | } 4 | 5 | .cardContainer { 6 | max-width: 300px; 7 | } -------------------------------------------------------------------------------- /portal/src/components/Conversation.module.scss: -------------------------------------------------------------------------------- 1 | .conversationTitle { 2 | flex: 1; 3 | } 4 | 5 | .messageImageContainer { 6 | max-width: 300px; 7 | } 8 | 9 | .messageImage { 10 | cursor: pointer; 11 | } -------------------------------------------------------------------------------- /portal/src/components/Conversations.tsx: -------------------------------------------------------------------------------- 1 | import { useQuery } from "@tanstack/react-query"; 2 | import * as React from "react"; 3 | import { getConversations } from "../state/api"; 4 | import { QueryWrapper } from "./QueryWrapper"; 5 | import type { Conversation } from "../state/types"; 6 | import { Card, Text, Group, Button, Stack, Badge } from "@mantine/core"; 7 | import { Link } from "@tanstack/react-router"; 8 | import styles from "../styles/layout.module.scss"; 9 | import { formatHumanDate } from "../util/date"; 10 | 11 | export const Conversations: React.FC = () => { 12 | const result = useQuery({ 13 | queryKey: ["conversations"], 14 | queryFn: getConversations, 15 | }); 16 | 17 | return ; 18 | }; 19 | 20 | const ConversationData: React.FC<{ 21 | data: Conversation[]; 22 | }> = ({ data }) => { 23 | return ( 24 | 25 | 26 | 27 | Conversations 28 | 29 | 30 | {data.length > 0 ? ( 31 | data.map((conversation) => ( 32 | 38 | 45 | 46 |
47 | 48 | 49 | {conversation.title} 50 | 51 | {conversation.isActive && ( 52 | 53 | Active 54 | 55 | )} 56 | 57 | 62 | {formatHumanDate(conversation.lastActivity)} 63 | 64 |
65 | 68 | 69 | 70 | 71 | )) 72 | ) : ( 73 | 74 | No conversations yet 75 | 76 | )} 77 | 78 | ); 79 | }; 80 | -------------------------------------------------------------------------------- /portal/src/components/ImageGallery.module.scss: -------------------------------------------------------------------------------- 1 | .galleryContainer { 2 | position: relative; 3 | width: 100%; 4 | height: 80vh; 5 | } 6 | 7 | .closeButton { 8 | position: absolute; 9 | top: 16px; 10 | right: 16px; 11 | z-index: 1000; 12 | background-color: rgba(0, 0, 0, 0.7); 13 | } 14 | 15 | .navButton { 16 | position: absolute; 17 | top: 50%; 18 | transform: translateY(-50%); 19 | z-index: 1000; 20 | background-color: rgba(0, 0, 0, 0.7); 21 | 22 | &.navLeft { 23 | left: 16px; 24 | } 25 | 26 | &.navRight { 27 | right: 16px; 28 | } 29 | } 30 | 31 | .imageContainer { 32 | width: 100%; 33 | height: 100%; 34 | } 35 | 36 | .infoOverlay { 37 | position: absolute; 38 | bottom: 0; 39 | left: 0; 40 | right: 0; 41 | background: linear-gradient(transparent, rgba(0, 0, 0, 0.8)); 42 | padding: 2rem 1rem 1rem; 43 | color: white; 44 | } 45 | 46 | .infoContent { 47 | display: flex; 48 | justify-content: space-between; 49 | align-items: flex-end; 50 | } 51 | 52 | .infoTitle { 53 | font-size: 1.125rem; 54 | font-weight: 600; 55 | color: white; 56 | } 57 | 58 | .infoMetadata { 59 | font-size: 0.875rem; 60 | color: #c1c2c5; 61 | } 62 | 63 | .infoCounter { 64 | font-size: 0.875rem; 65 | color: #c1c2c5; 66 | } -------------------------------------------------------------------------------- /portal/src/components/Navigation.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { Group, Button } from "@mantine/core"; 3 | import { Link, useLocation } from "@tanstack/react-router"; 4 | 5 | export const Navigation: React.FC = () => { 6 | const location = useLocation(); 7 | 8 | return ( 9 | 10 | 11 | 17 | 18 | 19 | 25 | 26 | 27 | ); 28 | }; 29 | -------------------------------------------------------------------------------- /portal/src/components/QueryWrapper.module.scss: -------------------------------------------------------------------------------- 1 | .loadingContainer { 2 | display: flex; 3 | justify-content: center; 4 | align-items: center; 5 | height: 50vh; 6 | } -------------------------------------------------------------------------------- /portal/src/components/QueryWrapper.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | 3 | import { Alert, Loader } from "@mantine/core"; 4 | import type { UseQueryResult } from "@tanstack/react-query"; 5 | import { IconAlertTriangle } from "@tabler/icons-react"; 6 | import styles from "./QueryWrapper.module.scss"; 7 | 8 | type ExtractData = T extends { data: unknown } ? T["data"] : never; 9 | 10 | type Props = { 11 | result: UseQueryResult, Error>; 12 | DataComponent: React.FC; 13 | } & Omit; 14 | 15 | export const QueryWrapper = ({ 16 | result, 17 | DataComponent, 18 | ...props 19 | }: Props) => { 20 | const { isPending, isError, data, error } = result; 21 | 22 | if (isPending) { 23 | return ( 24 |
25 | 26 |
27 | ); 28 | } 29 | 30 | if (isError) { 31 | return ( 32 | } 35 | title={`Error: ${error.name}`} 36 | > 37 | {error.message} 38 | 39 | ); 40 | } 41 | 42 | return ; 43 | }; 44 | -------------------------------------------------------------------------------- /portal/src/hooks/useImageGallery.ts: -------------------------------------------------------------------------------- 1 | import { useState, useCallback } from "react"; 2 | import type { GalleryImage } from "../components/ImageGallery"; 3 | 4 | export const useImageGallery = () => { 5 | const [opened, setOpened] = useState(false); 6 | const [images, setImages] = useState([]); 7 | const [initialIndex, setInitialIndex] = useState(0); 8 | 9 | const openGallery = useCallback( 10 | (galleryImages: GalleryImage[], startIndex: number = 0) => { 11 | setImages(galleryImages); 12 | setInitialIndex(startIndex); 13 | setOpened(true); 14 | }, 15 | [] 16 | ); 17 | 18 | const closeGallery = useCallback(() => { 19 | setOpened(false); 20 | }, []); 21 | 22 | return { 23 | opened, 24 | images, 25 | initialIndex, 26 | openGallery, 27 | closeGallery, 28 | }; 29 | }; 30 | -------------------------------------------------------------------------------- /portal/src/main.tsx: -------------------------------------------------------------------------------- 1 | import { StrictMode } from "react"; 2 | import { createRoot } from "react-dom/client"; 3 | import { App } from "./App.tsx"; 4 | import { MantineProvider } from "@mantine/core"; 5 | 6 | import "@mantine/core/styles.css"; 7 | import { QueryClientProvider } from "@tanstack/react-query"; 8 | import { queryClient } from "./state/query.ts"; 9 | 10 | createRoot(document.getElementById("root")!).render( 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | ); 19 | -------------------------------------------------------------------------------- /portal/src/routeTree.gen.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | 3 | // @ts-nocheck 4 | 5 | // noinspection JSUnusedGlobalSymbols 6 | 7 | // This file was automatically generated by TanStack Router. 8 | // You should NOT make any changes in this file as it will be overwritten. 9 | // Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. 10 | 11 | import { Route as rootRouteImport } from './routes/__root' 12 | import { Route as CameraRollRouteImport } from './routes/camera-roll' 13 | import { Route as IndexRouteImport } from './routes/index' 14 | import { Route as ConversationConversationIdRouteImport } from './routes/conversation.$conversationId' 15 | 16 | const CameraRollRoute = CameraRollRouteImport.update({ 17 | id: '/camera-roll', 18 | path: '/camera-roll', 19 | getParentRoute: () => rootRouteImport, 20 | } as any) 21 | const IndexRoute = IndexRouteImport.update({ 22 | id: '/', 23 | path: '/', 24 | getParentRoute: () => rootRouteImport, 25 | } as any) 26 | const ConversationConversationIdRoute = 27 | ConversationConversationIdRouteImport.update({ 28 | id: '/conversation/$conversationId', 29 | path: '/conversation/$conversationId', 30 | getParentRoute: () => rootRouteImport, 31 | } as any) 32 | 33 | export interface FileRoutesByFullPath { 34 | '/': typeof IndexRoute 35 | '/camera-roll': typeof CameraRollRoute 36 | '/conversation/$conversationId': typeof ConversationConversationIdRoute 37 | } 38 | export interface FileRoutesByTo { 39 | '/': typeof IndexRoute 40 | '/camera-roll': typeof CameraRollRoute 41 | '/conversation/$conversationId': typeof ConversationConversationIdRoute 42 | } 43 | export interface FileRoutesById { 44 | __root__: typeof rootRouteImport 45 | '/': typeof IndexRoute 46 | '/camera-roll': typeof CameraRollRoute 47 | '/conversation/$conversationId': typeof ConversationConversationIdRoute 48 | } 49 | export interface FileRouteTypes { 50 | fileRoutesByFullPath: FileRoutesByFullPath 51 | fullPaths: '/' | '/camera-roll' | '/conversation/$conversationId' 52 | fileRoutesByTo: FileRoutesByTo 53 | to: '/' | '/camera-roll' | '/conversation/$conversationId' 54 | id: '__root__' | '/' | '/camera-roll' | '/conversation/$conversationId' 55 | fileRoutesById: FileRoutesById 56 | } 57 | export interface RootRouteChildren { 58 | IndexRoute: typeof IndexRoute 59 | CameraRollRoute: typeof CameraRollRoute 60 | ConversationConversationIdRoute: typeof ConversationConversationIdRoute 61 | } 62 | 63 | declare module '@tanstack/react-router' { 64 | interface FileRoutesByPath { 65 | '/camera-roll': { 66 | id: '/camera-roll' 67 | path: '/camera-roll' 68 | fullPath: '/camera-roll' 69 | preLoaderRoute: typeof CameraRollRouteImport 70 | parentRoute: typeof rootRouteImport 71 | } 72 | '/': { 73 | id: '/' 74 | path: '/' 75 | fullPath: '/' 76 | preLoaderRoute: typeof IndexRouteImport 77 | parentRoute: typeof rootRouteImport 78 | } 79 | '/conversation/$conversationId': { 80 | id: '/conversation/$conversationId' 81 | path: '/conversation/$conversationId' 82 | fullPath: '/conversation/$conversationId' 83 | preLoaderRoute: typeof ConversationConversationIdRouteImport 84 | parentRoute: typeof rootRouteImport 85 | } 86 | } 87 | } 88 | 89 | const rootRouteChildren: RootRouteChildren = { 90 | IndexRoute: IndexRoute, 91 | CameraRollRoute: CameraRollRoute, 92 | ConversationConversationIdRoute: ConversationConversationIdRoute, 93 | } 94 | export const routeTree = rootRouteImport 95 | ._addFileChildren(rootRouteChildren) 96 | ._addFileTypes() 97 | -------------------------------------------------------------------------------- /portal/src/routes/__root.tsx: -------------------------------------------------------------------------------- 1 | import { createRootRoute, Outlet } from "@tanstack/react-router"; 2 | import { TanStackRouterDevtools } from "@tanstack/react-router-devtools"; 3 | import { Navigation } from "../components/Navigation"; 4 | 5 | export const Route = createRootRoute({ 6 | component: () => ( 7 | <> 8 | 9 | 10 | 11 | 12 | ), 13 | }); 14 | -------------------------------------------------------------------------------- /portal/src/routes/camera-roll.tsx: -------------------------------------------------------------------------------- 1 | import { createFileRoute } from "@tanstack/react-router"; 2 | import { CameraRoll } from "../components/CameraRoll"; 3 | 4 | export const Route = createFileRoute("/camera-roll")({ 5 | component: CameraRollPage, 6 | }); 7 | 8 | function CameraRollPage() { 9 | return ; 10 | } -------------------------------------------------------------------------------- /portal/src/routes/conversation.$conversationId.tsx: -------------------------------------------------------------------------------- 1 | import { createFileRoute, useNavigate } from "@tanstack/react-router"; 2 | import { Conversation } from "../components/Conversation"; 3 | 4 | export const Route = createFileRoute("/conversation/$conversationId")({ 5 | component: ConversationPage, 6 | }); 7 | 8 | function ConversationPage() { 9 | const { conversationId } = Route.useParams(); 10 | const navigate = useNavigate(); 11 | 12 | const handleBack = () => { 13 | navigate({ to: "/" }); 14 | }; 15 | 16 | return ; 17 | } 18 | -------------------------------------------------------------------------------- /portal/src/routes/index.tsx: -------------------------------------------------------------------------------- 1 | import { createFileRoute } from "@tanstack/react-router"; 2 | import { Conversations } from "../components/Conversations"; 3 | 4 | export const Route = createFileRoute("/")({ 5 | component: ConversationsPage, 6 | }); 7 | 8 | function ConversationsPage() { 9 | return ; 10 | } 11 | -------------------------------------------------------------------------------- /portal/src/state/api.ts: -------------------------------------------------------------------------------- 1 | import type { Conversation, ConversationWithMessages, CameraRollImage } from "./types"; 2 | 3 | type HTTPMethod = "GET" | "POST" | "PUT" | "DELETE"; 4 | 5 | const hostname = () => import.meta.env.VITE_HOSTNAME ?? "localhost:8080"; 6 | 7 | const queryFn = async ( 8 | method: HTTPMethod = "GET", 9 | route: string, 10 | body?: string 11 | ) => { 12 | const url = `http://${hostname()}${route}`; 13 | const response = await fetch(url, { 14 | method, 15 | body, 16 | }); 17 | 18 | if (!response.ok) { 19 | throw new Error("Network response was not ok"); 20 | } 21 | return response.json(); 22 | }; 23 | 24 | export const getConversations = async (): Promise => 25 | queryFn("GET", "/api/conversation"); 26 | 27 | export const getConversationById = async ( 28 | id: string 29 | ): Promise => 30 | queryFn("GET", `/api/conversation/${id}`); 31 | 32 | export const getImageUrl = (fileName: string): string => 33 | `http://${hostname()}/api/image/${fileName}`; 34 | 35 | export const getCameraRollImages = async ( 36 | limit: number = 50, 37 | offset: number = 0 38 | ): Promise => 39 | queryFn("GET", `/api/camera-roll?limit=${limit}&offset=${offset}`); 40 | 41 | export const getCameraRollImageById = async ( 42 | imageId: number 43 | ): Promise => 44 | queryFn("GET", `/api/camera-roll/${imageId}`); 45 | 46 | export const getCameraRollImageUrl = (imageId: number): string => 47 | `http://${hostname()}/api/camera-roll/${imageId}/file`; 48 | -------------------------------------------------------------------------------- /portal/src/state/query.ts: -------------------------------------------------------------------------------- 1 | import { QueryClient } from "@tanstack/react-query"; 2 | 3 | export const queryClient = new QueryClient(); 4 | -------------------------------------------------------------------------------- /portal/src/state/types.ts: -------------------------------------------------------------------------------- 1 | export interface Conversation { 2 | id: string; 3 | title: string; 4 | createdAt: number; 5 | lastActivity: number; 6 | isActive: boolean; 7 | } 8 | 9 | export interface ConversationMessage { 10 | conversationId: string; 11 | type: "user" | "assistant" | "tool"; 12 | content: string; 13 | // TODO: Deserialize JSON 14 | toolCalls: unknown; 15 | toolCallsId: string; 16 | timestamp: number; 17 | images?: ConversationImage[]; 18 | } 19 | 20 | export interface ConversationImage { 21 | id: number; 22 | messageId: number; 23 | fileName: string; 24 | mimeType: string; 25 | fileSizeBytes: number; 26 | width?: number; 27 | height?: number; 28 | timestamp: number; 29 | } 30 | 31 | export type ConversationWithMessages = Conversation & { 32 | messages: ConversationMessage[]; 33 | }; 34 | 35 | export interface CameraRollImage { 36 | id: number; 37 | fileName: string; 38 | filePath: string; 39 | mimeType: string; 40 | dateAdded: number; 41 | dateTaken: number; 42 | width: number; 43 | height: number; 44 | size: number; 45 | } 46 | -------------------------------------------------------------------------------- /portal/src/styles/layout.module.scss: -------------------------------------------------------------------------------- 1 | .pageContainer { 2 | max-width: 900px; 3 | margin: 0 auto; 4 | } -------------------------------------------------------------------------------- /portal/src/util/date.ts: -------------------------------------------------------------------------------- 1 | export const formatHumanDate = (timestamp: number) => { 2 | const date = new Date(timestamp); 3 | const now = new Date(); 4 | 5 | const today = new Date(now.getFullYear(), now.getMonth(), now.getDate()); 6 | const messageDate = new Date( 7 | date.getFullYear(), 8 | date.getMonth(), 9 | date.getDate() 10 | ); 11 | 12 | const diffTime = today.getTime() - messageDate.getTime(); 13 | const diffDays = Math.floor(diffTime / (1000 * 60 * 60 * 24)); 14 | 15 | if (diffDays === 0) { 16 | return date.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" }); 17 | } else if (diffDays === 1) { 18 | return "Yesterday"; 19 | } else if (diffDays < 7) { 20 | return `${diffDays} days ago`; 21 | } else { 22 | return date.toLocaleDateString(); 23 | } 24 | }; 25 | -------------------------------------------------------------------------------- /portal/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /portal/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", 4 | "target": "ES2022", 5 | "useDefineForClassFields": true, 6 | "lib": ["ES2022", "DOM", "DOM.Iterable"], 7 | "module": "ESNext", 8 | "skipLibCheck": true, 9 | 10 | /* Bundler mode */ 11 | "moduleResolution": "bundler", 12 | "allowImportingTsExtensions": true, 13 | "verbatimModuleSyntax": true, 14 | "moduleDetection": "force", 15 | "noEmit": true, 16 | "jsx": "react-jsx", 17 | 18 | /* Linting */ 19 | "strict": true, 20 | "noUnusedLocals": true, 21 | "noUnusedParameters": true, 22 | "erasableSyntaxOnly": true, 23 | "noFallthroughCasesInSwitch": true, 24 | "noUncheckedSideEffectImports": true 25 | }, 26 | "include": ["src"] 27 | } 28 | -------------------------------------------------------------------------------- /portal/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": [], 3 | "references": [ 4 | { "path": "./tsconfig.app.json" }, 5 | { "path": "./tsconfig.node.json" } 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /portal/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", 4 | "target": "ES2023", 5 | "lib": ["ES2023"], 6 | "module": "ESNext", 7 | "skipLibCheck": true, 8 | 9 | /* Bundler mode */ 10 | "moduleResolution": "bundler", 11 | "allowImportingTsExtensions": true, 12 | "verbatimModuleSyntax": true, 13 | "moduleDetection": "force", 14 | "noEmit": true, 15 | 16 | /* Linting */ 17 | "strict": true, 18 | "noUnusedLocals": true, 19 | "noUnusedParameters": true, 20 | "erasableSyntaxOnly": true, 21 | "noFallthroughCasesInSwitch": true, 22 | "noUncheckedSideEffectImports": true 23 | }, 24 | "include": ["vite.config.ts"] 25 | } 26 | -------------------------------------------------------------------------------- /portal/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "vite"; 2 | import react from "@vitejs/plugin-react"; 3 | import { tanstackRouter } from "@tanstack/router-plugin/vite"; 4 | 5 | // https://vite.dev/config/ 6 | export default defineConfig({ 7 | plugins: [ 8 | tanstackRouter({ 9 | target: "react", 10 | autoCodeSplitting: true, 11 | }), 12 | react(), 13 | ], 14 | }); 15 | -------------------------------------------------------------------------------- /sdk/.gitignore: -------------------------------------------------------------------------------- 1 | /build -------------------------------------------------------------------------------- /sdk/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | alias(libs.plugins.android.library) 3 | alias(libs.plugins.kotlin.android) 4 | } 5 | 6 | android { 7 | namespace = "com.penumbraos.mabl.sdk" 8 | compileSdk = 35 9 | 10 | defaultConfig { 11 | minSdk = 32 12 | } 13 | 14 | buildTypes { 15 | release { 16 | isMinifyEnabled = false 17 | signingConfig = signingConfigs.getByName("debug") 18 | } 19 | } 20 | compileOptions { 21 | sourceCompatibility = JavaVersion.VERSION_11 22 | targetCompatibility = JavaVersion.VERSION_11 23 | } 24 | kotlinOptions { 25 | jvmTarget = "11" 26 | } 27 | buildFeatures { 28 | aidl = true 29 | } 30 | } 31 | 32 | dependencies { 33 | implementation(libs.androidx.core.ktx) 34 | implementation(libs.androidx.appcompat) 35 | implementation(libs.material) 36 | } -------------------------------------------------------------------------------- /sdk/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /sdk/src/main/aidl/com/penumbraos/mabl/sdk/BinderConversationMessage.aidl: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk; 2 | 3 | import com.penumbraos.mabl.sdk.ToolCall; 4 | 5 | parcelable BinderConversationMessage { 6 | // "user", "assistant", "tool" 7 | String type; 8 | String content; 9 | // Optional 10 | ParcelFileDescriptor imageFile; 11 | ToolCall[] toolCalls; 12 | String toolCallId; 13 | } -------------------------------------------------------------------------------- /sdk/src/main/aidl/com/penumbraos/mabl/sdk/ILlmCallback.aidl: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk; 2 | 3 | import com.penumbraos.mabl.sdk.LlmResponse; 4 | 5 | interface ILlmCallback { 6 | void onPartialResponse(String newToken); 7 | void onCompleteResponse(in LlmResponse response); 8 | void onError(String error); 9 | } -------------------------------------------------------------------------------- /sdk/src/main/aidl/com/penumbraos/mabl/sdk/ILlmService.aidl: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk; 2 | 3 | import com.penumbraos.mabl.sdk.ILlmCallback; 4 | import com.penumbraos.mabl.sdk.ToolDefinition; 5 | import com.penumbraos.mabl.sdk.BinderConversationMessage; 6 | 7 | interface ILlmService { 8 | void generateResponse(in BinderConversationMessage[] messages, in ToolDefinition[] tools, ILlmCallback callback); 9 | void setAvailableTools(in ToolDefinition[] tools); 10 | } -------------------------------------------------------------------------------- /sdk/src/main/aidl/com/penumbraos/mabl/sdk/ISttCallback.aidl: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk; 2 | 3 | interface ISttCallback { 4 | void onPartialTranscription(String partialText); 5 | void onFinalTranscription(String finalText); 6 | void onError(String errorMessage); 7 | } -------------------------------------------------------------------------------- /sdk/src/main/aidl/com/penumbraos/mabl/sdk/ISttService.aidl: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk; 2 | 3 | import com.penumbraos.mabl.sdk.ISttCallback; 4 | 5 | interface ISttService { 6 | void startListening(in ISttCallback callback); 7 | void stopListening(); 8 | } -------------------------------------------------------------------------------- /sdk/src/main/aidl/com/penumbraos/mabl/sdk/ISystemServiceRegistry.aidl: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk; 2 | 3 | import com.penumbraos.mabl.sdk.ITtsService; 4 | 5 | interface ISystemServiceRegistry { 6 | ITtsService getTtsService(); 7 | } -------------------------------------------------------------------------------- /sdk/src/main/aidl/com/penumbraos/mabl/sdk/IToolCallback.aidl: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk; 2 | 3 | interface IToolCallback { 4 | void onSuccess(String result); 5 | void onError(String error); 6 | } -------------------------------------------------------------------------------- /sdk/src/main/aidl/com/penumbraos/mabl/sdk/IToolService.aidl: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk; 2 | 3 | import com.penumbraos.mabl.sdk.ToolCall; 4 | import com.penumbraos.mabl.sdk.IToolCallback; 5 | import com.penumbraos.mabl.sdk.ToolDefinition; 6 | import com.penumbraos.mabl.sdk.ISystemServiceRegistry; 7 | 8 | interface IToolService { 9 | void executeTool(in ToolCall call, IToolCallback callback); 10 | ToolDefinition[] getToolDefinitions(); 11 | void setSystemServices(ISystemServiceRegistry systemServices); 12 | } -------------------------------------------------------------------------------- /sdk/src/main/aidl/com/penumbraos/mabl/sdk/ITtsCallback.aidl: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk; 2 | 3 | interface ITtsCallback { 4 | void onSpeechStarted(); 5 | void onSpeechFinished(); 6 | void onError(String errorMessage); 7 | } -------------------------------------------------------------------------------- /sdk/src/main/aidl/com/penumbraos/mabl/sdk/ITtsService.aidl: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk; 2 | 3 | import com.penumbraos.mabl.sdk.ITtsCallback; 4 | 5 | interface ITtsService { 6 | void registerCallback(in ITtsCallback callback); 7 | /** 8 | * Stop any utterances currently being spoken and immediately begin speaking the given text 9 | */ 10 | void speakImmediately(String text); 11 | /** 12 | * Appends to the current utterance and continues speaking it. Can receive single or multiple words 13 | */ 14 | void speakIncremental(String text); 15 | /** 16 | * Stop any utterances currently being spoken 17 | */ 18 | void stopSpeaking(); 19 | } -------------------------------------------------------------------------------- /sdk/src/main/aidl/com/penumbraos/mabl/sdk/LlmResponse.aidl: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk; 2 | 3 | import com.penumbraos.mabl.sdk.ToolCall; 4 | 5 | parcelable LlmResponse { 6 | String text; 7 | ToolCall[] toolCalls; 8 | } -------------------------------------------------------------------------------- /sdk/src/main/aidl/com/penumbraos/mabl/sdk/ToolCall.aidl: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk; 2 | 3 | parcelable ToolCall { 4 | String id; 5 | String name; 6 | String parameters; 7 | /** 8 | * If true, result will be given to LLM. If false, result will be given to TTS directly 9 | */ 10 | boolean isLLM; 11 | } -------------------------------------------------------------------------------- /sdk/src/main/aidl/com/penumbraos/mabl/sdk/ToolDefinition.aidl: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk; 2 | 3 | import com.penumbraos.mabl.sdk.ToolParameter; 4 | 5 | parcelable ToolDefinition { 6 | String name; 7 | String description; 8 | ToolParameter[] parameters; 9 | /** 10 | * Example user utterances for offline intent classification. Presence of examples implies 11 | * the tool can be matched without cloud LLM support. 12 | */ 13 | String[] examples; 14 | /** 15 | * Whether the tool is a priority tool. Priority tools will always attempt to be included in the tool list passed to the LLM 16 | */ 17 | boolean isPriority; 18 | } -------------------------------------------------------------------------------- /sdk/src/main/aidl/com/penumbraos/mabl/sdk/ToolParameter.aidl: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk; 2 | 3 | parcelable ToolParameter { 4 | String name; 5 | String type; 6 | String description; 7 | boolean required; 8 | String[] enumValues; 9 | } -------------------------------------------------------------------------------- /sdk/src/main/java/com/penumbraos/mabl/sdk/DeviceUtils.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk 2 | 3 | import android.os.Build 4 | 5 | /** 6 | * Utility class for detecting device type and capabilities in MABL ecosystem 7 | */ 8 | object DeviceUtils { 9 | 10 | /** 11 | * Detects if the current device is a real AI Pin device (not simulator) 12 | * @return true if running on actual AI Pin hardware, false otherwise 13 | */ 14 | fun isAiPin(): Boolean { 15 | return try { 16 | Build.MANUFACTURER.equals("Humane", ignoreCase = true) || 17 | Build.PRODUCT.contains("humane", ignoreCase = true) 18 | } catch (e: Exception) { 19 | false 20 | } 21 | } 22 | 23 | /** 24 | * Detects if the current device is an AI Pin simulator 25 | * This is a best-effort detection for when simulator-specific behavior is needed 26 | * @return true if likely running in simulator mode, false otherwise 27 | */ 28 | fun isSimulator(): Boolean { 29 | // Taken from Flutter 30 | return (Build.BRAND.startsWith("generic") && Build.DEVICE.startsWith("generic")) 31 | || Build.FINGERPRINT.startsWith("generic") 32 | || Build.FINGERPRINT.startsWith("unknown") 33 | || Build.HARDWARE.contains("goldfish") 34 | || Build.HARDWARE.contains("ranchu") 35 | || Build.MODEL.contains("google_sdk") 36 | || Build.MODEL.contains("Emulator") 37 | || Build.MODEL.contains("Android SDK built for x86") 38 | || Build.MANUFACTURER.contains("Genymotion") 39 | || Build.PRODUCT.contains("sdk_google") 40 | || Build.PRODUCT.contains("google_sdk") 41 | || Build.PRODUCT.contains("sdk") 42 | || Build.PRODUCT.contains("sdk_x86") 43 | || Build.PRODUCT.contains("sdk_gphone64_arm64") 44 | || Build.PRODUCT.contains("vbox86p") 45 | || Build.PRODUCT.contains("emulator") 46 | || Build.PRODUCT.contains("simulator") 47 | } 48 | 49 | /** 50 | * Gets a string description of the detected device type 51 | * @return "Ai Pin", "Simulator", or "Unknown" 52 | */ 53 | fun getDeviceTypeDescription(): String { 54 | return when { 55 | isAiPin() -> "Ai Pin" 56 | isSimulator() -> "Simulator" 57 | else -> "Unknown" 58 | } 59 | } 60 | } -------------------------------------------------------------------------------- /sdk/src/main/java/com/penumbraos/mabl/sdk/MablService.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk 2 | 3 | import android.app.Notification 4 | import android.app.NotificationChannel 5 | import android.app.NotificationManager 6 | import android.app.Service 7 | import androidx.core.app.NotificationCompat 8 | 9 | abstract class MablService(private val name: String) : Service() { 10 | override fun onCreate() { 11 | super.onCreate() 12 | 13 | // TODO: Remove this requirement entirely 14 | createNotificationChannel() 15 | startForeground(1001, createNotification()) 16 | } 17 | 18 | private fun createNotificationChannel() { 19 | val channel = NotificationChannel( 20 | name, 21 | name, 22 | NotificationManager.IMPORTANCE_LOW 23 | ) 24 | val notificationManager = getSystemService(NotificationManager::class.java) 25 | notificationManager.createNotificationChannel(channel) 26 | } 27 | 28 | private fun createNotification(): Notification { 29 | return NotificationCompat.Builder(this, name) 30 | .setContentTitle(name) 31 | .setSmallIcon(android.R.drawable.ic_dialog_info) 32 | .build() 33 | } 34 | } -------------------------------------------------------------------------------- /sdk/src/main/java/com/penumbraos/mabl/sdk/PluginConstants.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk 2 | 3 | enum class PluginType(val action: String) { 4 | STT(PluginConstants.ACTION_STT_SERVICE), 5 | TTS(PluginConstants.ACTION_TTS_SERVICE), 6 | LLM(PluginConstants.ACTION_LLM_SERVICE), 7 | TOOL(PluginConstants.ACTION_TOOL_SERVICE); 8 | 9 | companion object { 10 | fun fromAction(action: String): PluginType? { 11 | return entries.firstOrNull { it.action == action } 12 | } 13 | } 14 | } 15 | 16 | object PluginConstants { 17 | // Intent Actions for Service Discovery 18 | const val ACTION_STT_SERVICE = "com.penumbraos.mabl.sdk.action.STT_SERVICE" 19 | const val ACTION_TTS_SERVICE = "com.penumbraos.mabl.sdk.action.TTS_SERVICE" 20 | const val ACTION_LLM_SERVICE = "com.penumbraos.mabl.sdk.action.LLM_SERVICE" 21 | const val ACTION_TOOL_SERVICE = "com.penumbraos.mabl.sdk.action.TOOL_SERVICE" 22 | 23 | // Metadata Keys for Capability Declaration 24 | const val METADATA_DISPLAY_NAME = "com.penumbraos.mabl.sdk.metadata.DISPLAY_NAME" 25 | const val METADATA_DESCRIPTION = "com.penumbraos.mabl.sdk.metadata.DESCRIPTION" 26 | } -------------------------------------------------------------------------------- /sdk/src/main/java/com/penumbraos/mabl/sdk/ToolService.kt: -------------------------------------------------------------------------------- 1 | package com.penumbraos.mabl.sdk 2 | 3 | import android.content.Intent 4 | import android.os.IBinder 5 | import org.json.JSONObject 6 | 7 | abstract class ToolService(name: String) : MablService(name) { 8 | private var systemServices: ISystemServiceRegistry? = null 9 | 10 | private val binder = object : IToolService.Stub() { 11 | override fun executeTool( 12 | call: ToolCall, 13 | callback: IToolCallback 14 | ) { 15 | val params = if (call.parameters != "") { 16 | JSONObject(call.parameters) 17 | } else { 18 | null 19 | } 20 | 21 | executeTool(call, params, callback) 22 | } 23 | 24 | override fun getToolDefinitions(): Array? { 25 | return this@ToolService.getToolDefinitions() 26 | } 27 | 28 | override fun setSystemServices(systemServices: ISystemServiceRegistry?) { 29 | this@ToolService.systemServices = systemServices 30 | } 31 | } 32 | 33 | override fun onBind(intent: Intent?): IBinder? = binder 34 | 35 | abstract fun executeTool(call: ToolCall, params: JSONObject?, callback: IToolCallback) 36 | abstract fun getToolDefinitions(): Array 37 | } -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | pluginManagement { 2 | repositories { 3 | google { 4 | content { 5 | includeGroupByRegex("com\\.android.*") 6 | includeGroupByRegex("com\\.google.*") 7 | includeGroupByRegex("androidx.*") 8 | } 9 | } 10 | mavenCentral() 11 | gradlePluginPortal() 12 | } 13 | } 14 | dependencyResolutionManagement { 15 | repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) 16 | repositories { 17 | maven { 18 | url = uri("https://jitpack.io") 19 | } 20 | google() 21 | mavenCentral() 22 | mavenLocal() 23 | } 24 | } 25 | 26 | rootProject.name = "MABL" 27 | include(":mabl") 28 | include(":sdk") 29 | include(":plugins:demo") 30 | include(":plugins:openai") 31 | include(":plugins:aipinsystem") 32 | include(":plugins:system") 33 | include(":plugins:searxng") 34 | include(":plugins:googlesearch") 35 | --------------------------------------------------------------------------------