├── android ├── gradle.properties ├── settings.gradle.kts ├── src │ └── main │ │ ├── res │ │ └── values │ │ │ └── strings.xml │ │ ├── AndroidManifest.xml │ │ └── kotlin │ │ └── com │ │ └── jamesward │ │ └── airdraw │ │ ├── BaseApplication.kt │ │ └── MainActivity.kt └── build.gradle.kts ├── web ├── settings.gradle.kts ├── build.gradle.kts └── src │ └── main │ └── kotlin │ └── com │ └── jamesward │ └── airdaw │ └── Main.kt ├── common ├── settings.gradle.kts ├── build.gradle.kts └── src │ └── commonMain │ └── kotlin │ └── com │ └── jamesward │ └── airdraw │ └── data │ └── Data.kt ├── server ├── settings.gradle.kts ├── src │ └── main │ │ ├── resources │ │ ├── application.properties │ │ ├── logback.xml │ │ └── assets │ │ │ └── index.css │ │ └── kotlin │ │ └── com │ │ └── jamesward │ │ └── airdraw │ │ └── WebApp.kt └── build.gradle.kts ├── .gitignore ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── cloudbuild.yaml ├── settings.gradle.kts ├── CONTRIBUTING.md ├── README.md ├── gradlew.bat ├── gradlew └── LICENSE /android/gradle.properties: -------------------------------------------------------------------------------- 1 | android.useAndroidX=true 2 | -------------------------------------------------------------------------------- /web/settings.gradle.kts: -------------------------------------------------------------------------------- 1 | rootProject.name = "air-draw-web" 2 | -------------------------------------------------------------------------------- /common/settings.gradle.kts: -------------------------------------------------------------------------------- 1 | rootProject.name = "air-draw-common" 2 | -------------------------------------------------------------------------------- /server/settings.gradle.kts: -------------------------------------------------------------------------------- 1 | rootProject.name = "air-draw-server" 2 | -------------------------------------------------------------------------------- /android/settings.gradle.kts: -------------------------------------------------------------------------------- 1 | rootProject.name = "air-draw-android" 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .gradle/ 2 | .idea/ 3 | build/ 4 | *.iml 5 | /local.properties 6 | /gradle.properties 7 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/air-draw-demo/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /common/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | kotlin("multiplatform") 3 | } 4 | 5 | kotlin { 6 | jvm() 7 | 8 | js { 9 | browser() 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.7-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /android/src/main/res/values/strings.xml: -------------------------------------------------------------------------------- 1 | 2 | Air Draw 3 | Identify It! 4 | Draw! 5 | Air Draw 6 | 7 | -------------------------------------------------------------------------------- /common/src/commonMain/kotlin/com/jamesward/airdraw/data/Data.kt: -------------------------------------------------------------------------------- 1 | package com.jamesward.airdraw.data 2 | 3 | data class Orientation(val azimuth: Float, val pitch: Float, val timestamp: Long) 4 | 5 | data class LabelAnnotation(val description: String, val score: Float) 6 | 7 | data class ImageResult(val image: ByteArray, val labelAnnotations: Array) 8 | -------------------------------------------------------------------------------- /server/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | micronaut.router.static-resources.assets.enabled=true 2 | micronaut.router.static-resources.assets.paths=classpath:assets 3 | micronaut.router.static-resources.assets.mapping=/assets/** 4 | 5 | micronaut.router.static-resources.resources.enabled=true 6 | micronaut.router.static-resources.resources.paths=classpath:META-INF/resources 7 | micronaut.router.static-resources.resources.mapping=/resources/** 8 | -------------------------------------------------------------------------------- /cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: 'gcr.io/cloud-builders/javac:8' 3 | entrypoint: './gradlew' 4 | args: ['--console=plain', '--no-daemon', ':server:jib', '-Djib.to.image=gcr.io/$PROJECT_ID/$REPO_NAME:$COMMIT_SHA'] 5 | 6 | - name: 'gcr.io/cloud-builders/gcloud' 7 | args: ['beta', 'run', 'deploy', '--image=gcr.io/$PROJECT_ID/$REPO_NAME:$COMMIT_SHA', '--platform=managed', '--project=$PROJECT_ID', '--region=us-central1', '--allow-unauthenticated', '--memory=512Mi', '$REPO_NAME'] 8 | -------------------------------------------------------------------------------- /server/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{YYYY-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /web/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | kotlin("js") 3 | } 4 | 5 | kotlin { 6 | js { 7 | browser() 8 | /* 9 | { 10 | dceTask { 11 | dceOptions.devMode = true 12 | } 13 | 14 | } 15 | */ 16 | binaries.executable() 17 | } 18 | sourceSets["main"].dependencies { 19 | implementation(kotlin("stdlib-js")) 20 | implementation(project(":common")) 21 | implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core-js:1.3.9") 22 | implementation("org.jetbrains.kotlinx:kotlinx-html-js:0.7.2") 23 | } 24 | } 25 | 26 | /* 27 | tasks.named("jsJar") { 28 | from("build/distributions") 29 | into("META-INF/resources") 30 | dependsOn("browserDevelopmentWebpack") 31 | } 32 | */ 33 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | rootProject.name = "air-draw" 2 | 3 | // when running the root jib task, ignore the android subproject 4 | if (startParameter.taskRequests.find { it.args.contains(":server:jib") } == null) { 5 | include("common", "web", "server", "android") 6 | } else { 7 | include("common", "web", "server") 8 | } 9 | 10 | pluginManagement { 11 | repositories { 12 | gradlePluginPortal() 13 | jcenter() 14 | google() 15 | } 16 | resolutionStrategy { 17 | eachPlugin { 18 | if (requested.id.namespace == "com.android" || requested.id.name == "kotlin-android-extensions") { 19 | useModule("com.android.tools.build:gradle:${requested.version}") 20 | } 21 | if (requested.id.id == "kotlinx-serialization") { 22 | useModule("org.jetbrains.kotlin:kotlin-serialization:${requested.version}") 23 | } 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /android/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 13 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /server/src/main/resources/assets/index.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Google LLC. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | html { 17 | height: 100%; 18 | width: 100%; 19 | } 20 | 21 | body { 22 | height: 100%; 23 | width: 100%; 24 | 25 | background-repeat: no-repeat; 26 | background-position-x: center; 27 | background-position-y: center; 28 | background-size: contain; 29 | 30 | font-family: monospace; 31 | font-size: 36px; 32 | color: hotpink; 33 | 34 | margin: 0; 35 | 36 | display: flex; 37 | align-items: center; 38 | justify-content: center; 39 | } 40 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows [Google's Open Source Community 28 | Guidelines](https://opensource.google.com/conduct/). 29 | -------------------------------------------------------------------------------- /web/src/main/kotlin/com/jamesward/airdaw/Main.kt: -------------------------------------------------------------------------------- 1 | import kotlinx.browser.document 2 | import kotlinx.browser.window 3 | import kotlinx.coroutines.* 4 | import kotlinx.html.* 5 | import kotlinx.html.dom.* 6 | import kotlinx.dom.clear 7 | import kotlin.math.round 8 | import com.jamesward.airdraw.data.ImageResult 9 | 10 | 11 | fun main() { 12 | GlobalScope.launch { 13 | poll() 14 | } 15 | } 16 | 17 | suspend fun poll() { 18 | val res = window.fetch("/events").await() 19 | when (res.status.toInt()) { 20 | 200 -> { 21 | val imageResult = res.json().await().unsafeCast() 22 | 23 | val urlImage = "url('data:image/png;base64,${imageResult.image}')" 24 | document.body?.style?.backgroundImage = urlImage 25 | 26 | if (imageResult.labelAnnotations.isNotEmpty()) { 27 | document.body?.clear() 28 | 29 | val div = document.create.div() 30 | 31 | imageResult.labelAnnotations.forEach { labelAnnotation -> 32 | div.append { 33 | p { 34 | +"${labelAnnotation.description} = ${round((labelAnnotation.score.toDouble()) * 100)}%" 35 | } 36 | } 37 | } 38 | 39 | document.body?.append(div) 40 | } 41 | else { 42 | document.body?.clear() 43 | } 44 | } 45 | } 46 | 47 | window.setTimeout({ GlobalScope.launch { poll() } }, 1000) 48 | } 49 | -------------------------------------------------------------------------------- /android/src/main/kotlin/com/jamesward/airdraw/BaseApplication.kt: -------------------------------------------------------------------------------- 1 | package com.jamesward.airdraw 2 | 3 | import android.app.Activity 4 | import android.app.Application 5 | import android.content.pm.PackageManager 6 | import android.os.Bundle 7 | import io.micronaut.context.ApplicationContext 8 | import io.micronaut.context.env.Environment 9 | import io.micronaut.context.env.PropertySource 10 | 11 | class BaseApplication : Application() { 12 | 13 | private var ctx: ApplicationContext? = null 14 | 15 | override fun onCreate() { 16 | super.onCreate() 17 | 18 | val ai = applicationContext.packageManager.getApplicationInfo(packageName, PackageManager.GET_META_DATA) 19 | val propertySource = AndroidMetadataPropertySource(ai.metaData) 20 | 21 | ctx = ApplicationContext.build(MainActivity::class.java, Environment.ANDROID).propertySources(propertySource).start() 22 | 23 | registerActivityLifecycleCallbacks(object: ActivityLifecycleCallbacks { 24 | override fun onActivityCreated(activity: Activity, savedInstanceState: Bundle?) { 25 | ctx?.inject(activity) 26 | } 27 | 28 | override fun onActivityStarted(activity: Activity) {} 29 | override fun onActivityResumed(activity: Activity) {} 30 | override fun onActivityPaused(activity: Activity) {} 31 | override fun onActivityStopped(activity: Activity) {} 32 | override fun onActivitySaveInstanceState(activity: Activity, outState: Bundle) {} 33 | override fun onActivityDestroyed(activity: Activity) {} 34 | }) 35 | } 36 | 37 | override fun onTerminate() { 38 | super.onTerminate() 39 | ctx?.let { ctx -> 40 | if (ctx.isRunning) ctx.stop() 41 | } 42 | } 43 | } 44 | 45 | class AndroidMetadataPropertySource(private val bundle: Bundle): PropertySource { 46 | override fun getName(): String { 47 | return javaClass.simpleName 48 | } 49 | 50 | override fun iterator(): MutableIterator { 51 | return bundle.keySet().iterator() 52 | } 53 | 54 | override fun get(key: String?): Any? { 55 | return bundle.get(key) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /server/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | application 3 | kotlin("jvm") 4 | kotlin("kapt") 5 | kotlin("plugin.allopen") 6 | id("com.google.cloud.tools.jib") version "2.6.0" 7 | } 8 | 9 | dependencies { 10 | implementation(project(":common")) 11 | //implementation(project(":web")) 12 | //runtimeOnly(files("../web/build/libs/web.jar")) 13 | 14 | implementation("com.github.haifengl:smile-plot:1.5.2") 15 | implementation("com.github.haifengl:smile-interpolation:1.5.3") 16 | implementation("com.github.haifengl:smile-netlib:1.5.3") 17 | 18 | implementation("io.micronaut.kotlin:micronaut-kotlin-runtime:2.1.1") 19 | implementation("io.micronaut:micronaut-runtime:2.1.2") 20 | implementation("io.micronaut:micronaut-http-server-netty:2.1.2") 21 | 22 | runtimeOnly("com.fasterxml.jackson.module:jackson-module-kotlin:2.11.3") 23 | runtimeOnly("ch.qos.logback:logback-classic:1.2.3") 24 | 25 | implementation("com.google.cloud:google-cloud-vision:1.100.6") 26 | implementation("com.google.cloud:google-cloud-pubsub:1.108.7") 27 | implementation("com.google.cloud:google-cloud-core:1.93.10") 28 | implementation("io.netty:netty-tcnative-boringssl-static:2.0.20.Final") 29 | 30 | implementation("org.jetbrains.kotlinx:kotlinx-html-jvm:0.7.2") 31 | 32 | kapt("io.micronaut:micronaut-inject-java:2.1.2") 33 | } 34 | 35 | java { 36 | sourceCompatibility = JavaVersion.VERSION_1_8 37 | } 38 | 39 | tasks.compileKotlin { 40 | kotlinOptions { 41 | jvmTarget = JavaVersion.VERSION_1_8.toString() 42 | javaParameters = true 43 | } 44 | } 45 | 46 | application { 47 | mainClass.set("com.jamesward.airdraw.WebAppKt") 48 | } 49 | 50 | allOpen { 51 | annotation("io.micronaut.aop.Around") 52 | } 53 | 54 | kapt { 55 | arguments { 56 | arg("micronaut.processing.incremental", true) 57 | arg("micronaut.processing.annotations", "com.jamesward.airdraw.*") 58 | } 59 | } 60 | 61 | tasks.withType { 62 | jvmArgs = listOf("-XX:TieredStopAtLevel=1", "-Dcom.sun.management.jmxremote") 63 | 64 | if (gradle.startParameter.isContinuous) { 65 | systemProperties = mapOf( 66 | "micronaut.io.watch.restart" to "true", 67 | "micronaut.io.watch.enabled" to "true", 68 | "micronaut.io.watch.paths" to "src/main" 69 | ) 70 | } 71 | } 72 | 73 | jib { 74 | container.mainClass = "com.jamesward.airdraw.WebAppKt" 75 | } 76 | 77 | /* 78 | tasks { 79 | classes { 80 | dependsOn(":web:jsJar") 81 | } 82 | } 83 | */ 84 | -------------------------------------------------------------------------------- /android/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | id("com.android.application") 3 | kotlin("android") 4 | kotlin("android.extensions") 5 | kotlin("kapt") 6 | } 7 | 8 | val composeVersion = "1.0.0-alpha03" 9 | 10 | dependencies { 11 | implementation(kotlin("reflect")) 12 | implementation(project(":common")) 13 | implementation("androidx.appcompat:appcompat:1.2.0") 14 | implementation("androidx.compose.foundation:foundation-layout:$composeVersion") 15 | implementation("androidx.compose.material:material:$composeVersion") 16 | implementation("androidx.compose.runtime:runtime:$composeVersion") 17 | implementation("androidx.compose.ui:ui:$composeVersion") 18 | 19 | implementation("io.micronaut.kotlin:micronaut-kotlin-runtime:2.1.1") 20 | implementation("io.micronaut:micronaut-http-client:2.1.2") 21 | implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.11.3") 22 | implementation("uk.uuid.slf4j:slf4j-android:1.7.28-0") 23 | 24 | kapt("io.micronaut:micronaut-inject-java:2.1.2") 25 | } 26 | 27 | android { 28 | compileSdkVersion(30) 29 | buildToolsVersion = "30.0.2" 30 | 31 | defaultConfig { 32 | applicationId = "com.jamesward.airdraw" 33 | minSdkVersion(23) 34 | targetSdkVersion(30) 35 | versionCode = 1 36 | versionName = "1.0" 37 | 38 | val drawUrl: String? by project 39 | if (drawUrl != null) { 40 | manifestPlaceholders = mapOf("drawurl" to drawUrl) 41 | } 42 | else { 43 | // 10.0.2.2 is the IP for your machine from the Android emulator 44 | manifestPlaceholders = mapOf("drawurl" to "http://10.0.2.2:8080") 45 | } 46 | } 47 | 48 | sourceSets["main"].java.srcDir("src/main/kotlin") 49 | 50 | compileOptions { 51 | sourceCompatibility = JavaVersion.VERSION_1_8 52 | targetCompatibility = JavaVersion.VERSION_1_8 53 | } 54 | 55 | kotlinOptions { 56 | jvmTarget = JavaVersion.VERSION_1_8.toString() 57 | useIR = true 58 | } 59 | 60 | packagingOptions { 61 | exclude("META-INF/main.kotlin_module") 62 | exclude("META-INF/INDEX.LIST") 63 | exclude("META-INF/config-properties.adoc") 64 | exclude("META-INF/io.netty.versions.properties") 65 | exclude("META-INF/spring-configuration-metadata.json") 66 | } 67 | 68 | lintOptions { 69 | isAbortOnError = false 70 | } 71 | 72 | buildFeatures { 73 | compose = true 74 | } 75 | 76 | composeOptions { 77 | kotlinCompilerVersion = org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION.toString() 78 | kotlinCompilerExtensionVersion = composeVersion 79 | } 80 | } 81 | 82 | tasks.withType { 83 | kotlinOptions { 84 | jvmTarget = JavaVersion.VERSION_1_8.toString() 85 | freeCompilerArgs = listOf("-Xallow-jvm-ir-dependencies", "-Xskip-prerelease-check") 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Air Draw 2 | 3 | 4 | 5 | ## Local Dev 6 | 7 | Run the web server: 8 | ``` 9 | ./gradlew -t :server:run 10 | ``` 11 | 12 | Run the web asset server: 13 | ``` 14 | ./gradlew -t :web:run 15 | ``` 16 | 17 | Visit: [http://localhost:8080](http://localhost:8080) 18 | 19 | Run the client: 20 | 21 | 1. [Download Android Command Line Tools:](https://developer.android.com/studio) 22 | 23 | 1. Install the SDK: 24 | ``` 25 | mkdir android-sdk 26 | cd android-sdk 27 | unzip PATH_TO_SDK_ZIP/sdk-tools-linux-VERSION.zip 28 | tools/bin/sdkmanager --update 29 | tools/bin/sdkmanager "platforms;android-30" "build-tools;30.0.2" "extras;google;m2repository" "extras;android;m2repository" 30 | tools/bin/sdkmanager --licenses 31 | ``` 32 | 33 | 1. Add the following to your ~/.bashrc 34 | ``` 35 | export ANDROID_SDK_ROOT=PATH_TO_SDK/android-sdk 36 | ``` 37 | 38 | 1. Source the new profile: 39 | ``` 40 | source ~/.bashrc 41 | ``` 42 | 43 | 1. Run the build from this project's dir: 44 | ``` 45 | ./gradlew :android:build 46 | ``` 47 | 48 | 1. For a physical device, [setup adb](https://developer.android.com/studio/run/device) 49 | 50 | 1. Run on a device using an external server: 51 | ``` 52 | ./gradlew android:installDebug -PdrawUrl=https://YOUR_URL/draw 53 | ``` 54 | 55 | 1. Or to run from Android Studio / IntelliJ, create a `gradle.properties` file in your root project directory containing: 56 | ``` 57 | drawUrl=http://YOUR_DRAW_SERVER:8080/draw 58 | ``` 59 | 60 | And setup the activity to first run *Gradle-aware Make* with a task of `:android:assembleDebug` 61 | 62 | Use GCP for Pub/Sub & Vision API: 63 | 64 | 1. Go to: https://console.cloud.google.com/apis/library/vision.googleapis.com 65 | 66 | 1. Enable the Vision API 67 | 68 | 1. Click "Create Credentials" 69 | 70 | 1. Select "Cloud Vision API" at the API you are using 71 | 72 | 1. Select "No, I’m not using them" 73 | 74 | 1. Click "What Credentials do I need" 75 | 76 | 1. Give the Service Account a name 77 | 78 | 1. Select a role, like Project Editor 79 | 80 | 1. Leave the JSON option selected and press "Continue" 81 | 82 | 1. A JSON file will be downloaded to your machine 83 | 84 | 1. Go to: https://console.cloud.google.com/cloudpubsub/topicList 85 | 86 | 1. Create a topic named `air-draw` 87 | 88 | 1. Create a subscription named `air-draw` 89 | 90 | 1. Select "Never Expire" 91 | 92 | 1. Press "Create" 93 | 94 | 1. Run the app locally connecting to Pub/Sub and the Vision API: 95 | ``` 96 | GOOGLE_APPLICATION_CREDENTIALS=YOUR_CREDS.json ./gradlew -t run 97 | ``` 98 | 99 | Build and Deploy manually: 100 | 101 | ``` 102 | gcloud config set run/region us-central1 103 | gcloud services enable run.googleapis.com 104 | gcloud builds submit --tag=gcr.io/$(gcloud config get-value project)/air-draw 105 | gcloud beta run deploy air-draw --image gcr.io/$(gcloud config get-value project)/air-draw --allow-unauthenticated --memory=512Mi 106 | ``` 107 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem http://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 33 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 34 | 35 | @rem Find java.exe 36 | if defined JAVA_HOME goto findJavaFromJavaHome 37 | 38 | set JAVA_EXE=java.exe 39 | %JAVA_EXE% -version >NUL 2>&1 40 | if "%ERRORLEVEL%" == "0" goto init 41 | 42 | echo. 43 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 44 | echo. 45 | echo Please set the JAVA_HOME variable in your environment to match the 46 | echo location of your Java installation. 47 | 48 | goto fail 49 | 50 | :findJavaFromJavaHome 51 | set JAVA_HOME=%JAVA_HOME:"=% 52 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 53 | 54 | if exist "%JAVA_EXE%" goto init 55 | 56 | echo. 57 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 58 | echo. 59 | echo Please set the JAVA_HOME variable in your environment to match the 60 | echo location of your Java installation. 61 | 62 | goto fail 63 | 64 | :init 65 | @rem Get command-line arguments, handling Windows variants 66 | 67 | if not "%OS%" == "Windows_NT" goto win9xME_args 68 | 69 | :win9xME_args 70 | @rem Slurp the command line arguments. 71 | set CMD_LINE_ARGS= 72 | set _SKIP=2 73 | 74 | :win9xME_args_slurp 75 | if "x%~1" == "x" goto execute 76 | 77 | set CMD_LINE_ARGS=%* 78 | 79 | :execute 80 | @rem Setup the command line 81 | 82 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 83 | 84 | @rem Execute Gradle 85 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 86 | 87 | :end 88 | @rem End local scope for the variables with windows NT shell 89 | if "%ERRORLEVEL%"=="0" goto mainEnd 90 | 91 | :fail 92 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 93 | rem the _cmd.exe /c_ return code! 94 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 95 | exit /b 1 96 | 97 | :mainEnd 98 | if "%OS%"=="Windows_NT" endlocal 99 | 100 | :omega 101 | -------------------------------------------------------------------------------- /android/src/main/kotlin/com/jamesward/airdraw/MainActivity.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Google LLC. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jamesward.airdraw 17 | 18 | 19 | import android.content.Context 20 | import android.hardware.Sensor 21 | import android.hardware.SensorEvent 22 | import android.hardware.SensorEventListener 23 | import android.hardware.SensorManager 24 | import android.os.Bundle 25 | import androidx.appcompat.app.AppCompatActivity 26 | import androidx.compose.foundation.Text 27 | import androidx.compose.foundation.layout.* 28 | import androidx.compose.material.Button 29 | import androidx.compose.material.MaterialTheme 30 | import androidx.compose.material.Surface 31 | import androidx.compose.runtime.Composable 32 | import androidx.compose.runtime.mutableStateOf 33 | import androidx.compose.runtime.onActive 34 | import androidx.compose.runtime.remember 35 | import androidx.compose.ui.Alignment 36 | import androidx.compose.ui.Modifier 37 | import androidx.compose.ui.platform.setContent 38 | import androidx.compose.ui.unit.dp 39 | import com.jamesward.airdraw.data.Orientation 40 | import io.micronaut.context.annotation.Value 41 | import io.micronaut.http.HttpResponse 42 | import io.micronaut.http.annotation.Body 43 | import io.micronaut.http.annotation.Post 44 | import io.micronaut.http.client.annotation.Client 45 | import kotlinx.coroutines.* 46 | import javax.inject.Inject 47 | 48 | class MainActivity : AppCompatActivity() { 49 | 50 | @Inject 51 | var drawService: DrawService? = null 52 | 53 | @Value("\${drawurl}") 54 | var drawUrl: String? = null 55 | 56 | private var orientationSensorMaybe: OrientationSensor? = null 57 | 58 | @Client("\${drawurl}") 59 | interface DrawService { 60 | @Post("/draw") 61 | suspend fun draw(@Body readings: List): HttpResponse 62 | } 63 | 64 | class OrientationSensor: SensorEventListener { 65 | val readings: MutableList = ArrayList() 66 | 67 | override fun onAccuracyChanged(sensor: Sensor?, accuracy: Int) { } 68 | 69 | override fun onSensorChanged(event: SensorEvent?) { 70 | event?.let { e -> 71 | val rotationMatrix = FloatArray(9) 72 | SensorManager.getRotationMatrixFromVector(rotationMatrix, e.values) 73 | val orientationAngles = FloatArray(3) 74 | SensorManager.getOrientation(rotationMatrix, orientationAngles) 75 | // the azimuth goes from -PI to PI potentially causing orientations to "cross over" from -PI to PI 76 | // to avoid this we convert negative readings to positive resulting in a range 0 to PI*2 77 | 78 | val absAzimuth = if (orientationAngles[0] < 0) 79 | orientationAngles[0] + (Math.PI.toFloat() * 2) 80 | else 81 | orientationAngles[0] 82 | 83 | val pitch = if (orientationAngles[1].isNaN()) 84 | 0f 85 | else 86 | orientationAngles[1] 87 | 88 | val orientation = Orientation(absAzimuth, pitch, e.timestamp) 89 | readings.add(orientation) 90 | } 91 | } 92 | } 93 | 94 | private val sensorManager: SensorManager by lazy { 95 | getSystemService(Context.SENSOR_SERVICE) as SensorManager 96 | } 97 | 98 | suspend fun drawClick(on: Boolean) { 99 | if (on) { 100 | orientationSensorMaybe = OrientationSensor() 101 | 102 | sensorManager.registerListener( 103 | orientationSensorMaybe, 104 | sensorManager.getDefaultSensor(Sensor.TYPE_ROTATION_VECTOR), 105 | SensorManager.SENSOR_DELAY_FASTEST 106 | ) 107 | } 108 | else { 109 | orientationSensorMaybe?.let { orientationSensor -> 110 | sensorManager.unregisterListener(orientationSensorMaybe) 111 | 112 | val status = drawService?.draw(orientationSensor.readings)?.status() 113 | println("Server Response: $status") 114 | 115 | orientationSensorMaybe = null 116 | } 117 | } 118 | } 119 | 120 | override fun onCreate(savedInstanceState: Bundle?) { 121 | super.onCreate(savedInstanceState) 122 | 123 | println("drawurl = $drawUrl") 124 | 125 | setContent { 126 | Surface(color = MaterialTheme.colors.background) { 127 | Column(modifier = Modifier.fillMaxWidth().fillMaxHeight(), verticalArrangement = Arrangement.Center, horizontalAlignment = Alignment.CenterHorizontally) { 128 | ToggleButton(::drawClick) 129 | } 130 | } 131 | } 132 | } 133 | } 134 | 135 | @Composable 136 | fun ToggleButton(onChange: suspend (Boolean) -> Unit) { 137 | val scope = remember { CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate) } 138 | 139 | onActive { onDispose { scope.cancel() } } 140 | 141 | val on = remember { mutableStateOf(false) } 142 | 143 | @Composable 144 | fun color() = if (on.value) MaterialTheme.colors.secondary else MaterialTheme.colors.primary 145 | 146 | @Composable 147 | fun text() = if (on.value) "Stop Drawing" else "Start Drawing" 148 | 149 | Button(onClick = { 150 | on.value = !on.value 151 | 152 | scope.launch { 153 | onChange(on.value) 154 | } 155 | }, modifier = Modifier.fillMaxWidth(0.75f).height(128.dp), backgroundColor = color()) { 156 | Text(text()) 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | # Determine the Java command to use to start the JVM. 86 | if [ -n "$JAVA_HOME" ] ; then 87 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 88 | # IBM's JDK on AIX uses strange locations for the executables 89 | JAVACMD="$JAVA_HOME/jre/sh/java" 90 | else 91 | JAVACMD="$JAVA_HOME/bin/java" 92 | fi 93 | if [ ! -x "$JAVACMD" ] ; then 94 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 95 | 96 | Please set the JAVA_HOME variable in your environment to match the 97 | location of your Java installation." 98 | fi 99 | else 100 | JAVACMD="java" 101 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 102 | 103 | Please set the JAVA_HOME variable in your environment to match the 104 | location of your Java installation." 105 | fi 106 | 107 | # Increase the maximum file descriptors if we can. 108 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 109 | MAX_FD_LIMIT=`ulimit -H -n` 110 | if [ $? -eq 0 ] ; then 111 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 112 | MAX_FD="$MAX_FD_LIMIT" 113 | fi 114 | ulimit -n $MAX_FD 115 | if [ $? -ne 0 ] ; then 116 | warn "Could not set maximum file descriptor limit: $MAX_FD" 117 | fi 118 | else 119 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 120 | fi 121 | fi 122 | 123 | # For Darwin, add options to specify how the application appears in the dock 124 | if $darwin; then 125 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 126 | fi 127 | 128 | # For Cygwin, switch paths to Windows format before running java 129 | if $cygwin ; then 130 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 131 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 132 | JAVACMD=`cygpath --unix "$JAVACMD"` 133 | 134 | # We build the pattern for arguments to be converted via cygpath 135 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 136 | SEP="" 137 | for dir in $ROOTDIRSRAW ; do 138 | ROOTDIRS="$ROOTDIRS$SEP$dir" 139 | SEP="|" 140 | done 141 | OURCYGPATTERN="(^($ROOTDIRS))" 142 | # Add a user-defined pattern to the cygpath arguments 143 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 144 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 145 | fi 146 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 147 | i=0 148 | for arg in "$@" ; do 149 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 150 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 151 | 152 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 153 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 154 | else 155 | eval `echo args$i`="\"$arg\"" 156 | fi 157 | i=$((i+1)) 158 | done 159 | case $i in 160 | (0) set -- ;; 161 | (1) set -- "$args0" ;; 162 | (2) set -- "$args0" "$args1" ;; 163 | (3) set -- "$args0" "$args1" "$args2" ;; 164 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 165 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 166 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 167 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 168 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 169 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 170 | esac 171 | fi 172 | 173 | # Escape application args 174 | save () { 175 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 176 | echo " " 177 | } 178 | APP_ARGS=$(save "$@") 179 | 180 | # Collect all arguments for the java command, following the shell quoting and substitution rules 181 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 182 | 183 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 184 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 185 | cd "$(dirname "$0")" 186 | fi 187 | 188 | exec "$JAVACMD" "$@" 189 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /server/src/main/kotlin/com/jamesward/airdraw/WebApp.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Google LLC. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jamesward.airdraw 17 | 18 | import com.jamesward.airdraw.data.* 19 | import com.google.cloud.ServiceOptions 20 | import com.google.cloud.pubsub.v1.Publisher 21 | import com.google.cloud.pubsub.v1.stub.GrpcSubscriberStub 22 | import com.google.cloud.pubsub.v1.stub.SubscriberStubSettings 23 | import com.google.cloud.vision.v1.* 24 | import com.google.cloud.vision.v1.Feature.Type 25 | import com.google.protobuf.ByteString 26 | import com.google.pubsub.v1.* 27 | import io.micronaut.context.annotation.Requires 28 | import io.micronaut.context.env.Environment 29 | import io.micronaut.http.HttpResponse 30 | import io.micronaut.http.MediaType 31 | import io.micronaut.http.annotation.Body 32 | import io.micronaut.http.annotation.Controller 33 | import io.micronaut.http.annotation.Get 34 | import io.micronaut.http.annotation.Post 35 | import io.micronaut.jackson.serialize.JacksonObjectSerializer 36 | import io.micronaut.runtime.Micronaut 37 | import io.reactivex.Maybe 38 | import io.reactivex.Single 39 | import kotlinx.html.dom.* 40 | import kotlinx.html.* 41 | import smile.interpolation.KrigingInterpolation1D 42 | import smile.plot.Headless 43 | import smile.plot.LinePlot 44 | import smile.plot.PlotCanvas 45 | import java.awt.BasicStroke 46 | import java.awt.Dimension 47 | import java.awt.GridLayout 48 | import java.awt.image.BufferedImage 49 | import java.io.ByteArrayOutputStream 50 | import java.util.concurrent.ArrayBlockingQueue 51 | import javax.annotation.PreDestroy 52 | import javax.imageio.ImageIO 53 | import javax.inject.Singleton 54 | import javax.swing.JFrame 55 | import javax.swing.JPanel 56 | import javax.swing.WindowConstants 57 | import kotlin.math.abs 58 | 59 | 60 | fun main() { 61 | Micronaut.build().packages("com.jamesward.airdraw").start() 62 | } 63 | 64 | fun List.toLabelAnnotation(): List { 65 | return this.map { entityAnnotation -> 66 | LabelAnnotation(entityAnnotation.description, entityAnnotation.score) 67 | } 68 | } 69 | 70 | @Controller 71 | class WebApp(private val airDraw: AirDraw, private val bus: Bus) { 72 | 73 | @Get("/") 74 | fun index(): HttpResponse { 75 | val html = createHTMLDocument().html { 76 | head { 77 | title("Air Draw") 78 | link("assets/index.css", "stylesheet") 79 | script(ScriptType.textJavaScript) { 80 | src = "http://localhost:8081/web.js" 81 | } 82 | } 83 | 84 | body { 85 | +"waiting for drawings..." 86 | } 87 | } 88 | 89 | val body = html.serialize(true) 90 | return HttpResponse.ok(body).contentType(MediaType.TEXT_HTML) 91 | } 92 | 93 | @Post("/draw") 94 | fun draw(@Body readingsSingle: Single>): Single> { 95 | return readingsSingle.map { readings -> 96 | airDraw.run(readings)?.let { bus.put(it) } 97 | HttpResponse.ok("") 98 | } 99 | } 100 | 101 | @Post("/show") 102 | fun show(@Body imageResult: ImageResult): HttpResponse { 103 | bus.put(imageResult) 104 | return HttpResponse.ok("") 105 | } 106 | 107 | @Get("/events") 108 | fun events(): Maybe { 109 | val maybe = bus.take() 110 | return if (maybe != null) 111 | Maybe.just(maybe) 112 | else 113 | Maybe.empty() 114 | } 115 | 116 | } 117 | 118 | @Singleton 119 | @Requires(beans = [MyImageAnnotatorClient::class]) 120 | class Vision(private val myImageAnnotatorClient: MyImageAnnotatorClient) { 121 | 122 | fun label(bytes: ByteArray): AnnotateImageResponse? { 123 | val imgBytes = ByteString.copyFrom(bytes) 124 | val img = Image.newBuilder().setContent(imgBytes).build() 125 | val feature = Feature.newBuilder().setType(Type.LABEL_DETECTION).build() 126 | val request = AnnotateImageRequest.newBuilder().addFeatures(feature).setImage(img).build() 127 | 128 | val response = myImageAnnotatorClient.imageAnnotatorClient.batchAnnotateImages(arrayListOf(request)) 129 | return response.responsesList.firstOrNull() 130 | } 131 | 132 | } 133 | 134 | // used to provide either a GCP or local ImageAnnotatorClient 135 | interface MyImageAnnotatorClient { 136 | val imageAnnotatorClient: ImageAnnotatorClient 137 | } 138 | 139 | @Singleton 140 | @Requires(property = "google.application.credentials") 141 | class LocalImageAnnotatorClient: MyImageAnnotatorClient { 142 | override val imageAnnotatorClient = ImageAnnotatorClient.create() 143 | } 144 | 145 | @Singleton 146 | @Requires(env = [Environment.GOOGLE_COMPUTE]) 147 | class GCPImageAnnotatorClient: MyImageAnnotatorClient { 148 | override val imageAnnotatorClient = ImageAnnotatorClient.create() 149 | } 150 | 151 | interface Bus { 152 | fun put(imageResult: ImageResult) 153 | fun take(): ImageResult? 154 | } 155 | 156 | interface CloudBusConfig { 157 | val projectId: String 158 | get() = ServiceOptions.getDefaultProjectId() 159 | 160 | val topic: String 161 | get() = "air-draw" 162 | 163 | val subsciption: String 164 | get() = "air-draw" 165 | } 166 | 167 | @Singleton 168 | @Requires(property = "google.application.credentials") 169 | class PropCloudBusConfig: CloudBusConfig 170 | 171 | @Singleton 172 | @Requires(env = [Environment.GOOGLE_COMPUTE]) 173 | class GcpCloudBusConfig: CloudBusConfig 174 | 175 | @Singleton 176 | @Requires(beans = [CloudBusConfig::class]) 177 | class CloudBus(cloudBusConfig: CloudBusConfig, private val objectSerializer: JacksonObjectSerializer): Bus, AutoCloseable { 178 | 179 | val topicName = ProjectTopicName.of(cloudBusConfig.projectId, cloudBusConfig.topic) 180 | val publisher = Publisher.newBuilder(topicName).build() 181 | 182 | val subscriptionName = ProjectSubscriptionName.format(cloudBusConfig.projectId, cloudBusConfig.subsciption) 183 | 184 | val subscriberStubSettings = SubscriberStubSettings.newBuilder() 185 | .setTransportChannelProvider(SubscriberStubSettings.defaultGrpcTransportProviderBuilder().build()) 186 | .build() 187 | 188 | val subscriber = GrpcSubscriberStub.create(subscriberStubSettings) 189 | val pullRequest = PullRequest.newBuilder() 190 | .setMaxMessages(1) 191 | .setReturnImmediately(true) 192 | .setSubscription(subscriptionName) 193 | .build() 194 | 195 | override fun put(imageResult: ImageResult) { 196 | objectSerializer.serialize(imageResult).map { bytes -> 197 | val data: ByteString = ByteString.copyFrom(bytes) 198 | 199 | val pubsubMessage = PubsubMessage.newBuilder() 200 | .setData(data) 201 | .build() 202 | 203 | // block 204 | publisher.publish(pubsubMessage).get() 205 | } 206 | } 207 | 208 | override fun take(): ImageResult? { 209 | val pullResponse = subscriber.pullCallable().call(pullRequest) 210 | 211 | return pullResponse.receivedMessagesList.firstOrNull()?.let { receivedMessage -> 212 | val acknowledgeRequest = AcknowledgeRequest.newBuilder() 213 | .setSubscription(subscriptionName) 214 | .addAckIds(receivedMessage.ackId) 215 | .build() 216 | 217 | subscriber.acknowledgeCallable().call(acknowledgeRequest) 218 | 219 | objectSerializer.deserialize(receivedMessage.message.data.toByteArray(), ImageResult::class.java).get() 220 | } 221 | } 222 | 223 | @PreDestroy 224 | override fun close() { 225 | publisher.shutdown() 226 | subscriber.shutdown() 227 | } 228 | 229 | } 230 | 231 | @Singleton 232 | @Requires(missingBeans = [CloudBus::class]) 233 | class LocalBus: Bus { 234 | 235 | private val queue = ArrayBlockingQueue(256) 236 | 237 | override fun put(imageResult: ImageResult) { 238 | queue.add(imageResult) 239 | } 240 | 241 | override fun take(): ImageResult? { 242 | val maybe = queue.firstOrNull() 243 | if (maybe != null) 244 | queue.remove(maybe) 245 | 246 | return maybe 247 | } 248 | } 249 | 250 | interface AirDraw { 251 | fun run(readings: List): ImageResult? 252 | } 253 | 254 | @Singleton 255 | @Requires(beans = [Vision::class]) 256 | class CloudAirDraw(private val vision: Vision): AirDraw { 257 | override fun run(readings: List): ImageResult? { 258 | val canvas = AirDrawSmileViewer.draw(readings) 259 | 260 | canvas.getAxis(0).isGridVisible = false 261 | canvas.getAxis(0).isFrameVisible = false 262 | canvas.getAxis(0).isLabelVisible = false 263 | canvas.getAxis(1).isGridVisible = false 264 | canvas.getAxis(1).isFrameVisible = false 265 | canvas.getAxis(1).isLabelVisible = false 266 | canvas.margin = 0.0 267 | 268 | val headless = Headless(canvas) 269 | headless.pack() 270 | headless.isVisible = true 271 | headless.setSize(1024, 1024) 272 | 273 | val bi = BufferedImage(canvas.width, canvas.height, BufferedImage.TYPE_INT_ARGB) 274 | val g2d = bi.createGraphics() 275 | canvas.print(g2d) 276 | 277 | val outputStream = ByteArrayOutputStream() 278 | ImageIO.write(bi, "png", outputStream) 279 | 280 | val bytes = outputStream.toByteArray() 281 | 282 | return vision.label(bytes)?.let { annotateImageResponse -> 283 | ImageResult(bytes, annotateImageResponse.labelAnnotationsList.toLabelAnnotation().toTypedArray()) 284 | } 285 | } 286 | } 287 | 288 | @Singleton 289 | @Requires(missingBeans = [Vision::class]) 290 | class LocalAirDraw: AirDraw { 291 | override fun run(readings: List): ImageResult? { 292 | val canvas = AirDrawSmileViewer.draw(readings) 293 | AirDrawSmileViewer.show(canvas) 294 | return null 295 | } 296 | } 297 | 298 | object AirDrawSmileViewer { 299 | fun show(jPanel: JPanel) { 300 | val frame = JFrame() 301 | frame.defaultCloseOperation = WindowConstants.DISPOSE_ON_CLOSE 302 | frame.contentPane.add(JPanel(GridLayout(4, 4))) 303 | frame.size = Dimension(1000, 1000) 304 | frame.isVisible = true 305 | frame.add(jPanel) 306 | } 307 | 308 | fun draw(readings: List): PlotCanvas { 309 | val t = readings.map { it.timestamp.toDouble() }.toDoubleArray() 310 | val x = readings.map { it.azimuth.toDouble() }.toDoubleArray() 311 | val y = readings.map { it.pitch.toDouble() * -1 }.toDoubleArray() 312 | 313 | val xl = KrigingInterpolation1D(t, x) 314 | val yl = KrigingInterpolation1D(t, y) 315 | 316 | val minTimestamp = readings.minByOrNull { it.timestamp }!!.timestamp 317 | val maxTimestamp = readings.maxByOrNull { it.timestamp }!!.timestamp 318 | val time = maxTimestamp - minTimestamp 319 | 320 | val xy: Array = (minTimestamp..maxTimestamp step(time / 100)).map { timestamp -> 321 | val ix = xl.interpolate(timestamp.toDouble()) 322 | val iy = yl.interpolate(timestamp.toDouble()) 323 | doubleArrayOf(ix, iy) 324 | }.toTypedArray() 325 | 326 | val yBounds = doubleArrayOf(-0.5, 1.5) 327 | val defaultXWidth = 2 328 | 329 | val minX = xy.minByOrNull { it[0] }!![0] 330 | val maxX = xy.maxByOrNull { it[0] }!![0] 331 | 332 | val width = abs(minX) + abs(maxX) 333 | val xBounds = if (width < defaultXWidth) { 334 | val more = (defaultXWidth - width) / 2 335 | doubleArrayOf(minX - more, maxX + more) 336 | } else { 337 | doubleArrayOf(minX, maxX) 338 | } 339 | 340 | val linePlot = LinePlot(xy).setStroke(BasicStroke(20F, BasicStroke.CAP_BUTT, BasicStroke.JOIN_ROUND)) 341 | val canvas = PlotCanvas(doubleArrayOf(xBounds[0], yBounds[0]), doubleArrayOf(xBounds[1], yBounds[1])) 342 | canvas.add(linePlot) 343 | 344 | return canvas 345 | } 346 | } 347 | --------------------------------------------------------------------------------