├── .gitignore ├── 01_Model-Training.ipynb ├── 02_Model-Training-CNN.ipynb ├── 03_Model-Training-DataAugmentation.ipynb ├── Android ├── MobileExample.apk ├── tutorial_mobile_example │ ├── .gitignore │ ├── app │ │ ├── .gitignore │ │ ├── build.gradle.kts │ │ ├── proguard-rules.pro │ │ └── src │ │ │ ├── androidTest │ │ │ └── java │ │ │ │ └── io │ │ │ │ └── interactionlab │ │ │ │ └── tutorial_mobile_example │ │ │ │ └── ExampleInstrumentedTest.java │ │ │ ├── main │ │ │ ├── AndroidManifest.xml │ │ │ ├── assets │ │ │ │ └── model.tflite │ │ │ ├── java │ │ │ │ └── io │ │ │ │ │ └── interactionlab │ │ │ │ │ └── tutorial_mobile_example │ │ │ │ │ ├── Constants.java │ │ │ │ │ ├── MainActivity.java │ │ │ │ │ ├── NumberClassifier.java │ │ │ │ │ └── ui │ │ │ │ │ ├── DrawModel.java │ │ │ │ │ ├── DrawRenderer.java │ │ │ │ │ └── DrawView.java │ │ │ └── res │ │ │ │ ├── drawable │ │ │ │ ├── ic_launcher_background.xml │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ ├── layout │ │ │ │ └── activity_main.xml │ │ │ │ ├── mipmap-anydpi-v26 │ │ │ │ ├── ic_launcher.xml │ │ │ │ └── ic_launcher_round.xml │ │ │ │ ├── mipmap-hdpi │ │ │ │ ├── ic_launcher.webp │ │ │ │ └── ic_launcher_round.webp │ │ │ │ ├── mipmap-mdpi │ │ │ │ ├── ic_launcher.webp │ │ │ │ └── ic_launcher_round.webp │ │ │ │ ├── mipmap-xhdpi │ │ │ │ ├── ic_launcher.webp │ │ │ │ └── ic_launcher_round.webp │ │ │ │ ├── mipmap-xxhdpi │ │ │ │ ├── ic_launcher.webp │ │ │ │ └── ic_launcher_round.webp │ │ │ │ ├── mipmap-xxxhdpi │ │ │ │ ├── ic_launcher.webp │ │ │ │ └── ic_launcher_round.webp │ │ │ │ ├── values-night │ │ │ │ └── themes.xml │ │ │ │ ├── values │ │ │ │ ├── colors.xml │ │ │ │ ├── strings.xml │ │ │ │ ├── styles.xml │ │ │ │ └── themes.xml │ │ │ │ └── xml │ │ │ │ ├── backup_rules.xml │ │ │ │ └── data_extraction_rules.xml │ │ │ └── test │ │ │ └── java │ │ │ └── io │ │ │ └── interactionlab │ │ │ └── tutorial_mobile_example │ │ │ └── ExampleUnitTest.java │ ├── build.gradle.kts │ ├── gradle.properties │ ├── gradle │ │ └── wrapper │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ ├── gradlew │ ├── gradlew.bat │ └── settings.gradle.kts └── tutorial_mobile_example_offline │ ├── .gitignore │ ├── app │ ├── .gitignore │ ├── build.gradle.kts │ ├── proguard-rules.pro │ └── src │ │ ├── androidTest │ │ └── java │ │ │ └── io │ │ │ └── interactionlab │ │ │ └── tutorial_mobile_example │ │ │ └── ExampleInstrumentedTest.java │ │ ├── main │ │ ├── AndroidManifest.xml │ │ ├── assets │ │ │ └── model.tflite │ │ ├── java │ │ │ └── io │ │ │ │ └── interactionlab │ │ │ │ └── tutorial_mobile_example │ │ │ │ ├── MainActivity.java │ │ │ │ ├── NumberClassifier.java │ │ │ │ └── ui │ │ │ │ ├── DrawModel.java │ │ │ │ ├── DrawRenderer.java │ │ │ │ └── DrawView.java │ │ └── res │ │ │ ├── drawable │ │ │ ├── ic_launcher_background.xml │ │ │ └── ic_launcher_foreground.xml │ │ │ ├── layout │ │ │ └── activity_main.xml │ │ │ ├── mipmap-anydpi-v26 │ │ │ ├── ic_launcher.xml │ │ │ └── ic_launcher_round.xml │ │ │ ├── mipmap-hdpi │ │ │ ├── ic_launcher.webp │ │ │ └── ic_launcher_round.webp │ │ │ ├── mipmap-mdpi │ │ │ ├── ic_launcher.webp │ │ │ └── ic_launcher_round.webp │ │ │ ├── mipmap-xhdpi │ │ │ ├── ic_launcher.webp │ │ │ └── ic_launcher_round.webp │ │ │ ├── mipmap-xxhdpi │ │ │ ├── ic_launcher.webp │ │ │ └── ic_launcher_round.webp │ │ │ ├── mipmap-xxxhdpi │ │ │ ├── ic_launcher.webp │ │ │ └── ic_launcher_round.webp │ │ │ ├── values-night │ │ │ └── themes.xml │ │ │ ├── values │ │ │ ├── colors.xml │ │ │ ├── strings.xml │ │ │ ├── styles.xml │ │ │ └── themes.xml │ │ │ └── xml │ │ │ ├── backup_rules.xml │ │ │ └── data_extraction_rules.xml │ │ └── test │ │ └── java │ │ └── io │ │ └── interactionlab │ │ └── tutorial_mobile_example │ │ └── ExampleUnitTest.java │ ├── build.gradle.kts │ ├── gradle.properties │ ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties │ ├── gradlew │ ├── gradlew.bat │ └── settings.gradle.kts ├── HAR-01_data-preprocessing.ipynb ├── HAR-02_model-training.ipynb ├── LICENSE ├── README.md ├── Slides └── TensorFlow_Mobile.pdf └── img ├── screenshot_detect.png └── screenshot_download.png /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore MNIST data and model 2 | mnist_data/ 3 | mnist_model/ 4 | 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | env/ 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | .hypothesis/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # SageMath parsed files 84 | *.sage.py 85 | 86 | # dotenv 87 | .env 88 | 89 | # virtualenv 90 | .venv 91 | venv/ 92 | ENV/ 93 | 94 | # Spyder project settings 95 | .spyderproject 96 | .spyproject 97 | 98 | # Rope project settings 99 | .ropeproject 100 | 101 | # mkdocs documentation 102 | /site 103 | 104 | # mypy 105 | .mypy_cache/ 106 | 107 | 108 | 109 | #built application files 110 | *.apk 111 | *.ap_ 112 | 113 | # files for the dex VM 114 | *.dex 115 | 116 | # Java class files 117 | *.class 118 | 119 | # generated files 120 | bin/ 121 | gen/ 122 | 123 | # Local configuration file (sdk path, etc) 124 | local.properties 125 | 126 | # Windows thumbnail db 127 | Thumbs.db 128 | 129 | # OSX files 130 | .DS_Store 131 | 132 | # Eclipse project files 133 | .classpath 134 | .project 135 | 136 | # Android Studio 137 | *.iml 138 | .idea 139 | #.idea/workspace.xml - remove # and delete .idea if it better suit your needs. 140 | .gradle 141 | build/ 142 | 143 | #NDK 144 | obj/ 145 | -------------------------------------------------------------------------------- /Android/MobileExample.apk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/interactionlab/imui-tutorial/5080cb0f0aaac57113801e31985717dafc199727/Android/MobileExample.apk -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | .gradle 3 | /local.properties 4 | /.idea/caches 5 | /.idea/libraries 6 | /.idea/modules.xml 7 | /.idea/workspace.xml 8 | /.idea/navEditor.xml 9 | /.idea/assetWizardSettings.xml 10 | .DS_Store 11 | /build 12 | /captures 13 | .externalNativeBuild 14 | .cxx 15 | local.properties 16 | -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/app/.gitignore: -------------------------------------------------------------------------------- 1 | /build -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/app/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | id("com.android.application") 3 | } 4 | 5 | android { 6 | namespace = "io.interactionlab.tutorial_mobile_example" 7 | compileSdk = 34 8 | 9 | defaultConfig { 10 | applicationId = "io.interactionlab.tutorial_mobile_example" 11 | minSdk = 24 12 | targetSdk = 34 13 | versionCode = 1 14 | versionName = "1.0" 15 | 16 | testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" 17 | } 18 | 19 | buildTypes { 20 | release { 21 | isMinifyEnabled = false 22 | proguardFiles( 23 | getDefaultProguardFile("proguard-android-optimize.txt"), 24 | "proguard-rules.pro" 25 | ) 26 | } 27 | } 28 | compileOptions { 29 | sourceCompatibility = JavaVersion.VERSION_1_8 30 | targetCompatibility = JavaVersion.VERSION_1_8 31 | } 32 | } 33 | 34 | dependencies { 35 | 36 | implementation("androidx.appcompat:appcompat:1.6.1") 37 | implementation("com.google.android.material:material:1.11.0") 38 | testImplementation("junit:junit:4.13.2") 39 | androidTestImplementation("androidx.test.ext:junit:1.1.5") 40 | androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1") 41 | // Tensorflow Lite dependencies 42 | implementation("org.tensorflow:tensorflow-lite:+") 43 | implementation("org.tensorflow:tensorflow-lite-gpu:+") 44 | implementation("org.tensorflow:tensorflow-lite-support:+") 45 | } -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/app/proguard-rules.pro: -------------------------------------------------------------------------------- 1 | # Add project specific ProGuard rules here. 2 | # You can control the set of applied configuration files using the 3 | # proguardFiles setting in build.gradle. 4 | # 5 | # For more details, see 6 | # http://developer.android.com/guide/developing/tools/proguard.html 7 | 8 | # If your project uses WebView with JS, uncomment the following 9 | # and specify the fully qualified class name to the JavaScript interface 10 | # class: 11 | #-keepclassmembers class fqcn.of.javascript.interface.for.webview { 12 | # public *; 13 | #} 14 | 15 | # Uncomment this to preserve the line number information for 16 | # debugging stack traces. 17 | #-keepattributes SourceFile,LineNumberTable 18 | 19 | # If you keep the line number information, uncomment this to 20 | # hide the original source file name. 21 | #-renamesourcefileattribute SourceFile -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/app/src/androidTest/java/io/interactionlab/tutorial_mobile_example/ExampleInstrumentedTest.java: -------------------------------------------------------------------------------- 1 | package io.interactionlab.tutorial_mobile_example; 2 | 3 | import android.content.Context; 4 | 5 | import androidx.test.platform.app.InstrumentationRegistry; 6 | import androidx.test.ext.junit.runners.AndroidJUnit4; 7 | 8 | import org.junit.Test; 9 | import org.junit.runner.RunWith; 10 | 11 | import static org.junit.Assert.*; 12 | 13 | /** 14 | * Instrumented test, which will execute on an Android device. 15 | * 16 | * @see Testing documentation 17 | */ 18 | @RunWith(AndroidJUnit4.class) 19 | public class ExampleInstrumentedTest { 20 | @Test 21 | public void useAppContext() { 22 | // Context of the app under test. 23 | Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext(); 24 | assertEquals("io.interactionlab.tutorial_mobile_example", appContext.getPackageName()); 25 | } 26 | } -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/app/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 18 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/app/src/main/assets/model.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/interactionlab/imui-tutorial/5080cb0f0aaac57113801e31985717dafc199727/Android/tutorial_mobile_example/app/src/main/assets/model.tflite -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/app/src/main/java/io/interactionlab/tutorial_mobile_example/Constants.java: -------------------------------------------------------------------------------- 1 | package io.interactionlab.tutorial_mobile_example; 2 | 3 | /** 4 | * Created by Huy on 05/06/2018. 5 | */ 6 | 7 | public class Constants { 8 | public final static String SHARED_PREF_ID = "tutorial_mobile_example"; 9 | } 10 | -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/app/src/main/java/io/interactionlab/tutorial_mobile_example/MainActivity.java: -------------------------------------------------------------------------------- 1 | package io.interactionlab.tutorial_mobile_example; 2 | 3 | import android.app.Dialog; 4 | import android.app.ProgressDialog; 5 | import android.content.DialogInterface; 6 | import android.content.Intent; 7 | import android.content.SharedPreferences; 8 | import android.graphics.PointF; 9 | import android.net.Uri; 10 | import android.os.AsyncTask; 11 | import android.os.Bundle; 12 | import android.view.MotionEvent; 13 | import android.view.View; 14 | import android.widget.Button; 15 | import android.widget.EditText; 16 | import android.widget.LinearLayout; 17 | import android.widget.TextView; 18 | 19 | import java.io.BufferedInputStream; 20 | import java.io.File; 21 | import java.io.FileInputStream; 22 | import java.io.FileNotFoundException; 23 | import java.io.FileOutputStream; 24 | import java.io.InputStream; 25 | import java.io.OutputStream; 26 | import java.net.URL; 27 | import java.net.URLConnection; 28 | 29 | import io.interactionlab.tutorial_mobile_example.ui.DrawModel; 30 | import io.interactionlab.tutorial_mobile_example.ui.DrawView; 31 | 32 | import static io.interactionlab.tutorial_mobile_example.Constants.SHARED_PREF_ID; 33 | 34 | import androidx.appcompat.app.AlertDialog; 35 | import androidx.appcompat.app.AppCompatActivity; 36 | 37 | public class MainActivity extends AppCompatActivity implements View.OnTouchListener { 38 | 39 | // Variables for the draw view. 40 | private static final int PIXEL_WIDTH = 28; 41 | private TextView tvResult; 42 | private float lastX; 43 | private float lastY; 44 | private DrawModel drawModel; 45 | private DrawView drawView; 46 | private PointF tmpPoint = new PointF(); 47 | 48 | // Download view 49 | private ProgressDialog progrssDialog; 50 | private Button downloadButton; 51 | private Button infoButton; 52 | 53 | 54 | private EditText etOutput; 55 | private EditText etServer; 56 | private EditText etInput; 57 | 58 | // Classifier 59 | private NumberClassifier numberClassifier; 60 | 61 | // File download path 62 | 63 | private String MODEL_FILE = null; 64 | 65 | @Override 66 | protected void onCreate(Bundle savedInstanceState) { 67 | super.onCreate(savedInstanceState); 68 | setContentView(R.layout.activity_main); 69 | 70 | File path = MainActivity.this.getExternalFilesDir(null); 71 | MODEL_FILE = path.getAbsolutePath() + "/downloaded_model.pb"; 72 | 73 | 74 | drawModel = new DrawModel(PIXEL_WIDTH, PIXEL_WIDTH); 75 | 76 | drawView = (DrawView) findViewById(R.id.view_draw); 77 | drawView.setModel(drawModel); 78 | drawView.setOnTouchListener(this); 79 | 80 | Button classifyButton = (Button) findViewById(R.id.button_classify); 81 | classifyButton.setOnClickListener(new View.OnClickListener() { 82 | @Override 83 | public void onClick(View v) { 84 | int classification = classifyNumber(); 85 | 86 | if (classification == -1) { 87 | tvResult.setText("Please load model first."); 88 | } else { 89 | tvResult.setText("Detected = " + classification); 90 | } 91 | 92 | } 93 | }); 94 | 95 | Button clearButton = (Button) findViewById(R.id.button_clear); 96 | clearButton.setOnClickListener(new View.OnClickListener() { 97 | @Override 98 | public void onClick(View v) { 99 | onClearClicked(); 100 | } 101 | }); 102 | 103 | downloadButton = (Button) findViewById(R.id.button_download); 104 | downloadButton.setOnClickListener(new View.OnClickListener() { 105 | @Override 106 | public void onClick(View v) { 107 | // retrieve saved server url 108 | SharedPreferences prefs = getSharedPreferences(SHARED_PREF_ID, MODE_PRIVATE); 109 | String serverURL = prefs.getString("server", "http://"); 110 | 111 | new FileDownloadTask().execute(serverURL); 112 | } 113 | }); 114 | 115 | infoButton = (Button) findViewById(R.id.button_info); 116 | infoButton.setOnClickListener(new View.OnClickListener() { 117 | @Override 118 | public void onClick(View v) { 119 | AlertDialog alertDialog = new AlertDialog.Builder(MainActivity.this).create(); 120 | alertDialog.setTitle("Info"); 121 | alertDialog.setMessage("To use the app you need to train your own MNIST models. You can use the following python code to train a model: https://github.com/interactionlab/imui-tutorial/"); 122 | alertDialog.setButton(AlertDialog.BUTTON_POSITIVE, "OK", 123 | new DialogInterface.OnClickListener() { 124 | public void onClick(DialogInterface dialog, int which) { 125 | dialog.dismiss(); 126 | } 127 | }); 128 | alertDialog.setButton(AlertDialog.BUTTON_NEUTRAL, "Open Link", 129 | new DialogInterface.OnClickListener() { 130 | public void onClick(DialogInterface dialog, int which) { 131 | String url = "https://github.com/interactionlab/imui-tutorial/"; 132 | Intent i = new Intent(Intent.ACTION_VIEW); 133 | i.setData(Uri.parse(url)); 134 | startActivity(i); 135 | dialog.dismiss(); 136 | } 137 | }); 138 | 139 | 140 | alertDialog.show(); 141 | } 142 | }); 143 | 144 | 145 | tvResult = (TextView) findViewById(R.id.text_result); 146 | 147 | promptModelSettings(); 148 | } 149 | 150 | 151 | private void promptModelSettings() { 152 | SharedPreferences prefs = getSharedPreferences(SHARED_PREF_ID, MODE_PRIVATE); 153 | String servername = prefs.getString("server", "http://"); 154 | String inputNode = prefs.getString("input_node", "dense_1_input"); 155 | String outputNode = prefs.getString("output_node", "output_node0"); 156 | 157 | AlertDialog.Builder builder = new AlertDialog.Builder(this); 158 | //you should edit this to fit your needs 159 | builder.setTitle("Enter model settings:"); 160 | 161 | final EditText etServer = new EditText(this); 162 | etServer.setHint("Server Address"); 163 | etServer.setText(servername); 164 | final EditText etInput = new EditText(this); 165 | etInput.setHint("Input Node"); 166 | etInput.setText(inputNode); 167 | final EditText etOutput = new EditText(this); 168 | etOutput.setHint("Output Node"); 169 | etOutput.setText(outputNode); 170 | 171 | LinearLayout lay = new LinearLayout(this); 172 | lay.setOrientation(LinearLayout.VERTICAL); 173 | lay.addView(etServer); 174 | lay.addView(etInput); 175 | lay.addView(etOutput); 176 | builder.setView(lay); 177 | 178 | // Set up the buttons 179 | builder.setPositiveButton("Ok", new DialogInterface.OnClickListener() { 180 | public void onClick(DialogInterface dialog, int whichButton) { 181 | SharedPreferences.Editor editor = getSharedPreferences(SHARED_PREF_ID, MODE_PRIVATE).edit(); 182 | editor.putString("server", etServer.getText().toString()); 183 | editor.putString("input_node", etInput.getText().toString()); 184 | editor.putString("output_node", etOutput.getText().toString()); 185 | 186 | editor.apply(); 187 | dialog.cancel(); 188 | } 189 | }); 190 | 191 | builder.setNegativeButton("Cancel", new DialogInterface.OnClickListener() { 192 | public void onClick(DialogInterface dialog, int whichButton) { 193 | dialog.cancel(); 194 | } 195 | }); 196 | builder.create(); 197 | builder.show(); 198 | } 199 | 200 | 201 | private int classifyNumber() { 202 | // Retrieve 28x28 image. 203 | float pixels[] = drawView.getPixelData(); 204 | 205 | // Classify. 206 | 207 | int idx = -1; 208 | if (numberClassifier != null) { 209 | idx = numberClassifier.classify(pixels); 210 | } 211 | 212 | return idx; 213 | } 214 | 215 | @Override 216 | protected void onResume() { 217 | drawView.onResume(); 218 | super.onResume(); 219 | } 220 | 221 | @Override 222 | protected void onPause() { 223 | drawView.onPause(); 224 | super.onPause(); 225 | } 226 | 227 | 228 | /** 229 | * ############################################################################# 230 | * Handling the drawing view. 231 | * ############################################################################# 232 | */ 233 | @Override 234 | public boolean onTouch(View v, MotionEvent event) { 235 | int action = event.getAction() & MotionEvent.ACTION_MASK; 236 | 237 | if (action == MotionEvent.ACTION_DOWN) { 238 | processTouchDown(event); 239 | return true; 240 | 241 | } else if (action == MotionEvent.ACTION_MOVE) { 242 | processTouchMove(event); 243 | return true; 244 | 245 | } else if (action == MotionEvent.ACTION_UP) { 246 | processTouchUp(); 247 | return true; 248 | } 249 | return false; 250 | } 251 | 252 | private void processTouchDown(MotionEvent event) { 253 | lastX = event.getX(); 254 | lastY = event.getY(); 255 | drawView.calcPos(lastX, lastY, tmpPoint); 256 | float lastConvX = tmpPoint.x; 257 | float lastConvY = tmpPoint.y; 258 | drawModel.startLine(lastConvX, lastConvY); 259 | } 260 | 261 | private void processTouchMove(MotionEvent event) { 262 | float x = event.getX(); 263 | float y = event.getY(); 264 | 265 | drawView.calcPos(x, y, tmpPoint); 266 | float newConvX = tmpPoint.x; 267 | float newConvY = tmpPoint.y; 268 | drawModel.addLineElem(newConvX, newConvY); 269 | 270 | lastX = x; 271 | lastY = y; 272 | drawView.invalidate(); 273 | } 274 | 275 | private void processTouchUp() { 276 | drawModel.endLine(); 277 | } 278 | 279 | private void onClearClicked() { 280 | drawModel.clear(); 281 | drawView.reset(); 282 | drawView.invalidate(); 283 | 284 | tvResult.setText(""); 285 | } 286 | 287 | 288 | /** 289 | * ############################################################################# 290 | * Downloading model from the internet and loading it. 291 | * ############################################################################# 292 | */ 293 | private static final int progress_bar_type = 0; 294 | 295 | @Override 296 | protected Dialog onCreateDialog(int id) { 297 | switch (id) { 298 | case progress_bar_type: 299 | progrssDialog = new ProgressDialog(this); 300 | progrssDialog.setMessage("Downloading model. Please wait..."); 301 | progrssDialog.setIndeterminate(false); 302 | progrssDialog.setMax(100); 303 | progrssDialog.setCancelable(false); 304 | progrssDialog.setProgressStyle(ProgressDialog.STYLE_HORIZONTAL); 305 | progrssDialog.show(); 306 | return progrssDialog; 307 | default: 308 | return null; 309 | } 310 | } 311 | 312 | 313 | /** 314 | * Background Async Task to download file 315 | */ 316 | class FileDownloadTask extends AsyncTask { 317 | @Override 318 | protected void onPreExecute() { 319 | super.onPreExecute(); 320 | showDialog(progress_bar_type); 321 | } 322 | 323 | @Override 324 | protected String doInBackground(String... f_url) { 325 | int count; 326 | try { 327 | // delete previous model first if available 328 | File fdelete = new File(MODEL_FILE); 329 | if (fdelete.exists()) { 330 | if (fdelete.delete()) { 331 | System.out.println("file Deleted."); 332 | } else { 333 | System.out.println("file not Deleted."); 334 | } 335 | } 336 | 337 | 338 | URL url = new URL(f_url[0]); 339 | 340 | URLConnection connection = url.openConnection(); 341 | connection.connect(); 342 | int lenghtOfFile = connection.getContentLength(); 343 | InputStream input = new BufferedInputStream(url.openStream(), 8192); 344 | 345 | // Output stream to write file 346 | OutputStream output = new FileOutputStream(MODEL_FILE); 347 | 348 | byte data[] = new byte[1024]; 349 | long total = 0; 350 | while ((count = input.read(data)) != -1) { 351 | total += count; 352 | publishProgress("" + (int) ((total * 100) / lenghtOfFile)); 353 | output.write(data, 0, count); 354 | } 355 | 356 | output.flush(); 357 | output.close(); 358 | input.close(); 359 | 360 | } catch (Exception e) { 361 | MainActivity.this.runOnUiThread(new Runnable() { 362 | public void run() { 363 | AlertDialog alertDialog = new AlertDialog.Builder(MainActivity.this).create(); 364 | alertDialog.setTitle("Invalid file."); 365 | alertDialog.setMessage("Click OK to enter a new server URL."); 366 | alertDialog.setButton(AlertDialog.BUTTON_NEUTRAL, "OK", 367 | new DialogInterface.OnClickListener() { 368 | public void onClick(DialogInterface dialog, int which) { 369 | dialog.dismiss(); 370 | promptModelSettings(); 371 | } 372 | }); 373 | alertDialog.show(); 374 | } 375 | }); 376 | } 377 | 378 | return null; 379 | } 380 | 381 | protected void onProgressUpdate(String... progress) { 382 | // setting progress percentage 383 | progrssDialog.setProgress(Integer.parseInt(progress[0])); 384 | } 385 | 386 | @Override 387 | protected void onPostExecute(String file_url) { 388 | // dismiss the dialog after the file was downloaded 389 | dismissDialog(progress_bar_type); 390 | 391 | File pathBack = MainActivity.this.getExternalFilesDir(null); 392 | 393 | FileInputStream fileInputStream = null; 394 | try { 395 | fileInputStream = new FileInputStream(MODEL_FILE); 396 | } catch (FileNotFoundException e) { 397 | e.printStackTrace(); 398 | } 399 | 400 | if (fileInputStream != null) { 401 | numberClassifier = new NumberClassifier(fileInputStream, MainActivity.this); 402 | } 403 | 404 | } 405 | 406 | } 407 | } 408 | -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/app/src/main/java/io/interactionlab/tutorial_mobile_example/NumberClassifier.java: -------------------------------------------------------------------------------- 1 | package io.interactionlab.tutorial_mobile_example; 2 | 3 | import android.content.Context; 4 | import android.content.res.AssetFileDescriptor; 5 | import android.util.Log; 6 | 7 | import org.tensorflow.lite.Interpreter; 8 | import org.tensorflow.lite.support.common.FileUtil; 9 | 10 | import java.io.FileInputStream; 11 | import java.io.IOException; 12 | import java.nio.MappedByteBuffer; 13 | import java.nio.channels.FileChannel; 14 | 15 | /** 16 | * This class demonstrates the use of the inference interface of TensorFlow. 17 | * The model (protobuf file) can either be loaded from the assets folder of the APK, or using an InputStream. 18 | */ 19 | public class NumberClassifier { 20 | 21 | private Interpreter interpreter = null; 22 | 23 | public NumberClassifier(String modelPath, Context context) { 24 | MappedByteBuffer myMappedBuffer = null; 25 | 26 | try { 27 | myMappedBuffer = FileUtil.loadMappedFile(context, modelPath); 28 | } catch (IOException e) { 29 | Log.e("NumberClassifier", "Error #002: " + e.toString()); 30 | return; 31 | } 32 | 33 | try { 34 | interpreter = new Interpreter(myMappedBuffer); 35 | } catch (Exception e) { 36 | Log.e("NumberClassifier", "Error #001: " + e.toString()); 37 | return; 38 | } 39 | Log.v("NumberClassifier", "Load model successful."); 40 | log(); 41 | } 42 | 43 | public NumberClassifier(FileInputStream fileInputStream, MainActivity context) { 44 | MappedByteBuffer myMappedBuffer = null; 45 | try { 46 | myMappedBuffer = NumberClassifier.toMappedByteBuffer(fileInputStream);; 47 | } catch (IOException e) { 48 | Log.e("NumberClassifier", "Error #002: " + e.toString()); 49 | return; 50 | } 51 | 52 | try { 53 | interpreter = new Interpreter(myMappedBuffer); 54 | } catch (Exception e) { 55 | Log.e("NumberClassifier", "Error #001: " + e.toString()); 56 | return; 57 | } 58 | 59 | Log.v("NumberClassifier", "Load model successful."); 60 | log(); 61 | } 62 | 63 | private void log(){ 64 | 65 | for (String s : interpreter.getSignatureKeys()) { 66 | Log.d("NumberClassifier - Model Debug", "SignatureKeys: " + s); 67 | } 68 | Log.d("NumberClassifier - Model Debug", "InputTensorCount: " + interpreter.getInputTensorCount()); 69 | for (int dim : interpreter.getInputTensor(0).shape()) { 70 | Log.d("NumberClassifier - Model Debug", "Input Shape: " +dim); 71 | } 72 | Log.d("NumberClassifier - Model Debug", "OutputTensorCount: " + interpreter.getOutputTensorCount()); 73 | for (int dim : interpreter.getOutputTensor(0).shape()) { 74 | Log.d("NumberClassifier - Model Debug", "Output Shape: " +dim); 75 | } 76 | } 77 | 78 | public int classify(float[] pixels) { 79 | Log.d("NumberClassifier", "PixelData - Length:" + pixels.length); 80 | if (interpreter == null){ 81 | Log.w("NumberClassifier", "interpreter not ready."); 82 | return -1; 83 | } 84 | 85 | // Scale input from [0 - 255] to [-1 - 1], if the training is different, this needs to change too. 86 | for(int a = 0; a < pixels.length; a++) { 87 | pixels[a] = pixels[a]/127.5f-1.0f; 88 | } 89 | 90 | float[][][] input = new float[1][1][28*28]; 91 | input[0][0] = pixels; 92 | float[][][] output = new float[1][1][10]; 93 | 94 | Log.v("NumberClassifier", "Feeding into model."); 95 | try { 96 | interpreter.run(input, output); 97 | } catch (Exception e){ 98 | Log.e("NumberClassifier", "Error #003: " + e.toString()); 99 | return -1; 100 | } 101 | Log.v("NumberClassifier", "Inference done."); 102 | 103 | float[] outputs = output[0][0]; 104 | 105 | // Convert one-hot encoded result to an int (= detected class) 106 | float max = Float.MIN_VALUE; 107 | int idx = -1; 108 | for (int i = 0; i < outputs.length; i++) { 109 | if (outputs[i] > max) { 110 | max = outputs[i]; 111 | idx = i; 112 | } 113 | } 114 | return idx; 115 | } 116 | 117 | public static MappedByteBuffer toMappedByteBuffer (FileInputStream inputStream) throws IOException { 118 | //AssetFileDescriptor fileDescriptor = context.getAssets().openFd(filePath); 119 | FileChannel fileChannel = inputStream.getChannel(); 120 | 121 | //ToDo : get the numbers from the inputStream or save it first and load it then. 122 | long startOffset = 0; //fileDescriptor.getStartOffset(); 123 | long declaredLength = 0; //fileDescriptor.getDeclaredLength(); 124 | return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/app/src/main/java/io/interactionlab/tutorial_mobile_example/ui/DrawModel.java: -------------------------------------------------------------------------------- 1 | package io.interactionlab.tutorial_mobile_example.ui; 2 | 3 | /** 4 | * Created by Huy on 31/08/2017. 5 | */ 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | /** 11 | * This class handles the draw view and was adapted from miyosuda's TensorFlow Android MNIST example 12 | * for TensorFlow r.0.10. See: https://github.com/miyosuda/TensorFlowAndroidMNIST 13 | */ 14 | public class DrawModel { 15 | 16 | public static class LineElem { 17 | public float x; 18 | public float y; 19 | 20 | private LineElem(float x, float y) { 21 | this.x = x; 22 | this.y = y; 23 | } 24 | } 25 | 26 | public static class Line { 27 | private List elems = new ArrayList<>(); 28 | 29 | private Line() { 30 | } 31 | 32 | private void addElem(LineElem elem) { 33 | elems.add(elem); 34 | } 35 | 36 | public int getElemSize() { 37 | return elems.size(); 38 | } 39 | 40 | public LineElem getElem(int index) { 41 | return elems.get(index); 42 | } 43 | } 44 | 45 | private Line mCurrentLine; 46 | 47 | private int mWidth; // pixel width = 28 48 | private int mHeight; // pixel height = 28 49 | 50 | private List mLines = new ArrayList<>(); 51 | 52 | public DrawModel(int width, int height) { 53 | this.mWidth = width; 54 | this.mHeight = height; 55 | } 56 | 57 | public int getWidth() { 58 | return mWidth; 59 | } 60 | 61 | public int getHeight() { 62 | return mHeight; 63 | } 64 | 65 | public void startLine(float x, float y) { 66 | mCurrentLine = new Line(); 67 | mCurrentLine.addElem(new LineElem(x, y)); 68 | mLines.add(mCurrentLine); 69 | } 70 | 71 | public void endLine() { 72 | mCurrentLine = null; 73 | } 74 | 75 | public void addLineElem(float x, float y) { 76 | if (mCurrentLine != null) { 77 | mCurrentLine.addElem(new LineElem(x, y)); 78 | } 79 | } 80 | 81 | public int getLineSize() { 82 | return mLines.size(); 83 | } 84 | 85 | public Line getLine(int index) { 86 | return mLines.get(index); 87 | } 88 | 89 | public void clear() { 90 | mLines.clear(); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/app/src/main/java/io/interactionlab/tutorial_mobile_example/ui/DrawRenderer.java: -------------------------------------------------------------------------------- 1 | package io.interactionlab.tutorial_mobile_example.ui; 2 | 3 | /** 4 | * Created by Huy on 31/08/2017. 5 | */ 6 | 7 | import android.graphics.Canvas; 8 | import android.graphics.Color; 9 | import android.graphics.Paint; 10 | 11 | /** 12 | * This class handles the draw view and was adapted from miyosuda's TensorFlow Android MNIST example 13 | * for TensorFlow r.0.10. See: https://github.com/miyosuda/TensorFlowAndroidMNIST 14 | */ 15 | public class DrawRenderer { 16 | /** 17 | * Draw lines to canvas 18 | */ 19 | public static void renderModel(Canvas canvas, DrawModel model, Paint paint, 20 | int startLineIndex) { 21 | paint.setAntiAlias(true); 22 | 23 | int lineSize = model.getLineSize(); 24 | for (int i = startLineIndex; i < lineSize; ++i) { 25 | DrawModel.Line line = model.getLine(i); 26 | paint.setColor(Color.BLACK); 27 | int elemSize = line.getElemSize(); 28 | if (elemSize < 1) { 29 | continue; 30 | } 31 | DrawModel.LineElem elem = line.getElem(0); 32 | float lastX = elem.x; 33 | float lastY = elem.y; 34 | 35 | for (int j = 0; j < elemSize; ++j) { 36 | elem = line.getElem(j); 37 | float x = elem.x; 38 | float y = elem.y; 39 | canvas.drawLine(lastX, lastY, x, y, paint); 40 | lastX = x; 41 | lastY = y; 42 | } 43 | } 44 | } 45 | } -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/app/src/main/java/io/interactionlab/tutorial_mobile_example/ui/DrawView.java: -------------------------------------------------------------------------------- 1 | package io.interactionlab.tutorial_mobile_example.ui; 2 | 3 | /** 4 | * Created by Huy on 31/08/2017. 5 | */ 6 | 7 | import android.content.Context; 8 | import android.graphics.Bitmap; 9 | import android.graphics.Canvas; 10 | import android.graphics.Color; 11 | import android.graphics.Matrix; 12 | import android.graphics.Paint; 13 | import android.graphics.PointF; 14 | import android.graphics.Rect; 15 | import android.util.AttributeSet; 16 | import android.view.View; 17 | 18 | /** 19 | * This class handles the draw view and was adapted from miyosuda's TensorFlow Android MNIST example 20 | * for TensorFlow r.0.10. See: https://github.com/miyosuda/TensorFlowAndroidMNIST 21 | */ 22 | public class DrawView extends View { 23 | private Paint mPaint = new Paint(); 24 | private DrawModel mModel; 25 | // 28x28 pixel Bitmap 26 | private Bitmap mOffscreenBitmap; 27 | private Canvas mOffscreenCanvas; 28 | 29 | private Matrix mMatrix = new Matrix(); 30 | private Matrix mInvMatrix = new Matrix(); 31 | private int mDrawnLineSize = 0; 32 | private boolean mSetuped = false; 33 | 34 | private float mTmpPoints[] = new float[2]; 35 | 36 | public DrawView(Context context, AttributeSet attrs) { 37 | super(context, attrs); 38 | } 39 | 40 | public void setModel(DrawModel model) { 41 | this.mModel = model; 42 | } 43 | 44 | public void reset() { 45 | mDrawnLineSize = 0; 46 | if (mOffscreenBitmap != null) { 47 | mPaint.setColor(Color.WHITE); 48 | int width = mOffscreenBitmap.getWidth(); 49 | int height = mOffscreenBitmap.getHeight(); 50 | mOffscreenCanvas.drawRect(new Rect(0, 0, width, height), mPaint); 51 | } 52 | } 53 | 54 | private void setup() { 55 | mSetuped = true; 56 | 57 | // View size 58 | float width = getWidth(); 59 | float height = getHeight(); 60 | 61 | // Model (bitmap) size 62 | float modelWidth = mModel.getWidth(); 63 | float modelHeight = mModel.getHeight(); 64 | 65 | float scaleW = width / modelWidth; 66 | float scaleH = height / modelHeight; 67 | 68 | float scale = scaleW; 69 | if (scale > scaleH) { 70 | scale = scaleH; 71 | } 72 | 73 | float newCx = modelWidth * scale / 2; 74 | float newCy = modelHeight * scale / 2; 75 | float dx = width / 2 - newCx; 76 | float dy = height / 2 - newCy; 77 | 78 | mMatrix.setScale(scale, scale); 79 | mMatrix.postTranslate(dx, dy); 80 | mMatrix.invert(mInvMatrix); 81 | mSetuped = true; 82 | } 83 | 84 | @Override 85 | public void onDraw(Canvas canvas) { 86 | if (mModel == null) { 87 | return; 88 | } 89 | if (!mSetuped) { 90 | setup(); 91 | } 92 | if (mOffscreenBitmap == null) { 93 | return; 94 | } 95 | 96 | int startIndex = mDrawnLineSize - 1; 97 | if (startIndex < 0) { 98 | startIndex = 0; 99 | } 100 | 101 | DrawRenderer.renderModel(mOffscreenCanvas, mModel, mPaint, startIndex); 102 | canvas.drawBitmap(mOffscreenBitmap, mMatrix, mPaint); 103 | 104 | mDrawnLineSize = mModel.getLineSize(); 105 | } 106 | 107 | /** 108 | * Convert screen position to local pos (pos in bitmap) 109 | */ 110 | public void calcPos(float x, float y, PointF out) { 111 | mTmpPoints[0] = x; 112 | mTmpPoints[1] = y; 113 | mInvMatrix.mapPoints(mTmpPoints); 114 | out.x = mTmpPoints[0]; 115 | out.y = mTmpPoints[1]; 116 | } 117 | 118 | public void onResume() { 119 | createBitmap(); 120 | } 121 | 122 | public void onPause() { 123 | releaseBitmap(); 124 | } 125 | 126 | private void createBitmap() { 127 | if (mOffscreenBitmap != null) { 128 | mOffscreenBitmap.recycle(); 129 | } 130 | mOffscreenBitmap = Bitmap.createBitmap(mModel.getWidth(), mModel.getHeight(), Bitmap.Config.ARGB_8888); 131 | mOffscreenCanvas = new Canvas(mOffscreenBitmap); 132 | reset(); 133 | } 134 | 135 | private void releaseBitmap() { 136 | if (mOffscreenBitmap != null) { 137 | mOffscreenBitmap.recycle(); 138 | mOffscreenBitmap = null; 139 | mOffscreenCanvas = null; 140 | } 141 | reset(); 142 | } 143 | 144 | /** 145 | * Get 28x28 pixel data for tensorflow input. 146 | */ 147 | public float[] getPixelData() { 148 | if (mOffscreenBitmap == null) { 149 | return null; 150 | } 151 | 152 | int width = mOffscreenBitmap.getWidth(); 153 | int height = mOffscreenBitmap.getHeight(); 154 | 155 | // Get 28x28 pixel data from bitmap 156 | int[] pixels = new int[width * height]; 157 | mOffscreenBitmap.getPixels(pixels, 0, width, 0, 0, width, height); 158 | 159 | float[] retPixels = new float[pixels.length]; 160 | for (int i = 0; i < pixels.length; ++i) { 161 | // Set 0 for white and 255 for black pixel 162 | int pix = pixels[i]; 163 | int b = pix & 0xff; 164 | retPixels[i] = (float) (0xff - b); 165 | } 166 | return retPixels; 167 | } 168 | } -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/app/src/main/res/drawable/ic_launcher_background.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 10 | 15 | 20 | 25 | 30 | 35 | 40 | 45 | 50 | 55 | 60 | 65 | 70 | 75 | 80 | 85 | 90 | 95 | 100 | 105 | 110 | 115 | 120 | 125 | 130 | 135 | 140 | 145 | 150 | 155 | 160 | 165 | 170 | 171 | -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/app/src/main/res/drawable/ic_launcher_foreground.xml: -------------------------------------------------------------------------------- 1 | 7 | 8 | 9 | 15 | 18 | 21 | 22 | 23 | 24 | 30 | -------------------------------------------------------------------------------- /Android/tutorial_mobile_example/app/src/main/res/layout/activity_main.xml: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | 14 | 15 | 18 | 19 | 27 | 28 | 29 | 30 | 34 | 35 |