├── .gitignore ├── README.md ├── WhatsThis.iml ├── app ├── .gitignore ├── app.iml ├── build.gradle ├── proguard-rules.pro └── src │ ├── androidTest │ └── java │ │ └── com │ │ └── happen │ │ └── it │ │ └── make │ │ └── whatisit │ │ └── ApplicationTest.java │ └── main │ ├── AndroidManifest.xml │ ├── java │ ├── com │ │ └── happen │ │ │ └── it │ │ │ └── make │ │ │ └── whatisit │ │ │ ├── Constants.java │ │ │ ├── MxNetUtils.java │ │ │ ├── WhatsActivity.java │ │ │ └── WhatsApplication.java │ └── org │ │ └── dmlc │ │ └── mxnet │ │ ├── MxnetException.java │ │ └── Predictor.java │ ├── jniLibs │ └── armeabi │ │ └── libmxnet_predict.so │ └── res │ ├── drawable │ ├── round_add.png │ └── text_border.xml │ ├── layout │ └── activity_whats.xml │ ├── menu │ └── menu_whats.xml │ ├── mipmap-hdpi │ └── ic_launcher.png │ ├── mipmap-mdpi │ └── ic_launcher.png │ ├── mipmap-xhdpi │ └── ic_launcher.png │ ├── mipmap-xxhdpi │ └── ic_launcher.png │ ├── raw │ ├── mean.json │ ├── params │ ├── symbol.json │ └── synset.txt │ ├── values-w820dp │ └── dimens.xml │ └── values │ ├── dimens.xml │ ├── strings.xml │ └── styles.xml ├── build.gradle ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── settings.gradle /.gitignore: -------------------------------------------------------------------------------- 1 | # Built application files 2 | *.apk 3 | *.ap_ 4 | 5 | # Files for the Dalvik VM 6 | *.dex 7 | 8 | # Java class files 9 | *.class 10 | 11 | # Generated files 12 | bin/ 13 | gen/ 14 | 15 | # Gradle files 16 | .gradle/ 17 | build/ 18 | 19 | # Local configuration file (sdk path, etc) 20 | local.properties 21 | 22 | # Proguard folder generated by Eclipse 23 | proguard/ 24 | 25 | # Log Files 26 | *.log 27 | 28 | # Android Studio Navigation editor temp files 29 | .navigation/ 30 | 31 | # Android Studio captures folder 32 | captures/ 33 | 34 | .idea/ 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # WhatsThis 2 | 3 | "What is it?" -- asked Disco the talking budgie. 4 | 5 | This is an android example of using mxnet to classify pictures. 6 | ``` 7 | -main 8 | | 9 | ----java 10 | | | 11 | | ----com.happen.it.make.whatisit -- the UI module 12 | | | 13 | | ----org.dmlc.mxnet -- the java interface 14 | | 15 | ----jniLibs 16 | | | 17 | | -----armeabi -- the directory containing dynamic link lib for android 18 | | | 19 | | -------------- libmxnet_predict.so --- the prediction lib 20 | | 21 | ----res 22 | | 23 | -----raw 24 | | 25 | ------params 26 | | 27 | ------symbol.json 28 | | 29 | ------synset.txt 30 | | 31 | ------mean.json 32 | 33 | ``` 34 | 35 | The example in mxnet doesn't include the model and pre-compiled native library for repo-size consideration. 36 | 37 | To compile the android lib by yourself, have a look at mxnet/amalgamation. 38 | 39 | To download a complete example with precompiled lib and a simple model, clone https://github.com/Leliana/WhatsThis.git . 40 | 41 | NOTE: This example doesn't run with emulator unless you build a native lib for android emulator. 42 | -------------------------------------------------------------------------------- /WhatsThis.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /app/.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | -------------------------------------------------------------------------------- /app/app.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 8 | 9 | 10 | 11 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /app/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'com.android.application' 2 | 3 | android { 4 | compileSdkVersion 22 5 | buildToolsVersion "22.0.1" 6 | 7 | defaultConfig { 8 | applicationId "com.happen.it.make.whatsthis" 9 | minSdkVersion 21 10 | targetSdkVersion 22 11 | versionCode 1 12 | versionName "1.0" 13 | } 14 | buildTypes { 15 | release { 16 | minifyEnabled false 17 | proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' 18 | } 19 | } 20 | } 21 | 22 | dependencies { 23 | compile fileTree(dir: 'libs', include: ['*.jar', '*.so']) 24 | compile 'com.android.support:appcompat-v7:22.2.1' 25 | } 26 | -------------------------------------------------------------------------------- /app/proguard-rules.pro: -------------------------------------------------------------------------------- 1 | # Add project specific ProGuard rules here. 2 | # By default, the flags in this file are appended to flags specified 3 | # in /home/leliana/Android/Sdk/tools/proguard/proguard-android.txt 4 | # You can edit the include path and order by changing the proguardFiles 5 | # directive in build.gradle. 6 | # 7 | # For more details, see 8 | # http://developer.android.com/guide/developing/tools/proguard.html 9 | 10 | # Add any project specific keep options here: 11 | 12 | # If your project uses WebView with JS, uncomment the following 13 | # and specify the fully qualified class name to the JavaScript interface 14 | # class: 15 | #-keepclassmembers class fqcn.of.javascript.interface.for.webview { 16 | # public *; 17 | #} 18 | -------------------------------------------------------------------------------- /app/src/androidTest/java/com/happen/it/make/whatisit/ApplicationTest.java: -------------------------------------------------------------------------------- 1 | package com.happen.it.make.whatisit; 2 | 3 | import android.app.Application; 4 | import android.test.ApplicationTestCase; 5 | 6 | /** 7 | * Testing Fundamentals 8 | */ 9 | public class ApplicationTest extends ApplicationTestCase { 10 | public ApplicationTest() { 11 | super(Application.class); 12 | } 13 | } -------------------------------------------------------------------------------- /app/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 12 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /app/src/main/java/com/happen/it/make/whatisit/Constants.java: -------------------------------------------------------------------------------- 1 | package com.happen.it.make.whatisit; 2 | 3 | /** 4 | * Created by leliana on 8/5/15. 5 | */ 6 | public class Constants { 7 | public static final int SELECT_PHOTO_CODE = 1; 8 | public static final int CAPTURE_PHOTO_CODE = 2; 9 | } 10 | -------------------------------------------------------------------------------- /app/src/main/java/com/happen/it/make/whatisit/MxNetUtils.java: -------------------------------------------------------------------------------- 1 | package com.happen.it.make.whatisit; 2 | 3 | import android.graphics.Bitmap; 4 | 5 | import org.dmlc.mxnet.Predictor; 6 | 7 | import java.nio.ByteBuffer; 8 | 9 | /** 10 | * Created by leliana on 11/6/15. 11 | */ 12 | public class MxNetUtils { 13 | private static boolean libLoaded = false; 14 | private MxNetUtils() {} 15 | 16 | public static String identifyImage(final Bitmap bitmap) { 17 | ByteBuffer byteBuffer = ByteBuffer.allocate(bitmap.getByteCount()); 18 | bitmap.copyPixelsToBuffer(byteBuffer); 19 | byte[] bytes = byteBuffer.array(); 20 | float[] colors = new float[bytes.length / 4 * 3]; 21 | 22 | float mean_b = WhatsApplication.getMean().get("b"); 23 | float mean_g = WhatsApplication.getMean().get("g"); 24 | float mean_r = WhatsApplication.getMean().get("r"); 25 | for (int i = 0; i < bytes.length; i += 4) { 26 | int j = i / 4; 27 | colors[0 * 224 * 224 + j] = (float)(((int)(bytes[i + 0])) & 0xFF) - mean_r; 28 | colors[1 * 224 * 224 + j] = (float)(((int)(bytes[i + 1])) & 0xFF) - mean_g; 29 | colors[2 * 224 * 224 + j] = (float)(((int)(bytes[i + 2])) & 0xFF) - mean_b; 30 | } 31 | Predictor predictor = WhatsApplication.getPredictor(); 32 | predictor.forward("data", colors); 33 | final float[] result = predictor.getOutput(0); 34 | 35 | int index = 0; 36 | for (int i = 0; i < result.length; ++i) { 37 | if (result[index] < result[i]) index = i; 38 | } 39 | String tag = WhatsApplication.getName(index); 40 | String [] arr = tag.split(" ", 2); 41 | return arr[1]; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /app/src/main/java/com/happen/it/make/whatisit/WhatsActivity.java: -------------------------------------------------------------------------------- 1 | package com.happen.it.make.whatisit; 2 | 3 | import android.content.Context; 4 | import android.content.Intent; 5 | import android.content.SharedPreferences; 6 | import android.graphics.Bitmap; 7 | import android.graphics.BitmapFactory; 8 | import android.net.Uri; 9 | import android.os.AsyncTask; 10 | import android.os.Environment; 11 | import android.provider.MediaStore; 12 | import android.support.v7.app.AppCompatActivity; 13 | import android.os.Bundle; 14 | import android.view.Menu; 15 | import android.view.MenuItem; 16 | import android.view.View; 17 | import android.widget.Button; 18 | import android.widget.ImageView; 19 | import android.widget.TextView; 20 | 21 | import java.io.File; 22 | import java.io.FileNotFoundException; 23 | import java.io.IOException; 24 | import java.io.InputStream; 25 | import java.text.SimpleDateFormat; 26 | import java.util.Date; 27 | 28 | public class WhatsActivity extends AppCompatActivity { 29 | 30 | private TextView resultTextView; 31 | private ImageView inputImageView; 32 | private Bitmap bitmap; 33 | private Bitmap processedBitmap; 34 | private Button identifyButton; 35 | private SharedPreferences sharedPreferences; 36 | private String currentPhotoPath; 37 | private static final String PREF_USE_CAMERA_KEY = "USE_CAMERA"; 38 | 39 | @Override 40 | protected void onCreate(Bundle savedInstanceState) { 41 | super.onCreate(savedInstanceState); 42 | setContentView(R.layout.activity_whats); 43 | identifyButton = (Button)findViewById(R.id.identify_button); 44 | inputImageView = (ImageView)findViewById(R.id.tap_to_add_image); 45 | resultTextView = (TextView)findViewById(R.id.result_text); 46 | sharedPreferences = getSharedPreferences("Picture Pref", Context.MODE_PRIVATE); 47 | 48 | identifyButton.setOnClickListener(new View.OnClickListener() { 49 | @Override 50 | public void onClick(View v) { 51 | if (v != identifyButton) { 52 | return; 53 | } 54 | if (processedBitmap == null) { 55 | return; 56 | } 57 | 58 | new AsyncTask(){ 59 | @Override 60 | protected void onPreExecute() { 61 | resultTextView.setText("Calculating..."); 62 | } 63 | 64 | @Override 65 | protected String doInBackground(Bitmap... bitmaps) { 66 | synchronized (identifyButton) { 67 | String tag = MxNetUtils.identifyImage(bitmaps[0]); 68 | return tag; 69 | } 70 | } 71 | @Override 72 | protected void onPostExecute(String tag) { 73 | resultTextView.setText(tag); 74 | } 75 | }.execute(processedBitmap); 76 | } 77 | }); 78 | inputImageView.setOnClickListener(new View.OnClickListener() { 79 | @Override 80 | public void onClick(View v) { 81 | if (v != inputImageView) { 82 | return; 83 | } 84 | final boolean useCamera = sharedPreferences.getBoolean(PREF_USE_CAMERA_KEY, false); 85 | if (useCamera) { 86 | dispatchTakePictureIntent(); 87 | } else { 88 | final Intent photoPickerIntent = new Intent(Intent.ACTION_PICK); 89 | photoPickerIntent.setType("image/*"); 90 | startActivityForResult(photoPickerIntent, Constants.SELECT_PHOTO_CODE); 91 | } 92 | } 93 | }); 94 | } 95 | 96 | private void dispatchTakePictureIntent() { 97 | Intent takePictureIntent = new Intent(MediaStore.ACTION_IMAGE_CAPTURE); 98 | // Ensure that there's a camera activity to handle the intent 99 | if (takePictureIntent.resolveActivity(getPackageManager()) != null) { 100 | // Create the File where the photo should go 101 | File photoFile = null; 102 | try { 103 | photoFile = createImageFile(); 104 | } catch (IOException ex) { 105 | ex.printStackTrace(); 106 | return; 107 | } 108 | // Continue only if the File was successfully created 109 | if (photoFile != null) { 110 | takePictureIntent.putExtra(MediaStore.EXTRA_OUTPUT, 111 | Uri.fromFile(photoFile)); 112 | startActivityForResult(takePictureIntent, Constants.CAPTURE_PHOTO_CODE); 113 | } 114 | } 115 | } 116 | 117 | private File createImageFile() throws IOException { 118 | // Create an image file name 119 | String timeStamp = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()); 120 | String imageFileName = "JPEG_" + timeStamp + "_"; 121 | File storageDir = Environment.getExternalStoragePublicDirectory( 122 | Environment.DIRECTORY_PICTURES); 123 | File image = File.createTempFile( 124 | imageFileName, /* prefix */ 125 | ".jpg", /* suffix */ 126 | storageDir /* directory */ 127 | ); 128 | 129 | // Save a file: path for use with ACTION_VIEW intents 130 | currentPhotoPath = image.getAbsolutePath(); 131 | return image; 132 | } 133 | 134 | @Override 135 | protected void onResume() { 136 | super.onResume(); 137 | if (processedBitmap != null) { 138 | inputImageView.setImageBitmap(processedBitmap); 139 | } 140 | } 141 | 142 | @Override 143 | public boolean onCreateOptionsMenu(Menu menu) { 144 | // Inflate the menu; this adds items to the action bar if it is present. 145 | getMenuInflater().inflate(R.menu.menu_whats, menu); 146 | return true; 147 | } 148 | 149 | @Override 150 | public boolean onOptionsItemSelected(MenuItem item) { 151 | // Handle action bar item clicks here. The action bar will 152 | // automatically handle clicks on the Home/Up button, so long 153 | // as you specify a parent activity in AndroidManifest.xml. 154 | int id = item.getItemId(); 155 | 156 | //noinspection SimplifiableIfStatement 157 | if (id == R.id.action_use_camera) { 158 | sharedPreferences.edit().putBoolean(PREF_USE_CAMERA_KEY, true).apply(); 159 | return true; 160 | } else if (id == R.id.action_use_gallery) { 161 | sharedPreferences.edit().putBoolean(PREF_USE_CAMERA_KEY, false).apply(); 162 | return true; 163 | } 164 | 165 | return super.onOptionsItemSelected(item); 166 | } 167 | 168 | @Override 169 | protected void onActivityResult(int requestCode, int resultCode, Intent imageReturnedIntent) { 170 | super.onActivityResult(requestCode, resultCode, imageReturnedIntent); 171 | 172 | switch(requestCode) { 173 | case Constants.SELECT_PHOTO_CODE: 174 | if(resultCode == RESULT_OK){ 175 | try { 176 | final Uri imageUri = imageReturnedIntent.getData(); 177 | final InputStream imageStream = getContentResolver().openInputStream(imageUri); 178 | bitmap = BitmapFactory.decodeStream(imageStream); 179 | processedBitmap = processBitmap(bitmap); 180 | inputImageView.setImageBitmap(processedBitmap); 181 | } catch (FileNotFoundException e) { 182 | e.printStackTrace(); 183 | } 184 | 185 | } 186 | break; 187 | case Constants.CAPTURE_PHOTO_CODE: 188 | if (resultCode == RESULT_OK) { 189 | bitmap = BitmapFactory.decodeFile(currentPhotoPath); 190 | processedBitmap = processBitmap(bitmap); 191 | inputImageView.setImageBitmap(bitmap); 192 | } 193 | break; 194 | } 195 | } 196 | 197 | static final int SHORTER_SIDE = 256; 198 | static final int DESIRED_SIDE = 224; 199 | 200 | private static Bitmap processBitmap(final Bitmap origin) { 201 | //TODO: error handling 202 | final int originWidth = origin.getWidth(); 203 | final int originHeight = origin.getHeight(); 204 | int height = SHORTER_SIDE; 205 | int width = SHORTER_SIDE; 206 | if (originWidth < originHeight) { 207 | height = (int)((float)originHeight / originWidth * width); 208 | } else { 209 | width = (int)((float)originWidth / originHeight * height); 210 | } 211 | final Bitmap scaled = Bitmap.createScaledBitmap(origin, width, height, false); 212 | int y = (height - DESIRED_SIDE) / 2; 213 | int x = (width - DESIRED_SIDE) / 2; 214 | return Bitmap.createBitmap(scaled, x, y, DESIRED_SIDE, DESIRED_SIDE); 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /app/src/main/java/com/happen/it/make/whatisit/WhatsApplication.java: -------------------------------------------------------------------------------- 1 | package com.happen.it.make.whatisit; 2 | 3 | import android.app.Application; 4 | import android.content.Context; 5 | 6 | import org.dmlc.mxnet.Predictor; 7 | import org.json.JSONException; 8 | import org.json.JSONObject; 9 | 10 | import java.io.BufferedReader; 11 | import java.io.ByteArrayOutputStream; 12 | import java.io.IOException; 13 | import java.io.InputStream; 14 | import java.io.InputStreamReader; 15 | import java.util.ArrayList; 16 | import java.util.HashMap; 17 | import java.util.List; 18 | import java.util.Map; 19 | 20 | /** 21 | * Created by leliana on 8/5/15. 22 | */ 23 | public class WhatsApplication extends Application{ 24 | private static Predictor predictor; 25 | public static Predictor getPredictor() {return predictor;} 26 | private static List dict; 27 | private static Map mean; 28 | public static String getName(int i) { 29 | if (i >= dict.size()) { 30 | return "Shit"; 31 | } 32 | return dict.get(i); 33 | } 34 | public static Map getMean() { 35 | return mean; 36 | } 37 | @Override 38 | public void onCreate() { 39 | super.onCreate(); 40 | 41 | final byte[] symbol = readRawFile(this, R.raw.symbol); 42 | final byte[] params = readRawFile(this, R.raw.params); 43 | final Predictor.Device device = new Predictor.Device(Predictor.Device.Type.CPU, 0); 44 | final int[] shape = {1, 3, 224, 224}; 45 | final String key = "data"; 46 | final Predictor.InputNode node = new Predictor.InputNode(key, shape); 47 | 48 | predictor = new Predictor(symbol, params, device, new Predictor.InputNode[]{node}); 49 | dict = readRawTextFile(this, R.raw.synset); 50 | try { 51 | final StringBuilder sb = new StringBuilder(); 52 | final List lines = readRawTextFile(this, R.raw.mean); 53 | for (final String line : lines) { 54 | sb.append(line); 55 | } 56 | final JSONObject meanJson = new JSONObject(sb.toString()); 57 | mean = new HashMap<>(); 58 | mean.put("b", (float) meanJson.optDouble("b")); 59 | mean.put("g", (float) meanJson.optDouble("g")); 60 | mean.put("r", (float) meanJson.optDouble("r")); 61 | } catch (JSONException e) { 62 | e.printStackTrace(); 63 | } 64 | } 65 | 66 | public static byte[] readRawFile(Context ctx, int resId) 67 | { 68 | ByteArrayOutputStream outputStream=new ByteArrayOutputStream(); 69 | int size = 0; 70 | byte[] buffer = new byte[1024]; 71 | try (InputStream ins = ctx.getResources().openRawResource(resId)) { 72 | while((size=ins.read(buffer,0,1024))>=0){ 73 | outputStream.write(buffer,0,size); 74 | } 75 | } catch (IOException e) { 76 | e.printStackTrace(); 77 | } 78 | return outputStream.toByteArray(); 79 | } 80 | 81 | public static List readRawTextFile(Context ctx, int resId) 82 | { 83 | List result = new ArrayList<>(); 84 | InputStream inputStream = ctx.getResources().openRawResource(resId); 85 | 86 | InputStreamReader inputreader = new InputStreamReader(inputStream); 87 | BufferedReader buffreader = new BufferedReader(inputreader); 88 | String line; 89 | 90 | try { 91 | while (( line = buffreader.readLine()) != null) { 92 | result.add(line); 93 | } 94 | } catch (IOException e) { 95 | return null; 96 | } 97 | return result; 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /app/src/main/java/org/dmlc/mxnet/MxnetException.java: -------------------------------------------------------------------------------- 1 | package org.dmlc.mxnet; 2 | 3 | public class MxnetException extends Exception { 4 | public MxnetException(){} 5 | public MxnetException(String txt) { 6 | super(txt); 7 | } 8 | } 9 | 10 | -------------------------------------------------------------------------------- /app/src/main/java/org/dmlc/mxnet/Predictor.java: -------------------------------------------------------------------------------- 1 | package org.dmlc.mxnet; 2 | 3 | public class Predictor { 4 | static { 5 | System.loadLibrary("mxnet_predict"); 6 | } 7 | 8 | public static class InputNode { 9 | String key; 10 | int[] shape; 11 | public InputNode(String key, int[] shape) { 12 | this.key = key; 13 | this.shape = shape; 14 | } 15 | } 16 | 17 | public static class Device { 18 | public enum Type { 19 | CPU, GPU, CPU_PINNED 20 | } 21 | 22 | public Device(Type t, int i) { 23 | this.type = t; 24 | this.id = i; 25 | } 26 | 27 | Type type; 28 | int id; 29 | int ctype() { 30 | return this.type == Type.CPU? 1: this.type == Type.GPU? 2: 3; 31 | } 32 | } 33 | 34 | private long handle = 0; 35 | 36 | public Predictor(byte[] symbol, byte[] params, Device dev, InputNode[] input) { 37 | String[] keys = new String[input.length]; 38 | int[][] shapes = new int[input.length][]; 39 | for (int i=0; i 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /app/src/main/res/layout/activity_whats.xml: -------------------------------------------------------------------------------- 1 | 10 | 11 | 18 | 19 |