├── .gitignore ├── .idea ├── caches │ └── build_file_checksums.ser ├── codeStyles │ └── Project.xml ├── compiler.xml ├── encodings.xml ├── gradle.xml ├── misc.xml ├── runConfigurations.xml └── vcs.xml ├── .travis.yml ├── LICENSE ├── README.md ├── app ├── .gitignore ├── build.gradle ├── proguard-rules.pro └── src │ └── main │ ├── AndroidManifest.xml │ ├── assets │ └── deeplabv3_257_mv_gpu.tflite │ ├── java │ └── pp │ │ └── imagesegmenter │ │ ├── AutoFitTextureView.java │ │ ├── CameraActivity.java │ │ ├── CameraConnectionFragment.java │ │ ├── Deeplab.java │ │ ├── MainActivity.java │ │ ├── OverlayView.java │ │ ├── env │ │ ├── BorderedText.java │ │ ├── ImageUtils.java │ │ ├── Logger.java │ │ └── Size.java │ │ └── tracking │ │ ├── MultiBoxTracker.java │ │ └── ObjectTracker.java │ ├── jni │ ├── CMakeLists.txt │ ├── imageutils_jni.cc │ ├── object_tracking │ │ ├── config.h │ │ ├── flow_cache.h │ │ ├── frame_pair.cc │ │ ├── frame_pair.h │ │ ├── geom.h │ │ ├── gl_utils.h │ │ ├── image-inl.h │ │ ├── image.h │ │ ├── image_data.h │ │ ├── image_neon.cc │ │ ├── image_utils.h │ │ ├── integral_image.h │ │ ├── jni_utils.h │ │ ├── keypoint.h │ │ ├── keypoint_detector.cc │ │ ├── keypoint_detector.h │ │ ├── logging.cc │ │ ├── logging.h │ │ ├── object_detector.cc │ │ ├── object_detector.h │ │ ├── object_model.h │ │ ├── object_tracker.cc │ │ ├── object_tracker.h │ │ ├── object_tracker_jni.cc │ │ ├── optical_flow.cc │ │ ├── optical_flow.h │ │ ├── sprite.h │ │ ├── time_log.cc │ │ ├── time_log.h │ │ ├── tracked_object.cc │ │ ├── tracked_object.h │ │ ├── utils.h │ │ └── utils_neon.cc │ ├── rgb2yuv.cc │ ├── rgb2yuv.h │ ├── yuv2rgb.cc │ └── yuv2rgb.h │ └── res │ ├── drawable-v24 │ └── ic_launcher_foreground.xml │ ├── drawable │ └── ic_launcher_background.xml │ ├── layout │ ├── activity_camera.xml │ └── camera_connection_fragment_tracking.xml │ ├── mipmap-anydpi-v26 │ ├── ic_launcher.xml │ └── ic_launcher_round.xml │ ├── mipmap-hdpi │ ├── ic_launcher.png │ └── ic_launcher_round.png │ ├── mipmap-mdpi │ ├── ic_launcher.png │ └── ic_launcher_round.png │ ├── mipmap-xhdpi │ ├── ic_launcher.png │ └── ic_launcher_round.png │ ├── mipmap-xxhdpi │ ├── ic_launcher.png │ └── ic_launcher_round.png │ ├── mipmap-xxxhdpi │ ├── ic_launcher.png │ └── ic_launcher_round.png │ └── values │ ├── strings.xml │ └── styles.xml ├── build.gradle ├── demo.gif ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── settings.gradle /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | .gradle 3 | /local.properties 4 | /.idea/caches 5 | /.idea/caches/build_file_checksums.ser 6 | /.idea/dictionaries 7 | /.idea/libraries 8 | /.idea/assetWizardSettings.xml 9 | /.idea/gradle.xml 10 | /.idea/modules.xml 11 | /.idea/tasks.xml 12 | /.idea/workspace.xml 13 | .DS_Store 14 | /build 15 | /captures 16 | .externalNativeBuild 17 | -------------------------------------------------------------------------------- /.idea/caches/build_file_checksums.ser: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pillarpond/image-segmenter-android/36e1dd245dca25c8f44e9e217ca280a3a1bf9f1c/.idea/caches/build_file_checksums.ser -------------------------------------------------------------------------------- /.idea/codeStyles/Project.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 15 | 16 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/gradle.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 14 | 15 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 14 | -------------------------------------------------------------------------------- /.idea/runConfigurations.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 11 | 12 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: android 2 | 3 | jdk: oraclejdk8 4 | 5 | android: 6 | components: 7 | - tools 8 | - platform-tools 9 | - tools 10 | - build-tools-28.0.3 11 | - android-28 12 | - extra-android-m2repository 13 | - extra-google-m2repository 14 | install: 15 | - echo y | sdkmanager "ndk-bundle" 16 | - echo y | sdkmanager "cmake;3.10.2.4988404" 17 | - echo y | sdkmanager "lldb;3.1" 18 | # - sdkmanager --update 19 | before_script: 20 | - export ANDROID_NDK_HOME=$ANDROID_HOME/ndk-bundle 21 | 22 | script: 23 | - "./gradlew assembleDebug" -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Realtime Image Segmenter 2 | 3 | [![Build Status](https://travis-ci.org/pillarpond/image-segmenter-android.svg?branch=master)](https://travis-ci.org/pillarpond/image-segmenter-android) 4 | 5 | This sample demonstrates realtime image segmentation on Android. The project is based on the [Deeplab](http://liangchiehchen.com/projects/DeepLab.html) 6 | 7 | ## Model 8 | Tensorflow provide deeplab models pretrained several datasets. In this project, I used [mobilenetv2_coco_voc_trainaug](http://download.tensorflow.org/models/deeplabv3_mnv2_pascal_train_aug_2018_01_29.tar.gz) 9 | 10 | ## Inspiration 11 | The project is heavily inspired by 12 | * [Deeplab](https://github.com/tensorflow/models/tree/master/research/deeplab) 13 | * [DeepLab on Android](https://github.com/dailystudio/ml/tree/master/deeplab) 14 | * [Tensorflow Android Camera Demo](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android) 15 | 16 | ## Screenshots 17 | ![demo](./demo.gif) 18 | 19 | ## Pre-trained model 20 | [DeepLab segmentation](https://ai.googleblog.com/2018/03/semantic-image-segmentation-with.html) (257x257) [[download]](https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/deeplabv3_257_mv_gpu.tflite) 21 | 22 | ## License 23 | [Apache License 2.0](./LICENSE) 24 | -------------------------------------------------------------------------------- /app/.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | -------------------------------------------------------------------------------- /app/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'com.android.application' 2 | 3 | android { 4 | compileSdkVersion 28 5 | defaultConfig { 6 | applicationId "pp.imagesegmenter" 7 | minSdkVersion 25 8 | targetSdkVersion 28 9 | versionCode 1 10 | versionName "1.0" 11 | } 12 | buildTypes { 13 | release { 14 | minifyEnabled false 15 | proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' 16 | } 17 | } 18 | externalNativeBuild { 19 | cmake { 20 | path 'src/main/jni/CMakeLists.txt' 21 | } 22 | } 23 | aaptOptions { 24 | noCompress "tflite" 25 | } 26 | compileOptions { 27 | sourceCompatibility JavaVersion.VERSION_1_8 28 | targetCompatibility JavaVersion.VERSION_1_8 29 | } 30 | } 31 | 32 | dependencies { 33 | implementation 'androidx.annotation:annotation:1.1.0' 34 | implementation 'androidx.appcompat:appcompat:1.0.2' 35 | implementation 'com.google.android.material:material:1.1.0-alpha07' 36 | implementation 'org.tensorflow:tensorflow-lite:0.0.0-nightly' 37 | implementation 'org.tensorflow:tensorflow-lite-gpu:0.0.0-nightly' 38 | } 39 | -------------------------------------------------------------------------------- /app/proguard-rules.pro: -------------------------------------------------------------------------------- 1 | # Add project specific ProGuard rules here. 2 | # You can control the set of applied configuration files using the 3 | # proguardFiles setting in build.gradle. 4 | # 5 | # For more details, see 6 | # http://developer.android.com/guide/developing/tools/proguard.html 7 | 8 | # If your project uses WebView with JS, uncomment the following 9 | # and specify the fully qualified class name to the JavaScript interface 10 | # class: 11 | #-keepclassmembers class fqcn.of.javascript.interface.for.webview { 12 | # public *; 13 | #} 14 | 15 | # Uncomment this to preserve the line number information for 16 | # debugging stack traces. 17 | #-keepattributes SourceFile,LineNumberTable 18 | 19 | # If you keep the line number information, uncomment this to 20 | # hide the original source file name. 21 | #-renamesourcefileattribute SourceFile 22 | -------------------------------------------------------------------------------- /app/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 18 | 19 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /app/src/main/assets/deeplabv3_257_mv_gpu.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pillarpond/image-segmenter-android/36e1dd245dca25c8f44e9e217ca280a3a1bf9f1c/app/src/main/assets/deeplabv3_257_mv_gpu.tflite -------------------------------------------------------------------------------- /app/src/main/java/pp/imagesegmenter/AutoFitTextureView.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 The TensorFlow Authors. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package pp.imagesegmenter; 18 | 19 | import android.content.Context; 20 | import android.util.AttributeSet; 21 | import android.view.TextureView; 22 | 23 | /** 24 | * A {@link TextureView} that can be adjusted to a specified aspect ratio. 25 | */ 26 | public class AutoFitTextureView extends TextureView { 27 | private int ratioWidth = 0; 28 | private int ratioHeight = 0; 29 | 30 | public AutoFitTextureView(final Context context) { 31 | this(context, null); 32 | } 33 | 34 | public AutoFitTextureView(final Context context, final AttributeSet attrs) { 35 | this(context, attrs, 0); 36 | } 37 | 38 | public AutoFitTextureView(final Context context, final AttributeSet attrs, final int defStyle) { 39 | super(context, attrs, defStyle); 40 | } 41 | 42 | /** 43 | * Sets the aspect ratio for this view. The size of the view will be measured based on the ratio 44 | * calculated from the parameters. Note that the actual sizes of parameters don't matter, that 45 | * is, calling setAspectRatio(2, 3) and setAspectRatio(4, 6) make the same result. 46 | * 47 | * @param width Relative horizontal size 48 | * @param height Relative vertical size 49 | */ 50 | public void setAspectRatio(final int width, final int height) { 51 | if (width < 0 || height < 0) { 52 | throw new IllegalArgumentException("Size cannot be negative."); 53 | } 54 | ratioWidth = width; 55 | ratioHeight = height; 56 | requestLayout(); 57 | } 58 | 59 | @Override 60 | protected void onMeasure(final int widthMeasureSpec, final int heightMeasureSpec) { 61 | super.onMeasure(widthMeasureSpec, heightMeasureSpec); 62 | final int width = MeasureSpec.getSize(widthMeasureSpec); 63 | final int height = MeasureSpec.getSize(heightMeasureSpec); 64 | if (0 == ratioWidth || 0 == ratioHeight) { 65 | setMeasuredDimension(width, height); 66 | } else { 67 | if (width < height * ratioWidth / ratioHeight) { 68 | setMeasuredDimension(width, width * ratioHeight / ratioWidth); 69 | } else { 70 | setMeasuredDimension(height * ratioWidth / ratioHeight, height); 71 | } 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /app/src/main/java/pp/imagesegmenter/MainActivity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 The TensorFlow Authors. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package pp.imagesegmenter; 18 | 19 | import android.graphics.Bitmap; 20 | import android.graphics.Bitmap.Config; 21 | import android.graphics.Canvas; 22 | import android.graphics.Color; 23 | import android.graphics.Matrix; 24 | import android.graphics.Paint; 25 | import android.graphics.Typeface; 26 | import android.media.ImageReader.OnImageAvailableListener; 27 | import android.os.SystemClock; 28 | import android.util.Log; 29 | import android.util.Size; 30 | import android.util.TypedValue; 31 | import android.widget.FrameLayout; 32 | 33 | import com.google.android.material.snackbar.Snackbar; 34 | 35 | import pp.imagesegmenter.env.BorderedText; 36 | import pp.imagesegmenter.env.ImageUtils; 37 | import pp.imagesegmenter.env.Logger; 38 | import pp.imagesegmenter.tracking.MultiBoxTracker; 39 | 40 | import java.util.Collections; 41 | import java.util.List; 42 | import java.util.Vector; 43 | 44 | /** 45 | * An activity that uses a Deeplab and ObjectTracker to segment and then track objects. 46 | */ 47 | public class MainActivity extends CameraActivity implements OnImageAvailableListener { 48 | private static final Logger LOGGER = new Logger(); 49 | 50 | private static final int CROP_SIZE = 257; 51 | 52 | private static final Size DESIRED_PREVIEW_SIZE = new Size(640, 480); 53 | 54 | private static final boolean SAVE_PREVIEW_BITMAP = false; 55 | private static final float TEXT_SIZE_DIP = 10; 56 | 57 | private Integer sensorOrientation; 58 | 59 | private Deeplab deeplab; 60 | 61 | private long lastProcessingTimeMs; 62 | private Bitmap rgbFrameBitmap = null; 63 | private Bitmap croppedBitmap = null; 64 | private Bitmap cropCopyBitmap = null; 65 | 66 | private boolean computingDetection = false; 67 | 68 | private long timestamp = 0; 69 | 70 | private Matrix frameToCropTransform; 71 | private Matrix cropToFrameTransform; 72 | 73 | private MultiBoxTracker tracker; 74 | 75 | private byte[] luminanceCopy; 76 | 77 | private BorderedText borderedText; 78 | 79 | private Snackbar initSnackbar; 80 | 81 | private boolean initialized = false; 82 | 83 | @Override 84 | public void onPreviewSizeChosen(final Size size, final int rotation) { 85 | sensorOrientation = rotation - getScreenOrientation(); 86 | LOGGER.i("Camera orientation relative to screen canvas: %d", sensorOrientation); 87 | 88 | FrameLayout container = findViewById(R.id.container); 89 | initSnackbar = Snackbar.make(container, "Initializing...", Snackbar.LENGTH_INDEFINITE); 90 | 91 | init(); 92 | 93 | final float textSizePx = 94 | TypedValue.applyDimension( 95 | TypedValue.COMPLEX_UNIT_DIP, TEXT_SIZE_DIP, getResources().getDisplayMetrics()); 96 | borderedText = new BorderedText(textSizePx); 97 | borderedText.setTypeface(Typeface.MONOSPACE); 98 | 99 | tracker = new MultiBoxTracker(this); 100 | 101 | previewWidth = size.getWidth(); 102 | previewHeight = size.getHeight(); 103 | 104 | LOGGER.i("Initializing at size %dx%d", previewWidth, previewHeight); 105 | rgbFrameBitmap = Bitmap.createBitmap(previewWidth, previewHeight, Config.ARGB_8888); 106 | croppedBitmap = Bitmap.createBitmap(CROP_SIZE, CROP_SIZE, Config.ARGB_8888); 107 | 108 | frameToCropTransform = 109 | ImageUtils.getTransformationMatrix( 110 | previewWidth, previewHeight, 111 | CROP_SIZE, CROP_SIZE, 112 | sensorOrientation, false); 113 | 114 | cropToFrameTransform = new Matrix(); 115 | frameToCropTransform.invert(cropToFrameTransform); 116 | 117 | trackingOverlay = findViewById(R.id.tracking_overlay); 118 | trackingOverlay.addCallback( 119 | canvas -> { 120 | tracker.draw(canvas); 121 | if (isDebug()) { 122 | tracker.drawDebug(canvas); 123 | } 124 | }); 125 | 126 | addCallback( 127 | canvas -> { 128 | if (!isDebug()) { 129 | return; 130 | } 131 | final Bitmap copy = cropCopyBitmap; 132 | if (copy == null) { 133 | return; 134 | } 135 | 136 | final int backgroundColor = Color.argb(100, 0, 0, 0); 137 | canvas.drawColor(backgroundColor); 138 | 139 | final Matrix matrix = new Matrix(); 140 | final float scaleFactor = 2; 141 | matrix.postScale(scaleFactor, scaleFactor); 142 | matrix.postTranslate( 143 | canvas.getWidth() - copy.getWidth() * scaleFactor, 144 | canvas.getHeight() - copy.getHeight() * scaleFactor); 145 | canvas.drawBitmap(copy, matrix, new Paint()); 146 | 147 | final Vector lines = new Vector(); 148 | lines.add("Frame: " + previewWidth + "x" + previewHeight); 149 | lines.add("Crop: " + copy.getWidth() + "x" + copy.getHeight()); 150 | lines.add("View: " + canvas.getWidth() + "x" + canvas.getHeight()); 151 | lines.add("Rotation: " + sensorOrientation); 152 | lines.add("Inference time: " + lastProcessingTimeMs + "ms"); 153 | 154 | borderedText.drawLines(canvas, 10, canvas.getHeight() - 10, lines); 155 | }); 156 | } 157 | 158 | OverlayView trackingOverlay; 159 | 160 | void init() { 161 | runInBackground(() -> { 162 | runOnUiThread(()->initSnackbar.show()); 163 | try { 164 | deeplab = Deeplab.create(getAssets(), CROP_SIZE, CROP_SIZE, sensorOrientation); 165 | } catch (Exception e) { 166 | LOGGER.e("Exception initializing classifier!", e); 167 | finish(); 168 | } 169 | runOnUiThread(()->initSnackbar.dismiss()); 170 | initialized = true; 171 | }); 172 | } 173 | 174 | @Override 175 | protected void processImage() { 176 | ++timestamp; 177 | final long currTimestamp = timestamp; 178 | byte[] originalLuminance = getLuminance(); 179 | tracker.onFrame( 180 | previewWidth, 181 | previewHeight, 182 | getLuminanceStride(), 183 | sensorOrientation, 184 | originalLuminance, 185 | timestamp); 186 | trackingOverlay.postInvalidate(); 187 | 188 | // No mutex needed as this method is not reentrant. 189 | if (computingDetection || !initialized) { 190 | readyForNextImage(); 191 | return; 192 | } 193 | computingDetection = true; 194 | LOGGER.i("Preparing image " + currTimestamp + " for detection in bg thread."); 195 | 196 | rgbFrameBitmap.setPixels(getRgbBytes(), 0, previewWidth, 0, 0, previewWidth, previewHeight); 197 | 198 | if (luminanceCopy == null) { 199 | luminanceCopy = new byte[originalLuminance.length]; 200 | } 201 | System.arraycopy(originalLuminance, 0, luminanceCopy, 0, originalLuminance.length); 202 | readyForNextImage(); 203 | 204 | final Canvas canvas = new Canvas(croppedBitmap); 205 | canvas.drawBitmap(rgbFrameBitmap, frameToCropTransform, null); 206 | // For examining the actual TF input. 207 | if (SAVE_PREVIEW_BITMAP) { 208 | ImageUtils.saveBitmap(croppedBitmap); 209 | } 210 | 211 | runInBackground( 212 | () -> { 213 | LOGGER.i("Running detection on image " + currTimestamp); 214 | final long startTime = SystemClock.uptimeMillis(); 215 | 216 | cropCopyBitmap = Bitmap.createBitmap(croppedBitmap); 217 | List mappedRecognitions = 218 | deeplab.segment(croppedBitmap,cropToFrameTransform); 219 | 220 | lastProcessingTimeMs = SystemClock.uptimeMillis() - startTime; 221 | tracker.trackResults(mappedRecognitions, luminanceCopy, currTimestamp); 222 | trackingOverlay.postInvalidate(); 223 | 224 | requestRender(); 225 | computingDetection = false; 226 | }); 227 | } 228 | 229 | @Override 230 | protected int getLayoutId() { 231 | return R.layout.camera_connection_fragment_tracking; 232 | } 233 | 234 | @Override 235 | protected Size getDesiredPreviewFrameSize() { 236 | return DESIRED_PREVIEW_SIZE; 237 | } 238 | } 239 | -------------------------------------------------------------------------------- /app/src/main/java/pp/imagesegmenter/OverlayView.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | package pp.imagesegmenter; 17 | 18 | import android.content.Context; 19 | import android.graphics.Canvas; 20 | import android.util.AttributeSet; 21 | import android.view.View; 22 | import java.util.LinkedList; 23 | import java.util.List; 24 | 25 | /** 26 | * A simple View providing a render callback to other classes. 27 | */ 28 | public class OverlayView extends View { 29 | private final List callbacks = new LinkedList(); 30 | 31 | public OverlayView(final Context context, final AttributeSet attrs) { 32 | super(context, attrs); 33 | } 34 | 35 | /** 36 | * Interface defining the callback for client classes. 37 | */ 38 | public interface DrawCallback { 39 | public void drawCallback(final Canvas canvas); 40 | } 41 | 42 | public void addCallback(final DrawCallback callback) { 43 | callbacks.add(callback); 44 | } 45 | 46 | @Override 47 | public synchronized void draw(final Canvas canvas) { 48 | for (final DrawCallback callback : callbacks) { 49 | callback.drawCallback(canvas); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /app/src/main/java/pp/imagesegmenter/env/BorderedText.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | package pp.imagesegmenter.env; 17 | 18 | import android.graphics.Canvas; 19 | import android.graphics.Color; 20 | import android.graphics.Paint; 21 | import android.graphics.Paint.Align; 22 | import android.graphics.Paint.Style; 23 | import android.graphics.Rect; 24 | import android.graphics.Typeface; 25 | 26 | import java.util.Vector; 27 | 28 | /** 29 | * A class that encapsulates the tedious bits of rendering legible, bordered text onto a canvas. 30 | */ 31 | public class BorderedText { 32 | private final Paint interiorPaint; 33 | private final Paint exteriorPaint; 34 | 35 | private final float textSize; 36 | 37 | /** 38 | * Creates a left-aligned bordered text object with a white interior, and a black exterior with 39 | * the specified text size. 40 | * 41 | * @param textSize text size in pixels 42 | */ 43 | public BorderedText(final float textSize) { 44 | this(Color.WHITE, Color.BLACK, textSize); 45 | } 46 | 47 | /** 48 | * Create a bordered text object with the specified interior and exterior colors, text size and 49 | * alignment. 50 | * 51 | * @param interiorColor the interior text color 52 | * @param exteriorColor the exterior text color 53 | * @param textSize text size in pixels 54 | */ 55 | public BorderedText(final int interiorColor, final int exteriorColor, final float textSize) { 56 | interiorPaint = new Paint(); 57 | interiorPaint.setTextSize(textSize); 58 | interiorPaint.setColor(interiorColor); 59 | interiorPaint.setStyle(Style.FILL); 60 | interiorPaint.setAntiAlias(false); 61 | interiorPaint.setAlpha(255); 62 | 63 | exteriorPaint = new Paint(); 64 | exteriorPaint.setTextSize(textSize); 65 | exteriorPaint.setColor(exteriorColor); 66 | exteriorPaint.setStyle(Style.FILL_AND_STROKE); 67 | exteriorPaint.setStrokeWidth(textSize / 8); 68 | exteriorPaint.setAntiAlias(false); 69 | exteriorPaint.setAlpha(255); 70 | 71 | this.textSize = textSize; 72 | } 73 | 74 | public void setTypeface(Typeface typeface) { 75 | interiorPaint.setTypeface(typeface); 76 | exteriorPaint.setTypeface(typeface); 77 | } 78 | 79 | public void drawText(final Canvas canvas, final float posX, final float posY, final String text) { 80 | canvas.drawText(text, posX, posY, exteriorPaint); 81 | canvas.drawText(text, posX, posY, interiorPaint); 82 | } 83 | 84 | public void drawLines(Canvas canvas, final float posX, final float posY, Vector lines) { 85 | int lineNum = 0; 86 | for (final String line : lines) { 87 | drawText(canvas, posX, posY - getTextSize() * (lines.size() - lineNum - 1), line); 88 | ++lineNum; 89 | } 90 | } 91 | 92 | public void setInteriorColor(final int color) { 93 | interiorPaint.setColor(color); 94 | } 95 | 96 | public void setExteriorColor(final int color) { 97 | exteriorPaint.setColor(color); 98 | } 99 | 100 | public float getTextSize() { 101 | return textSize; 102 | } 103 | 104 | public void setAlpha(final int alpha) { 105 | interiorPaint.setAlpha(alpha); 106 | exteriorPaint.setAlpha(alpha); 107 | } 108 | 109 | public void getTextBounds( 110 | final String line, final int index, final int count, final Rect lineBounds) { 111 | interiorPaint.getTextBounds(line, index, count, lineBounds); 112 | } 113 | 114 | public void setTextAlign(final Align align) { 115 | interiorPaint.setTextAlign(align); 116 | exteriorPaint.setTextAlign(align); 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /app/src/main/java/pp/imagesegmenter/env/Logger.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | package pp.imagesegmenter.env; 17 | 18 | import android.util.Log; 19 | 20 | import java.util.HashSet; 21 | import java.util.Set; 22 | 23 | /** 24 | * Wrapper for the platform log function, allows convenient message prefixing and log disabling. 25 | */ 26 | public final class Logger { 27 | private static final String DEFAULT_TAG = "imagesegmenter"; 28 | private static final int DEFAULT_MIN_LOG_LEVEL = Log.DEBUG; 29 | 30 | // Classes to be ignored when examining the stack trace 31 | private static final Set IGNORED_CLASS_NAMES; 32 | 33 | static { 34 | IGNORED_CLASS_NAMES = new HashSet<>(3); 35 | IGNORED_CLASS_NAMES.add("dalvik.system.VMStack"); 36 | IGNORED_CLASS_NAMES.add("java.lang.Thread"); 37 | IGNORED_CLASS_NAMES.add(Logger.class.getCanonicalName()); 38 | } 39 | 40 | private final String tag; 41 | private final String messagePrefix; 42 | private int minLogLevel = DEFAULT_MIN_LOG_LEVEL; 43 | 44 | /** 45 | * Creates a Logger using the class name as the message prefix. 46 | * 47 | * @param clazz the simple name of this class is used as the message prefix. 48 | */ 49 | public Logger(final Class clazz) { 50 | this(clazz.getSimpleName()); 51 | } 52 | 53 | /** 54 | * Creates a Logger using the specified message prefix. 55 | * 56 | * @param messagePrefix is prepended to the text of every message. 57 | */ 58 | public Logger(final String messagePrefix) { 59 | this(DEFAULT_TAG, messagePrefix); 60 | } 61 | 62 | /** 63 | * Creates a Logger with a custom tag and a custom message prefix. If the message prefix 64 | * is set to 
null
, the caller's class name is used as the prefix. 65 | * 66 | * @param tag identifies the source of a log message. 67 | * @param messagePrefix prepended to every message if non-null. If null, the name of the caller is 68 | * being used 69 | */ 70 | public Logger(final String tag, final String messagePrefix) { 71 | this.tag = tag; 72 | final String prefix = messagePrefix == null ? getCallerSimpleName() : messagePrefix; 73 | this.messagePrefix = (prefix.length() > 0) ? prefix + ": " : prefix; 74 | } 75 | 76 | /** 77 | * Creates a Logger using the caller's class name as the message prefix. 78 | */ 79 | public Logger() { 80 | this(DEFAULT_TAG, null); 81 | } 82 | 83 | /** 84 | * Creates a Logger using the caller's class name as the message prefix. 85 | */ 86 | public Logger(final int minLogLevel) { 87 | this(DEFAULT_TAG, null); 88 | this.minLogLevel = minLogLevel; 89 | } 90 | 91 | public void setMinLogLevel(final int minLogLevel) { 92 | this.minLogLevel = minLogLevel; 93 | } 94 | 95 | public boolean isLoggable(final int logLevel) { 96 | return logLevel >= minLogLevel || Log.isLoggable(tag, logLevel); 97 | } 98 | 99 | /** 100 | * Return caller's simple name. 101 | * 102 | * Android getStackTrace() returns an array that looks like this: 103 | * stackTrace[0]: dalvik.system.VMStack 104 | * stackTrace[1]: java.lang.Thread 105 | * stackTrace[2]: com.google.android.apps.unveil.env.UnveilLogger 106 | * stackTrace[3]: com.google.android.apps.unveil.BaseApplication 107 | * 108 | * This function returns the simple version of the first non-filtered name. 109 | * 110 | * @return caller's simple name 111 | */ 112 | private static String getCallerSimpleName() { 113 | // Get the current callstack so we can pull the class of the caller off of it. 114 | final StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace(); 115 | 116 | for (final StackTraceElement elem : stackTrace) { 117 | final String className = elem.getClassName(); 118 | if (!IGNORED_CLASS_NAMES.contains(className)) { 119 | // We're only interested in the simple name of the class, not the complete package. 120 | final String[] classParts = className.split("\\."); 121 | return classParts[classParts.length - 1]; 122 | } 123 | } 124 | 125 | return Logger.class.getSimpleName(); 126 | } 127 | 128 | private String toMessage(final String format, final Object... args) { 129 | return messagePrefix + (args.length > 0 ? String.format(format, args) : format); 130 | } 131 | 132 | public void v(final String format, final Object... args) { 133 | if (isLoggable(Log.VERBOSE)) { 134 | Log.v(tag, toMessage(format, args)); 135 | } 136 | } 137 | 138 | public void v(final Throwable t, final String format, final Object... args) { 139 | if (isLoggable(Log.VERBOSE)) { 140 | Log.v(tag, toMessage(format, args), t); 141 | } 142 | } 143 | 144 | public void d(final String format, final Object... args) { 145 | if (isLoggable(Log.DEBUG)) { 146 | Log.d(tag, toMessage(format, args)); 147 | } 148 | } 149 | 150 | public void d(final Throwable t, final String format, final Object... args) { 151 | if (isLoggable(Log.DEBUG)) { 152 | Log.d(tag, toMessage(format, args), t); 153 | } 154 | } 155 | 156 | public void i(final String format, final Object... args) { 157 | if (isLoggable(Log.INFO)) { 158 | Log.i(tag, toMessage(format, args)); 159 | } 160 | } 161 | 162 | public void i(final Throwable t, final String format, final Object... args) { 163 | if (isLoggable(Log.INFO)) { 164 | Log.i(tag, toMessage(format, args), t); 165 | } 166 | } 167 | 168 | public void w(final String format, final Object... args) { 169 | if (isLoggable(Log.WARN)) { 170 | Log.w(tag, toMessage(format, args)); 171 | } 172 | } 173 | 174 | public void w(final Throwable t, final String format, final Object... args) { 175 | if (isLoggable(Log.WARN)) { 176 | Log.w(tag, toMessage(format, args), t); 177 | } 178 | } 179 | 180 | public void e(final String format, final Object... args) { 181 | if (isLoggable(Log.ERROR)) { 182 | Log.e(tag, toMessage(format, args)); 183 | } 184 | } 185 | 186 | public void e(final Throwable t, final String format, final Object... args) { 187 | if (isLoggable(Log.ERROR)) { 188 | Log.e(tag, toMessage(format, args), t); 189 | } 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /app/src/main/java/pp/imagesegmenter/env/Size.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | package pp.imagesegmenter.env; 17 | 18 | import android.graphics.Bitmap; 19 | import android.text.TextUtils; 20 | import java.io.Serializable; 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | /** 25 | * Size class independent of a Camera object. 26 | */ 27 | public class Size implements Comparable, Serializable { 28 | 29 | // 1.4 went out with this UID so we'll need to maintain it to preserve pending queries when 30 | // upgrading. 31 | public static final long serialVersionUID = 7689808733290872361L; 32 | 33 | public final int width; 34 | public final int height; 35 | 36 | public Size(final int width, final int height) { 37 | this.width = width; 38 | this.height = height; 39 | } 40 | 41 | public Size(final Bitmap bmp) { 42 | this.width = bmp.getWidth(); 43 | this.height = bmp.getHeight(); 44 | } 45 | 46 | /** 47 | * Rotate a size by the given number of degrees. 48 | * @param size Size to rotate. 49 | * @param rotation Degrees {0, 90, 180, 270} to rotate the size. 50 | * @return Rotated size. 51 | */ 52 | public static Size getRotatedSize(final Size size, final int rotation) { 53 | if (rotation % 180 != 0) { 54 | // The phone is portrait, therefore the camera is sideways and frame should be rotated. 55 | return new Size(size.height, size.width); 56 | } 57 | return size; 58 | } 59 | 60 | public static Size parseFromString(String sizeString) { 61 | if (TextUtils.isEmpty(sizeString)) { 62 | return null; 63 | } 64 | 65 | sizeString = sizeString.trim(); 66 | 67 | // The expected format is "x". 68 | final String[] components = sizeString.split("x"); 69 | if (components.length == 2) { 70 | try { 71 | final int width = Integer.parseInt(components[0]); 72 | final int height = Integer.parseInt(components[1]); 73 | return new Size(width, height); 74 | } catch (final NumberFormatException e) { 75 | return null; 76 | } 77 | } else { 78 | return null; 79 | } 80 | } 81 | 82 | public static List sizeStringToList(final String sizes) { 83 | final List sizeList = new ArrayList(); 84 | if (sizes != null) { 85 | final String[] pairs = sizes.split(","); 86 | for (final String pair : pairs) { 87 | final Size size = Size.parseFromString(pair); 88 | if (size != null) { 89 | sizeList.add(size); 90 | } 91 | } 92 | } 93 | return sizeList; 94 | } 95 | 96 | public static String sizeListToString(final List sizes) { 97 | String sizesString = ""; 98 | if (sizes != null && sizes.size() > 0) { 99 | sizesString = sizes.get(0).toString(); 100 | for (int i = 1; i < sizes.size(); i++) { 101 | sizesString += "," + sizes.get(i).toString(); 102 | } 103 | } 104 | return sizesString; 105 | } 106 | 107 | public final float aspectRatio() { 108 | return (float) width / (float) height; 109 | } 110 | 111 | @Override 112 | public int compareTo(final Size other) { 113 | return width * height - other.width * other.height; 114 | } 115 | 116 | @Override 117 | public boolean equals(final Object other) { 118 | if (other == null) { 119 | return false; 120 | } 121 | 122 | if (!(other instanceof Size)) { 123 | return false; 124 | } 125 | 126 | final Size otherSize = (Size) other; 127 | return (width == otherSize.width && height == otherSize.height); 128 | } 129 | 130 | @Override 131 | public int hashCode() { 132 | return width * 32713 + height; 133 | } 134 | 135 | @Override 136 | public String toString() { 137 | return dimensionsAsString(width, height); 138 | } 139 | 140 | public static final String dimensionsAsString(final int width, final int height) { 141 | return width + "x" + height; 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /app/src/main/jni/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2016 The Android Open Source Project 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | cmake_minimum_required(VERSION 3.4.1) 18 | set(CMAKE_VERBOSE_MAKEFILE on) 19 | 20 | get_filename_component(SRC_DIR ${CMAKE_SOURCE_DIR}/.. ABSOLUTE) 21 | 22 | 23 | project(TENSORFLOW_DEMO) 24 | 25 | if (ANDROID_ABI MATCHES "^armeabi-v7a$") 26 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfloat-abi=softfp -mfpu=neon") 27 | elseif(ANDROID_ABI MATCHES "^arm64-v8a") 28 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -ftree-vectorize") 29 | endif() 30 | 31 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSTANDALONE_DEMO_LIB \ 32 | -std=c++11 -fno-exceptions -fno-rtti -O2 -Wno-narrowing \ 33 | -fPIE") 34 | set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} \ 35 | -Wl,--allow-multiple-definition \ 36 | -Wl,--whole-archive -fPIE -v") 37 | 38 | file(GLOB_RECURSE tensorflow_demo_sources ${SRC_DIR}/jni/*.*) 39 | add_library(tensorflow_demo SHARED 40 | ${tensorflow_demo_sources}) 41 | target_include_directories(tensorflow_demo PRIVATE 42 | ${CMAKE_SOURCE_DIR}) 43 | 44 | target_link_libraries(tensorflow_demo 45 | android 46 | log 47 | jnigraphics 48 | m 49 | atomic 50 | z) 51 | -------------------------------------------------------------------------------- /app/src/main/jni/imageutils_jni.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // This file binds the native image utility code to the Java class 17 | // which exposes them. 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "rgb2yuv.h" 24 | #include "yuv2rgb.h" 25 | 26 | #define IMAGEUTILS_METHOD(METHOD_NAME) \ 27 | Java_pp_imagesegmenter_env_ImageUtils_##METHOD_NAME // NOLINT 28 | 29 | #ifdef __cplusplus 30 | extern "C" { 31 | #endif 32 | 33 | JNIEXPORT void JNICALL 34 | IMAGEUTILS_METHOD(convertYUV420SPToARGB8888)( 35 | JNIEnv* env, jclass clazz, jbyteArray input, jintArray output, 36 | jint width, jint height, jboolean halfSize); 37 | 38 | JNIEXPORT void JNICALL IMAGEUTILS_METHOD(convertYUV420ToARGB8888)( 39 | JNIEnv* env, jclass clazz, jbyteArray y, jbyteArray u, jbyteArray v, 40 | jintArray output, jint width, jint height, jint y_row_stride, 41 | jint uv_row_stride, jint uv_pixel_stride, jboolean halfSize); 42 | 43 | JNIEXPORT void JNICALL IMAGEUTILS_METHOD(convertYUV420SPToRGB565)( 44 | JNIEnv* env, jclass clazz, jbyteArray input, jbyteArray output, jint width, 45 | jint height); 46 | 47 | JNIEXPORT void JNICALL 48 | IMAGEUTILS_METHOD(convertARGB8888ToYUV420SP)( 49 | JNIEnv* env, jclass clazz, jintArray input, jbyteArray output, 50 | jint width, jint height); 51 | 52 | JNIEXPORT void JNICALL 53 | IMAGEUTILS_METHOD(convertRGB565ToYUV420SP)( 54 | JNIEnv* env, jclass clazz, jbyteArray input, jbyteArray output, 55 | jint width, jint height); 56 | 57 | #ifdef __cplusplus 58 | } 59 | #endif 60 | 61 | JNIEXPORT void JNICALL 62 | IMAGEUTILS_METHOD(convertYUV420SPToARGB8888)( 63 | JNIEnv* env, jclass clazz, jbyteArray input, jintArray output, 64 | jint width, jint height, jboolean halfSize) { 65 | jboolean inputCopy = JNI_FALSE; 66 | jbyte* const i = env->GetByteArrayElements(input, &inputCopy); 67 | 68 | jboolean outputCopy = JNI_FALSE; 69 | jint* const o = env->GetIntArrayElements(output, &outputCopy); 70 | 71 | if (halfSize) { 72 | ConvertYUV420SPToARGB8888HalfSize(reinterpret_cast(i), 73 | reinterpret_cast(o), width, 74 | height); 75 | } else { 76 | ConvertYUV420SPToARGB8888(reinterpret_cast(i), 77 | reinterpret_cast(i) + width * height, 78 | reinterpret_cast(o), width, height); 79 | } 80 | 81 | env->ReleaseByteArrayElements(input, i, JNI_ABORT); 82 | env->ReleaseIntArrayElements(output, o, 0); 83 | } 84 | 85 | JNIEXPORT void JNICALL IMAGEUTILS_METHOD(convertYUV420ToARGB8888)( 86 | JNIEnv* env, jclass clazz, jbyteArray y, jbyteArray u, jbyteArray v, 87 | jintArray output, jint width, jint height, jint y_row_stride, 88 | jint uv_row_stride, jint uv_pixel_stride, jboolean halfSize) { 89 | jboolean inputCopy = JNI_FALSE; 90 | jbyte* const y_buff = env->GetByteArrayElements(y, &inputCopy); 91 | jboolean outputCopy = JNI_FALSE; 92 | jint* const o = env->GetIntArrayElements(output, &outputCopy); 93 | 94 | if (halfSize) { 95 | ConvertYUV420SPToARGB8888HalfSize(reinterpret_cast(y_buff), 96 | reinterpret_cast(o), width, 97 | height); 98 | } else { 99 | jbyte* const u_buff = env->GetByteArrayElements(u, &inputCopy); 100 | jbyte* const v_buff = env->GetByteArrayElements(v, &inputCopy); 101 | 102 | ConvertYUV420ToARGB8888( 103 | reinterpret_cast(y_buff), reinterpret_cast(u_buff), 104 | reinterpret_cast(v_buff), reinterpret_cast(o), 105 | width, height, y_row_stride, uv_row_stride, uv_pixel_stride); 106 | 107 | env->ReleaseByteArrayElements(u, u_buff, JNI_ABORT); 108 | env->ReleaseByteArrayElements(v, v_buff, JNI_ABORT); 109 | } 110 | 111 | env->ReleaseByteArrayElements(y, y_buff, JNI_ABORT); 112 | env->ReleaseIntArrayElements(output, o, 0); 113 | } 114 | 115 | JNIEXPORT void JNICALL IMAGEUTILS_METHOD(convertYUV420SPToRGB565)( 116 | JNIEnv* env, jclass clazz, jbyteArray input, jbyteArray output, jint width, 117 | jint height) { 118 | jboolean inputCopy = JNI_FALSE; 119 | jbyte* const i = env->GetByteArrayElements(input, &inputCopy); 120 | 121 | jboolean outputCopy = JNI_FALSE; 122 | jbyte* const o = env->GetByteArrayElements(output, &outputCopy); 123 | 124 | ConvertYUV420SPToRGB565(reinterpret_cast(i), 125 | reinterpret_cast(o), width, height); 126 | 127 | env->ReleaseByteArrayElements(input, i, JNI_ABORT); 128 | env->ReleaseByteArrayElements(output, o, 0); 129 | } 130 | 131 | JNIEXPORT void JNICALL 132 | IMAGEUTILS_METHOD(convertARGB8888ToYUV420SP)( 133 | JNIEnv* env, jclass clazz, jintArray input, jbyteArray output, 134 | jint width, jint height) { 135 | jboolean inputCopy = JNI_FALSE; 136 | jint* const i = env->GetIntArrayElements(input, &inputCopy); 137 | 138 | jboolean outputCopy = JNI_FALSE; 139 | jbyte* const o = env->GetByteArrayElements(output, &outputCopy); 140 | 141 | ConvertARGB8888ToYUV420SP(reinterpret_cast(i), 142 | reinterpret_cast(o), width, height); 143 | 144 | env->ReleaseIntArrayElements(input, i, JNI_ABORT); 145 | env->ReleaseByteArrayElements(output, o, 0); 146 | } 147 | 148 | JNIEXPORT void JNICALL 149 | IMAGEUTILS_METHOD(convertRGB565ToYUV420SP)( 150 | JNIEnv* env, jclass clazz, jbyteArray input, jbyteArray output, 151 | jint width, jint height) { 152 | jboolean inputCopy = JNI_FALSE; 153 | jbyte* const i = env->GetByteArrayElements(input, &inputCopy); 154 | 155 | jboolean outputCopy = JNI_FALSE; 156 | jbyte* const o = env->GetByteArrayElements(output, &outputCopy); 157 | 158 | ConvertRGB565ToYUV420SP(reinterpret_cast(i), 159 | reinterpret_cast(o), width, height); 160 | 161 | env->ReleaseByteArrayElements(input, i, JNI_ABORT); 162 | env->ReleaseByteArrayElements(output, o, 0); 163 | } 164 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/frame_pair.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_FRAME_PAIR_H_ 17 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_FRAME_PAIR_H_ 18 | 19 | #include "keypoint.h" 20 | 21 | namespace tf_tracking { 22 | 23 | // A class that records keypoint correspondences from pairs of 24 | // consecutive frames. 25 | class FramePair { 26 | public: 27 | FramePair() 28 | : start_time_(0), 29 | end_time_(0), 30 | number_of_keypoints_(0) {} 31 | 32 | // Cleans up the FramePair so that they can be reused. 33 | void Init(const int64_t start_time, const int64_t end_time); 34 | 35 | void AdjustBox(const BoundingBox box, 36 | float* const translation_x, 37 | float* const translation_y, 38 | float* const scale_x, 39 | float* const scale_y) const; 40 | 41 | private: 42 | // Returns the weighted median of the given deltas, computed independently on 43 | // x and y. Returns 0,0 in case of failure. The assumption is that a 44 | // translation of 0.0 in the degenerate case is the best that can be done, and 45 | // should not be considered an error. 46 | // 47 | // In the case of scale, a slight exception is made just to be safe and 48 | // there is a check for 0.0 explicitly, but that shouldn't ever be possible to 49 | // happen naturally because of the non-zero + parity checks in FillScales. 50 | Point2f GetWeightedMedian(const float* const weights, 51 | const Point2f* const deltas) const; 52 | 53 | float GetWeightedMedianScale(const float* const weights, 54 | const Point2f* const deltas) const; 55 | 56 | // Weights points based on the query_point and cutoff_dist. 57 | int FillWeights(const BoundingBox& box, 58 | float* const weights) const; 59 | 60 | // Fills in the array of deltas with the translations of the points 61 | // between frames. 62 | void FillTranslations(Point2f* const translations) const; 63 | 64 | // Fills in the array of deltas with the relative scale factor of points 65 | // relative to a given center. Has the ability to override the weight to 0 if 66 | // a degenerate scale is detected. 67 | // Translation is the amount the center of the box has moved from one frame to 68 | // the next. 69 | int FillScales(const Point2f& old_center, 70 | const Point2f& translation, 71 | float* const weights, 72 | Point2f* const scales) const; 73 | 74 | // TODO(andrewharp): Make these private. 75 | public: 76 | // The time at frame1. 77 | int64_t start_time_; 78 | 79 | // The time at frame2. 80 | int64_t end_time_; 81 | 82 | // This array will contain the keypoints found in frame 1. 83 | Keypoint frame1_keypoints_[kMaxKeypoints]; 84 | 85 | // Contain the locations of the keypoints from frame 1 in frame 2. 86 | Keypoint frame2_keypoints_[kMaxKeypoints]; 87 | 88 | // The number of keypoints in frame 1. 89 | int number_of_keypoints_; 90 | 91 | // Keeps track of which keypoint correspondences were actually found from one 92 | // frame to another. 93 | // The i-th element of this array will be non-zero if and only if the i-th 94 | // keypoint of frame 1 was found in frame 2. 95 | bool optical_flow_found_keypoint_[kMaxKeypoints]; 96 | 97 | private: 98 | TF_DISALLOW_COPY_AND_ASSIGN(FramePair); 99 | }; 100 | 101 | } // namespace tf_tracking 102 | 103 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_FRAME_PAIR_H_ 104 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/geom.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_GEOM_H_ 17 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_GEOM_H_ 18 | 19 | #include 20 | #include "utils.h" 21 | 22 | namespace tf_tracking { 23 | 24 | struct Size { 25 | Size(const int width, const int height) : width(width), height(height) {} 26 | 27 | int width; 28 | int height; 29 | }; 30 | 31 | 32 | class Point2f { 33 | public: 34 | Point2f() : x(0.0f), y(0.0f) {} 35 | Point2f(const float x, const float y) : x(x), y(y) {} 36 | 37 | inline Point2f operator- (const Point2f& that) const { 38 | return Point2f(this->x - that.x, this->y - that.y); 39 | } 40 | 41 | inline Point2f operator+ (const Point2f& that) const { 42 | return Point2f(this->x + that.x, this->y + that.y); 43 | } 44 | 45 | inline Point2f& operator+= (const Point2f& that) { 46 | this->x += that.x; 47 | this->y += that.y; 48 | return *this; 49 | } 50 | 51 | inline Point2f& operator-= (const Point2f& that) { 52 | this->x -= that.x; 53 | this->y -= that.y; 54 | return *this; 55 | } 56 | 57 | inline Point2f operator- (const Point2f& that) { 58 | return Point2f(this->x - that.x, this->y - that.y); 59 | } 60 | 61 | inline float LengthSquared() { 62 | return Square(this->x) + Square(this->y); 63 | } 64 | 65 | inline float Length() { 66 | return sqrtf(LengthSquared()); 67 | } 68 | 69 | inline float DistanceSquared(const Point2f& that) { 70 | return Square(this->x - that.x) + Square(this->y - that.y); 71 | } 72 | 73 | inline float Distance(const Point2f& that) { 74 | return sqrtf(DistanceSquared(that)); 75 | } 76 | 77 | float x; 78 | float y; 79 | }; 80 | 81 | inline std::ostream& operator<<(std::ostream& stream, const Point2f& point) { 82 | stream << point.x << "," << point.y; 83 | return stream; 84 | } 85 | 86 | class BoundingBox { 87 | public: 88 | BoundingBox() 89 | : left_(0), 90 | top_(0), 91 | right_(0), 92 | bottom_(0) {} 93 | 94 | BoundingBox(const BoundingBox& bounding_box) 95 | : left_(bounding_box.left_), 96 | top_(bounding_box.top_), 97 | right_(bounding_box.right_), 98 | bottom_(bounding_box.bottom_) { 99 | SCHECK(left_ < right_, "Bounds out of whack! %.2f vs %.2f!", left_, right_); 100 | SCHECK(top_ < bottom_, "Bounds out of whack! %.2f vs %.2f!", top_, bottom_); 101 | } 102 | 103 | BoundingBox(const float left, 104 | const float top, 105 | const float right, 106 | const float bottom) 107 | : left_(left), 108 | top_(top), 109 | right_(right), 110 | bottom_(bottom) { 111 | SCHECK(left_ < right_, "Bounds out of whack! %.2f vs %.2f!", left_, right_); 112 | SCHECK(top_ < bottom_, "Bounds out of whack! %.2f vs %.2f!", top_, bottom_); 113 | } 114 | 115 | BoundingBox(const Point2f& point1, const Point2f& point2) 116 | : left_(MIN(point1.x, point2.x)), 117 | top_(MIN(point1.y, point2.y)), 118 | right_(MAX(point1.x, point2.x)), 119 | bottom_(MAX(point1.y, point2.y)) {} 120 | 121 | inline void CopyToArray(float* const bounds_array) const { 122 | bounds_array[0] = left_; 123 | bounds_array[1] = top_; 124 | bounds_array[2] = right_; 125 | bounds_array[3] = bottom_; 126 | } 127 | 128 | inline float GetWidth() const { 129 | return right_ - left_; 130 | } 131 | 132 | inline float GetHeight() const { 133 | return bottom_ - top_; 134 | } 135 | 136 | inline float GetArea() const { 137 | const float width = GetWidth(); 138 | const float height = GetHeight(); 139 | if (width <= 0 || height <= 0) { 140 | return 0.0f; 141 | } 142 | 143 | return width * height; 144 | } 145 | 146 | inline Point2f GetCenter() const { 147 | return Point2f((left_ + right_) / 2.0f, 148 | (top_ + bottom_) / 2.0f); 149 | } 150 | 151 | inline bool ValidBox() const { 152 | return GetArea() > 0.0f; 153 | } 154 | 155 | // Returns a bounding box created from the overlapping area of these two. 156 | inline BoundingBox Intersect(const BoundingBox& that) const { 157 | const float new_left = MAX(this->left_, that.left_); 158 | const float new_right = MIN(this->right_, that.right_); 159 | 160 | if (new_left >= new_right) { 161 | return BoundingBox(); 162 | } 163 | 164 | const float new_top = MAX(this->top_, that.top_); 165 | const float new_bottom = MIN(this->bottom_, that.bottom_); 166 | 167 | if (new_top >= new_bottom) { 168 | return BoundingBox(); 169 | } 170 | 171 | return BoundingBox(new_left, new_top, new_right, new_bottom); 172 | } 173 | 174 | // Returns a bounding box that can contain both boxes. 175 | inline BoundingBox Union(const BoundingBox& that) const { 176 | return BoundingBox(MIN(this->left_, that.left_), 177 | MIN(this->top_, that.top_), 178 | MAX(this->right_, that.right_), 179 | MAX(this->bottom_, that.bottom_)); 180 | } 181 | 182 | inline float PascalScore(const BoundingBox& that) const { 183 | SCHECK(GetArea() > 0.0f, "Empty bounding box!"); 184 | SCHECK(that.GetArea() > 0.0f, "Empty bounding box!"); 185 | 186 | const float intersect_area = this->Intersect(that).GetArea(); 187 | 188 | if (intersect_area <= 0) { 189 | return 0; 190 | } 191 | 192 | const float score = 193 | intersect_area / (GetArea() + that.GetArea() - intersect_area); 194 | SCHECK(InRange(score, 0.0f, 1.0f), "Invalid score! %.2f", score); 195 | return score; 196 | } 197 | 198 | inline bool Intersects(const BoundingBox& that) const { 199 | return InRange(that.left_, left_, right_) 200 | || InRange(that.right_, left_, right_) 201 | || InRange(that.top_, top_, bottom_) 202 | || InRange(that.bottom_, top_, bottom_); 203 | } 204 | 205 | // Returns whether another bounding box is completely inside of this bounding 206 | // box. Sharing edges is ok. 207 | inline bool Contains(const BoundingBox& that) const { 208 | return that.left_ >= left_ && 209 | that.right_ <= right_ && 210 | that.top_ >= top_ && 211 | that.bottom_ <= bottom_; 212 | } 213 | 214 | inline bool Contains(const Point2f& point) const { 215 | return InRange(point.x, left_, right_) && InRange(point.y, top_, bottom_); 216 | } 217 | 218 | inline void Shift(const Point2f shift_amount) { 219 | left_ += shift_amount.x; 220 | top_ += shift_amount.y; 221 | right_ += shift_amount.x; 222 | bottom_ += shift_amount.y; 223 | } 224 | 225 | inline void ScaleOrigin(const float scale_x, const float scale_y) { 226 | left_ *= scale_x; 227 | right_ *= scale_x; 228 | top_ *= scale_y; 229 | bottom_ *= scale_y; 230 | } 231 | 232 | inline void Scale(const float scale_x, const float scale_y) { 233 | const Point2f center = GetCenter(); 234 | const float half_width = GetWidth() / 2.0f; 235 | const float half_height = GetHeight() / 2.0f; 236 | 237 | left_ = center.x - half_width * scale_x; 238 | right_ = center.x + half_width * scale_x; 239 | 240 | top_ = center.y - half_height * scale_y; 241 | bottom_ = center.y + half_height * scale_y; 242 | } 243 | 244 | float left_; 245 | float top_; 246 | float right_; 247 | float bottom_; 248 | }; 249 | inline std::ostream& operator<<(std::ostream& stream, const BoundingBox& box) { 250 | stream << "[" << box.left_ << " - " << box.right_ 251 | << ", " << box.top_ << " - " << box.bottom_ 252 | << ", w:" << box.GetWidth() << " h:" << box.GetHeight() << "]"; 253 | return stream; 254 | } 255 | 256 | 257 | class BoundingSquare { 258 | public: 259 | BoundingSquare(const float x, const float y, const float size) 260 | : x_(x), y_(y), size_(size) {} 261 | 262 | explicit BoundingSquare(const BoundingBox& box) 263 | : x_(box.left_), y_(box.top_), size_(box.GetWidth()) { 264 | #ifdef SANITY_CHECKS 265 | if (std::abs(box.GetWidth() - box.GetHeight()) > 0.1f) { 266 | LOG(WARNING) << "This is not a square: " << box << std::endl; 267 | } 268 | #endif 269 | } 270 | 271 | inline BoundingBox ToBoundingBox() const { 272 | return BoundingBox(x_, y_, x_ + size_, y_ + size_); 273 | } 274 | 275 | inline bool ValidBox() { 276 | return size_ > 0.0f; 277 | } 278 | 279 | inline void Shift(const Point2f shift_amount) { 280 | x_ += shift_amount.x; 281 | y_ += shift_amount.y; 282 | } 283 | 284 | inline void Scale(const float scale) { 285 | const float new_size = size_ * scale; 286 | const float position_diff = (new_size - size_) / 2.0f; 287 | x_ -= position_diff; 288 | y_ -= position_diff; 289 | size_ = new_size; 290 | } 291 | 292 | float x_; 293 | float y_; 294 | float size_; 295 | }; 296 | inline std::ostream& operator<<(std::ostream& stream, 297 | const BoundingSquare& square) { 298 | stream << "[" << square.x_ << "," << square.y_ << " " << square.size_ << "]"; 299 | return stream; 300 | } 301 | 302 | 303 | inline BoundingSquare GetCenteredSquare(const BoundingBox& original_box, 304 | const float size) { 305 | const float width_diff = (original_box.GetWidth() - size) / 2.0f; 306 | const float height_diff = (original_box.GetHeight() - size) / 2.0f; 307 | return BoundingSquare(original_box.left_ + width_diff, 308 | original_box.top_ + height_diff, 309 | size); 310 | } 311 | 312 | inline BoundingSquare GetCenteredSquare(const BoundingBox& original_box) { 313 | return GetCenteredSquare( 314 | original_box, MIN(original_box.GetWidth(), original_box.GetHeight())); 315 | } 316 | 317 | } // namespace tf_tracking 318 | 319 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_GEOM_H_ 320 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/gl_utils.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_GL_UTILS_H_ 17 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_GL_UTILS_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include "tensorflow/examples/android/jni/object_tracking/geom.h" 23 | 24 | namespace tf_tracking { 25 | 26 | // Draws a box at the given position. 27 | inline static void DrawBox(const BoundingBox& bounding_box) { 28 | const GLfloat line[] = { 29 | bounding_box.left_, bounding_box.bottom_, 30 | bounding_box.left_, bounding_box.top_, 31 | bounding_box.left_, bounding_box.top_, 32 | bounding_box.right_, bounding_box.top_, 33 | bounding_box.right_, bounding_box.top_, 34 | bounding_box.right_, bounding_box.bottom_, 35 | bounding_box.right_, bounding_box.bottom_, 36 | bounding_box.left_, bounding_box.bottom_ 37 | }; 38 | 39 | glVertexPointer(2, GL_FLOAT, 0, line); 40 | glEnableClientState(GL_VERTEX_ARRAY); 41 | 42 | glDrawArrays(GL_LINES, 0, 8); 43 | } 44 | 45 | 46 | // Changes the coordinate system such that drawing to an arbitrary square in 47 | // the world can thereafter be drawn to using coordinates 0 - 1. 48 | inline static void MapWorldSquareToUnitSquare(const BoundingSquare& square) { 49 | glScalef(square.size_, square.size_, 1.0f); 50 | glTranslatef(square.x_ / square.size_, square.y_ / square.size_, 0.0f); 51 | } 52 | 53 | } // namespace tf_tracking 54 | 55 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_GL_UTILS_H_ 56 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/image_data.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_IMAGE_DATA_H_ 17 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_IMAGE_DATA_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include "image-inl.h" 23 | #include "image.h" 24 | #include "image_utils.h" 25 | #include "integral_image.h" 26 | #include "time_log.h" 27 | #include "utils.h" 28 | 29 | #include "config.h" 30 | 31 | namespace tf_tracking { 32 | 33 | // Class that encapsulates all bulky processed data for a frame. 34 | class ImageData { 35 | public: 36 | explicit ImageData(const int width, const int height) 37 | : uv_frame_width_(width << 1), 38 | uv_frame_height_(height << 1), 39 | timestamp_(0), 40 | image_(width, height) { 41 | InitPyramid(width, height); 42 | ResetComputationCache(); 43 | } 44 | 45 | private: 46 | void ResetComputationCache() { 47 | uv_data_computed_ = false; 48 | integral_image_computed_ = false; 49 | for (int i = 0; i < kNumPyramidLevels; ++i) { 50 | spatial_x_computed_[i] = false; 51 | spatial_y_computed_[i] = false; 52 | pyramid_sqrt2_computed_[i * 2] = false; 53 | pyramid_sqrt2_computed_[i * 2 + 1] = false; 54 | } 55 | } 56 | 57 | void InitPyramid(const int width, const int height) { 58 | int level_width = width; 59 | int level_height = height; 60 | 61 | for (int i = 0; i < kNumPyramidLevels; ++i) { 62 | pyramid_sqrt2_[i * 2] = NULL; 63 | pyramid_sqrt2_[i * 2 + 1] = NULL; 64 | spatial_x_[i] = NULL; 65 | spatial_y_[i] = NULL; 66 | 67 | level_width /= 2; 68 | level_height /= 2; 69 | } 70 | 71 | // Alias the first pyramid level to image_. 72 | pyramid_sqrt2_[0] = &image_; 73 | } 74 | 75 | public: 76 | ~ImageData() { 77 | // The first pyramid level is actually an alias to image_, 78 | // so make sure it doesn't get deleted here. 79 | pyramid_sqrt2_[0] = NULL; 80 | 81 | for (int i = 0; i < kNumPyramidLevels; ++i) { 82 | SAFE_DELETE(pyramid_sqrt2_[i * 2]); 83 | SAFE_DELETE(pyramid_sqrt2_[i * 2 + 1]); 84 | SAFE_DELETE(spatial_x_[i]); 85 | SAFE_DELETE(spatial_y_[i]); 86 | } 87 | } 88 | 89 | void SetData(const uint8_t* const new_frame, const int stride, 90 | const int64_t timestamp, const int downsample_factor) { 91 | SetData(new_frame, NULL, stride, timestamp, downsample_factor); 92 | } 93 | 94 | void SetData(const uint8_t* const new_frame, const uint8_t* const uv_frame, 95 | const int stride, const int64_t timestamp, 96 | const int downsample_factor) { 97 | ResetComputationCache(); 98 | 99 | timestamp_ = timestamp; 100 | 101 | TimeLog("SetData!"); 102 | 103 | pyramid_sqrt2_[0]->FromArray(new_frame, stride, downsample_factor); 104 | pyramid_sqrt2_computed_[0] = true; 105 | TimeLog("Downsampled image"); 106 | 107 | if (uv_frame != NULL) { 108 | if (u_data_.get() == NULL) { 109 | u_data_.reset(new Image(uv_frame_width_, uv_frame_height_)); 110 | v_data_.reset(new Image(uv_frame_width_, uv_frame_height_)); 111 | } 112 | 113 | GetUV(uv_frame, u_data_.get(), v_data_.get()); 114 | uv_data_computed_ = true; 115 | TimeLog("Copied UV data"); 116 | } else { 117 | LOGV("No uv data!"); 118 | } 119 | 120 | #ifdef LOG_TIME 121 | // If profiling is enabled, precompute here to make it easier to distinguish 122 | // total costs. 123 | Precompute(); 124 | #endif 125 | } 126 | 127 | inline const uint64_t GetTimestamp() const { return timestamp_; } 128 | 129 | inline const Image* GetImage() const { 130 | SCHECK(pyramid_sqrt2_computed_[0], "image not set!"); 131 | return pyramid_sqrt2_[0]; 132 | } 133 | 134 | const Image* GetPyramidSqrt2Level(const int level) const { 135 | if (!pyramid_sqrt2_computed_[level]) { 136 | SCHECK(level != 0, "Level equals 0!"); 137 | if (level == 1) { 138 | const Image& upper_level = *GetPyramidSqrt2Level(0); 139 | if (pyramid_sqrt2_[level] == NULL) { 140 | const int new_width = 141 | (static_cast(upper_level.GetWidth() / sqrtf(2)) + 1) / 2 * 2; 142 | const int new_height = 143 | (static_cast(upper_level.GetHeight() / sqrtf(2)) + 1) / 2 * 144 | 2; 145 | 146 | pyramid_sqrt2_[level] = new Image(new_width, new_height); 147 | } 148 | pyramid_sqrt2_[level]->DownsampleInterpolateLinear(upper_level); 149 | } else { 150 | const Image& upper_level = *GetPyramidSqrt2Level(level - 2); 151 | if (pyramid_sqrt2_[level] == NULL) { 152 | pyramid_sqrt2_[level] = new Image( 153 | upper_level.GetWidth() / 2, upper_level.GetHeight() / 2); 154 | } 155 | pyramid_sqrt2_[level]->DownsampleAveraged( 156 | upper_level.data(), upper_level.stride(), 2); 157 | } 158 | pyramid_sqrt2_computed_[level] = true; 159 | } 160 | return pyramid_sqrt2_[level]; 161 | } 162 | 163 | inline const Image* GetSpatialX(const int level) const { 164 | if (!spatial_x_computed_[level]) { 165 | const Image& src = *GetPyramidSqrt2Level(level * 2); 166 | if (spatial_x_[level] == NULL) { 167 | spatial_x_[level] = new Image(src.GetWidth(), src.GetHeight()); 168 | } 169 | spatial_x_[level]->DerivativeX(src); 170 | spatial_x_computed_[level] = true; 171 | } 172 | return spatial_x_[level]; 173 | } 174 | 175 | inline const Image* GetSpatialY(const int level) const { 176 | if (!spatial_y_computed_[level]) { 177 | const Image& src = *GetPyramidSqrt2Level(level * 2); 178 | if (spatial_y_[level] == NULL) { 179 | spatial_y_[level] = new Image(src.GetWidth(), src.GetHeight()); 180 | } 181 | spatial_y_[level]->DerivativeY(src); 182 | spatial_y_computed_[level] = true; 183 | } 184 | return spatial_y_[level]; 185 | } 186 | 187 | // The integral image is currently only used for object detection, so lazily 188 | // initialize it on request. 189 | inline const IntegralImage* GetIntegralImage() const { 190 | if (integral_image_.get() == NULL) { 191 | integral_image_.reset(new IntegralImage(image_)); 192 | } else if (!integral_image_computed_) { 193 | integral_image_->Recompute(image_); 194 | } 195 | integral_image_computed_ = true; 196 | return integral_image_.get(); 197 | } 198 | 199 | inline const Image* GetU() const { 200 | SCHECK(uv_data_computed_, "UV data not provided!"); 201 | return u_data_.get(); 202 | } 203 | 204 | inline const Image* GetV() const { 205 | SCHECK(uv_data_computed_, "UV data not provided!"); 206 | return v_data_.get(); 207 | } 208 | 209 | private: 210 | void Precompute() { 211 | // Create the smoothed pyramids. 212 | for (int i = 0; i < kNumPyramidLevels * 2; i += 2) { 213 | (void) GetPyramidSqrt2Level(i); 214 | } 215 | TimeLog("Created smoothed pyramids"); 216 | 217 | // Create the smoothed pyramids. 218 | for (int i = 1; i < kNumPyramidLevels * 2; i += 2) { 219 | (void) GetPyramidSqrt2Level(i); 220 | } 221 | TimeLog("Created smoothed sqrt pyramids"); 222 | 223 | // Create the spatial derivatives for frame 1. 224 | for (int i = 0; i < kNumPyramidLevels; ++i) { 225 | (void) GetSpatialX(i); 226 | (void) GetSpatialY(i); 227 | } 228 | TimeLog("Created spatial derivatives"); 229 | 230 | (void) GetIntegralImage(); 231 | TimeLog("Got integral image!"); 232 | } 233 | 234 | const int uv_frame_width_; 235 | const int uv_frame_height_; 236 | 237 | int64_t timestamp_; 238 | 239 | Image image_; 240 | 241 | bool uv_data_computed_; 242 | std::unique_ptr > u_data_; 243 | std::unique_ptr > v_data_; 244 | 245 | mutable bool spatial_x_computed_[kNumPyramidLevels]; 246 | mutable Image* spatial_x_[kNumPyramidLevels]; 247 | 248 | mutable bool spatial_y_computed_[kNumPyramidLevels]; 249 | mutable Image* spatial_y_[kNumPyramidLevels]; 250 | 251 | // Mutable so the lazy initialization can work when this class is const. 252 | // Whether or not the integral image has been computed for the current image. 253 | mutable bool integral_image_computed_; 254 | mutable std::unique_ptr integral_image_; 255 | 256 | mutable bool pyramid_sqrt2_computed_[kNumPyramidLevels * 2]; 257 | mutable Image* pyramid_sqrt2_[kNumPyramidLevels * 2]; 258 | 259 | TF_DISALLOW_COPY_AND_ASSIGN(ImageData); 260 | }; 261 | 262 | } // namespace tf_tracking 263 | 264 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_IMAGE_DATA_H_ 265 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/image_neon.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // NEON implementations of Image methods for compatible devices. Control 17 | // should never enter this compilation unit on incompatible devices. 18 | 19 | #ifdef __ARM_NEON 20 | 21 | #include 22 | 23 | #include 24 | 25 | #include "image-inl.h" 26 | #include "image.h" 27 | #include "image_utils.h" 28 | #include "utils.h" 29 | 30 | namespace tf_tracking { 31 | 32 | // This function does the bulk of the work. 33 | template <> 34 | void Image::Downsample2x32ColumnsNeon(const uint8_t* const original, 35 | const int stride, 36 | const int orig_x) { 37 | // Divide input x offset by 2 to find output offset. 38 | const int new_x = orig_x >> 1; 39 | 40 | // Initial offset into top row. 41 | const uint8_t* offset = original + orig_x; 42 | 43 | // This points to the leftmost pixel of our 8 horizontally arranged 44 | // pixels in the destination data. 45 | uint8_t* ptr_dst = (*this)[0] + new_x; 46 | 47 | // Sum along vertical columns. 48 | // Process 32x2 input pixels and 16x1 output pixels per iteration. 49 | for (int new_y = 0; new_y < height_; ++new_y) { 50 | uint16x8_t accum1 = vdupq_n_u16(0); 51 | uint16x8_t accum2 = vdupq_n_u16(0); 52 | 53 | // Go top to bottom across the four rows of input pixels that make up 54 | // this output row. 55 | for (int row_num = 0; row_num < 2; ++row_num) { 56 | // First 16 bytes. 57 | { 58 | // Load 16 bytes of data from current offset. 59 | const uint8x16_t curr_data1 = vld1q_u8(offset); 60 | 61 | // Pairwise add and accumulate into accum vectors (16 bit to account 62 | // for values above 255). 63 | accum1 = vpadalq_u8(accum1, curr_data1); 64 | } 65 | 66 | // Second 16 bytes. 67 | { 68 | // Load 16 bytes of data from current offset. 69 | const uint8x16_t curr_data2 = vld1q_u8(offset + 16); 70 | 71 | // Pairwise add and accumulate into accum vectors (16 bit to account 72 | // for values above 255). 73 | accum2 = vpadalq_u8(accum2, curr_data2); 74 | } 75 | 76 | // Move offset down one row. 77 | offset += stride; 78 | } 79 | 80 | // Divide by 4 (number of input pixels per output 81 | // pixel) and narrow data from 16 bits per pixel to 8 bpp. 82 | const uint8x8_t tmp_pix1 = vqshrn_n_u16(accum1, 2); 83 | const uint8x8_t tmp_pix2 = vqshrn_n_u16(accum2, 2); 84 | 85 | // Concatenate 8x1 pixel strips into 16x1 pixel strip. 86 | const uint8x16_t allpixels = vcombine_u8(tmp_pix1, tmp_pix2); 87 | 88 | // Copy all pixels from composite 16x1 vector into output strip. 89 | vst1q_u8(ptr_dst, allpixels); 90 | 91 | ptr_dst += stride_; 92 | } 93 | } 94 | 95 | // This function does the bulk of the work. 96 | template <> 97 | void Image::Downsample4x32ColumnsNeon(const uint8_t* const original, 98 | const int stride, 99 | const int orig_x) { 100 | // Divide input x offset by 4 to find output offset. 101 | const int new_x = orig_x >> 2; 102 | 103 | // Initial offset into top row. 104 | const uint8_t* offset = original + orig_x; 105 | 106 | // This points to the leftmost pixel of our 8 horizontally arranged 107 | // pixels in the destination data. 108 | uint8_t* ptr_dst = (*this)[0] + new_x; 109 | 110 | // Sum along vertical columns. 111 | // Process 32x4 input pixels and 8x1 output pixels per iteration. 112 | for (int new_y = 0; new_y < height_; ++new_y) { 113 | uint16x8_t accum1 = vdupq_n_u16(0); 114 | uint16x8_t accum2 = vdupq_n_u16(0); 115 | 116 | // Go top to bottom across the four rows of input pixels that make up 117 | // this output row. 118 | for (int row_num = 0; row_num < 4; ++row_num) { 119 | // First 16 bytes. 120 | { 121 | // Load 16 bytes of data from current offset. 122 | const uint8x16_t curr_data1 = vld1q_u8(offset); 123 | 124 | // Pairwise add and accumulate into accum vectors (16 bit to account 125 | // for values above 255). 126 | accum1 = vpadalq_u8(accum1, curr_data1); 127 | } 128 | 129 | // Second 16 bytes. 130 | { 131 | // Load 16 bytes of data from current offset. 132 | const uint8x16_t curr_data2 = vld1q_u8(offset + 16); 133 | 134 | // Pairwise add and accumulate into accum vectors (16 bit to account 135 | // for values above 255). 136 | accum2 = vpadalq_u8(accum2, curr_data2); 137 | } 138 | 139 | // Move offset down one row. 140 | offset += stride; 141 | } 142 | 143 | // Add and widen, then divide by 16 (number of input pixels per output 144 | // pixel) and narrow data from 32 bits per pixel to 16 bpp. 145 | const uint16x4_t tmp_pix1 = vqshrn_n_u32(vpaddlq_u16(accum1), 4); 146 | const uint16x4_t tmp_pix2 = vqshrn_n_u32(vpaddlq_u16(accum2), 4); 147 | 148 | // Combine 4x1 pixel strips into 8x1 pixel strip and narrow from 149 | // 16 bits to 8 bits per pixel. 150 | const uint8x8_t allpixels = vmovn_u16(vcombine_u16(tmp_pix1, tmp_pix2)); 151 | 152 | // Copy all pixels from composite 8x1 vector into output strip. 153 | vst1_u8(ptr_dst, allpixels); 154 | 155 | ptr_dst += stride_; 156 | } 157 | } 158 | 159 | 160 | // Hardware accelerated downsampling method for supported devices. 161 | // Requires that image size be a multiple of 16 pixels in each dimension, 162 | // and that downsampling be by a factor of 2 or 4. 163 | template <> 164 | void Image::DownsampleAveragedNeon(const uint8_t* const original, 165 | const int stride, 166 | const int factor) { 167 | // TODO(andrewharp): stride is a bad approximation for the src image's width. 168 | // Better to pass that in directly. 169 | SCHECK(width_ * factor <= stride, "Uh oh!"); 170 | const int last_starting_index = width_ * factor - 32; 171 | 172 | // We process 32 input pixels lengthwise at a time. 173 | // The output per pass of this loop is an 8 wide by downsampled height tall 174 | // pixel strip. 175 | int orig_x = 0; 176 | for (; orig_x <= last_starting_index; orig_x += 32) { 177 | if (factor == 2) { 178 | Downsample2x32ColumnsNeon(original, stride, orig_x); 179 | } else { 180 | Downsample4x32ColumnsNeon(original, stride, orig_x); 181 | } 182 | } 183 | 184 | // If a last pass is required, push it to the left enough so that it never 185 | // goes out of bounds. This will result in some extra computation on devices 186 | // whose frame widths are multiples of 16 and not 32. 187 | if (orig_x < last_starting_index + 32) { 188 | if (factor == 2) { 189 | Downsample2x32ColumnsNeon(original, stride, last_starting_index); 190 | } else { 191 | Downsample4x32ColumnsNeon(original, stride, last_starting_index); 192 | } 193 | } 194 | } 195 | 196 | 197 | // Puts the image gradient matrix about a pixel into the 2x2 float array G. 198 | // vals_x should be an array of the window x gradient values, whose indices 199 | // can be in any order but are parallel to the vals_y entries. 200 | // See http://robots.stanford.edu/cs223b04/algo_tracking.pdf for more details. 201 | void CalculateGNeon(const float* const vals_x, const float* const vals_y, 202 | const int num_vals, float* const G) { 203 | const float32_t* const arm_vals_x = (const float32_t*) vals_x; 204 | const float32_t* const arm_vals_y = (const float32_t*) vals_y; 205 | 206 | // Running sums. 207 | float32x4_t xx = vdupq_n_f32(0.0f); 208 | float32x4_t xy = vdupq_n_f32(0.0f); 209 | float32x4_t yy = vdupq_n_f32(0.0f); 210 | 211 | // Maximum index we can load 4 consecutive values from. 212 | // e.g. if there are 81 values, our last full pass can be from index 77: 213 | // 81-4=>77 (77, 78, 79, 80) 214 | const int max_i = num_vals - 4; 215 | 216 | // Defined here because we want to keep track of how many values were 217 | // processed by NEON, so that we can finish off the remainder the normal 218 | // way. 219 | int i = 0; 220 | 221 | // Process values 4 at a time, accumulating the sums of 222 | // the pixel-wise x*x, x*y, and y*y values. 223 | for (; i <= max_i; i += 4) { 224 | // Load xs 225 | float32x4_t x = vld1q_f32(arm_vals_x + i); 226 | 227 | // Multiply x*x and accumulate. 228 | xx = vmlaq_f32(xx, x, x); 229 | 230 | // Load ys 231 | float32x4_t y = vld1q_f32(arm_vals_y + i); 232 | 233 | // Multiply x*y and accumulate. 234 | xy = vmlaq_f32(xy, x, y); 235 | 236 | // Multiply y*y and accumulate. 237 | yy = vmlaq_f32(yy, y, y); 238 | } 239 | 240 | static float32_t xx_vals[4]; 241 | static float32_t xy_vals[4]; 242 | static float32_t yy_vals[4]; 243 | 244 | vst1q_f32(xx_vals, xx); 245 | vst1q_f32(xy_vals, xy); 246 | vst1q_f32(yy_vals, yy); 247 | 248 | // Accumulated values are store in sets of 4, we have to manually add 249 | // the last bits together. 250 | for (int j = 0; j < 4; ++j) { 251 | G[0] += xx_vals[j]; 252 | G[1] += xy_vals[j]; 253 | G[3] += yy_vals[j]; 254 | } 255 | 256 | // Finishes off last few values (< 4) from above. 257 | for (; i < num_vals; ++i) { 258 | G[0] += Square(vals_x[i]); 259 | G[1] += vals_x[i] * vals_y[i]; 260 | G[3] += Square(vals_y[i]); 261 | } 262 | 263 | // The matrix is symmetric, so this is a given. 264 | G[2] = G[1]; 265 | } 266 | 267 | } // namespace tf_tracking 268 | 269 | #endif 270 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/image_utils.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_IMAGE_UTILS_H_ 17 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_IMAGE_UTILS_H_ 18 | 19 | #include 20 | 21 | #include "geom.h" 22 | #include "image-inl.h" 23 | #include "image.h" 24 | #include "utils.h" 25 | 26 | 27 | namespace tf_tracking { 28 | 29 | inline void GetUV(const uint8_t* const input, Image* const u, 30 | Image* const v) { 31 | const uint8_t* pUV = input; 32 | 33 | for (int row = 0; row < u->GetHeight(); ++row) { 34 | uint8_t* u_curr = (*u)[row]; 35 | uint8_t* v_curr = (*v)[row]; 36 | for (int col = 0; col < u->GetWidth(); ++col) { 37 | #ifdef __APPLE__ 38 | *u_curr++ = *pUV++; 39 | *v_curr++ = *pUV++; 40 | #else 41 | *v_curr++ = *pUV++; 42 | *u_curr++ = *pUV++; 43 | #endif 44 | } 45 | } 46 | } 47 | 48 | // Marks every point within a circle of a given radius on the given boolean 49 | // image true. 50 | template 51 | inline static void MarkImage(const int x, const int y, const int radius, 52 | Image* const img) { 53 | SCHECK(img->ValidPixel(x, y), "Marking invalid pixel in image! %d, %d", x, y); 54 | 55 | // Precomputed for efficiency. 56 | const int squared_radius = Square(radius); 57 | 58 | // Mark every row in the circle. 59 | for (int d_y = 0; d_y <= radius; ++d_y) { 60 | const int squared_y_dist = Square(d_y); 61 | 62 | const int min_y = MAX(y - d_y, 0); 63 | const int max_y = MIN(y + d_y, img->height_less_one_); 64 | 65 | // The max d_x of the circle must be strictly greater or equal to 66 | // radius - d_y for any positive d_y. Thus, starting from radius - d_y will 67 | // reduce the number of iterations required as compared to starting from 68 | // either 0 and counting up or radius and counting down. 69 | for (int d_x = radius - d_y; d_x <= radius; ++d_x) { 70 | // The first time this criteria is met, we know the width of the circle at 71 | // this row (without using sqrt). 72 | if (squared_y_dist + Square(d_x) >= squared_radius) { 73 | const int min_x = MAX(x - d_x, 0); 74 | const int max_x = MIN(x + d_x, img->width_less_one_); 75 | 76 | // Mark both above and below the center row. 77 | bool* const top_row_start = (*img)[min_y] + min_x; 78 | bool* const bottom_row_start = (*img)[max_y] + min_x; 79 | 80 | const int x_width = max_x - min_x + 1; 81 | memset(top_row_start, true, sizeof(*top_row_start) * x_width); 82 | memset(bottom_row_start, true, sizeof(*bottom_row_start) * x_width); 83 | 84 | // This row is marked, time to move on to the next row. 85 | break; 86 | } 87 | } 88 | } 89 | } 90 | 91 | #ifdef __ARM_NEON 92 | void CalculateGNeon( 93 | const float* const vals_x, const float* const vals_y, 94 | const int num_vals, float* const G); 95 | #endif 96 | 97 | // Puts the image gradient matrix about a pixel into the 2x2 float array G. 98 | // vals_x should be an array of the window x gradient values, whose indices 99 | // can be in any order but are parallel to the vals_y entries. 100 | // See http://robots.stanford.edu/cs223b04/algo_tracking.pdf for more details. 101 | inline void CalculateG(const float* const vals_x, const float* const vals_y, 102 | const int num_vals, float* const G) { 103 | #ifdef __ARM_NEON 104 | CalculateGNeon(vals_x, vals_y, num_vals, G); 105 | return; 106 | #endif 107 | 108 | // Non-accelerated version. 109 | for (int i = 0; i < num_vals; ++i) { 110 | G[0] += Square(vals_x[i]); 111 | G[1] += vals_x[i] * vals_y[i]; 112 | G[3] += Square(vals_y[i]); 113 | } 114 | 115 | // The matrix is symmetric, so this is a given. 116 | G[2] = G[1]; 117 | } 118 | 119 | inline void CalculateGInt16(const int16_t* const vals_x, 120 | const int16_t* const vals_y, const int num_vals, 121 | int* const G) { 122 | // Non-accelerated version. 123 | for (int i = 0; i < num_vals; ++i) { 124 | G[0] += Square(vals_x[i]); 125 | G[1] += vals_x[i] * vals_y[i]; 126 | G[3] += Square(vals_y[i]); 127 | } 128 | 129 | // The matrix is symmetric, so this is a given. 130 | G[2] = G[1]; 131 | } 132 | 133 | 134 | // Puts the image gradient matrix about a pixel into the 2x2 float array G. 135 | // Looks up interpolated pixels, then calls above method for implementation. 136 | inline void CalculateG(const int window_radius, const float center_x, 137 | const float center_y, const Image& I_x, 138 | const Image& I_y, float* const G) { 139 | SCHECK(I_x.ValidPixel(center_x, center_y), "Problem in calculateG!"); 140 | 141 | // Hardcoded to allow for a max window radius of 5 (9 pixels x 9 pixels). 142 | static const int kMaxWindowRadius = 5; 143 | SCHECK(window_radius <= kMaxWindowRadius, 144 | "Window %d > %d!", window_radius, kMaxWindowRadius); 145 | 146 | // Diameter of window is 2 * radius + 1 for center pixel. 147 | static const int kWindowBufferSize = 148 | (kMaxWindowRadius * 2 + 1) * (kMaxWindowRadius * 2 + 1); 149 | 150 | // Preallocate buffers statically for efficiency. 151 | static int16_t vals_x[kWindowBufferSize]; 152 | static int16_t vals_y[kWindowBufferSize]; 153 | 154 | const int src_left_fixed = RealToFixed1616(center_x - window_radius); 155 | const int src_top_fixed = RealToFixed1616(center_y - window_radius); 156 | 157 | int16_t* vals_x_ptr = vals_x; 158 | int16_t* vals_y_ptr = vals_y; 159 | 160 | const int window_size = 2 * window_radius + 1; 161 | for (int y = 0; y < window_size; ++y) { 162 | const int fp_y = src_top_fixed + (y << 16); 163 | 164 | for (int x = 0; x < window_size; ++x) { 165 | const int fp_x = src_left_fixed + (x << 16); 166 | 167 | *vals_x_ptr++ = I_x.GetPixelInterpFixed1616(fp_x, fp_y); 168 | *vals_y_ptr++ = I_y.GetPixelInterpFixed1616(fp_x, fp_y); 169 | } 170 | } 171 | 172 | int32_t g_temp[] = {0, 0, 0, 0}; 173 | CalculateGInt16(vals_x, vals_y, window_size * window_size, g_temp); 174 | 175 | for (int i = 0; i < 4; ++i) { 176 | G[i] = g_temp[i]; 177 | } 178 | } 179 | 180 | inline float ImageCrossCorrelation(const Image& image1, 181 | const Image& image2, 182 | const int x_offset, const int y_offset) { 183 | SCHECK(image1.GetWidth() == image2.GetWidth() && 184 | image1.GetHeight() == image2.GetHeight(), 185 | "Dimension mismatch! %dx%d vs %dx%d", 186 | image1.GetWidth(), image1.GetHeight(), 187 | image2.GetWidth(), image2.GetHeight()); 188 | 189 | const int num_pixels = image1.GetWidth() * image1.GetHeight(); 190 | const float* data1 = image1.data(); 191 | const float* data2 = image2.data(); 192 | return ComputeCrossCorrelation(data1, data2, num_pixels); 193 | } 194 | 195 | // Copies an arbitrary region of an image to another (floating point) 196 | // image, scaling as it goes using bilinear interpolation. 197 | inline void CopyArea(const Image& image, 198 | const BoundingBox& area_to_copy, 199 | Image* const patch_image) { 200 | VLOG(2) << "Copying from: " << area_to_copy << std::endl; 201 | 202 | const int patch_width = patch_image->GetWidth(); 203 | const int patch_height = patch_image->GetHeight(); 204 | 205 | const float x_dist_between_samples = patch_width > 0 ? 206 | area_to_copy.GetWidth() / (patch_width - 1) : 0; 207 | 208 | const float y_dist_between_samples = patch_height > 0 ? 209 | area_to_copy.GetHeight() / (patch_height - 1) : 0; 210 | 211 | for (int y_index = 0; y_index < patch_height; ++y_index) { 212 | const float sample_y = 213 | y_index * y_dist_between_samples + area_to_copy.top_; 214 | 215 | for (int x_index = 0; x_index < patch_width; ++x_index) { 216 | const float sample_x = 217 | x_index * x_dist_between_samples + area_to_copy.left_; 218 | 219 | if (image.ValidInterpPixel(sample_x, sample_y)) { 220 | // TODO(andrewharp): Do area averaging when downsampling. 221 | (*patch_image)[y_index][x_index] = 222 | image.GetPixelInterp(sample_x, sample_y); 223 | } else { 224 | (*patch_image)[y_index][x_index] = -1.0f; 225 | } 226 | } 227 | } 228 | } 229 | 230 | 231 | // Takes a floating point image and normalizes it in-place. 232 | // 233 | // First, negative values will be set to the mean of the non-negative pixels 234 | // in the image. 235 | // 236 | // Then, the resulting will be normalized such that it has mean value of 0.0 and 237 | // a standard deviation of 1.0. 238 | inline void NormalizeImage(Image* const image) { 239 | const float* const data_ptr = image->data(); 240 | 241 | // Copy only the non-negative values to some temp memory. 242 | float running_sum = 0.0f; 243 | int num_data_gte_zero = 0; 244 | { 245 | float* const curr_data = (*image)[0]; 246 | for (int i = 0; i < image->data_size_; ++i) { 247 | if (curr_data[i] >= 0.0f) { 248 | running_sum += curr_data[i]; 249 | ++num_data_gte_zero; 250 | } else { 251 | curr_data[i] = -1.0f; 252 | } 253 | } 254 | } 255 | 256 | // If none of the pixels are valid, just set the entire thing to 0.0f. 257 | if (num_data_gte_zero == 0) { 258 | image->Clear(0.0f); 259 | return; 260 | } 261 | 262 | const float corrected_mean = running_sum / num_data_gte_zero; 263 | 264 | float* curr_data = (*image)[0]; 265 | for (int i = 0; i < image->data_size_; ++i) { 266 | const float curr_val = *curr_data; 267 | *curr_data++ = curr_val < 0 ? 0 : curr_val - corrected_mean; 268 | } 269 | 270 | const float std_dev = ComputeStdDev(data_ptr, image->data_size_, 0.0f); 271 | 272 | if (std_dev > 0.0f) { 273 | curr_data = (*image)[0]; 274 | for (int i = 0; i < image->data_size_; ++i) { 275 | *curr_data++ /= std_dev; 276 | } 277 | 278 | #ifdef SANITY_CHECKS 279 | LOGV("corrected_mean: %1.2f std_dev: %1.2f", corrected_mean, std_dev); 280 | const float correlation = 281 | ComputeCrossCorrelation(image->data(), 282 | image->data(), 283 | image->data_size_); 284 | 285 | if (std::abs(correlation - 1.0f) > EPSILON) { 286 | LOG(ERROR) << "Bad image!" << std::endl; 287 | LOG(ERROR) << *image << std::endl; 288 | } 289 | 290 | SCHECK(std::abs(correlation - 1.0f) < EPSILON, 291 | "Correlation wasn't 1.0f: %.10f", correlation); 292 | #endif 293 | } 294 | } 295 | 296 | } // namespace tf_tracking 297 | 298 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_IMAGE_UTILS_H_ 299 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/integral_image.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_INTEGRAL_IMAGE_H_ 17 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_INTEGRAL_IMAGE_H_ 18 | 19 | #include "geom.h" 20 | #include "image-inl.h" 21 | #include "image.h" 22 | #include "utils.h" 23 | 24 | namespace tf_tracking { 25 | 26 | typedef uint8_t Code; 27 | 28 | class IntegralImage : public Image { 29 | public: 30 | explicit IntegralImage(const Image& image_base) 31 | : Image(image_base.GetWidth(), image_base.GetHeight()) { 32 | Recompute(image_base); 33 | } 34 | 35 | IntegralImage(const int width, const int height) 36 | : Image(width, height) {} 37 | 38 | void Recompute(const Image& image_base) { 39 | SCHECK(image_base.GetWidth() == GetWidth() && 40 | image_base.GetHeight() == GetHeight(), "Dimensions don't match!"); 41 | 42 | // Sum along first row. 43 | { 44 | int x_sum = 0; 45 | for (int x = 0; x < image_base.GetWidth(); ++x) { 46 | x_sum += image_base[0][x]; 47 | (*this)[0][x] = x_sum; 48 | } 49 | } 50 | 51 | // Sum everything else. 52 | for (int y = 1; y < image_base.GetHeight(); ++y) { 53 | uint32_t* curr_sum = (*this)[y]; 54 | 55 | // Previously summed pointers. 56 | const uint32_t* up_one = (*this)[y - 1]; 57 | 58 | // Current value pointer. 59 | const uint8_t* curr_delta = image_base[y]; 60 | 61 | uint32_t row_till_now = 0; 62 | 63 | for (int x = 0; x < GetWidth(); ++x) { 64 | // Add the one above and the one to the left. 65 | row_till_now += *curr_delta; 66 | *curr_sum = *up_one + row_till_now; 67 | 68 | // Scoot everything along. 69 | ++curr_sum; 70 | ++up_one; 71 | ++curr_delta; 72 | } 73 | } 74 | 75 | SCHECK(VerifyData(image_base), "Images did not match!"); 76 | } 77 | 78 | bool VerifyData(const Image& image_base) { 79 | for (int y = 0; y < GetHeight(); ++y) { 80 | for (int x = 0; x < GetWidth(); ++x) { 81 | uint32_t curr_val = (*this)[y][x]; 82 | 83 | if (x > 0) { 84 | curr_val -= (*this)[y][x - 1]; 85 | } 86 | 87 | if (y > 0) { 88 | curr_val -= (*this)[y - 1][x]; 89 | } 90 | 91 | if (x > 0 && y > 0) { 92 | curr_val += (*this)[y - 1][x - 1]; 93 | } 94 | 95 | if (curr_val != image_base[y][x]) { 96 | LOGE("Mismatch! %d vs %d", curr_val, image_base[y][x]); 97 | return false; 98 | } 99 | 100 | if (GetRegionSum(x, y, x, y) != curr_val) { 101 | LOGE("Mismatch!"); 102 | } 103 | } 104 | } 105 | 106 | return true; 107 | } 108 | 109 | // Returns the sum of all pixels in the specified region. 110 | inline uint32_t GetRegionSum(const int x1, const int y1, const int x2, 111 | const int y2) const { 112 | SCHECK(x1 >= 0 && y1 >= 0 && 113 | x2 >= x1 && y2 >= y1 && x2 < GetWidth() && y2 < GetHeight(), 114 | "indices out of bounds! %d-%d / %d, %d-%d / %d, ", 115 | x1, x2, GetWidth(), y1, y2, GetHeight()); 116 | 117 | const uint32_t everything = (*this)[y2][x2]; 118 | 119 | uint32_t sum = everything; 120 | if (x1 > 0 && y1 > 0) { 121 | // Most common case. 122 | const uint32_t left = (*this)[y2][x1 - 1]; 123 | const uint32_t top = (*this)[y1 - 1][x2]; 124 | const uint32_t top_left = (*this)[y1 - 1][x1 - 1]; 125 | 126 | sum = everything - left - top + top_left; 127 | SCHECK(sum >= 0, "Both: %d - %d - %d + %d => %d! indices: %d %d %d %d", 128 | everything, left, top, top_left, sum, x1, y1, x2, y2); 129 | } else if (x1 > 0) { 130 | // Flush against top of image. 131 | // Subtract out the region to the left only. 132 | const uint32_t top = (*this)[y2][x1 - 1]; 133 | sum = everything - top; 134 | SCHECK(sum >= 0, "Top: %d - %d => %d!", everything, top, sum); 135 | } else if (y1 > 0) { 136 | // Flush against left side of image. 137 | // Subtract out the region above only. 138 | const uint32_t left = (*this)[y1 - 1][x2]; 139 | sum = everything - left; 140 | SCHECK(sum >= 0, "Left: %d - %d => %d!", everything, left, sum); 141 | } 142 | 143 | SCHECK(sum >= 0, "Negative sum!"); 144 | 145 | return sum; 146 | } 147 | 148 | // Returns the 2bit code associated with this region, which represents 149 | // the overall gradient. 150 | inline Code GetCode(const BoundingBox& bounding_box) const { 151 | return GetCode(bounding_box.left_, bounding_box.top_, 152 | bounding_box.right_, bounding_box.bottom_); 153 | } 154 | 155 | inline Code GetCode(const int x1, const int y1, 156 | const int x2, const int y2) const { 157 | SCHECK(x1 < x2 && y1 < y2, "Bounds out of order!! TL:%d,%d BR:%d,%d", 158 | x1, y1, x2, y2); 159 | 160 | // Gradient computed vertically. 161 | const int box_height = (y2 - y1) / 2; 162 | const int top_sum = GetRegionSum(x1, y1, x2, y1 + box_height); 163 | const int bottom_sum = GetRegionSum(x1, y2 - box_height, x2, y2); 164 | const bool vertical_code = top_sum > bottom_sum; 165 | 166 | // Gradient computed horizontally. 167 | const int box_width = (x2 - x1) / 2; 168 | const int left_sum = GetRegionSum(x1, y1, x1 + box_width, y2); 169 | const int right_sum = GetRegionSum(x2 - box_width, y1, x2, y2); 170 | const bool horizontal_code = left_sum > right_sum; 171 | 172 | const Code final_code = (vertical_code << 1) | horizontal_code; 173 | 174 | SCHECK(InRange(final_code, static_cast(0), static_cast(3)), 175 | "Invalid code! %d", final_code); 176 | 177 | // Returns a value 0-3. 178 | return final_code; 179 | } 180 | 181 | private: 182 | TF_DISALLOW_COPY_AND_ASSIGN(IntegralImage); 183 | }; 184 | 185 | } // namespace tf_tracking 186 | 187 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_INTEGRAL_IMAGE_H_ 188 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/jni_utils.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_JNI_UTILS_H_ 17 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_JNI_UTILS_H_ 18 | 19 | #include 20 | 21 | #include "utils.h" 22 | 23 | // The JniLongField class is used to access Java fields from native code. This 24 | // technique of hiding pointers to native objects in opaque Java fields is how 25 | // the Android hardware libraries work. This reduces the amount of static 26 | // native methods and makes it easier to manage the lifetime of native objects. 27 | class JniLongField { 28 | public: 29 | JniLongField(const char* field_name) 30 | : field_name_(field_name), field_ID_(0) {} 31 | 32 | int64_t get(JNIEnv* env, jobject thiz) { 33 | if (field_ID_ == 0) { 34 | jclass cls = env->GetObjectClass(thiz); 35 | CHECK_ALWAYS(cls != 0, "Unable to find class"); 36 | field_ID_ = env->GetFieldID(cls, field_name_, "J"); 37 | CHECK_ALWAYS(field_ID_ != 0, 38 | "Unable to find field %s. (Check proguard cfg)", field_name_); 39 | } 40 | 41 | return env->GetLongField(thiz, field_ID_); 42 | } 43 | 44 | void set(JNIEnv* env, jobject thiz, int64_t value) { 45 | if (field_ID_ == 0) { 46 | jclass cls = env->GetObjectClass(thiz); 47 | CHECK_ALWAYS(cls != 0, "Unable to find class"); 48 | field_ID_ = env->GetFieldID(cls, field_name_, "J"); 49 | CHECK_ALWAYS(field_ID_ != 0, 50 | "Unable to find field %s (Check proguard cfg)", field_name_); 51 | } 52 | 53 | env->SetLongField(thiz, field_ID_, value); 54 | } 55 | 56 | private: 57 | const char* const field_name_; 58 | 59 | // This is just a cache 60 | jfieldID field_ID_; 61 | }; 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/keypoint.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_KEYPOINT_H_ 17 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_KEYPOINT_H_ 18 | 19 | #include "geom.h" 20 | #include "image-inl.h" 21 | #include "image.h" 22 | #include "utils.h" 23 | 24 | #include "config.h" 25 | 26 | namespace tf_tracking { 27 | 28 | // For keeping track of keypoints. 29 | struct Keypoint { 30 | Keypoint() : pos_(0.0f, 0.0f), score_(0.0f), type_(0) {} 31 | Keypoint(const float x, const float y) 32 | : pos_(x, y), score_(0.0f), type_(0) {} 33 | 34 | Point2f pos_; 35 | float score_; 36 | uint8_t type_; 37 | }; 38 | 39 | inline std::ostream& operator<<(std::ostream& stream, const Keypoint keypoint) { 40 | return stream << "[" << keypoint.pos_ << ", " 41 | << keypoint.score_ << ", " << keypoint.type_ << "]"; 42 | } 43 | 44 | } // namespace tf_tracking 45 | 46 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_KEYPOINT_H_ 47 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/keypoint_detector.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_KEYPOINT_DETECTOR_H_ 17 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_KEYPOINT_DETECTOR_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include "image-inl.h" 23 | #include "image.h" 24 | #include "image_data.h" 25 | #include "optical_flow.h" 26 | 27 | namespace tf_tracking { 28 | 29 | struct Keypoint; 30 | 31 | class KeypointDetector { 32 | public: 33 | explicit KeypointDetector(const KeypointDetectorConfig* const config) 34 | : config_(config), 35 | keypoint_scratch_(new Image(config_->image_size)), 36 | interest_map_(new Image(config_->image_size)), 37 | fast_quadrant_(0) { 38 | interest_map_->Clear(false); 39 | } 40 | 41 | ~KeypointDetector() {} 42 | 43 | // Finds a new set of keypoints for the current frame, picked from the current 44 | // set of keypoints and also from a set discovered via a keypoint detector. 45 | // Special attention is applied to make sure that keypoints are distributed 46 | // within the supplied ROIs. 47 | void FindKeypoints(const ImageData& image_data, 48 | const std::vector& rois, 49 | const FramePair& prev_change, 50 | FramePair* const curr_change); 51 | 52 | private: 53 | // Compute the corneriness of a point in the image. 54 | float HarrisFilter(const Image& I_x, const Image& I_y, 55 | const float x, const float y) const; 56 | 57 | // Adds a grid of candidate keypoints to the given box, up to 58 | // max_num_keypoints or kNumToAddAsCandidates^2, whichever is lower. 59 | int AddExtraCandidatesForBoxes( 60 | const std::vector& boxes, 61 | const int max_num_keypoints, 62 | Keypoint* const keypoints) const; 63 | 64 | // Scan the frame for potential keypoints using the FAST keypoint detector. 65 | // Quadrant is an argument 0-3 which refers to the quadrant of the image in 66 | // which to detect keypoints. 67 | int FindFastKeypoints(const Image& frame, const int quadrant, 68 | const int downsample_factor, 69 | const int max_num_keypoints, Keypoint* const keypoints); 70 | 71 | int FindFastKeypoints(const ImageData& image_data, 72 | const int max_num_keypoints, 73 | Keypoint* const keypoints); 74 | 75 | // Score a bunch of candidate keypoints. Assigns the scores to the input 76 | // candidate_keypoints array entries. 77 | void ScoreKeypoints(const ImageData& image_data, 78 | const int num_candidates, 79 | Keypoint* const candidate_keypoints); 80 | 81 | void SortKeypoints(const int num_candidates, 82 | Keypoint* const candidate_keypoints) const; 83 | 84 | // Selects a set of keypoints falling within the supplied box such that the 85 | // most highly rated keypoints are picked first, and so that none of them are 86 | // too close together. 87 | int SelectKeypointsInBox( 88 | const BoundingBox& box, 89 | const Keypoint* const candidate_keypoints, 90 | const int num_candidates, 91 | const int max_keypoints, 92 | const int num_existing_keypoints, 93 | const Keypoint* const existing_keypoints, 94 | Keypoint* const final_keypoints) const; 95 | 96 | // Selects from the supplied sorted keypoint pool a set of keypoints that will 97 | // best cover the given set of boxes, such that each box is covered at a 98 | // resolution proportional to its size. 99 | void SelectKeypoints( 100 | const std::vector& boxes, 101 | const Keypoint* const candidate_keypoints, 102 | const int num_candidates, 103 | FramePair* const frame_change) const; 104 | 105 | // Copies and compacts the found keypoints in the second frame of prev_change 106 | // into the array at new_keypoints. 107 | static int CopyKeypoints(const FramePair& prev_change, 108 | Keypoint* const new_keypoints); 109 | 110 | const KeypointDetectorConfig* const config_; 111 | 112 | // Scratch memory for keypoint candidacy detection and non-max suppression. 113 | std::unique_ptr > keypoint_scratch_; 114 | 115 | // Regions of the image to pay special attention to. 116 | std::unique_ptr > interest_map_; 117 | 118 | // The current quadrant of the image to detect FAST keypoints in. 119 | // Keypoint detection is staggered for performance reasons. Every four frames 120 | // a full scan of the frame will have been performed. 121 | int fast_quadrant_; 122 | 123 | Keypoint tmp_keypoints_[kMaxTempKeypoints]; 124 | }; 125 | 126 | } // namespace tf_tracking 127 | 128 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_KEYPOINT_DETECTOR_H_ 129 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/logging.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "logging.h" 17 | 18 | #ifdef STANDALONE_DEMO_LIB 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | LogMessage::LogMessage(const char* fname, int line, int severity) 27 | : fname_(fname), line_(line), severity_(severity) {} 28 | 29 | void LogMessage::GenerateLogMessage() { 30 | int android_log_level; 31 | switch (severity_) { 32 | case INFO: 33 | android_log_level = ANDROID_LOG_INFO; 34 | break; 35 | case WARNING: 36 | android_log_level = ANDROID_LOG_WARN; 37 | break; 38 | case ERROR: 39 | android_log_level = ANDROID_LOG_ERROR; 40 | break; 41 | case FATAL: 42 | android_log_level = ANDROID_LOG_FATAL; 43 | break; 44 | default: 45 | if (severity_ < INFO) { 46 | android_log_level = ANDROID_LOG_VERBOSE; 47 | } else { 48 | android_log_level = ANDROID_LOG_ERROR; 49 | } 50 | break; 51 | } 52 | 53 | std::stringstream ss; 54 | const char* const partial_name = strrchr(fname_, '/'); 55 | ss << (partial_name != nullptr ? partial_name + 1 : fname_) << ":" << line_ 56 | << " " << str(); 57 | __android_log_write(android_log_level, "native", ss.str().c_str()); 58 | 59 | // Also log to stderr (for standalone Android apps). 60 | std::cerr << "native : " << ss.str() << std::endl; 61 | 62 | // Android logging at level FATAL does not terminate execution, so abort() 63 | // is still required to stop the program. 64 | if (severity_ == FATAL) { 65 | abort(); 66 | } 67 | } 68 | 69 | namespace { 70 | 71 | // Parse log level (int64) from environment variable (char*) 72 | int64_t LogLevelStrToInt(const char* tf_env_var_val) { 73 | if (tf_env_var_val == nullptr) { 74 | return 0; 75 | } 76 | 77 | // Ideally we would use env_var / safe_strto64, but it is 78 | // hard to use here without pulling in a lot of dependencies, 79 | // so we use std:istringstream instead 80 | std::string min_log_level(tf_env_var_val); 81 | std::istringstream ss(min_log_level); 82 | int64_t level; 83 | if (!(ss >> level)) { 84 | // Invalid vlog level setting, set level to default (0) 85 | level = 0; 86 | } 87 | 88 | return level; 89 | } 90 | 91 | int64_t MinLogLevelFromEnv() { 92 | const char* tf_env_var_val = getenv("TF_CPP_MIN_LOG_LEVEL"); 93 | return LogLevelStrToInt(tf_env_var_val); 94 | } 95 | 96 | int64_t MinVLogLevelFromEnv() { 97 | const char* tf_env_var_val = getenv("TF_CPP_MIN_VLOG_LEVEL"); 98 | return LogLevelStrToInt(tf_env_var_val); 99 | } 100 | 101 | } // namespace 102 | 103 | LogMessage::~LogMessage() { 104 | // Read the min log level once during the first call to logging. 105 | static int64_t min_log_level = MinLogLevelFromEnv(); 106 | if (TF_PREDICT_TRUE(severity_ >= min_log_level)) GenerateLogMessage(); 107 | } 108 | 109 | int64_t LogMessage::MinVLogLevel() { 110 | static const int64_t min_vlog_level = MinVLogLevelFromEnv(); 111 | return min_vlog_level; 112 | } 113 | 114 | LogMessageFatal::LogMessageFatal(const char* file, int line) 115 | : LogMessage(file, line, ANDROID_LOG_FATAL) {} 116 | LogMessageFatal::~LogMessageFatal() { 117 | // abort() ensures we don't return (we promised we would not via 118 | // ATTRIBUTE_NORETURN). 119 | GenerateLogMessage(); 120 | abort(); 121 | } 122 | 123 | void LogString(const char* fname, int line, int severity, 124 | const std::string& message) { 125 | LogMessage(fname, line, severity) << message; 126 | } 127 | 128 | void LogPrintF(const int severity, const char* format, ...) { 129 | char message[1024]; 130 | va_list argptr; 131 | va_start(argptr, format); 132 | vsnprintf(message, 1024, format, argptr); 133 | va_end(argptr); 134 | __android_log_write(severity, "native", message); 135 | 136 | // Also log to stderr (for standalone Android apps). 137 | std::cerr << "native : " << message << std::endl; 138 | } 139 | 140 | #endif 141 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/logging.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_LOG_STREAMING_H_ 17 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_LOG_STREAMING_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | // Allow this library to be built without depending on TensorFlow by 26 | // defining STANDALONE_DEMO_LIB. Otherwise TensorFlow headers will be 27 | // used. 28 | #ifdef STANDALONE_DEMO_LIB 29 | 30 | // A macro to disallow the copy constructor and operator= functions 31 | // This is usually placed in the private: declarations for a class. 32 | #define TF_DISALLOW_COPY_AND_ASSIGN(TypeName) \ 33 | TypeName(const TypeName&) = delete; \ 34 | void operator=(const TypeName&) = delete 35 | 36 | #if defined(COMPILER_GCC3) 37 | #define TF_PREDICT_FALSE(x) (__builtin_expect(x, 0)) 38 | #define TF_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) 39 | #else 40 | #define TF_PREDICT_FALSE(x) (x) 41 | #define TF_PREDICT_TRUE(x) (x) 42 | #endif 43 | 44 | // Log levels equivalent to those defined by 45 | // third_party/tensorflow/core/platform/logging.h 46 | const int INFO = 0; // base_logging::INFO; 47 | const int WARNING = 1; // base_logging::WARNING; 48 | const int ERROR = 2; // base_logging::ERROR; 49 | const int FATAL = 3; // base_logging::FATAL; 50 | const int NUM_SEVERITIES = 4; // base_logging::NUM_SEVERITIES; 51 | 52 | class LogMessage : public std::basic_ostringstream { 53 | public: 54 | LogMessage(const char* fname, int line, int severity); 55 | ~LogMessage(); 56 | 57 | // Returns the minimum log level for VLOG statements. 58 | // E.g., if MinVLogLevel() is 2, then VLOG(2) statements will produce output, 59 | // but VLOG(3) will not. Defaults to 0. 60 | static int64_t MinVLogLevel(); 61 | 62 | protected: 63 | void GenerateLogMessage(); 64 | 65 | private: 66 | const char* fname_; 67 | int line_; 68 | int severity_; 69 | }; 70 | 71 | // LogMessageFatal ensures the process will exit in failure after 72 | // logging this message. 73 | class LogMessageFatal : public LogMessage { 74 | public: 75 | LogMessageFatal(const char* file, int line); 76 | ~LogMessageFatal(); 77 | }; 78 | 79 | #define _TF_LOG_INFO \ 80 | ::tensorflow::internal::LogMessage(__FILE__, __LINE__, tensorflow::INFO) 81 | #define _TF_LOG_WARNING \ 82 | ::tensorflow::internal::LogMessage(__FILE__, __LINE__, tensorflow::WARNING) 83 | #define _TF_LOG_ERROR \ 84 | ::tensorflow::internal::LogMessage(__FILE__, __LINE__, tensorflow::ERROR) 85 | #define _TF_LOG_FATAL \ 86 | ::tensorflow::internal::LogMessageFatal(__FILE__, __LINE__) 87 | 88 | #define _TF_LOG_QFATAL _TF_LOG_FATAL 89 | 90 | #define LOG(severity) _TF_LOG_##severity 91 | 92 | #define VLOG_IS_ON(lvl) ((lvl) <= LogMessage::MinVLogLevel()) 93 | 94 | #define VLOG(lvl) \ 95 | if (TF_PREDICT_FALSE(VLOG_IS_ON(lvl))) \ 96 | LogMessage(__FILE__, __LINE__, ANDROID_LOG_INFO) 97 | 98 | void LogPrintF(const int severity, const char* format, ...); 99 | 100 | // Support for printf style logging. 101 | #define LOGV(...) 102 | #define LOGD(...) 103 | #define LOGI(...) LogPrintF(ANDROID_LOG_INFO, __VA_ARGS__); 104 | #define LOGW(...) LogPrintF(ANDROID_LOG_INFO, __VA_ARGS__); 105 | #define LOGE(...) LogPrintF(ANDROID_LOG_ERROR, __VA_ARGS__); 106 | 107 | #else 108 | 109 | #include "tensorflow/core/lib/strings/stringprintf.h" 110 | #include "tensorflow/core/platform/logging.h" 111 | 112 | // Support for printf style logging. 113 | #define LOGV(...) 114 | #define LOGD(...) 115 | #define LOGI(...) LOG(INFO) << tensorflow::strings::Printf(__VA_ARGS__); 116 | #define LOGW(...) LOG(INFO) << tensorflow::strings::Printf(__VA_ARGS__); 117 | #define LOGE(...) LOG(INFO) << tensorflow::strings::Printf(__VA_ARGS__); 118 | 119 | #endif 120 | 121 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_LOG_STREAMING_H_ 122 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/object_detector.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // NOTE: no native object detectors are currently provided or used by the code 17 | // in this directory. This class remains mainly for historical reasons. 18 | // Detection in the TF demo is done through TensorFlowMultiBoxDetector.java. 19 | 20 | #include "object_detector.h" 21 | 22 | namespace tf_tracking { 23 | 24 | // This is here so that the vtable gets created properly. 25 | ObjectDetectorBase::~ObjectDetectorBase() {} 26 | 27 | } // namespace tf_tracking 28 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/object_detector.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // NOTE: no native object detectors are currently provided or used by the code 17 | // in this directory. This class remains mainly for historical reasons. 18 | // Detection in the TF demo is done through TensorFlowMultiBoxDetector.java. 19 | 20 | // Defines the ObjectDetector class that is the main interface for detecting 21 | // ObjectModelBases in frames. 22 | 23 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OBJECT_DETECTOR_H_ 24 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OBJECT_DETECTOR_H_ 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | #include "geom.h" 34 | #include "image-inl.h" 35 | #include "image.h" 36 | #include "integral_image.h" 37 | #ifdef __RENDER_OPENGL__ 38 | #include "tensorflow/examples/android/jni/object_tracking/sprite.h" 39 | #endif 40 | #include "utils.h" 41 | 42 | #include "config.h" 43 | #include "image_data.h" 44 | #include "object_model.h" 45 | 46 | namespace tf_tracking { 47 | 48 | // Adds BoundingSquares to a vector such that the first square added is centered 49 | // in the position given and of square_size, and the remaining squares are added 50 | // concentrentically, scaling down by scale_factor until the minimum threshold 51 | // size is passed. 52 | // Squares that do not fall completely within image_bounds will not be added. 53 | static inline void FillWithSquares( 54 | const BoundingBox& image_bounds, 55 | const BoundingBox& position, 56 | const float starting_square_size, 57 | const float smallest_square_size, 58 | const float scale_factor, 59 | std::vector* const squares) { 60 | BoundingSquare descriptor_area = 61 | GetCenteredSquare(position, starting_square_size); 62 | 63 | SCHECK(scale_factor < 1.0f, "Scale factor too large at %.2f!", scale_factor); 64 | 65 | // Use a do/while loop to ensure that at least one descriptor is created. 66 | do { 67 | if (image_bounds.Contains(descriptor_area.ToBoundingBox())) { 68 | squares->push_back(descriptor_area); 69 | } 70 | descriptor_area.Scale(scale_factor); 71 | } while (descriptor_area.size_ >= smallest_square_size - EPSILON); 72 | LOGV("Created %zu squares starting from size %.2f to min size %.2f " 73 | "using scale factor: %.2f", 74 | squares->size(), starting_square_size, smallest_square_size, 75 | scale_factor); 76 | } 77 | 78 | 79 | // Represents a potential detection of a specific ObjectExemplar and Descriptor 80 | // at a specific position in the image. 81 | class Detection { 82 | public: 83 | explicit Detection(const ObjectModelBase* const object_model, 84 | const MatchScore match_score, 85 | const BoundingBox& bounding_box) 86 | : object_model_(object_model), 87 | match_score_(match_score), 88 | bounding_box_(bounding_box) {} 89 | 90 | Detection(const Detection& other) 91 | : object_model_(other.object_model_), 92 | match_score_(other.match_score_), 93 | bounding_box_(other.bounding_box_) {} 94 | 95 | virtual ~Detection() {} 96 | 97 | inline BoundingBox GetObjectBoundingBox() const { 98 | return bounding_box_; 99 | } 100 | 101 | inline MatchScore GetMatchScore() const { 102 | return match_score_; 103 | } 104 | 105 | inline const ObjectModelBase* GetObjectModel() const { 106 | return object_model_; 107 | } 108 | 109 | inline bool Intersects(const Detection& other) { 110 | // Check if any of the four axes separates us, there must be at least one. 111 | return bounding_box_.Intersects(other.bounding_box_); 112 | } 113 | 114 | struct Comp { 115 | inline bool operator()(const Detection& a, const Detection& b) const { 116 | return a.match_score_ > b.match_score_; 117 | } 118 | }; 119 | 120 | // TODO(andrewharp): add accessors to update these instead. 121 | const ObjectModelBase* object_model_; 122 | MatchScore match_score_; 123 | BoundingBox bounding_box_; 124 | }; 125 | 126 | inline std::ostream& operator<<(std::ostream& stream, 127 | const Detection& detection) { 128 | const BoundingBox actual_area = detection.GetObjectBoundingBox(); 129 | stream << actual_area; 130 | return stream; 131 | } 132 | 133 | class ObjectDetectorBase { 134 | public: 135 | explicit ObjectDetectorBase(const ObjectDetectorConfig* const config) 136 | : config_(config), 137 | image_data_(NULL) {} 138 | 139 | virtual ~ObjectDetectorBase(); 140 | 141 | // Sets the current image data. All calls to ObjectDetector other than 142 | // FillDescriptors use the image data last set. 143 | inline void SetImageData(const ImageData* const image_data) { 144 | image_data_ = image_data; 145 | } 146 | 147 | // Main entry point into the detection algorithm. 148 | // Scans the frame for candidates, tweaks them, and fills in the 149 | // given std::vector of Detection objects with acceptable matches. 150 | virtual void Detect(const std::vector& positions, 151 | std::vector* const detections) const = 0; 152 | 153 | virtual ObjectModelBase* CreateObjectModel(const std::string& name) = 0; 154 | 155 | virtual void DeleteObjectModel(const std::string& name) = 0; 156 | 157 | virtual void GetObjectModels( 158 | std::vector* models) const = 0; 159 | 160 | // Creates a new ObjectExemplar from the given position in the context of 161 | // the last frame passed to NextFrame. 162 | // Will return null in the case that there's no room for a descriptor to be 163 | // created in the example area, or the example area is not completely 164 | // contained within the frame. 165 | virtual void UpdateModel(const Image& base_image, 166 | const IntegralImage& integral_image, 167 | const BoundingBox& bounding_box, const bool locked, 168 | ObjectModelBase* model) const = 0; 169 | 170 | virtual void Draw() const = 0; 171 | 172 | virtual bool AllowSpontaneousDetections() = 0; 173 | 174 | protected: 175 | const std::unique_ptr config_; 176 | 177 | // The latest frame data, upon which all detections will be performed. 178 | // Not owned by this object, just provided for reference by ObjectTracker 179 | // via SetImageData(). 180 | const ImageData* image_data_; 181 | 182 | private: 183 | TF_DISALLOW_COPY_AND_ASSIGN(ObjectDetectorBase); 184 | }; 185 | 186 | template 187 | class ObjectDetector : public ObjectDetectorBase { 188 | public: 189 | explicit ObjectDetector(const ObjectDetectorConfig* const config) 190 | : ObjectDetectorBase(config) {} 191 | 192 | virtual ~ObjectDetector() { 193 | typename std::map::const_iterator it = 194 | object_models_.begin(); 195 | for (; it != object_models_.end(); ++it) { 196 | ModelType* model = it->second; 197 | delete model; 198 | } 199 | } 200 | 201 | virtual void DeleteObjectModel(const std::string& name) { 202 | ModelType* model = object_models_[name]; 203 | CHECK_ALWAYS(model != NULL, "Model was null!"); 204 | object_models_.erase(name); 205 | SAFE_DELETE(model); 206 | } 207 | 208 | virtual void GetObjectModels( 209 | std::vector* models) const { 210 | typename std::map::const_iterator it = 211 | object_models_.begin(); 212 | for (; it != object_models_.end(); ++it) { 213 | models->push_back(it->second); 214 | } 215 | } 216 | 217 | virtual bool AllowSpontaneousDetections() { 218 | return false; 219 | } 220 | 221 | protected: 222 | std::map object_models_; 223 | 224 | private: 225 | TF_DISALLOW_COPY_AND_ASSIGN(ObjectDetector); 226 | }; 227 | 228 | } // namespace tf_tracking 229 | 230 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OBJECT_DETECTOR_H_ 231 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/object_model.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // NOTE: no native object detectors are currently provided or used by the code 17 | // in this directory. This class remains mainly for historical reasons. 18 | // Detection in the TF demo is done through TensorFlowMultiBoxDetector.java. 19 | 20 | // Contains ObjectModelBase declaration. 21 | 22 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_DETECTION_OBJECT_MODEL_H_ 23 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_DETECTION_OBJECT_MODEL_H_ 24 | 25 | #ifdef __RENDER_OPENGL__ 26 | #include 27 | #include 28 | #endif 29 | 30 | #include 31 | 32 | #include "geom.h" 33 | #include "image-inl.h" 34 | #include "image.h" 35 | #include "integral_image.h" 36 | #ifdef __RENDER_OPENGL__ 37 | #include "tensorflow/examples/android/jni/object_tracking/sprite.h" 38 | #endif 39 | #include "utils.h" 40 | 41 | #include "config.h" 42 | #include "image_data.h" 43 | #include "keypoint.h" 44 | 45 | namespace tf_tracking { 46 | 47 | // The ObjectModelBase class represents all the known appearance information for 48 | // an object. It is not a specific instance of the object in the world, 49 | // but just the general appearance information that enables detection. An 50 | // ObjectModelBase can be reused across multiple-instances of TrackedObjects. 51 | class ObjectModelBase { 52 | public: 53 | ObjectModelBase(const std::string& name) : name_(name) {} 54 | 55 | virtual ~ObjectModelBase() {} 56 | 57 | // Called when the next step in an ongoing track occurs. 58 | virtual void TrackStep(const BoundingBox& position, 59 | const Image& image, 60 | const IntegralImage& integral_image, 61 | const bool authoritative) {} 62 | 63 | // Called when an object track is lost. 64 | virtual void TrackLost() {} 65 | 66 | // Called when an object track is confirmed as legitimate. 67 | virtual void TrackConfirmed() {} 68 | 69 | virtual float GetMaxCorrelation(const Image& patch_image) const = 0; 70 | 71 | virtual MatchScore GetMatchScore( 72 | const BoundingBox& position, const ImageData& image_data) const = 0; 73 | 74 | virtual void Draw(float* const depth) const = 0; 75 | 76 | inline const std::string& GetName() const { 77 | return name_; 78 | } 79 | 80 | protected: 81 | const std::string name_; 82 | 83 | private: 84 | TF_DISALLOW_COPY_AND_ASSIGN(ObjectModelBase); 85 | }; 86 | 87 | template 88 | class ObjectModel : public ObjectModelBase { 89 | public: 90 | ObjectModel(const DetectorType* const detector, 91 | const std::string& name) 92 | : ObjectModelBase(name), detector_(detector) {} 93 | 94 | protected: 95 | const DetectorType* const detector_; 96 | 97 | TF_DISALLOW_COPY_AND_ASSIGN(ObjectModel); 98 | }; 99 | 100 | } // namespace tf_tracking 101 | 102 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_DETECTION_OBJECT_MODEL_H_ 103 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/object_tracker.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OBJECT_TRACKER_H_ 17 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OBJECT_TRACKER_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include "geom.h" 23 | #include "integral_image.h" 24 | #include "utils.h" 25 | 26 | #include "config.h" 27 | #include "flow_cache.h" 28 | #include "keypoint_detector.h" 29 | #include "object_model.h" 30 | #include "optical_flow.h" 31 | #include "tracked_object.h" 32 | 33 | namespace tf_tracking { 34 | 35 | typedef std::map TrackedObjectMap; 36 | 37 | inline std::ostream& operator<<(std::ostream& stream, 38 | const TrackedObjectMap& map) { 39 | for (TrackedObjectMap::const_iterator iter = map.begin(); 40 | iter != map.end(); ++iter) { 41 | const TrackedObject& tracked_object = *iter->second; 42 | const std::string& key = iter->first; 43 | stream << key << ": " << tracked_object; 44 | } 45 | return stream; 46 | } 47 | 48 | 49 | // ObjectTracker is the highest-level class in the tracking/detection framework. 50 | // It handles basic image processing, keypoint detection, keypoint tracking, 51 | // object tracking, and object detection/relocalization. 52 | class ObjectTracker { 53 | public: 54 | ObjectTracker(const TrackerConfig* const config, 55 | ObjectDetectorBase* const detector); 56 | virtual ~ObjectTracker(); 57 | 58 | virtual void NextFrame(const uint8_t* const new_frame, 59 | const int64_t timestamp, 60 | const float* const alignment_matrix_2x3) { 61 | NextFrame(new_frame, NULL, timestamp, alignment_matrix_2x3); 62 | } 63 | 64 | // Called upon the arrival of a new frame of raw data. 65 | // Does all image processing, keypoint detection, and object 66 | // tracking/detection for registered objects. 67 | // Argument alignment_matrix_2x3 is a 2x3 matrix (stored row-wise) that 68 | // represents the main transformation that has happened between the last 69 | // and the current frame. 70 | // Argument align_level is the pyramid level (where 0 == finest) that 71 | // the matrix is valid for. 72 | virtual void NextFrame(const uint8_t* const new_frame, 73 | const uint8_t* const uv_frame, const int64_t timestamp, 74 | const float* const alignment_matrix_2x3); 75 | 76 | virtual void RegisterNewObjectWithAppearance(const std::string& id, 77 | const uint8_t* const new_frame, 78 | const BoundingBox& bounding_box); 79 | 80 | // Updates the position of a tracked object, given that it was known to be at 81 | // a certain position at some point in the past. 82 | virtual void SetPreviousPositionOfObject(const std::string& id, 83 | const BoundingBox& bounding_box, 84 | const int64_t timestamp); 85 | 86 | // Sets the current position of the object in the most recent frame provided. 87 | virtual void SetCurrentPositionOfObject(const std::string& id, 88 | const BoundingBox& bounding_box); 89 | 90 | // Tells the ObjectTracker to stop tracking a target. 91 | void ForgetTarget(const std::string& id); 92 | 93 | // Fills the given out_data buffer with the latest detected keypoint 94 | // correspondences, first scaled by scale_factor (to adjust for downsampling 95 | // that may have occurred elsewhere), then packed in a fixed-point format. 96 | int GetKeypointsPacked(uint16_t* const out_data, 97 | const float scale_factor) const; 98 | 99 | // Copy the keypoint arrays after computeFlow is called. 100 | // out_data should be at least kMaxKeypoints * kKeypointStep long. 101 | // Currently, its format is [x1 y1 found x2 y2 score] repeated N times, 102 | // where N is the number of keypoints tracked. N is returned as the result. 103 | int GetKeypoints(const bool only_found, float* const out_data) const; 104 | 105 | // Returns the current position of a box, given that it was at a certain 106 | // position at the given time. 107 | BoundingBox TrackBox(const BoundingBox& region, 108 | const int64_t timestamp) const; 109 | 110 | // Returns the number of frames that have been passed to NextFrame(). 111 | inline int GetNumFrames() const { 112 | return num_frames_; 113 | } 114 | 115 | inline bool HaveObject(const std::string& id) const { 116 | return objects_.find(id) != objects_.end(); 117 | } 118 | 119 | // Returns the TrackedObject associated with the given id. 120 | inline const TrackedObject* GetObject(const std::string& id) const { 121 | TrackedObjectMap::const_iterator iter = objects_.find(id); 122 | CHECK_ALWAYS(iter != objects_.end(), 123 | "Unknown object key! \"%s\"", id.c_str()); 124 | TrackedObject* const object = iter->second; 125 | return object; 126 | } 127 | 128 | // Returns the TrackedObject associated with the given id. 129 | inline TrackedObject* GetObject(const std::string& id) { 130 | TrackedObjectMap::iterator iter = objects_.find(id); 131 | CHECK_ALWAYS(iter != objects_.end(), 132 | "Unknown object key! \"%s\"", id.c_str()); 133 | TrackedObject* const object = iter->second; 134 | return object; 135 | } 136 | 137 | bool IsObjectVisible(const std::string& id) const { 138 | SCHECK(HaveObject(id), "Don't have this object."); 139 | 140 | const TrackedObject* object = GetObject(id); 141 | return object->IsVisible(); 142 | } 143 | 144 | virtual void Draw(const int canvas_width, const int canvas_height, 145 | const float* const frame_to_canvas) const; 146 | 147 | protected: 148 | // Creates a new tracked object at the given position. 149 | // If an object model is provided, then that model will be associated with the 150 | // object. If not, a new model may be created from the appearance at the 151 | // initial position and registered with the object detector. 152 | virtual TrackedObject* MaybeAddObject(const std::string& id, 153 | const Image& image, 154 | const BoundingBox& bounding_box, 155 | const ObjectModelBase* object_model); 156 | 157 | // Find the keypoints in the frame before the current frame. 158 | // If only one frame exists, keypoints will be found in that frame. 159 | void ComputeKeypoints(const bool cached_ok = false); 160 | 161 | // Finds the correspondences for all the points in the current pair of frames. 162 | // Stores the results in the given FramePair. 163 | void FindCorrespondences(FramePair* const curr_change) const; 164 | 165 | inline int GetNthIndexFromEnd(const int offset) const { 166 | return GetNthIndexFromStart(curr_num_frame_pairs_ - 1 - offset); 167 | } 168 | 169 | BoundingBox TrackBox(const BoundingBox& region, 170 | const FramePair& frame_pair) const; 171 | 172 | inline void IncrementFrameIndex() { 173 | // Move the current framechange index up. 174 | ++num_frames_; 175 | ++curr_num_frame_pairs_; 176 | 177 | // If we've got too many, push up the start of the queue. 178 | if (curr_num_frame_pairs_ > kNumFrames) { 179 | first_frame_index_ = GetNthIndexFromStart(1); 180 | --curr_num_frame_pairs_; 181 | } 182 | } 183 | 184 | inline int GetNthIndexFromStart(const int offset) const { 185 | SCHECK(offset >= 0 && offset < curr_num_frame_pairs_, 186 | "Offset out of range! %d out of %d.", offset, curr_num_frame_pairs_); 187 | return (first_frame_index_ + offset) % kNumFrames; 188 | } 189 | 190 | void TrackObjects(); 191 | 192 | const std::unique_ptr config_; 193 | 194 | const int frame_width_; 195 | const int frame_height_; 196 | 197 | int64_t curr_time_; 198 | 199 | int num_frames_; 200 | 201 | TrackedObjectMap objects_; 202 | 203 | FlowCache flow_cache_; 204 | 205 | KeypointDetector keypoint_detector_; 206 | 207 | int curr_num_frame_pairs_; 208 | int first_frame_index_; 209 | 210 | std::unique_ptr frame1_; 211 | std::unique_ptr frame2_; 212 | 213 | FramePair frame_pairs_[kNumFrames]; 214 | 215 | std::unique_ptr detector_; 216 | 217 | int num_detected_; 218 | 219 | private: 220 | void TrackTarget(TrackedObject* const object); 221 | 222 | bool GetBestObjectForDetection( 223 | const Detection& detection, TrackedObject** match) const; 224 | 225 | void ProcessDetections(std::vector* const detections); 226 | 227 | void DetectTargets(); 228 | 229 | // Temp object used in ObjectTracker::CreateNewExample. 230 | mutable std::vector squares; 231 | 232 | friend std::ostream& operator<<(std::ostream& stream, 233 | const ObjectTracker& tracker); 234 | 235 | TF_DISALLOW_COPY_AND_ASSIGN(ObjectTracker); 236 | }; 237 | 238 | inline std::ostream& operator<<(std::ostream& stream, 239 | const ObjectTracker& tracker) { 240 | stream << "Frame size: " << tracker.frame_width_ << "x" 241 | << tracker.frame_height_ << std::endl; 242 | 243 | stream << "Num frames: " << tracker.num_frames_ << std::endl; 244 | 245 | stream << "Curr time: " << tracker.curr_time_ << std::endl; 246 | 247 | const int first_frame_index = tracker.GetNthIndexFromStart(0); 248 | const FramePair& first_frame_pair = tracker.frame_pairs_[first_frame_index]; 249 | 250 | const int last_frame_index = tracker.GetNthIndexFromEnd(0); 251 | const FramePair& last_frame_pair = tracker.frame_pairs_[last_frame_index]; 252 | 253 | stream << "first frame: " << first_frame_index << "," 254 | << first_frame_pair.end_time_ << " " 255 | << "last frame: " << last_frame_index << "," 256 | << last_frame_pair.end_time_ << " diff: " 257 | << last_frame_pair.end_time_ - first_frame_pair.end_time_ << "ms" 258 | << std::endl; 259 | 260 | stream << "Tracked targets:"; 261 | stream << tracker.objects_; 262 | 263 | return stream; 264 | } 265 | 266 | } // namespace tf_tracking 267 | 268 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OBJECT_TRACKER_H_ 269 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/optical_flow.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OPTICAL_FLOW_H_ 17 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OPTICAL_FLOW_H_ 18 | 19 | #include "geom.h" 20 | #include "image-inl.h" 21 | #include "image.h" 22 | #include "utils.h" 23 | 24 | #include "config.h" 25 | #include "frame_pair.h" 26 | #include "image_data.h" 27 | #include "keypoint.h" 28 | 29 | namespace tf_tracking { 30 | 31 | class FlowCache; 32 | 33 | // Class encapsulating all the data and logic necessary for performing optical 34 | // flow. 35 | class OpticalFlow { 36 | public: 37 | explicit OpticalFlow(const OpticalFlowConfig* const config); 38 | 39 | // Add a new frame to the optical flow. Will update all the non-keypoint 40 | // related member variables. 41 | // 42 | // new_frame should be a buffer of grayscale values, one byte per pixel, 43 | // at the original frame_width and frame_height used to initialize the 44 | // OpticalFlow object. Downsampling will be handled internally. 45 | // 46 | // time_stamp should be a time in milliseconds that later calls to this and 47 | // other methods will be relative to. 48 | void NextFrame(const ImageData* const image_data); 49 | 50 | // An implementation of the Lucas-Kanade Optical Flow algorithm. 51 | static bool FindFlowAtPoint_LK(const Image& img_I, 52 | const Image& img_J, 53 | const Image& I_x, 54 | const Image& I_y, const float p_x, 55 | const float p_y, float* out_g_x, 56 | float* out_g_y); 57 | 58 | // Pointwise flow using translational 2dof ESM. 59 | static bool FindFlowAtPoint_ESM( 60 | const Image& img_I, const Image& img_J, 61 | const Image& I_x, const Image& I_y, 62 | const Image& J_x, const Image& J_y, const float p_x, 63 | const float p_y, float* out_g_x, float* out_g_y); 64 | 65 | // Finds the flow using a specific level, in either direction. 66 | // If reversed, the coordinates are in the context of the latest 67 | // frame, not the frame before it. 68 | // All coordinates used in parameters are global, not scaled. 69 | bool FindFlowAtPointReversible( 70 | const int level, const float u_x, const float u_y, 71 | const bool reverse_flow, 72 | float* final_x, float* final_y) const; 73 | 74 | // Finds the flow using a specific level, filterable by forward-backward 75 | // error. All coordinates used in parameters are global, not scaled. 76 | bool FindFlowAtPointSingleLevel(const int level, 77 | const float u_x, const float u_y, 78 | const bool filter_by_fb_error, 79 | float* flow_x, float* flow_y) const; 80 | 81 | // Pyramidal optical-flow using all levels. 82 | bool FindFlowAtPointPyramidal(const float u_x, const float u_y, 83 | const bool filter_by_fb_error, 84 | float* flow_x, float* flow_y) const; 85 | 86 | private: 87 | const OpticalFlowConfig* const config_; 88 | 89 | const ImageData* frame1_; 90 | const ImageData* frame2_; 91 | 92 | // Size of the internally allocated images (after original is downsampled). 93 | const Size working_size_; 94 | 95 | TF_DISALLOW_COPY_AND_ASSIGN(OpticalFlow); 96 | }; 97 | 98 | } // namespace tf_tracking 99 | 100 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OPTICAL_FLOW_H_ 101 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/sprite.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_SPRITE_H_ 17 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_SPRITE_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include "tensorflow/examples/android/jni/object_tracking/image-inl.h" 23 | #include "tensorflow/examples/android/jni/object_tracking/image.h" 24 | 25 | #ifndef __RENDER_OPENGL__ 26 | #error sprite.h should not included if OpenGL is not enabled by platform.h 27 | #endif 28 | 29 | namespace tf_tracking { 30 | 31 | // This class encapsulates the logic necessary to load an render image data 32 | // at the same aspect ratio as the original source. 33 | class Sprite { 34 | public: 35 | // Only create Sprites when you have an OpenGl context. 36 | explicit Sprite(const Image& image) { LoadTexture(image, NULL); } 37 | 38 | Sprite(const Image& image, const BoundingBox* const area) { 39 | LoadTexture(image, area); 40 | } 41 | 42 | // Also, try to only delete a Sprite when holding an OpenGl context. 43 | ~Sprite() { 44 | glDeleteTextures(1, &texture_); 45 | } 46 | 47 | inline int GetWidth() const { 48 | return actual_width_; 49 | } 50 | 51 | inline int GetHeight() const { 52 | return actual_height_; 53 | } 54 | 55 | // Draw the sprite at 0,0 - original width/height in the current reference 56 | // frame. Any transformations desired must be applied before calling this 57 | // function. 58 | void Draw() const { 59 | const float float_width = static_cast(actual_width_); 60 | const float float_height = static_cast(actual_height_); 61 | 62 | // Where it gets rendered to. 63 | const float vertices[] = { 0.0f, 0.0f, 0.0f, 64 | 0.0f, float_height, 0.0f, 65 | float_width, 0.0f, 0.0f, 66 | float_width, float_height, 0.0f, 67 | }; 68 | 69 | // The coordinates the texture gets drawn from. 70 | const float max_x = float_width / texture_width_; 71 | const float max_y = float_height / texture_height_; 72 | const float textureVertices[] = { 73 | 0, 0, 74 | 0, max_y, 75 | max_x, 0, 76 | max_x, max_y, 77 | }; 78 | 79 | glEnable(GL_TEXTURE_2D); 80 | glBindTexture(GL_TEXTURE_2D, texture_); 81 | 82 | glEnableClientState(GL_VERTEX_ARRAY); 83 | glEnableClientState(GL_TEXTURE_COORD_ARRAY); 84 | 85 | glVertexPointer(3, GL_FLOAT, 0, vertices); 86 | glTexCoordPointer(2, GL_FLOAT, 0, textureVertices); 87 | 88 | glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); 89 | 90 | glDisableClientState(GL_VERTEX_ARRAY); 91 | glDisableClientState(GL_TEXTURE_COORD_ARRAY); 92 | } 93 | 94 | private: 95 | inline int GetNextPowerOfTwo(const int number) const { 96 | int power_of_two = 1; 97 | while (power_of_two < number) { 98 | power_of_two *= 2; 99 | } 100 | return power_of_two; 101 | } 102 | 103 | // TODO(andrewharp): Allow sprites to have their textures reloaded. 104 | void LoadTexture(const Image& texture_source, 105 | const BoundingBox* const area) { 106 | glEnable(GL_TEXTURE_2D); 107 | 108 | glGenTextures(1, &texture_); 109 | 110 | glBindTexture(GL_TEXTURE_2D, texture_); 111 | 112 | int left = 0; 113 | int top = 0; 114 | 115 | if (area != NULL) { 116 | // If a sub-region was provided to pull the texture from, use that. 117 | left = area->left_; 118 | top = area->top_; 119 | actual_width_ = area->GetWidth(); 120 | actual_height_ = area->GetHeight(); 121 | } else { 122 | actual_width_ = texture_source.GetWidth(); 123 | actual_height_ = texture_source.GetHeight(); 124 | } 125 | 126 | // The textures must be a power of two, so find the sizes that are large 127 | // enough to contain the image data. 128 | texture_width_ = GetNextPowerOfTwo(actual_width_); 129 | texture_height_ = GetNextPowerOfTwo(actual_height_); 130 | 131 | bool allocated_data = false; 132 | uint8_t* texture_data; 133 | 134 | // Except in the lucky case where we're not using a sub-region of the 135 | // original image AND the source data has dimensions that are power of two, 136 | // care must be taken to copy data at the appropriate source and destination 137 | // strides so that the final block can be copied directly into texture 138 | // memory. 139 | // TODO(andrewharp): Figure out if data can be pulled directly from the 140 | // source image with some alignment modifications. 141 | if (left != 0 || top != 0 || 142 | actual_width_ != texture_source.GetWidth() || 143 | actual_height_ != texture_source.GetHeight()) { 144 | texture_data = new uint8_t[actual_width_ * actual_height_]; 145 | 146 | for (int y = 0; y < actual_height_; ++y) { 147 | memcpy(texture_data + actual_width_ * y, texture_source[top + y] + left, 148 | actual_width_ * sizeof(uint8_t)); 149 | } 150 | allocated_data = true; 151 | } else { 152 | // Cast away const-ness because for some reason glTexSubImage2D wants 153 | // a non-const data pointer. 154 | texture_data = const_cast(texture_source.data()); 155 | } 156 | 157 | glTexImage2D(GL_TEXTURE_2D, 158 | 0, 159 | GL_LUMINANCE, 160 | texture_width_, 161 | texture_height_, 162 | 0, 163 | GL_LUMINANCE, 164 | GL_UNSIGNED_BYTE, 165 | NULL); 166 | 167 | glPixelStorei(GL_UNPACK_ALIGNMENT, 1); 168 | glTexSubImage2D(GL_TEXTURE_2D, 169 | 0, 170 | 0, 171 | 0, 172 | actual_width_, 173 | actual_height_, 174 | GL_LUMINANCE, 175 | GL_UNSIGNED_BYTE, 176 | texture_data); 177 | 178 | if (allocated_data) { 179 | delete(texture_data); 180 | } 181 | 182 | glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); 183 | } 184 | 185 | // The id for the texture on the GPU. 186 | GLuint texture_; 187 | 188 | // The width and height to be used for display purposes, referring to the 189 | // dimensions of the original texture. 190 | int actual_width_; 191 | int actual_height_; 192 | 193 | // The allocated dimensions of the texture data, which must be powers of 2. 194 | int texture_width_; 195 | int texture_height_; 196 | 197 | TF_DISALLOW_COPY_AND_ASSIGN(Sprite); 198 | }; 199 | 200 | } // namespace tf_tracking 201 | 202 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_SPRITE_H_ 203 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/time_log.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "time_log.h" 17 | 18 | #ifdef LOG_TIME 19 | // Storage for logging functionality. 20 | int num_time_logs = 0; 21 | LogEntry time_logs[NUM_LOGS]; 22 | 23 | int num_avg_entries = 0; 24 | AverageEntry avg_entries[NUM_LOGS]; 25 | #endif 26 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/time_log.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // Utility functions for performance profiling. 17 | 18 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_TIME_LOG_H_ 19 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_TIME_LOG_H_ 20 | 21 | #include 22 | 23 | #include "logging.h" 24 | #include "utils.h" 25 | 26 | #ifdef LOG_TIME 27 | 28 | // Blend constant for running average. 29 | #define ALPHA 0.98f 30 | #define NUM_LOGS 100 31 | 32 | struct LogEntry { 33 | const char* id; 34 | int64_t time_stamp; 35 | }; 36 | 37 | struct AverageEntry { 38 | const char* id; 39 | float average_duration; 40 | }; 41 | 42 | // Storage for keeping track of this frame's values. 43 | extern int num_time_logs; 44 | extern LogEntry time_logs[NUM_LOGS]; 45 | 46 | // Storage for keeping track of average values (each entry may not be printed 47 | // out each frame). 48 | extern AverageEntry avg_entries[NUM_LOGS]; 49 | extern int num_avg_entries; 50 | 51 | // Call this at the start of a logging phase. 52 | inline static void ResetTimeLog() { 53 | num_time_logs = 0; 54 | } 55 | 56 | 57 | // Log a message to be printed out when printTimeLog is called, along with the 58 | // amount of time in ms that has passed since the last call to this function. 59 | inline static void TimeLog(const char* const str) { 60 | LOGV("%s", str); 61 | if (num_time_logs >= NUM_LOGS) { 62 | LOGE("Out of log entries!"); 63 | return; 64 | } 65 | 66 | time_logs[num_time_logs].id = str; 67 | time_logs[num_time_logs].time_stamp = CurrentThreadTimeNanos(); 68 | ++num_time_logs; 69 | } 70 | 71 | 72 | inline static float Blend(float old_val, float new_val) { 73 | return ALPHA * old_val + (1.0f - ALPHA) * new_val; 74 | } 75 | 76 | 77 | inline static float UpdateAverage(const char* str, const float new_val) { 78 | for (int entry_num = 0; entry_num < num_avg_entries; ++entry_num) { 79 | AverageEntry* const entry = avg_entries + entry_num; 80 | if (str == entry->id) { 81 | entry->average_duration = Blend(entry->average_duration, new_val); 82 | return entry->average_duration; 83 | } 84 | } 85 | 86 | if (num_avg_entries >= NUM_LOGS) { 87 | LOGE("Too many log entries!"); 88 | } 89 | 90 | // If it wasn't there already, add it. 91 | avg_entries[num_avg_entries].id = str; 92 | avg_entries[num_avg_entries].average_duration = new_val; 93 | ++num_avg_entries; 94 | 95 | return new_val; 96 | } 97 | 98 | 99 | // Prints out all the timeLog statements in chronological order with the 100 | // interval that passed between subsequent statements. The total time between 101 | // the first and last statements is printed last. 102 | inline static void PrintTimeLog() { 103 | LogEntry* last_time = time_logs; 104 | 105 | float average_running_total = 0.0f; 106 | 107 | for (int i = 0; i < num_time_logs; ++i) { 108 | LogEntry* const this_time = time_logs + i; 109 | 110 | const float curr_time = 111 | (this_time->time_stamp - last_time->time_stamp) / 1000000.0f; 112 | 113 | const float avg_time = UpdateAverage(this_time->id, curr_time); 114 | average_running_total += avg_time; 115 | 116 | LOGD("%32s: %6.3fms %6.4fms", this_time->id, curr_time, avg_time); 117 | last_time = this_time; 118 | } 119 | 120 | const float total_time = 121 | (last_time->time_stamp - time_logs->time_stamp) / 1000000.0f; 122 | 123 | LOGD("TOTAL TIME: %6.3fms %6.4fms\n", 124 | total_time, average_running_total); 125 | LOGD(" "); 126 | } 127 | #else 128 | inline static void ResetTimeLog() {} 129 | 130 | inline static void TimeLog(const char* const str) { 131 | LOGV("%s", str); 132 | } 133 | 134 | inline static void PrintTimeLog() {} 135 | #endif 136 | 137 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_TIME_LOG_H_ 138 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/tracked_object.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tracked_object.h" 17 | 18 | namespace tf_tracking { 19 | 20 | static const float kInitialDistance = 20.0f; 21 | 22 | static void InitNormalized(const Image& src_image, 23 | const BoundingBox& position, 24 | Image* const dst_image) { 25 | BoundingBox scaled_box(position); 26 | CopyArea(src_image, scaled_box, dst_image); 27 | NormalizeImage(dst_image); 28 | } 29 | 30 | TrackedObject::TrackedObject(const std::string& id, const Image& image, 31 | const BoundingBox& bounding_box, 32 | ObjectModelBase* const model) 33 | : id_(id), 34 | last_known_position_(bounding_box), 35 | last_detection_position_(bounding_box), 36 | position_last_computed_time_(-1), 37 | object_model_(model), 38 | last_detection_thumbnail_(kNormalizedThumbnailSize, 39 | kNormalizedThumbnailSize), 40 | last_frame_thumbnail_(kNormalizedThumbnailSize, kNormalizedThumbnailSize), 41 | tracked_correlation_(0.0f), 42 | tracked_match_score_(0.0), 43 | num_consecutive_frames_below_threshold_(0), 44 | allowable_detection_distance_(Square(kInitialDistance)) { 45 | InitNormalized(image, bounding_box, &last_detection_thumbnail_); 46 | } 47 | 48 | TrackedObject::~TrackedObject() {} 49 | 50 | void TrackedObject::UpdatePosition(const BoundingBox& new_position, 51 | const int64_t timestamp, 52 | const ImageData& image_data, 53 | const bool authoratative) { 54 | last_known_position_ = new_position; 55 | position_last_computed_time_ = timestamp; 56 | 57 | InitNormalized(*image_data.GetImage(), new_position, &last_frame_thumbnail_); 58 | 59 | const float last_localization_correlation = ComputeCrossCorrelation( 60 | last_detection_thumbnail_.data(), 61 | last_frame_thumbnail_.data(), 62 | last_frame_thumbnail_.data_size_); 63 | LOGV("Tracked correlation to last localization: %.6f", 64 | last_localization_correlation); 65 | 66 | // Correlation to object model, if it exists. 67 | if (object_model_ != NULL) { 68 | tracked_correlation_ = 69 | object_model_->GetMaxCorrelation(last_frame_thumbnail_); 70 | LOGV("Tracked correlation to model: %.6f", 71 | tracked_correlation_); 72 | 73 | tracked_match_score_ = 74 | object_model_->GetMatchScore(new_position, image_data); 75 | LOGV("Tracked match score with model: %.6f", 76 | tracked_match_score_.value); 77 | } else { 78 | // If there's no model to check against, set the tracked correlation to 79 | // simply be the correlation to the last set position. 80 | tracked_correlation_ = last_localization_correlation; 81 | tracked_match_score_ = MatchScore(0.0f); 82 | } 83 | 84 | // Determine if it's still being tracked. 85 | if (tracked_correlation_ >= kMinimumCorrelationForTracking && 86 | tracked_match_score_ >= kMinimumMatchScore) { 87 | num_consecutive_frames_below_threshold_ = 0; 88 | 89 | if (object_model_ != NULL) { 90 | object_model_->TrackStep(last_known_position_, *image_data.GetImage(), 91 | *image_data.GetIntegralImage(), authoratative); 92 | } 93 | } else if (tracked_match_score_ < kMatchScoreForImmediateTermination) { 94 | if (num_consecutive_frames_below_threshold_ < 1000) { 95 | LOGD("Tracked match score is way too low (%.6f), aborting track.", 96 | tracked_match_score_.value); 97 | } 98 | 99 | // Add an absurd amount of missed frames so that all heuristics will 100 | // consider it a lost track. 101 | num_consecutive_frames_below_threshold_ += 1000; 102 | 103 | if (object_model_ != NULL) { 104 | object_model_->TrackLost(); 105 | } 106 | } else { 107 | ++num_consecutive_frames_below_threshold_; 108 | allowable_detection_distance_ *= 1.1f; 109 | } 110 | } 111 | 112 | void TrackedObject::OnDetection(ObjectModelBase* const model, 113 | const BoundingBox& detection_position, 114 | const MatchScore match_score, 115 | const int64_t timestamp, 116 | const ImageData& image_data) { 117 | const float overlap = detection_position.PascalScore(last_known_position_); 118 | if (overlap > kPositionOverlapThreshold) { 119 | // If the position agreement with the current tracked position is good 120 | // enough, lock all the current unlocked examples. 121 | object_model_->TrackConfirmed(); 122 | num_consecutive_frames_below_threshold_ = 0; 123 | } 124 | 125 | // Before relocalizing, make sure the new proposed position is better than 126 | // the existing position by a small amount to prevent thrashing. 127 | if (match_score <= tracked_match_score_ + kMatchScoreBuffer) { 128 | LOGI("Not relocalizing since new match is worse: %.6f < %.6f + %.6f", 129 | match_score.value, tracked_match_score_.value, 130 | kMatchScoreBuffer.value); 131 | return; 132 | } 133 | 134 | LOGI("Relocalizing! From (%.1f, %.1f)[%.1fx%.1f] to " 135 | "(%.1f, %.1f)[%.1fx%.1f]: %.6f > %.6f", 136 | last_known_position_.left_, last_known_position_.top_, 137 | last_known_position_.GetWidth(), last_known_position_.GetHeight(), 138 | detection_position.left_, detection_position.top_, 139 | detection_position.GetWidth(), detection_position.GetHeight(), 140 | match_score.value, tracked_match_score_.value); 141 | 142 | if (overlap < kPositionOverlapThreshold) { 143 | // The path might be good, it might be bad, but it's no longer a path 144 | // since we're moving the box to a new position, so just nuke it from 145 | // orbit to be safe. 146 | object_model_->TrackLost(); 147 | } 148 | 149 | object_model_ = model; 150 | 151 | // Reset the last detected appearance. 152 | InitNormalized( 153 | *image_data.GetImage(), detection_position, &last_detection_thumbnail_); 154 | 155 | num_consecutive_frames_below_threshold_ = 0; 156 | last_detection_position_ = detection_position; 157 | 158 | UpdatePosition(detection_position, timestamp, image_data, false); 159 | allowable_detection_distance_ = Square(kInitialDistance); 160 | } 161 | 162 | } // namespace tf_tracking 163 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/tracked_object.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_TRACKED_OBJECT_H_ 17 | #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_TRACKED_OBJECT_H_ 18 | 19 | #ifdef __RENDER_OPENGL__ 20 | #include "tensorflow/examples/android/jni/object_tracking/gl_utils.h" 21 | #endif 22 | #include "object_detector.h" 23 | 24 | namespace tf_tracking { 25 | 26 | // A TrackedObject is a specific instance of an ObjectModel, with a known 27 | // position in the world. 28 | // It provides the last known position and number of recent detection failures, 29 | // in addition to the more general appearance data associated with the object 30 | // class (which is in ObjectModel). 31 | // TODO(andrewharp): Make getters/setters follow styleguide. 32 | class TrackedObject { 33 | public: 34 | TrackedObject(const std::string& id, const Image& image, 35 | const BoundingBox& bounding_box, ObjectModelBase* const model); 36 | 37 | ~TrackedObject(); 38 | 39 | void UpdatePosition(const BoundingBox& new_position, const int64_t timestamp, 40 | const ImageData& image_data, const bool authoratative); 41 | 42 | // This method is called when the tracked object is detected at a 43 | // given position, and allows the associated Model to grow and/or prune 44 | // itself based on where the detection occurred. 45 | void OnDetection(ObjectModelBase* const model, 46 | const BoundingBox& detection_position, 47 | const MatchScore match_score, const int64_t timestamp, 48 | const ImageData& image_data); 49 | 50 | // Called when there's no detection of the tracked object. This will cause 51 | // a tracking failure after enough consecutive failures if the area under 52 | // the current bounding box also doesn't meet a minimum correlation threshold 53 | // with the model. 54 | void OnDetectionFailure() {} 55 | 56 | inline bool IsVisible() const { 57 | return tracked_correlation_ >= kMinimumCorrelationForTracking || 58 | num_consecutive_frames_below_threshold_ < kMaxNumDetectionFailures; 59 | } 60 | 61 | inline float GetCorrelation() { 62 | return tracked_correlation_; 63 | } 64 | 65 | inline MatchScore GetMatchScore() { 66 | return tracked_match_score_; 67 | } 68 | 69 | inline BoundingBox GetPosition() const { 70 | return last_known_position_; 71 | } 72 | 73 | inline BoundingBox GetLastDetectionPosition() const { 74 | return last_detection_position_; 75 | } 76 | 77 | inline const ObjectModelBase* GetModel() const { 78 | return object_model_; 79 | } 80 | 81 | inline const std::string& GetName() const { 82 | return id_; 83 | } 84 | 85 | inline void Draw() const { 86 | #ifdef __RENDER_OPENGL__ 87 | if (tracked_correlation_ < kMinimumCorrelationForTracking) { 88 | glColor4f(MAX(0.0f, -tracked_correlation_), 89 | MAX(0.0f, tracked_correlation_), 90 | 0.0f, 91 | 1.0f); 92 | } else { 93 | glColor4f(MAX(0.0f, -tracked_correlation_), 94 | MAX(0.0f, tracked_correlation_), 95 | 1.0f, 96 | 1.0f); 97 | } 98 | 99 | // Render the box itself. 100 | BoundingBox temp_box(last_known_position_); 101 | DrawBox(temp_box); 102 | 103 | // Render a box inside this one (in case the actual box is hidden). 104 | const float kBufferSize = 1.0f; 105 | temp_box.left_ -= kBufferSize; 106 | temp_box.top_ -= kBufferSize; 107 | temp_box.right_ += kBufferSize; 108 | temp_box.bottom_ += kBufferSize; 109 | DrawBox(temp_box); 110 | 111 | // Render one outside as well. 112 | temp_box.left_ -= -2.0f * kBufferSize; 113 | temp_box.top_ -= -2.0f * kBufferSize; 114 | temp_box.right_ += -2.0f * kBufferSize; 115 | temp_box.bottom_ += -2.0f * kBufferSize; 116 | DrawBox(temp_box); 117 | #endif 118 | } 119 | 120 | // Get current object's num_consecutive_frames_below_threshold_. 121 | inline int64_t GetNumConsecutiveFramesBelowThreshold() { 122 | return num_consecutive_frames_below_threshold_; 123 | } 124 | 125 | // Reset num_consecutive_frames_below_threshold_ to 0. 126 | inline void resetNumConsecutiveFramesBelowThreshold() { 127 | num_consecutive_frames_below_threshold_ = 0; 128 | } 129 | 130 | inline float GetAllowableDistanceSquared() const { 131 | return allowable_detection_distance_; 132 | } 133 | 134 | private: 135 | // The unique id used throughout the system to identify this 136 | // tracked object. 137 | const std::string id_; 138 | 139 | // The last known position of the object. 140 | BoundingBox last_known_position_; 141 | 142 | // The last known position of the object. 143 | BoundingBox last_detection_position_; 144 | 145 | // When the position was last computed. 146 | int64_t position_last_computed_time_; 147 | 148 | // The object model this tracked object is representative of. 149 | ObjectModelBase* object_model_; 150 | 151 | Image last_detection_thumbnail_; 152 | 153 | Image last_frame_thumbnail_; 154 | 155 | // The correlation of the object model with the preview frame at its last 156 | // tracked position. 157 | float tracked_correlation_; 158 | 159 | MatchScore tracked_match_score_; 160 | 161 | // The number of consecutive frames that the tracked position for this object 162 | // has been under the correlation threshold. 163 | int num_consecutive_frames_below_threshold_; 164 | 165 | float allowable_detection_distance_; 166 | 167 | friend std::ostream& operator<<(std::ostream& stream, 168 | const TrackedObject& tracked_object); 169 | 170 | TF_DISALLOW_COPY_AND_ASSIGN(TrackedObject); 171 | }; 172 | 173 | inline std::ostream& operator<<(std::ostream& stream, 174 | const TrackedObject& tracked_object) { 175 | stream << tracked_object.id_ 176 | << " " << tracked_object.last_known_position_ 177 | << " " << tracked_object.position_last_computed_time_ 178 | << " " << tracked_object.num_consecutive_frames_below_threshold_ 179 | << " " << tracked_object.object_model_ 180 | << " " << tracked_object.tracked_correlation_; 181 | return stream; 182 | } 183 | 184 | } // namespace tf_tracking 185 | 186 | #endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_TRACKED_OBJECT_H_ 187 | -------------------------------------------------------------------------------- /app/src/main/jni/object_tracking/utils_neon.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // NEON implementations of Image methods for compatible devices. Control 17 | // should never enter this compilation unit on incompatible devices. 18 | 19 | #ifdef __ARM_NEON 20 | 21 | #include 22 | 23 | #include "geom.h" 24 | #include "image-inl.h" 25 | #include "image.h" 26 | #include "utils.h" 27 | 28 | namespace tf_tracking { 29 | 30 | inline static float GetSum(const float32x4_t& values) { 31 | static float32_t summed_values[4]; 32 | vst1q_f32(summed_values, values); 33 | return summed_values[0] 34 | + summed_values[1] 35 | + summed_values[2] 36 | + summed_values[3]; 37 | } 38 | 39 | 40 | float ComputeMeanNeon(const float* const values, const int num_vals) { 41 | SCHECK(num_vals >= 8, "Not enough values to merit NEON: %d", num_vals); 42 | 43 | const float32_t* const arm_vals = (const float32_t* const) values; 44 | float32x4_t accum = vdupq_n_f32(0.0f); 45 | 46 | int offset = 0; 47 | for (; offset <= num_vals - 4; offset += 4) { 48 | accum = vaddq_f32(accum, vld1q_f32(&arm_vals[offset])); 49 | } 50 | 51 | // Pull the accumulated values into a single variable. 52 | float sum = GetSum(accum); 53 | 54 | // Get the remaining 1 to 3 values. 55 | for (; offset < num_vals; ++offset) { 56 | sum += values[offset]; 57 | } 58 | 59 | const float mean_neon = sum / static_cast(num_vals); 60 | 61 | #ifdef SANITY_CHECKS 62 | const float mean_cpu = ComputeMeanCpu(values, num_vals); 63 | SCHECK(NearlyEqual(mean_neon, mean_cpu, EPSILON * num_vals), 64 | "Neon mismatch with CPU mean! %.10f vs %.10f", 65 | mean_neon, mean_cpu); 66 | #endif 67 | 68 | return mean_neon; 69 | } 70 | 71 | 72 | float ComputeStdDevNeon(const float* const values, 73 | const int num_vals, const float mean) { 74 | SCHECK(num_vals >= 8, "Not enough values to merit NEON: %d", num_vals); 75 | 76 | const float32_t* const arm_vals = (const float32_t* const) values; 77 | const float32x4_t mean_vec = vdupq_n_f32(-mean); 78 | 79 | float32x4_t accum = vdupq_n_f32(0.0f); 80 | 81 | int offset = 0; 82 | for (; offset <= num_vals - 4; offset += 4) { 83 | const float32x4_t deltas = 84 | vaddq_f32(mean_vec, vld1q_f32(&arm_vals[offset])); 85 | 86 | accum = vmlaq_f32(accum, deltas, deltas); 87 | } 88 | 89 | // Pull the accumulated values into a single variable. 90 | float squared_sum = GetSum(accum); 91 | 92 | // Get the remaining 1 to 3 values. 93 | for (; offset < num_vals; ++offset) { 94 | squared_sum += Square(values[offset] - mean); 95 | } 96 | 97 | const float std_dev_neon = sqrt(squared_sum / static_cast(num_vals)); 98 | 99 | #ifdef SANITY_CHECKS 100 | const float std_dev_cpu = ComputeStdDevCpu(values, num_vals, mean); 101 | SCHECK(NearlyEqual(std_dev_neon, std_dev_cpu, EPSILON * num_vals), 102 | "Neon mismatch with CPU std dev! %.10f vs %.10f", 103 | std_dev_neon, std_dev_cpu); 104 | #endif 105 | 106 | return std_dev_neon; 107 | } 108 | 109 | 110 | float ComputeCrossCorrelationNeon(const float* const values1, 111 | const float* const values2, 112 | const int num_vals) { 113 | SCHECK(num_vals >= 8, "Not enough values to merit NEON: %d", num_vals); 114 | 115 | const float32_t* const arm_vals1 = (const float32_t* const) values1; 116 | const float32_t* const arm_vals2 = (const float32_t* const) values2; 117 | 118 | float32x4_t accum = vdupq_n_f32(0.0f); 119 | 120 | int offset = 0; 121 | for (; offset <= num_vals - 4; offset += 4) { 122 | accum = vmlaq_f32(accum, 123 | vld1q_f32(&arm_vals1[offset]), 124 | vld1q_f32(&arm_vals2[offset])); 125 | } 126 | 127 | // Pull the accumulated values into a single variable. 128 | float sxy = GetSum(accum); 129 | 130 | // Get the remaining 1 to 3 values. 131 | for (; offset < num_vals; ++offset) { 132 | sxy += values1[offset] * values2[offset]; 133 | } 134 | 135 | const float cross_correlation_neon = sxy / num_vals; 136 | 137 | #ifdef SANITY_CHECKS 138 | const float cross_correlation_cpu = 139 | ComputeCrossCorrelationCpu(values1, values2, num_vals); 140 | SCHECK(NearlyEqual(cross_correlation_neon, cross_correlation_cpu, 141 | EPSILON * num_vals), 142 | "Neon mismatch with CPU cross correlation! %.10f vs %.10f", 143 | cross_correlation_neon, cross_correlation_cpu); 144 | #endif 145 | 146 | return cross_correlation_neon; 147 | } 148 | 149 | } // namespace tf_tracking 150 | 151 | #endif // __ARM_NEON 152 | -------------------------------------------------------------------------------- /app/src/main/jni/rgb2yuv.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // These utility functions allow for the conversion of RGB data to YUV data. 17 | 18 | #include "rgb2yuv.h" 19 | 20 | static inline void WriteYUV(const int x, const int y, const int width, 21 | const int r8, const int g8, const int b8, 22 | uint8_t* const pY, uint8_t* const pUV) { 23 | // Using formulas from http://msdn.microsoft.com/en-us/library/ms893078 24 | *pY = ((66 * r8 + 129 * g8 + 25 * b8 + 128) >> 8) + 16; 25 | 26 | // Odd widths get rounded up so that UV blocks on the side don't get cut off. 27 | const int blocks_per_row = (width + 1) / 2; 28 | 29 | // 2 bytes per UV block 30 | const int offset = 2 * (((y / 2) * blocks_per_row + (x / 2))); 31 | 32 | // U and V are the average values of all 4 pixels in the block. 33 | if (!(x & 1) && !(y & 1)) { 34 | // Explicitly clear the block if this is the first pixel in it. 35 | pUV[offset] = 0; 36 | pUV[offset + 1] = 0; 37 | } 38 | 39 | // V (with divide by 4 factored in) 40 | #ifdef __APPLE__ 41 | const int u_offset = 0; 42 | const int v_offset = 1; 43 | #else 44 | const int u_offset = 1; 45 | const int v_offset = 0; 46 | #endif 47 | pUV[offset + v_offset] += ((112 * r8 - 94 * g8 - 18 * b8 + 128) >> 10) + 32; 48 | 49 | // U (with divide by 4 factored in) 50 | pUV[offset + u_offset] += ((-38 * r8 - 74 * g8 + 112 * b8 + 128) >> 10) + 32; 51 | } 52 | 53 | void ConvertARGB8888ToYUV420SP(const uint32_t* const input, 54 | uint8_t* const output, int width, int height) { 55 | uint8_t* pY = output; 56 | uint8_t* pUV = output + (width * height); 57 | const uint32_t* in = input; 58 | 59 | for (int y = 0; y < height; y++) { 60 | for (int x = 0; x < width; x++) { 61 | const uint32_t rgb = *in++; 62 | #ifdef __APPLE__ 63 | const int nB = (rgb >> 8) & 0xFF; 64 | const int nG = (rgb >> 16) & 0xFF; 65 | const int nR = (rgb >> 24) & 0xFF; 66 | #else 67 | const int nR = (rgb >> 16) & 0xFF; 68 | const int nG = (rgb >> 8) & 0xFF; 69 | const int nB = rgb & 0xFF; 70 | #endif 71 | WriteYUV(x, y, width, nR, nG, nB, pY++, pUV); 72 | } 73 | } 74 | } 75 | 76 | void ConvertRGB565ToYUV420SP(const uint16_t* const input, uint8_t* const output, 77 | const int width, const int height) { 78 | uint8_t* pY = output; 79 | uint8_t* pUV = output + (width * height); 80 | const uint16_t* in = input; 81 | 82 | for (int y = 0; y < height; y++) { 83 | for (int x = 0; x < width; x++) { 84 | const uint32_t rgb = *in++; 85 | 86 | const int r5 = ((rgb >> 11) & 0x1F); 87 | const int g6 = ((rgb >> 5) & 0x3F); 88 | const int b5 = (rgb & 0x1F); 89 | 90 | // Shift left, then fill in the empty low bits with a copy of the high 91 | // bits so we can stretch across the entire 0 - 255 range. 92 | const int r8 = r5 << 3 | r5 >> 2; 93 | const int g8 = g6 << 2 | g6 >> 4; 94 | const int b8 = b5 << 3 | b5 >> 2; 95 | 96 | WriteYUV(x, y, width, r8, g8, b8, pY++, pUV); 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /app/src/main/jni/rgb2yuv.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef ORG_TENSORFLOW_JNI_IMAGEUTILS_RGB2YUV_H_ 17 | #define ORG_TENSORFLOW_JNI_IMAGEUTILS_RGB2YUV_H_ 18 | 19 | #include 20 | 21 | #ifdef __cplusplus 22 | extern "C" { 23 | #endif 24 | 25 | void ConvertARGB8888ToYUV420SP(const uint32_t* const input, 26 | uint8_t* const output, int width, int height); 27 | 28 | void ConvertRGB565ToYUV420SP(const uint16_t* const input, uint8_t* const output, 29 | const int width, const int height); 30 | 31 | #ifdef __cplusplus 32 | } 33 | #endif 34 | 35 | #endif // ORG_TENSORFLOW_JNI_IMAGEUTILS_RGB2YUV_H_ 36 | -------------------------------------------------------------------------------- /app/src/main/jni/yuv2rgb.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // This is a collection of routines which converts various YUV image formats 17 | // to ARGB. 18 | 19 | #include "yuv2rgb.h" 20 | 21 | #ifndef MAX 22 | #define MAX(a, b) ({__typeof__(a) _a = (a); __typeof__(b) _b = (b); _a > _b ? _a : _b; }) 23 | #define MIN(a, b) ({__typeof__(a) _a = (a); __typeof__(b) _b = (b); _a < _b ? _a : _b; }) 24 | #endif 25 | 26 | // This value is 2 ^ 18 - 1, and is used to clamp the RGB values before their ranges 27 | // are normalized to eight bits. 28 | static const int kMaxChannelValue = 262143; 29 | 30 | static inline uint32_t YUV2RGB(int nY, int nU, int nV) { 31 | nY -= 16; 32 | nU -= 128; 33 | nV -= 128; 34 | if (nY < 0) nY = 0; 35 | 36 | // This is the floating point equivalent. We do the conversion in integer 37 | // because some Android devices do not have floating point in hardware. 38 | // nR = (int)(1.164 * nY + 2.018 * nU); 39 | // nG = (int)(1.164 * nY - 0.813 * nV - 0.391 * nU); 40 | // nB = (int)(1.164 * nY + 1.596 * nV); 41 | 42 | int nR = 1192 * nY + 1634 * nV; 43 | int nG = 1192 * nY - 833 * nV - 400 * nU; 44 | int nB = 1192 * nY + 2066 * nU; 45 | 46 | nR = MIN(kMaxChannelValue, MAX(0, nR)); 47 | nG = MIN(kMaxChannelValue, MAX(0, nG)); 48 | nB = MIN(kMaxChannelValue, MAX(0, nB)); 49 | 50 | nR = (nR >> 10) & 0xff; 51 | nG = (nG >> 10) & 0xff; 52 | nB = (nB >> 10) & 0xff; 53 | 54 | return 0xff000000 | (nR << 16) | (nG << 8) | nB; 55 | } 56 | 57 | // Accepts a YUV 4:2:0 image with a plane of 8 bit Y samples followed by 58 | // separate u and v planes with arbitrary row and column strides, 59 | // containing 8 bit 2x2 subsampled chroma samples. 60 | // Converts to a packed ARGB 32 bit output of the same pixel dimensions. 61 | void ConvertYUV420ToARGB8888(const uint8_t* const yData, 62 | const uint8_t* const uData, 63 | const uint8_t* const vData, uint32_t* const output, 64 | const int width, const int height, 65 | const int y_row_stride, const int uv_row_stride, 66 | const int uv_pixel_stride) { 67 | uint32_t* out = output; 68 | 69 | for (int y = 0; y < height; y++) { 70 | const uint8_t* pY = yData + y_row_stride * y; 71 | 72 | const int uv_row_start = uv_row_stride * (y >> 1); 73 | const uint8_t* pU = uData + uv_row_start; 74 | const uint8_t* pV = vData + uv_row_start; 75 | 76 | for (int x = 0; x < width; x++) { 77 | const int uv_offset = (x >> 1) * uv_pixel_stride; 78 | *out++ = YUV2RGB(pY[x], pU[uv_offset], pV[uv_offset]); 79 | } 80 | } 81 | } 82 | 83 | // Accepts a YUV 4:2:0 image with a plane of 8 bit Y samples followed by an 84 | // interleaved U/V plane containing 8 bit 2x2 subsampled chroma samples, 85 | // except the interleave order of U and V is reversed. Converts to a packed 86 | // ARGB 32 bit output of the same pixel dimensions. 87 | void ConvertYUV420SPToARGB8888(const uint8_t* const yData, 88 | const uint8_t* const uvData, 89 | uint32_t* const output, const int width, 90 | const int height) { 91 | const uint8_t* pY = yData; 92 | const uint8_t* pUV = uvData; 93 | uint32_t* out = output; 94 | 95 | for (int y = 0; y < height; y++) { 96 | for (int x = 0; x < width; x++) { 97 | int nY = *pY++; 98 | int offset = (y >> 1) * width + 2 * (x >> 1); 99 | #ifdef __APPLE__ 100 | int nU = pUV[offset]; 101 | int nV = pUV[offset + 1]; 102 | #else 103 | int nV = pUV[offset]; 104 | int nU = pUV[offset + 1]; 105 | #endif 106 | 107 | *out++ = YUV2RGB(nY, nU, nV); 108 | } 109 | } 110 | } 111 | 112 | // The same as above, but downsamples each dimension to half size. 113 | void ConvertYUV420SPToARGB8888HalfSize(const uint8_t* const input, 114 | uint32_t* const output, int width, 115 | int height) { 116 | const uint8_t* pY = input; 117 | const uint8_t* pUV = input + (width * height); 118 | uint32_t* out = output; 119 | int stride = width; 120 | width >>= 1; 121 | height >>= 1; 122 | 123 | for (int y = 0; y < height; y++) { 124 | for (int x = 0; x < width; x++) { 125 | int nY = (pY[0] + pY[1] + pY[stride] + pY[stride + 1]) >> 2; 126 | pY += 2; 127 | #ifdef __APPLE__ 128 | int nU = *pUV++; 129 | int nV = *pUV++; 130 | #else 131 | int nV = *pUV++; 132 | int nU = *pUV++; 133 | #endif 134 | 135 | *out++ = YUV2RGB(nY, nU, nV); 136 | } 137 | pY += stride; 138 | } 139 | } 140 | 141 | // Accepts a YUV 4:2:0 image with a plane of 8 bit Y samples followed by an 142 | // interleaved U/V plane containing 8 bit 2x2 subsampled chroma samples, 143 | // except the interleave order of U and V is reversed. Converts to a packed 144 | // RGB 565 bit output of the same pixel dimensions. 145 | void ConvertYUV420SPToRGB565(const uint8_t* const input, uint16_t* const output, 146 | const int width, const int height) { 147 | const uint8_t* pY = input; 148 | const uint8_t* pUV = input + (width * height); 149 | uint16_t* out = output; 150 | 151 | for (int y = 0; y < height; y++) { 152 | for (int x = 0; x < width; x++) { 153 | int nY = *pY++; 154 | int offset = (y >> 1) * width + 2 * (x >> 1); 155 | #ifdef __APPLE__ 156 | int nU = pUV[offset]; 157 | int nV = pUV[offset + 1]; 158 | #else 159 | int nV = pUV[offset]; 160 | int nU = pUV[offset + 1]; 161 | #endif 162 | 163 | nY -= 16; 164 | nU -= 128; 165 | nV -= 128; 166 | if (nY < 0) nY = 0; 167 | 168 | // This is the floating point equivalent. We do the conversion in integer 169 | // because some Android devices do not have floating point in hardware. 170 | // nR = (int)(1.164 * nY + 2.018 * nU); 171 | // nG = (int)(1.164 * nY - 0.813 * nV - 0.391 * nU); 172 | // nB = (int)(1.164 * nY + 1.596 * nV); 173 | 174 | int nR = 1192 * nY + 1634 * nV; 175 | int nG = 1192 * nY - 833 * nV - 400 * nU; 176 | int nB = 1192 * nY + 2066 * nU; 177 | 178 | nR = MIN(kMaxChannelValue, MAX(0, nR)); 179 | nG = MIN(kMaxChannelValue, MAX(0, nG)); 180 | nB = MIN(kMaxChannelValue, MAX(0, nB)); 181 | 182 | // Shift more than for ARGB8888 and apply appropriate bitmask. 183 | nR = (nR >> 13) & 0x1f; 184 | nG = (nG >> 12) & 0x3f; 185 | nB = (nB >> 13) & 0x1f; 186 | 187 | // R is high 5 bits, G is middle 6 bits, and B is low 5 bits. 188 | *out++ = (nR << 11) | (nG << 5) | nB; 189 | } 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /app/src/main/jni/yuv2rgb.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // This is a collection of routines which converts various YUV image formats 17 | // to (A)RGB. 18 | 19 | #ifndef ORG_TENSORFLOW_JNI_IMAGEUTILS_YUV2RGB_H_ 20 | #define ORG_TENSORFLOW_JNI_IMAGEUTILS_YUV2RGB_H_ 21 | 22 | #include 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | void ConvertYUV420ToARGB8888(const uint8_t* const yData, 29 | const uint8_t* const uData, 30 | const uint8_t* const vData, uint32_t* const output, 31 | const int width, const int height, 32 | const int y_row_stride, const int uv_row_stride, 33 | const int uv_pixel_stride); 34 | 35 | // Converts YUV420 semi-planar data to ARGB 8888 data using the supplied width 36 | // and height. The input and output must already be allocated and non-null. 37 | // For efficiency, no error checking is performed. 38 | void ConvertYUV420SPToARGB8888(const uint8_t* const pY, 39 | const uint8_t* const pUV, uint32_t* const output, 40 | const int width, const int height); 41 | 42 | // The same as above, but downsamples each dimension to half size. 43 | void ConvertYUV420SPToARGB8888HalfSize(const uint8_t* const input, 44 | uint32_t* const output, int width, 45 | int height); 46 | 47 | // Converts YUV420 semi-planar data to RGB 565 data using the supplied width 48 | // and height. The input and output must already be allocated and non-null. 49 | // For efficiency, no error checking is performed. 50 | void ConvertYUV420SPToRGB565(const uint8_t* const input, uint16_t* const output, 51 | const int width, const int height); 52 | 53 | #ifdef __cplusplus 54 | } 55 | #endif 56 | 57 | #endif // ORG_TENSORFLOW_JNI_IMAGEUTILS_YUV2RGB_H_ 58 | -------------------------------------------------------------------------------- /app/src/main/res/drawable-v24/ic_launcher_foreground.xml: -------------------------------------------------------------------------------- 1 | 7 | 12 | 13 | 19 | 22 | 25 | 26 | 27 | 28 | 34 | 35 | -------------------------------------------------------------------------------- /app/src/main/res/drawable/ic_launcher_background.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 10 | 15 | 20 | 25 | 30 | 35 | 40 | 45 | 50 | 55 | 60 | 65 | 70 | 75 | 80 | 85 | 90 | 95 | 100 | 105 | 110 | 115 | 120 | 125 | 130 | 135 | 140 | 145 | 150 | 155 | 160 | 165 | 170 | 171 | -------------------------------------------------------------------------------- /app/src/main/res/layout/activity_camera.xml: -------------------------------------------------------------------------------- 1 | 16 | -------------------------------------------------------------------------------- /app/src/main/res/layout/camera_connection_fragment_tracking.xml: -------------------------------------------------------------------------------- 1 | 16 | 19 | 20 | 24 | 25 | 29 | 30 | 34 | 35 | -------------------------------------------------------------------------------- /app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /app/src/main/res/mipmap-hdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pillarpond/image-segmenter-android/36e1dd245dca25c8f44e9e217ca280a3a1bf9f1c/app/src/main/res/mipmap-hdpi/ic_launcher.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-hdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pillarpond/image-segmenter-android/36e1dd245dca25c8f44e9e217ca280a3a1bf9f1c/app/src/main/res/mipmap-hdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-mdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pillarpond/image-segmenter-android/36e1dd245dca25c8f44e9e217ca280a3a1bf9f1c/app/src/main/res/mipmap-mdpi/ic_launcher.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-mdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pillarpond/image-segmenter-android/36e1dd245dca25c8f44e9e217ca280a3a1bf9f1c/app/src/main/res/mipmap-mdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pillarpond/image-segmenter-android/36e1dd245dca25c8f44e9e217ca280a3a1bf9f1c/app/src/main/res/mipmap-xhdpi/ic_launcher.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xhdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pillarpond/image-segmenter-android/36e1dd245dca25c8f44e9e217ca280a3a1bf9f1c/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xxhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pillarpond/image-segmenter-android/36e1dd245dca25c8f44e9e217ca280a3a1bf9f1c/app/src/main/res/mipmap-xxhdpi/ic_launcher.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pillarpond/image-segmenter-android/36e1dd245dca25c8f44e9e217ca280a3a1bf9f1c/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xxxhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pillarpond/image-segmenter-android/36e1dd245dca25c8f44e9e217ca280a3a1bf9f1c/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pillarpond/image-segmenter-android/36e1dd245dca25c8f44e9e217ca280a3a1bf9f1c/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /app/src/main/res/values/strings.xml: -------------------------------------------------------------------------------- 1 | 2 | image segmenter 3 | This device doesn\'t support Camera2 API. 4 | 5 | -------------------------------------------------------------------------------- /app/src/main/res/values/styles.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | // Top-level build file where you can add configuration options common to all sub-projects/modules. 2 | 3 | buildscript { 4 | 5 | repositories { 6 | google() 7 | jcenter() 8 | } 9 | dependencies { 10 | classpath 'com.android.tools.build:gradle:3.3.2' 11 | 12 | // NOTE: Do not place your application dependencies here; they belong 13 | // in the individual module build.gradle files 14 | } 15 | } 16 | 17 | allprojects { 18 | repositories { 19 | google() 20 | jcenter() 21 | } 22 | } 23 | 24 | task clean(type: Delete) { 25 | delete rootProject.buildDir 26 | } 27 | -------------------------------------------------------------------------------- /demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pillarpond/image-segmenter-android/36e1dd245dca25c8f44e9e217ca280a3a1bf9f1c/demo.gif -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | # Project-wide Gradle settings. 2 | # IDE (e.g. Android Studio) users: 3 | # Gradle settings configured through the IDE *will override* 4 | # any settings specified in this file. 5 | # For more details on how to configure your build environment visit 6 | # http://www.gradle.org/docs/current/userguide/build_environment.html 7 | # Specifies the JVM arguments used for the daemon process. 8 | # The setting is particularly useful for tweaking memory settings. 9 | org.gradle.jvmargs=-Xmx1536m 10 | # When configured, Gradle will run in incubating parallel mode. 11 | # This option should only be used with decoupled projects. More details, visit 12 | # http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects 13 | # org.gradle.parallel=true 14 | android.useAndroidX=true 15 | android.enableJetifier=true -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pillarpond/image-segmenter-android/36e1dd245dca25c8f44e9e217ca280a3a1bf9f1c/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Tue Sep 04 20:44:15 KST 2018 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.1-all.zip 7 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn () { 37 | echo "$*" 38 | } 39 | 40 | die () { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save () { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | include ':app' 2 | --------------------------------------------------------------------------------