├── app ├── .gitignore ├── src │ ├── main │ │ ├── res │ │ │ ├── mipmap-hdpi │ │ │ │ ├── ic_launcher.png │ │ │ │ └── ic_launcher_round.png │ │ │ ├── mipmap-mdpi │ │ │ │ ├── ic_launcher.png │ │ │ │ └── ic_launcher_round.png │ │ │ ├── mipmap-xhdpi │ │ │ │ ├── ic_launcher.png │ │ │ │ └── ic_launcher_round.png │ │ │ ├── mipmap-xxhdpi │ │ │ │ ├── ic_launcher.png │ │ │ │ └── ic_launcher_round.png │ │ │ ├── mipmap-xxxhdpi │ │ │ │ ├── ic_launcher.png │ │ │ │ └── ic_launcher_round.png │ │ │ ├── values │ │ │ │ ├── strings.xml │ │ │ │ ├── colors.xml │ │ │ │ └── styles.xml │ │ │ ├── mipmap-anydpi-v26 │ │ │ │ ├── ic_launcher.xml │ │ │ │ └── ic_launcher_round.xml │ │ │ ├── layout │ │ │ │ ├── activity_main.xml │ │ │ │ └── camera_connection_fragment_stylize.xml │ │ │ ├── drawable-v24 │ │ │ │ └── ic_launcher_foreground.xml │ │ │ └── drawable │ │ │ │ └── ic_launcher_background.xml │ │ ├── java │ │ │ └── ai │ │ │ │ └── fritz │ │ │ │ └── camera │ │ │ │ ├── OverlayView.java │ │ │ │ ├── AutoFitTextureView.java │ │ │ │ ├── MainActivity.java │ │ │ │ ├── BaseCameraActivity.java │ │ │ │ └── CameraConnectionFragment.java │ │ └── AndroidManifest.xml │ ├── test │ │ └── java │ │ │ └── ai │ │ │ └── fritz │ │ │ └── camera │ │ │ └── ExampleUnitTest.java │ └── androidTest │ │ └── java │ │ └── ai │ │ └── fritz │ │ └── camera │ │ └── ExampleInstrumentedTest.java ├── proguard-rules.pro └── build.gradle ├── settings.gradle ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradle.properties ├── .gitignore ├── README.md ├── gradlew.bat └── gradlew /app/.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | include ':app' 2 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/todo/camera-sample-app/master/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /app/src/main/res/mipmap-hdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/todo/camera-sample-app/master/app/src/main/res/mipmap-hdpi/ic_launcher.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-mdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/todo/camera-sample-app/master/app/src/main/res/mipmap-mdpi/ic_launcher.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/todo/camera-sample-app/master/app/src/main/res/mipmap-xhdpi/ic_launcher.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xxhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/todo/camera-sample-app/master/app/src/main/res/mipmap-xxhdpi/ic_launcher.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xxxhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/todo/camera-sample-app/master/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-hdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/todo/camera-sample-app/master/app/src/main/res/mipmap-hdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-mdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/todo/camera-sample-app/master/app/src/main/res/mipmap-mdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xhdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/todo/camera-sample-app/master/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/todo/camera-sample-app/master/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/todo/camera-sample-app/master/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /app/src/main/res/values/strings.xml: -------------------------------------------------------------------------------- 1 | 2 | FritzCameraApp 3 | This device doesn\'t support Camera2 API. 4 | 5 | -------------------------------------------------------------------------------- /app/src/main/res/values/colors.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | #3F51B5 4 | #303F9F 5 | #FF4081 6 | 7 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Mon Sep 10 19:55:33 EDT 2018 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-all.zip 7 | -------------------------------------------------------------------------------- /app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /app/src/main/res/layout/activity_main.xml: -------------------------------------------------------------------------------- 1 | 4 | 5 | 9 | -------------------------------------------------------------------------------- /app/src/main/res/values/styles.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /app/src/test/java/ai/fritz/camera/ExampleUnitTest.java: -------------------------------------------------------------------------------- 1 | package ai.fritz.camera; 2 | 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.*; 6 | 7 | /** 8 | * Example local unit test, which will execute on the development machine (host). 9 | * 10 | * @see Testing documentation 11 | */ 12 | public class ExampleUnitTest { 13 | @Test 14 | public void addition_isCorrect() { 15 | assertEquals(4, 2 + 2); 16 | } 17 | } -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | # Project-wide Gradle settings. 2 | # IDE (e.g. Android Studio) users: 3 | # Gradle settings configured through the IDE *will override* 4 | # any settings specified in this file. 5 | # For more details on how to configure your build environment visit 6 | # http://www.gradle.org/docs/current/userguide/build_environment.html 7 | # Specifies the JVM arguments used for the daemon process. 8 | # The setting is particularly useful for tweaking memory settings. 9 | org.gradle.jvmargs=-Xmx1536m 10 | # When configured, Gradle will run in incubating parallel mode. 11 | # This option should only be used with decoupled projects. More details, visit 12 | # http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects 13 | # org.gradle.parallel=true 14 | -------------------------------------------------------------------------------- /app/src/main/res/layout/camera_connection_fragment_stylize.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 12 | 13 | 18 | 19 | -------------------------------------------------------------------------------- /app/proguard-rules.pro: -------------------------------------------------------------------------------- 1 | # Add project specific ProGuard rules here. 2 | # You can control the set of applied configuration files using the 3 | # proguardFiles setting in build.gradle. 4 | # 5 | # For more details, see 6 | # http://developer.android.com/guide/developing/tools/proguard.html 7 | 8 | # If your project uses WebView with JS, uncomment the following 9 | # and specify the fully qualified class name to the JavaScript interface 10 | # class: 11 | #-keepclassmembers class fqcn.of.javascript.interface.for.webview { 12 | # public *; 13 | #} 14 | 15 | # Uncomment this to preserve the line number information for 16 | # debugging stack traces. 17 | #-keepattributes SourceFile,LineNumberTable 18 | 19 | # If you keep the line number information, uncomment this to 20 | # hide the original source file name. 21 | #-renamesourcefileattribute SourceFile 22 | -------------------------------------------------------------------------------- /app/src/androidTest/java/ai/fritz/camera/ExampleInstrumentedTest.java: -------------------------------------------------------------------------------- 1 | package ai.fritz.camera; 2 | 3 | import android.content.Context; 4 | import android.support.test.InstrumentationRegistry; 5 | import android.support.test.runner.AndroidJUnit4; 6 | 7 | import org.junit.Test; 8 | import org.junit.runner.RunWith; 9 | 10 | import static org.junit.Assert.*; 11 | 12 | /** 13 | * Instrumented test, which will execute on an Android device. 14 | * 15 | * @see Testing documentation 16 | */ 17 | @RunWith(AndroidJUnit4.class) 18 | public class ExampleInstrumentedTest { 19 | @Test 20 | public void useAppContext() { 21 | // Context of the app under test. 22 | Context appContext = InstrumentationRegistry.getTargetContext(); 23 | 24 | assertEquals("ai.fritz.camera", appContext.getPackageName()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Built application files 2 | *.apk 3 | *.ap_ 4 | 5 | # Files for the ART/Dalvik VM 6 | *.dex 7 | 8 | # Java class files 9 | *.class 10 | 11 | # Generated files 12 | bin/ 13 | gen/ 14 | out/ 15 | 16 | # Gradle files 17 | .gradle/ 18 | build/ 19 | 20 | # Local configuration file (sdk path, etc) 21 | local.properties 22 | 23 | # Proguard folder generated by Eclipse 24 | proguard/ 25 | 26 | # Log Files 27 | *.log 28 | 29 | # Android Studio Navigation editor temp files 30 | .navigation/ 31 | 32 | # Android Studio captures folder 33 | captures/ 34 | 35 | # IntelliJ 36 | *.iml 37 | .idea 38 | 39 | # Keystore files 40 | # Uncomment the following line if you do not want to check your keystore files in. 41 | #*.jks 42 | 43 | # External native build folder generated in Android Studio 2.2 and later 44 | .externalNativeBuild 45 | 46 | # Google Services (e.g. APIs or Firebase) 47 | google-services.json 48 | 49 | # Freeline 50 | freeline.py 51 | freeline/ 52 | freeline_project_description.json 53 | 54 | # fastlane 55 | fastlane/report.xml 56 | fastlane/Preview.html 57 | fastlane/screenshots 58 | fastlane/test_output 59 | fastlane/readme.md 60 | -------------------------------------------------------------------------------- /app/src/main/java/ai/fritz/camera/OverlayView.java: -------------------------------------------------------------------------------- 1 | package ai.fritz.camera; 2 | 3 | import android.content.Context; 4 | import android.graphics.Canvas; 5 | import android.util.AttributeSet; 6 | import android.view.View; 7 | 8 | import java.util.LinkedList; 9 | import java.util.List; 10 | 11 | /** 12 | * A simple View providing a render callback to other classes. 13 | */ 14 | public class OverlayView extends View { 15 | private final List callbacks = new LinkedList(); 16 | 17 | public OverlayView(final Context context, final AttributeSet attrs) { 18 | super(context, attrs); 19 | } 20 | 21 | /** 22 | * Interface defining the callback for client classes. 23 | */ 24 | public interface DrawCallback { 25 | public void drawCallback(final Canvas canvas); 26 | } 27 | 28 | public void addCallback(final DrawCallback callback) { 29 | callbacks.add(callback); 30 | } 31 | 32 | @Override 33 | public synchronized void draw(final Canvas canvas) { 34 | super.draw(canvas); 35 | for (final DrawCallback callback : callbacks) { 36 | callback.drawCallback(canvas); 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /app/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'com.android.application' 2 | 3 | android { 4 | compileSdkVersion 28 5 | defaultConfig { 6 | // MUST MATCH THE APPLICATION YOU CREATE IN FRITZ 7 | applicationId "ai.fritz.camera" 8 | minSdkVersion 21 9 | targetSdkVersion 28 10 | versionCode 1 11 | versionName "1.0" 12 | testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" 13 | } 14 | buildTypes { 15 | debug { 16 | debuggable true 17 | } 18 | release { 19 | minifyEnabled false 20 | proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' 21 | } 22 | } 23 | } 24 | 25 | dependencies { 26 | implementation 'com.android.support:appcompat-v7:28.0.0-rc02' 27 | implementation 'com.android.support.constraint:constraint-layout:1.1.2' 28 | 29 | implementation 'ai.fritz:core:1.2.0' 30 | implementation "ai.fritz:vision-style-model:1.2.0" 31 | 32 | testImplementation 'junit:junit:4.12' 33 | androidTestImplementation 'com.android.support.test:runner:1.0.2' 34 | androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2' 35 | } 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Sample Camera App 2 | A simple camera app for Android that allows you to run ML predictions on the video input. 3 | 4 | Get Started: 5 | ============= 6 | 7 | 1. Setup your Fritz account (https://fritz.ai/). 8 | 2. Fork/Clone this repo and run it in Android Studios. 9 | 3. Edit the AndroidManifest.xml andd add your API key. This is in the Fritz webapp under Project Settings > App > Show API Key. 10 | ``` 11 | 12 | 14 | 15 | 18 | 19 | 20 | ... 21 | ``` 22 | 4.Follow the documentation or a tutorial to add in specific Fritz feature or add your own custom model. 23 | 24 | * Object Detection: https://docs.fritz.ai/features/object-detection/about.html 25 | * Image Labeling: https://docs.fritz.ai/features/image-labeling/about.html 26 | * Style Transefer: https://docs.fritz.ai/features/style-transfer/about.html 27 | 28 | Tutorials: 29 | ============ 30 | Style Transfer: https://heartbeat.fritz.ai/real-time-style-transfer-for-android-6a9d238dfdb5 31 | -------------------------------------------------------------------------------- /app/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 31 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /app/src/main/java/ai/fritz/camera/AutoFitTextureView.java: -------------------------------------------------------------------------------- 1 | package ai.fritz.camera; 2 | 3 | import android.content.Context; 4 | import android.util.AttributeSet; 5 | import android.view.TextureView; 6 | 7 | /** 8 | * A {@link TextureView} that can be adjusted to a specified aspect ratio. 9 | */ 10 | public class AutoFitTextureView extends TextureView { 11 | private int ratioWidth = 0; 12 | private int ratioHeight = 0; 13 | 14 | public AutoFitTextureView(final Context context) { 15 | this(context, null); 16 | } 17 | 18 | public AutoFitTextureView(final Context context, final AttributeSet attrs) { 19 | this(context, attrs, 0); 20 | } 21 | 22 | public AutoFitTextureView(final Context context, final AttributeSet attrs, final int defStyle) { 23 | super(context, attrs, defStyle); 24 | } 25 | 26 | /** 27 | * Sets the aspect ratio for this view. The size of the view will be measured based on the ratio 28 | * calculated from the parameters. Note that the actual sizes of parameters don't matter, that 29 | * is, calling setAspectRatio(2, 3) and setAspectRatio(4, 6) make the same result. 30 | * 31 | * @param width Relative horizontal size 32 | * @param height Relative vertical size 33 | */ 34 | public void setAspectRatio(final int width, final int height) { 35 | if (width < 0 || height < 0) { 36 | throw new IllegalArgumentException("Size cannot be negative."); 37 | } 38 | ratioWidth = width; 39 | ratioHeight = height; 40 | requestLayout(); 41 | } 42 | } -------------------------------------------------------------------------------- /app/src/main/res/drawable-v24/ic_launcher_foreground.xml: -------------------------------------------------------------------------------- 1 | 7 | 12 | 13 | 19 | 22 | 25 | 26 | 27 | 28 | 34 | 35 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /app/src/main/java/ai/fritz/camera/MainActivity.java: -------------------------------------------------------------------------------- 1 | package ai.fritz.camera; 2 | 3 | import android.graphics.Bitmap; 4 | import android.graphics.Canvas; 5 | import android.graphics.Matrix; 6 | import android.graphics.Paint; 7 | import android.media.Image; 8 | import android.media.ImageReader; 9 | import android.os.SystemClock; 10 | import android.os.Bundle; 11 | import android.util.Log; 12 | import android.util.Size; 13 | import android.view.View; 14 | import android.widget.Toast; 15 | 16 | import java.util.concurrent.atomic.AtomicBoolean; 17 | 18 | import ai.fritz.core.Fritz; 19 | import ai.fritz.fritzvisionstylemodel.ArtisticStyle; 20 | import ai.fritz.fritzvisionstylemodel.FritzStyleResolution; 21 | import ai.fritz.fritzvisionstylemodel.FritzVisionStylePredictor; 22 | import ai.fritz.fritzvisionstylemodel.FritzVisionStylePredictorOptions; 23 | import ai.fritz.fritzvisionstylemodel.FritzVisionStyleTransfer; 24 | import ai.fritz.vision.inputs.FritzVisionImage; 25 | import ai.fritz.vision.inputs.FritzVisionOrientation; 26 | import ai.fritz.vision.predictors.FritzVisionPredictor; 27 | 28 | public class MainActivity extends BaseCameraActivity implements ImageReader.OnImageAvailableListener { 29 | 30 | private static final String TAG = MainActivity.class.getSimpleName(); 31 | 32 | private static final Size DESIRED_PREVIEW_SIZE = new Size(1280, 960); 33 | 34 | private AtomicBoolean computing = new AtomicBoolean(false); 35 | 36 | private FritzVisionImage styledImage; 37 | 38 | // STEP 1: 39 | // TODO: Define the predictor variable 40 | // private FritzVisionStylePredictor predictor; 41 | // END STEP 1 42 | 43 | private Size cameraViewSize; 44 | 45 | @Override 46 | public void onCreate(final Bundle savedInstanceState) { 47 | super.onCreate(savedInstanceState); 48 | 49 | // Initialize Fritz 50 | Fritz.configure(this); 51 | 52 | // STEP 1: Get the predictor and set the options. 53 | // ---------------------------------------------- 54 | // TODO: Add the predictor snippet here 55 | // predictor = FritzVisionStyleTransfer.getPredictor(this, ArtisticStyle.STARRY_NIGHT); 56 | // ---------------------------------------------- 57 | // END STEP 1 58 | } 59 | 60 | @Override 61 | protected int getLayoutId() { 62 | return R.layout.camera_connection_fragment_stylize; 63 | } 64 | 65 | @Override 66 | protected Size getDesiredPreviewFrameSize() { 67 | return DESIRED_PREVIEW_SIZE; 68 | } 69 | 70 | @Override 71 | public void onPreviewSizeChosen(final Size previewSize, final Size cameraViewSize, final int rotation) { 72 | 73 | this.cameraViewSize = cameraViewSize; 74 | 75 | // Callback draws a canvas on the OverlayView 76 | addCallback( 77 | new OverlayView.DrawCallback() { 78 | @Override 79 | public void drawCallback(final Canvas canvas) { 80 | // STEP 4: Draw the prediction result 81 | // ---------------------------------- 82 | if (styledImage != null) { 83 | // TODO: Draw or show the result here 84 | // styledImage.drawOnCanvas(canvas); 85 | } 86 | // ---------------------------------- 87 | // END STEP 4 88 | } 89 | }); 90 | } 91 | 92 | @Override 93 | public void onImageAvailable(final ImageReader reader) { 94 | Image image = reader.acquireLatestImage(); 95 | 96 | if (image == null) { 97 | return; 98 | } 99 | 100 | if (!computing.compareAndSet(false, true)) { 101 | image.close(); 102 | return; 103 | } 104 | 105 | // STEP 2: Create the FritzVisionImage object from media.Image 106 | // ------------------------------------------------------------------------ 107 | // TODO: Add code for creating FritzVisionImage from a media.Image object 108 | // int rotationFromCamera = FritzVisionOrientation.getImageRotationFromCamera(this, cameraId); 109 | // final FritzVisionImage fritzImage = FritzVisionImage.fromMediaImage(image, rotationFromCamera); 110 | // ------------------------------------------------------------------------ 111 | // END STEP 2 112 | 113 | image.close(); 114 | 115 | 116 | runInBackground( 117 | new Runnable() { 118 | @Override 119 | public void run() { 120 | // STEP 3: Run predict on the image 121 | // --------------------------------------------------- 122 | // TODO: Add code for running prediction on the image 123 | // final long startTime = SystemClock.uptimeMillis(); 124 | // styledImage = predictor.predict(fritzImage); 125 | // styledImage.scale(cameraViewSize.getWidth(), cameraViewSize.getHeight()); 126 | // Log.d(TAG, "INFERENCE TIME:" + (SystemClock.uptimeMillis() - startTime)); 127 | // ---------------------------------------------------- 128 | // END STEP 3 129 | 130 | // Fire callback to change the OverlayView 131 | requestRender(); 132 | computing.set(false); 133 | } 134 | }); 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn () { 37 | echo "$*" 38 | } 39 | 40 | die () { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save () { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /app/src/main/res/drawable/ic_launcher_background.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 10 | 15 | 20 | 25 | 30 | 35 | 40 | 45 | 50 | 55 | 60 | 65 | 70 | 75 | 80 | 85 | 90 | 95 | 100 | 105 | 110 | 115 | 120 | 125 | 130 | 135 | 140 | 145 | 150 | 155 | 160 | 165 | 170 | 171 | -------------------------------------------------------------------------------- /app/src/main/java/ai/fritz/camera/BaseCameraActivity.java: -------------------------------------------------------------------------------- 1 | package ai.fritz.camera; 2 | 3 | import android.Manifest; 4 | import android.app.Activity; 5 | import android.content.Context; 6 | import android.content.pm.PackageManager; 7 | import android.hardware.camera2.CameraAccessException; 8 | import android.hardware.camera2.CameraCharacteristics; 9 | import android.hardware.camera2.CameraManager; 10 | import android.hardware.camera2.params.StreamConfigurationMap; 11 | import android.media.ImageReader.OnImageAvailableListener; 12 | import android.os.Build; 13 | import android.os.Bundle; 14 | import android.os.Handler; 15 | import android.os.HandlerThread; 16 | import android.util.Log; 17 | import android.util.Size; 18 | import android.view.KeyEvent; 19 | import android.view.WindowManager; 20 | import android.widget.Toast; 21 | 22 | 23 | public abstract class BaseCameraActivity extends Activity implements OnImageAvailableListener { 24 | private static final String TAG = BaseCameraActivity.class.getSimpleName(); 25 | 26 | private static final int PERMISSIONS_REQUEST = 1; 27 | 28 | private static final String PERMISSION_CAMERA = Manifest.permission.CAMERA; 29 | private static final String PERMISSION_STORAGE = Manifest.permission.WRITE_EXTERNAL_STORAGE; 30 | private boolean useCamera2API; 31 | 32 | private boolean debug = false; 33 | 34 | private Handler handler; 35 | private HandlerThread handlerThread; 36 | 37 | protected String cameraId; 38 | 39 | @Override 40 | protected void onCreate(final Bundle savedInstanceState) { 41 | Log.d(TAG, "onCreate " + this); 42 | super.onCreate(null); 43 | getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); 44 | 45 | setContentView(R.layout.activity_main); 46 | 47 | if (hasPermission()) { 48 | setFragment(); 49 | } else { 50 | requestPermission(); 51 | } 52 | } 53 | 54 | @Override 55 | public synchronized void onStart() { 56 | Log.d(TAG, "onStart " + this); 57 | super.onStart(); 58 | } 59 | 60 | @Override 61 | public synchronized void onResume() { 62 | Log.d(TAG, "onResume " + this); 63 | super.onResume(); 64 | 65 | handlerThread = new HandlerThread("inference"); 66 | handlerThread.start(); 67 | handler = new Handler(handlerThread.getLooper()); 68 | } 69 | 70 | @Override 71 | public synchronized void onPause() { 72 | Log.d(TAG, "onPause " + this); 73 | 74 | if (!isFinishing()) { 75 | Log.d(TAG, "Requesting finish"); 76 | finish(); 77 | } 78 | 79 | handlerThread.quitSafely(); 80 | try { 81 | handlerThread.join(); 82 | handlerThread = null; 83 | handler = null; 84 | } catch (final InterruptedException e) { 85 | Log.e(TAG, "Exception!" + e); 86 | } 87 | 88 | super.onPause(); 89 | } 90 | 91 | @Override 92 | public synchronized void onStop() { 93 | Log.d(TAG, "onStop " + this); 94 | super.onStop(); 95 | } 96 | 97 | @Override 98 | public synchronized void onDestroy() { 99 | Log.d(TAG, "onDestroy " + this); 100 | super.onDestroy(); 101 | } 102 | 103 | protected synchronized void runInBackground(final Runnable r) { 104 | if (handler != null) { 105 | handler.post(r); 106 | } 107 | } 108 | 109 | @Override 110 | public void onRequestPermissionsResult( 111 | final int requestCode, final String[] permissions, final int[] grantResults) { 112 | switch (requestCode) { 113 | case PERMISSIONS_REQUEST: { 114 | if (grantResults.length > 0 115 | && grantResults[0] == PackageManager.PERMISSION_GRANTED 116 | && grantResults[1] == PackageManager.PERMISSION_GRANTED) { 117 | setFragment(); 118 | } else { 119 | requestPermission(); 120 | } 121 | } 122 | } 123 | } 124 | 125 | private boolean hasPermission() { 126 | if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { 127 | return checkSelfPermission(PERMISSION_CAMERA) == PackageManager.PERMISSION_GRANTED && checkSelfPermission(PERMISSION_STORAGE) == PackageManager.PERMISSION_GRANTED; 128 | } else { 129 | return true; 130 | } 131 | } 132 | 133 | private void requestPermission() { 134 | if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { 135 | if (shouldShowRequestPermissionRationale(PERMISSION_CAMERA) || shouldShowRequestPermissionRationale(PERMISSION_STORAGE)) { 136 | Toast.makeText(BaseCameraActivity.this, "Camera AND storage permission are required for this demo", Toast.LENGTH_LONG).show(); 137 | } 138 | requestPermissions(new String[]{PERMISSION_CAMERA, PERMISSION_STORAGE}, PERMISSIONS_REQUEST); 139 | } 140 | } 141 | 142 | protected void setFragment() { 143 | cameraId = chooseCamera(); 144 | final CameraConnectionFragment fragment = 145 | CameraConnectionFragment.newInstance( 146 | new CameraConnectionFragment.ConnectionCallback() { 147 | @Override 148 | public void onPreviewSizeChosen(final Size previewSize, final Size cameraViewSize, final int rotation) { 149 | BaseCameraActivity.this.onPreviewSizeChosen(previewSize, cameraViewSize, rotation); 150 | } 151 | }, 152 | this, 153 | getLayoutId(), 154 | getDesiredPreviewFrameSize()); 155 | 156 | fragment.setCamera(cameraId); 157 | 158 | getFragmentManager() 159 | .beginTransaction() 160 | .replace(R.id.camera_container, fragment) 161 | .commit(); 162 | } 163 | 164 | private String chooseCamera() { 165 | final CameraManager manager = (CameraManager) getSystemService(Context.CAMERA_SERVICE); 166 | try { 167 | for (final String cameraId : manager.getCameraIdList()) { 168 | final CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId); 169 | 170 | // We don't use a front facing camera in this sample. 171 | final Integer facing = characteristics.get(CameraCharacteristics.LENS_FACING); 172 | if (facing != null && facing == CameraCharacteristics.LENS_FACING_FRONT) { 173 | continue; 174 | } 175 | 176 | final StreamConfigurationMap map = 177 | characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 178 | 179 | if (map == null) { 180 | continue; 181 | } 182 | 183 | // Fallback to camera1 API for internal cameras that don't have full support. 184 | // This should help with legacy situations where using the camera2 API causes 185 | // distorted or otherwise broken previews. 186 | useCamera2API = (facing == CameraCharacteristics.LENS_FACING_EXTERNAL) 187 | || isHardwareLevelSupported(characteristics, 188 | CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_FULL); 189 | Log.i(TAG, "Camera API lv2?: " + useCamera2API); 190 | return cameraId; 191 | } 192 | } catch (CameraAccessException e) { 193 | Log.e(TAG, "Not allowed to access camera: " + e); 194 | } 195 | 196 | return null; 197 | } 198 | 199 | // Returns true if the device supports the required hardware level, or better. 200 | private boolean isHardwareLevelSupported( 201 | CameraCharacteristics characteristics, int requiredLevel) { 202 | int deviceLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL); 203 | if (deviceLevel == CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY) { 204 | return requiredLevel == deviceLevel; 205 | } 206 | // deviceLevel is not LEGACY, can use numerical sort 207 | return requiredLevel <= deviceLevel; 208 | } 209 | 210 | 211 | public boolean isDebug() { 212 | return debug; 213 | } 214 | 215 | public void requestRender() { 216 | final OverlayView overlay = (OverlayView) findViewById(R.id.debug_overlay); 217 | if (overlay != null) { 218 | overlay.postInvalidate(); 219 | } 220 | } 221 | 222 | public void addCallback(final OverlayView.DrawCallback callback) { 223 | final OverlayView overlay = (OverlayView) findViewById(R.id.debug_overlay); 224 | if (overlay != null) { 225 | overlay.addCallback(callback); 226 | } 227 | } 228 | 229 | public void onSetDebug(final boolean debug) { 230 | } 231 | 232 | @Override 233 | public boolean onKeyDown(final int keyCode, final KeyEvent event) { 234 | if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN || keyCode == KeyEvent.KEYCODE_VOLUME_UP) { 235 | debug = !debug; 236 | requestRender(); 237 | onSetDebug(debug); 238 | return true; 239 | } 240 | return super.onKeyDown(keyCode, event); 241 | } 242 | 243 | protected abstract void onPreviewSizeChosen(final Size previewSize, final Size cameraViewSize, final int rotation); 244 | 245 | protected abstract int getLayoutId(); 246 | 247 | protected abstract Size getDesiredPreviewFrameSize(); 248 | } 249 | 250 | 251 | 252 | -------------------------------------------------------------------------------- /app/src/main/java/ai/fritz/camera/CameraConnectionFragment.java: -------------------------------------------------------------------------------- 1 | package ai.fritz.camera; 2 | 3 | import android.app.Activity; 4 | import android.app.AlertDialog; 5 | import android.app.Dialog; 6 | import android.app.DialogFragment; 7 | import android.app.Fragment; 8 | import android.content.Context; 9 | import android.content.DialogInterface; 10 | import android.graphics.ImageFormat; 11 | import android.graphics.Matrix; 12 | import android.graphics.RectF; 13 | import android.graphics.SurfaceTexture; 14 | import android.hardware.camera2.CameraAccessException; 15 | import android.hardware.camera2.CameraCaptureSession; 16 | import android.hardware.camera2.CameraCharacteristics; 17 | import android.hardware.camera2.CameraDevice; 18 | import android.hardware.camera2.CameraManager; 19 | import android.hardware.camera2.CaptureRequest; 20 | import android.hardware.camera2.CaptureResult; 21 | import android.hardware.camera2.TotalCaptureResult; 22 | import android.hardware.camera2.params.StreamConfigurationMap; 23 | import android.media.ImageReader; 24 | import android.media.ImageReader.OnImageAvailableListener; 25 | import android.os.Bundle; 26 | import android.os.Handler; 27 | import android.os.HandlerThread; 28 | import android.text.TextUtils; 29 | import android.util.Log; 30 | import android.util.Size; 31 | import android.view.LayoutInflater; 32 | import android.view.Surface; 33 | import android.view.TextureView; 34 | import android.view.View; 35 | import android.view.ViewGroup; 36 | import android.widget.Toast; 37 | 38 | import java.util.ArrayList; 39 | import java.util.Arrays; 40 | import java.util.Collections; 41 | import java.util.Comparator; 42 | import java.util.List; 43 | import java.util.concurrent.Semaphore; 44 | import java.util.concurrent.TimeUnit; 45 | 46 | import ai.fritz.camera.R; 47 | import ai.fritz.camera.AutoFitTextureView; 48 | 49 | 50 | public class CameraConnectionFragment extends Fragment { 51 | private static final String TAG = CameraConnectionFragment.class.getSimpleName(); 52 | 53 | public CameraConnectionFragment() { 54 | 55 | } 56 | 57 | /** 58 | * The camera preview size will be chosen to be the smallest frame by pixel size capable of 59 | * containing a DESIRED_SIZE x DESIRED_SIZE square. 60 | */ 61 | private static final int MINIMUM_PREVIEW_SIZE = 320; 62 | 63 | /** 64 | * Conversion from screen rotation to JPEG orientation. 65 | */ 66 | private static final String FRAGMENT_DIALOG = "dialog"; 67 | 68 | /** 69 | * {@link android.view.TextureView.SurfaceTextureListener} handles several lifecycle events on a 70 | * {@link TextureView}. 71 | */ 72 | private final TextureView.SurfaceTextureListener surfaceTextureListener = 73 | new TextureView.SurfaceTextureListener() { 74 | @Override 75 | public void onSurfaceTextureAvailable( 76 | final SurfaceTexture texture, final int width, final int height) { 77 | openCamera(width, height); 78 | } 79 | 80 | @Override 81 | public void onSurfaceTextureSizeChanged( 82 | final SurfaceTexture texture, final int width, final int height) { 83 | configureTransform(width, height); 84 | } 85 | 86 | @Override 87 | public boolean onSurfaceTextureDestroyed(final SurfaceTexture texture) { 88 | return true; 89 | } 90 | 91 | @Override 92 | public void onSurfaceTextureUpdated(final SurfaceTexture texture) { 93 | } 94 | }; 95 | 96 | /** 97 | * Callback for Activities to use to initialize their data once the 98 | * selected preview size is known. 99 | */ 100 | public interface ConnectionCallback { 101 | void onPreviewSizeChosen(Size size, Size cameraViewSize, int cameraRotation); 102 | } 103 | 104 | /** 105 | * ID of the current {@link CameraDevice}. 106 | */ 107 | private String cameraId; 108 | 109 | /** 110 | * An {@link AutoFitTextureView} for camera preview. 111 | */ 112 | private AutoFitTextureView textureView; 113 | 114 | /** 115 | * A {@link CameraCaptureSession } for camera preview. 116 | */ 117 | private CameraCaptureSession captureSession; 118 | 119 | /** 120 | * A reference to the opened {@link CameraDevice}. 121 | */ 122 | private CameraDevice cameraDevice; 123 | 124 | /** 125 | * The rotation in degrees of the camera sensor from the display. 126 | */ 127 | private Integer sensorOrientation; 128 | 129 | /** 130 | * The {@link android.util.Size} of camera preview. 131 | */ 132 | private Size previewSize; 133 | 134 | /** 135 | * {@link android.hardware.camera2.CameraDevice.StateCallback} 136 | * is called when {@link CameraDevice} changes its state. 137 | */ 138 | private final CameraDevice.StateCallback stateCallback = 139 | new CameraDevice.StateCallback() { 140 | @Override 141 | public void onOpened(final CameraDevice cd) { 142 | // This method is called when the camera is opened. We start camera preview here. 143 | cameraOpenCloseLock.release(); 144 | cameraDevice = cd; 145 | createCameraPreviewSession(); 146 | } 147 | 148 | @Override 149 | public void onDisconnected(final CameraDevice cd) { 150 | cameraOpenCloseLock.release(); 151 | cd.close(); 152 | cameraDevice = null; 153 | } 154 | 155 | @Override 156 | public void onError(final CameraDevice cd, final int error) { 157 | cameraOpenCloseLock.release(); 158 | cd.close(); 159 | cameraDevice = null; 160 | final Activity activity = getActivity(); 161 | if (null != activity) { 162 | activity.finish(); 163 | } 164 | } 165 | }; 166 | 167 | /** 168 | * An additional thread for running tasks that shouldn't block the UI. 169 | */ 170 | private HandlerThread backgroundThread; 171 | 172 | /** 173 | * A {@link Handler} for running tasks in the background. 174 | */ 175 | private Handler backgroundHandler; 176 | 177 | /** 178 | * An {@link ImageReader} that handles preview frame capture. 179 | */ 180 | private ImageReader previewReader; 181 | 182 | /** 183 | * {@link android.hardware.camera2.CaptureRequest.Builder} for the camera preview 184 | */ 185 | private CaptureRequest.Builder previewRequestBuilder; 186 | 187 | /** 188 | * {@link CaptureRequest} generated by {@link #previewRequestBuilder} 189 | */ 190 | private CaptureRequest previewRequest; 191 | 192 | /** 193 | * A {@link Semaphore} to prevent the app from exiting before closing the camera. 194 | */ 195 | private final Semaphore cameraOpenCloseLock = new Semaphore(1); 196 | 197 | /** 198 | * A {@link OnImageAvailableListener} to receive frames as they are available. 199 | */ 200 | private OnImageAvailableListener imageListener = null; 201 | 202 | /** 203 | * The input size in pixels desired by TensorFlow (width and height of a square bitmap). 204 | */ 205 | private Size inputSize = null; 206 | 207 | /** 208 | * The layout identifier to inflate for this Fragment. 209 | */ 210 | private int layout = -1; 211 | 212 | 213 | private ConnectionCallback cameraConnectionCallback = null; 214 | 215 | private CameraConnectionFragment( 216 | final ConnectionCallback connectionCallback, 217 | final OnImageAvailableListener imageListener, 218 | final int layout, 219 | final Size inputSize) { 220 | this.cameraConnectionCallback = connectionCallback; 221 | this.imageListener = imageListener; 222 | this.layout = layout; 223 | this.inputSize = inputSize; 224 | } 225 | 226 | /** 227 | * Shows a {@link Toast} on the UI thread. 228 | * 229 | * @param text The message to show 230 | */ 231 | private void showToast(final String text) { 232 | final Activity activity = getActivity(); 233 | if (activity != null) { 234 | activity.runOnUiThread( 235 | new Runnable() { 236 | @Override 237 | public void run() { 238 | Toast.makeText(activity, text, Toast.LENGTH_SHORT).show(); 239 | } 240 | }); 241 | } 242 | } 243 | 244 | /** 245 | * Given {@code choices} of {@code Size}s supported by a camera, chooses the smallest one whose 246 | * width and height are at least as large as the minimum of both, or an exact match if possible. 247 | * 248 | * @param choices The list of sizes that the camera supports for the intended output class 249 | * @param width The minimum desired width 250 | * @param height The minimum desired height 251 | * @return The optimal {@code Size}, or an arbitrary one if none were big enough 252 | */ 253 | protected static Size chooseOptimalSize(final Size[] choices, final int width, final int height) { 254 | final int minSize = Math.max(Math.min(width, height), MINIMUM_PREVIEW_SIZE); 255 | final Size desiredSize = new Size(width, height); 256 | 257 | // Collect the supported resolutions that are at least as big as the preview Surface 258 | boolean exactSizeFound = false; 259 | final List bigEnough = new ArrayList(); 260 | final List tooSmall = new ArrayList(); 261 | for (final Size option : choices) { 262 | if (option.equals(desiredSize)) { 263 | // Set the size but don't return yet so that remaining sizes will still be logged. 264 | exactSizeFound = true; 265 | } 266 | 267 | if (option.getHeight() >= minSize && option.getWidth() >= minSize) { 268 | bigEnough.add(option); 269 | } else { 270 | tooSmall.add(option); 271 | } 272 | } 273 | 274 | Log.d(TAG, "Desired size: " + desiredSize + ", min size: " + minSize + "x" + minSize); 275 | Log.d(TAG, "Valid preview sizes: [" + TextUtils.join(", ", bigEnough) + "]"); 276 | Log.d(TAG, "Rejected preview sizes: [" + TextUtils.join(", ", tooSmall) + "]"); 277 | 278 | if (exactSizeFound) { 279 | Log.d(TAG, "Exact size match found."); 280 | return desiredSize; 281 | } 282 | 283 | // Pick the smallest of those, assuming we found any 284 | if (bigEnough.size() > 0) { 285 | final Size chosenSize = Collections.min(bigEnough, new CompareSizesByArea()); 286 | Log.d(TAG, "Chosen size: " + chosenSize.getWidth() + "x" + chosenSize.getHeight()); 287 | return chosenSize; 288 | } else { 289 | Log.e(TAG, "Couldn't find any suitable preview size"); 290 | return choices[0]; 291 | } 292 | } 293 | 294 | public static CameraConnectionFragment newInstance( 295 | final ConnectionCallback callback, 296 | final OnImageAvailableListener imageListener, 297 | final int layout, 298 | final Size inputSize) { 299 | return new CameraConnectionFragment(callback, imageListener, layout, inputSize); 300 | } 301 | 302 | @Override 303 | public View onCreateView( 304 | final LayoutInflater inflater, final ViewGroup container, final Bundle savedInstanceState) { 305 | return inflater.inflate(layout, container, false); 306 | } 307 | 308 | @Override 309 | public void onViewCreated(final View view, final Bundle savedInstanceState) { 310 | textureView = (AutoFitTextureView) view.findViewById(R.id.texture); 311 | } 312 | 313 | @Override 314 | public void onActivityCreated(final Bundle savedInstanceState) { 315 | super.onActivityCreated(savedInstanceState); 316 | } 317 | 318 | @Override 319 | public void onResume() { 320 | super.onResume(); 321 | startBackgroundThread(); 322 | 323 | // When the screen is turned off and turned back on, the SurfaceTexture is already 324 | // available, and "onSurfaceTextureAvailable" will not be called. In that case, we can open 325 | // a camera and start preview from here (otherwise, we wait until the surface is ready in 326 | // the SurfaceTextureListener). 327 | if (textureView.isAvailable()) { 328 | openCamera(textureView.getWidth(), textureView.getHeight()); 329 | } else { 330 | textureView.setSurfaceTextureListener(surfaceTextureListener); 331 | } 332 | } 333 | 334 | @Override 335 | public void onPause() { 336 | closeCamera(); 337 | stopBackgroundThread(); 338 | super.onPause(); 339 | } 340 | 341 | public void setCamera(String cameraId) { 342 | this.cameraId = cameraId; 343 | } 344 | 345 | /** 346 | * Sets up member variables related to camera. 347 | */ 348 | private void setUpCameraOutputs() { 349 | final Activity activity = getActivity(); 350 | final CameraManager manager = (CameraManager) activity.getSystemService(Context.CAMERA_SERVICE); 351 | try { 352 | final CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId); 353 | 354 | final StreamConfigurationMap map = 355 | characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 356 | 357 | // For still image captures, we use the largest available size. 358 | final Size largest = 359 | Collections.max( 360 | Arrays.asList(map.getOutputSizes(ImageFormat.YUV_420_888)), 361 | new CompareSizesByArea()); 362 | 363 | sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION); 364 | 365 | // Danger, W.R.! Attempting to use too large a preview size could exceed the camera 366 | // bus' bandwidth limitation, resulting in gorgeous previews but the storage of 367 | // garbage capture data. 368 | previewSize = 369 | chooseOptimalSize(map.getOutputSizes(SurfaceTexture.class), 370 | inputSize.getWidth(), 371 | inputSize.getHeight()); 372 | } catch (final CameraAccessException e) { 373 | Log.e(TAG, "Exception!" + e); 374 | } catch (final NullPointerException e) { 375 | // Currently an NPE is thrown when the Camera2API is used but not supported on the 376 | // device this code runs. 377 | // TODO(andrewharp): abstract ErrorDialog/RuntimeException handling out into new method and 378 | // reuse throughout app. 379 | ErrorDialog.newInstance(getString(R.string.camera_error)) 380 | .show(getChildFragmentManager(), FRAGMENT_DIALOG); 381 | throw new RuntimeException(getString(R.string.camera_error)); 382 | } 383 | 384 | Size textureViewSize = new Size(textureView.getWidth(), textureView.getHeight()); 385 | cameraConnectionCallback.onPreviewSizeChosen(previewSize, textureViewSize, sensorOrientation); 386 | } 387 | 388 | /** 389 | * Opens the camera specified by {@link CameraConnectionFragment#cameraId}. 390 | */ 391 | private void openCamera(final int width, final int height) { 392 | setUpCameraOutputs(); 393 | configureTransform(width, height); 394 | final Activity activity = getActivity(); 395 | final CameraManager manager = (CameraManager) activity.getSystemService(Context.CAMERA_SERVICE); 396 | try { 397 | if (!cameraOpenCloseLock.tryAcquire(2500, TimeUnit.MILLISECONDS)) { 398 | throw new RuntimeException("Time out waiting to lock camera opening."); 399 | } 400 | manager.openCamera(cameraId, stateCallback, backgroundHandler); 401 | } catch (final CameraAccessException | SecurityException e) { 402 | Log.e(TAG, "Exception!" + e); 403 | } catch (final InterruptedException e) { 404 | throw new RuntimeException("Interrupted while trying to lock camera opening.", e); 405 | } 406 | } 407 | 408 | /** 409 | * Closes the current {@link CameraDevice}. 410 | */ 411 | private void closeCamera() { 412 | try { 413 | cameraOpenCloseLock.acquire(); 414 | if (null != captureSession) { 415 | captureSession.close(); 416 | captureSession = null; 417 | } 418 | if (null != cameraDevice) { 419 | cameraDevice.close(); 420 | cameraDevice = null; 421 | } 422 | if (null != previewReader) { 423 | previewReader.close(); 424 | previewReader = null; 425 | } 426 | } catch (final InterruptedException e) { 427 | throw new RuntimeException("Interrupted while trying to lock camera closing.", e); 428 | } finally { 429 | cameraOpenCloseLock.release(); 430 | } 431 | } 432 | 433 | /** 434 | * Starts a background thread and its {@link Handler}. 435 | */ 436 | private void startBackgroundThread() { 437 | backgroundThread = new HandlerThread("ImageListener"); 438 | backgroundThread.start(); 439 | backgroundHandler = new Handler(backgroundThread.getLooper()); 440 | } 441 | 442 | /** 443 | * Stops the background thread and its {@link Handler}. 444 | */ 445 | private void stopBackgroundThread() { 446 | backgroundThread.quitSafely(); 447 | try { 448 | backgroundThread.join(); 449 | backgroundThread = null; 450 | backgroundHandler = null; 451 | } catch (final InterruptedException e) { 452 | Log.e(TAG, "Exception!" + e); 453 | } 454 | } 455 | 456 | private final CameraCaptureSession.CaptureCallback captureCallback = 457 | new CameraCaptureSession.CaptureCallback() { 458 | @Override 459 | public void onCaptureProgressed( 460 | final CameraCaptureSession session, 461 | final CaptureRequest request, 462 | final CaptureResult partialResult) { 463 | } 464 | 465 | @Override 466 | public void onCaptureCompleted( 467 | final CameraCaptureSession session, 468 | final CaptureRequest request, 469 | final TotalCaptureResult result) { 470 | } 471 | }; 472 | 473 | /** 474 | * Creates a new {@link CameraCaptureSession} for camera preview. 475 | */ 476 | private void createCameraPreviewSession() { 477 | try { 478 | final SurfaceTexture texture = textureView.getSurfaceTexture(); 479 | assert texture != null; 480 | 481 | // We configure the size of default buffer to be the size of camera preview we want. 482 | texture.setDefaultBufferSize(previewSize.getWidth(), previewSize.getHeight()); 483 | 484 | // This is the output Surface we need to start preview. 485 | final Surface surface = new Surface(texture); 486 | 487 | // We set up a CaptureRequest.Builder with the output Surface. 488 | previewRequestBuilder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW); 489 | previewRequestBuilder.addTarget(surface); 490 | 491 | Log.i(TAG, "Opening camera preview: " + previewSize.getWidth() + "x" + previewSize.getHeight()); 492 | 493 | // Create the reader for the preview frames. 494 | previewReader = 495 | ImageReader.newInstance( 496 | previewSize.getWidth(), previewSize.getHeight(), ImageFormat.YUV_420_888, 2); 497 | 498 | previewReader.setOnImageAvailableListener(imageListener, backgroundHandler); 499 | previewRequestBuilder.addTarget(previewReader.getSurface()); 500 | 501 | // Here, we create a CameraCaptureSession for camera preview. 502 | cameraDevice.createCaptureSession( 503 | Arrays.asList(surface, previewReader.getSurface()), 504 | new CameraCaptureSession.StateCallback() { 505 | 506 | @Override 507 | public void onConfigured(final CameraCaptureSession cameraCaptureSession) { 508 | // The camera is already closed 509 | if (null == cameraDevice) { 510 | return; 511 | } 512 | 513 | // When the session is ready, we start displaying the preview. 514 | captureSession = cameraCaptureSession; 515 | try { 516 | // Auto focus should be continuous for camera preview. 517 | previewRequestBuilder.set( 518 | CaptureRequest.CONTROL_AF_MODE, 519 | CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE); 520 | // Flash is automatically enabled when necessary. 521 | previewRequestBuilder.set( 522 | CaptureRequest.CONTROL_AE_MODE, CaptureRequest.CONTROL_AE_MODE_ON_AUTO_FLASH); 523 | 524 | // Finally, we start displaying the camera preview. 525 | previewRequest = previewRequestBuilder.build(); 526 | captureSession.setRepeatingRequest( 527 | previewRequest, captureCallback, backgroundHandler); 528 | } catch (final CameraAccessException e) { 529 | Log.e(TAG, "Exception!" + e); 530 | } 531 | } 532 | 533 | @Override 534 | public void onConfigureFailed(final CameraCaptureSession cameraCaptureSession) { 535 | showToast("Failed"); 536 | } 537 | }, 538 | null); 539 | } catch (final CameraAccessException e) { 540 | Log.e(TAG, "Exception!" + e); 541 | } 542 | } 543 | 544 | /** 545 | * Configures the necessary {@link android.graphics.Matrix} transformation to `mTextureView`. 546 | * This method should be called after the camera preview size is determined in 547 | * setUpCameraOutputs and also the size of `mTextureView` is fixed. 548 | * 549 | * @param viewWidth The width of `mTextureView` 550 | * @param viewHeight The height of `mTextureView` 551 | */ 552 | private void configureTransform(final int viewWidth, final int viewHeight) { 553 | final Activity activity = getActivity(); 554 | if (null == textureView || null == previewSize || null == activity) { 555 | return; 556 | } 557 | final int rotation = activity.getWindowManager().getDefaultDisplay().getRotation(); 558 | final Matrix matrix = new Matrix(); 559 | final RectF viewRect = new RectF(0, 0, viewWidth, viewHeight); 560 | final RectF bufferRect = new RectF(0, 0, previewSize.getHeight(), previewSize.getWidth()); 561 | final float centerX = viewRect.centerX(); 562 | final float centerY = viewRect.centerY(); 563 | if (Surface.ROTATION_90 == rotation || Surface.ROTATION_270 == rotation) { 564 | bufferRect.offset(centerX - bufferRect.centerX(), centerY - bufferRect.centerY()); 565 | matrix.setRectToRect(viewRect, bufferRect, Matrix.ScaleToFit.FILL); 566 | final float scale = 567 | Math.max( 568 | (float) viewHeight / previewSize.getHeight(), 569 | (float) viewWidth / previewSize.getWidth()); 570 | matrix.postScale(scale, scale, centerX, centerY); 571 | matrix.postRotate(90 * (rotation - 2), centerX, centerY); 572 | } else if (Surface.ROTATION_180 == rotation) { 573 | matrix.postRotate(180, centerX, centerY); 574 | } 575 | textureView.setTransform(matrix); 576 | } 577 | 578 | /** 579 | * Compares two {@code Size}s based on their areas. 580 | */ 581 | static class CompareSizesByArea implements Comparator { 582 | @Override 583 | public int compare(final Size lhs, final Size rhs) { 584 | // We cast here to ensure the multiplications won't overflow 585 | return Long.signum( 586 | (long) lhs.getWidth() * lhs.getHeight() - (long) rhs.getWidth() * rhs.getHeight()); 587 | } 588 | } 589 | 590 | /** 591 | * Shows an error message dialog. 592 | */ 593 | public static class ErrorDialog extends DialogFragment { 594 | private static final String ARG_MESSAGE = "message"; 595 | 596 | public static ErrorDialog newInstance(final String message) { 597 | final ErrorDialog dialog = new ErrorDialog(); 598 | final Bundle args = new Bundle(); 599 | args.putString(ARG_MESSAGE, message); 600 | dialog.setArguments(args); 601 | return dialog; 602 | } 603 | 604 | @Override 605 | public Dialog onCreateDialog(final Bundle savedInstanceState) { 606 | final Activity activity = getActivity(); 607 | return new AlertDialog.Builder(activity) 608 | .setMessage(getArguments().getString(ARG_MESSAGE)) 609 | .setPositiveButton( 610 | android.R.string.ok, 611 | new DialogInterface.OnClickListener() { 612 | @Override 613 | public void onClick(final DialogInterface dialogInterface, final int i) { 614 | activity.finish(); 615 | } 616 | }) 617 | .create(); 618 | } 619 | } 620 | } 621 | --------------------------------------------------------------------------------