├── .yaml ├── LICENSE ├── Model ├── style_predict_f16_256.tflite ├── style_predict_quantized_256.tflite ├── style_transfer_f16_384.tflite └── style_transfer_quantized_384.tflite ├── README.md ├── Ruse.ipynb ├── android ├── README.md ├── app │ ├── build.gradle │ ├── download_model.gradle │ ├── proguard-rules.pro │ └── src │ │ └── main │ │ ├── AndroidManifest.xml │ │ ├── assets │ │ └── thumbnails │ │ │ ├── style0.jpg │ │ │ ├── style1.jpg │ │ │ ├── style10.jpg │ │ │ ├── style11.jpg │ │ │ ├── style12.jpg │ │ │ ├── style13.jpg │ │ │ ├── style14.jpg │ │ │ ├── style15.jpg │ │ │ ├── style16.jpg │ │ │ ├── style17.jpg │ │ │ ├── style18.jpg │ │ │ ├── style19.jpg │ │ │ ├── style2.jpg │ │ │ ├── style20.jpg │ │ │ ├── style21.jpg │ │ │ ├── style22.jpg │ │ │ ├── style23.jpg │ │ │ ├── style24.jpg │ │ │ ├── style25.jpg │ │ │ ├── style3.jpg │ │ │ ├── style4.jpg │ │ │ ├── style5.jpg │ │ │ ├── style6.jpg │ │ │ ├── style7.jpg │ │ │ ├── style8.jpg │ │ │ └── style9.jpg │ │ ├── java │ │ └── org │ │ │ └── tensorflow │ │ │ └── lite │ │ │ └── examples │ │ │ └── styletransfer │ │ │ ├── ImageUtils.kt │ │ │ ├── MLExecutionViewModel.kt │ │ │ ├── MainActivity.kt │ │ │ ├── ModelExecutionResult.kt │ │ │ ├── StyleFragment.kt │ │ │ ├── StyleRecyclerViewAdapter.kt │ │ │ ├── StyleTransferModelExecutor.kt │ │ │ └── camera │ │ │ ├── AutoFitSurfaceView.kt │ │ │ ├── CameraFragment.kt │ │ │ ├── CameraSizes.kt │ │ │ └── OrientationLiveData.kt │ │ └── res │ │ ├── anim │ │ └── scale_anim.xml │ │ ├── drawable-hdpi │ │ └── ic_launcher.png │ │ ├── drawable-mdpi │ │ └── ic_launcher.png │ │ ├── drawable-v24 │ │ └── ic_launcher_foreground.xml │ │ ├── drawable-xxhdpi │ │ ├── ic_launcher.png │ │ ├── icn_chevron_down.png │ │ ├── icn_chevron_up.png │ │ ├── styles_square_thumb.jpg │ │ ├── tfl2_logo.png │ │ └── tfl2_logo_dark.png │ │ ├── drawable │ │ ├── ic_launcher_background.xml │ │ ├── ic_switchcam.xml │ │ └── rounded_edge.xml │ │ ├── layout │ │ ├── activity_main.xml │ │ ├── bottom_sheet_layout.xml │ │ ├── fragment_style_list.xml │ │ └── image_item.xml │ │ ├── mipmap-anydpi-v26 │ │ ├── ic_launcher.xml │ │ └── ic_launcher_round.xml │ │ ├── mipmap-hdpi │ │ ├── ic_launcher.png │ │ ├── ic_launcher_foreground.png │ │ └── ic_launcher_round.png │ │ ├── mipmap-mdpi │ │ ├── ic_launcher.png │ │ ├── ic_launcher_foreground.png │ │ └── ic_launcher_round.png │ │ ├── mipmap-xhdpi │ │ ├── ic_launcher.png │ │ ├── ic_launcher_foreground.png │ │ └── ic_launcher_round.png │ │ ├── mipmap-xxhdpi │ │ ├── ic_launcher.png │ │ ├── ic_launcher_foreground.png │ │ └── ic_launcher_round.png │ │ ├── mipmap-xxxhdpi │ │ ├── ic_launcher.png │ │ ├── ic_launcher_foreground.png │ │ └── ic_launcher_round.png │ │ └── values │ │ ├── colors.xml │ │ ├── dimens.xml │ │ ├── strings.xml │ │ └── styles.xml ├── build.gradle ├── gradle.properties ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── placeholder.txt └── settings.gradle └── ios ├── Assets.xcassets ├── AppIcon.appiconset │ ├── Contents.json │ ├── Ruse1024.png │ ├── Ruse120-1.png │ ├── Ruse120.png │ ├── Ruse152.png │ ├── Ruse167.png │ ├── Ruse180.png │ ├── Ruse20.png │ ├── Ruse29.png │ ├── Ruse40-1.png │ ├── Ruse40-2.png │ ├── Ruse40.png │ ├── Ruse58-1.png │ ├── Ruse58.png │ ├── Ruse60.png │ ├── Ruse76.png │ ├── Ruse80-1.png │ ├── Ruse80.png │ └── Ruse87.png ├── Contents.json ├── face1.imageset │ ├── Contents.json │ └── face1.jpg ├── face2.imageset │ ├── Contents.json │ └── face2.jpg ├── face3.imageset │ ├── Contents.json │ └── face3.jpg ├── face4.imageset │ ├── Contents.json │ └── face4.jpg ├── face5.imageset │ ├── Contents.json │ └── face5.jpg ├── photo_camera.imageset │ ├── Contents.json │ ├── photo_camera_2x.png │ └── photo_camera_3x.png ├── photo_library.imageset │ ├── Contents.json │ ├── photo_library_2x.png │ └── photo_library_3x.png ├── style0.imageset │ ├── Contents.json │ └── style0.jpg ├── style1.imageset │ ├── Contents.json │ └── style1.jpg ├── style10.imageset │ ├── Contents.json │ └── style10.jpg ├── style11.imageset │ ├── Contents.json │ └── style11.jpg ├── style12.imageset │ ├── Contents.json │ └── style12.jpg ├── style13.imageset │ ├── Contents.json │ └── style13.jpg ├── style14.imageset │ ├── Contents.json │ └── style14.jpg ├── style15.imageset │ ├── Contents.json │ └── style15.jpg ├── style16.imageset │ ├── Contents.json │ └── style16.jpg ├── style17.imageset │ ├── Contents.json │ └── style17.jpg ├── style18.imageset │ ├── Contents.json │ └── style18.jpg ├── style19.imageset │ ├── Contents.json │ └── style19.jpg ├── style2.imageset │ ├── Contents.json │ └── style2.jpg ├── style20.imageset │ ├── Contents.json │ └── style20.jpg ├── style21.imageset │ ├── Contents.json │ └── style21.jpg ├── style22.imageset │ ├── Contents.json │ └── style22.jpg ├── style23.imageset │ ├── Contents.json │ └── style23.jpg ├── style24.imageset │ ├── Contents.json │ └── style24.jpg ├── style25.imageset │ ├── Contents.json │ └── style25.jpg ├── style3.imageset │ ├── Contents.json │ └── style3.jpg ├── style4.imageset │ ├── Contents.json │ └── style4.jpg ├── style5.imageset │ ├── Contents.json │ └── style5.jpg ├── style6.imageset │ ├── Contents.json │ └── style6.jpg ├── style7.imageset │ ├── Contents.json │ └── style7.jpg ├── style8.imageset │ ├── Contents.json │ └── style8.jpg ├── style9.imageset │ ├── Contents.json │ └── style9.jpg ├── switch_camera.imageset │ ├── Contents.json │ ├── switch_camera_2x.png │ └── switch_camera_3x.png └── video_camera.imageset │ ├── Contents.json │ ├── video_camera_2x.png │ └── video_camera_3x.png ├── Podfile ├── README.md ├── Resources ├── face1.jpg ├── face2.jpg ├── face3.jpg ├── face4.jpg ├── face5.jpg ├── style1.jpg ├── style2.jpg ├── style3.jpg ├── style4.jpg ├── style5.jpg └── style6.jpg ├── Ruse.xcodeproj ├── project.pbxproj ├── project.xcworkspace │ ├── contents.xcworkspacedata │ ├── xcshareddata │ │ ├── IDEWorkspaceChecks.plist │ │ └── WorkspaceSettings.xcsettings │ └── xcuserdata │ │ └── mike.kiser.xcuserdatad │ │ ├── UserInterfaceState.xcuserstate │ │ └── WorkspaceSettings.xcsettings └── xcuserdata │ └── mike.kiser.xcuserdatad │ └── xcschemes │ ├── Ruse.xcscheme │ └── xcschememanagement.plist ├── Ruse.xcworkspace ├── contents.xcworkspacedata ├── xcshareddata │ └── IDEWorkspaceChecks.plist └── xcuserdata │ └── mike.kiser.xcuserdatad │ └── UserInterfaceState.xcuserstate └── Ruse ├── AppDelegate.swift ├── Base.lproj ├── LaunchScreen.storyboard └── Main.storyboard ├── CameraViewController.swift ├── Info.plist ├── MLKitExtensions.swift ├── SimplexNoise.swift ├── StylePickerViewController.swift ├── StyleTransferer.swift ├── TFLiteExtension.swift ├── UIKitExtension.swift ├── UIUtilities.swift └── ViewController.swift /.yaml: -------------------------------------------------------------------------------- 1 | os: osx 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 derrumbe 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Model/style_predict_f16_256.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/Model/style_predict_f16_256.tflite -------------------------------------------------------------------------------- /Model/style_predict_quantized_256.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/Model/style_predict_quantized_256.tflite -------------------------------------------------------------------------------- /Model/style_transfer_f16_384.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/Model/style_transfer_f16_384.tflite -------------------------------------------------------------------------------- /Model/style_transfer_quantized_384.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/Model/style_transfer_quantized_384.tflite -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ruse 2 | 3 | 4 | 5 | ![image](https://user-images.githubusercontent.com/12752489/116280198-0fd77c00-a74e-11eb-8093-642d38c6220a.png) 6 | ![image](https://img.shields.io/badge/platform-ios-blue) 7 | ![image](https://img.shields.io/badge/platform-android-blue) 8 | 9 | 10 | 11 | ![Alt text](https://irisar.com/Ruse/Ruse180.png) 12 | 13 | 14 | Mobile camera-based application that attempts to alter photos to preserve their utility to humans while making them unusable for facial recognition systems. 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | ![](header.png) 24 | 25 | ## Installation 26 | 27 | (1) Easy Method: Wait and download app from appropriate app store. 28 | 29 | (2) Download and run ios app via XCode (see Development setup for more detail) 30 | 31 | 32 | ## Usage example 33 | 34 | App is developed as a camera-based app, allowing for the modification of faces on new camera capture or current photos on camera roll with the goal of keeping them useful for social media and human consumption while making it difficult for facial recognition systems to utilize them accurately and effectively. 35 | 36 | This is done through a variety of methods based on previous research. Due to the limits of mobile and TensorFlow Lite, learning on the device itself is not possible—so some of the more advanced techniques are not yet possible (but research and development may yield future results.) 37 | 38 | Instructions on usage and a full video to come with first release. 39 | 40 | The Jupyter notebook illustrates the "arbitrary fast style" adversarial technique that is possible on mobile: 41 | ![image](https://user-images.githubusercontent.com/12752489/116135863-3f777d00-a697-11eb-8265-3e4f1dba64dd.png) 42 | 43 | In the long term, this technique will be applied selectively (likey to segments of the photographs), along with perlin/simplex noise generated on a per image basis, a la https://github.com/kieranbrowne/camera-adversaria. 44 | 45 | A variety of methods are used to conceal the faces from commerical recognition systems (e.g. arbitrary file transfer, perlin noise introduction). Before saving to the camera roll or being used for online purposes, an onboard facility checks to see if faces can be detected. 46 | 47 | 48 | 49 | The effect of these adversarial approaches may then be checked wihtout needing to have network access. 50 | 51 | (Future versions plan on including a similar onboard estimation of how a sample recognition system fairs against the modified image (classification as opposed to merely detection.)) 52 | 53 | 54 | ## Development setup 55 | 56 | Requirements: Xcode >12 57 | 58 | Pods installed as part of the process below: TensorFlow Lite (Swift nightly build) // GoogleMLKit // GPUImage3 59 | 60 | - Download ios and model (tflite models) directory 61 | - run "pod install" in the downloaded directory 62 | - Open Ruse.xcworkspace 63 | 64 | 65 | Installation to device is left as an excercise for the reader. 66 | 67 | Future Note: On-device learning is now possible with TFLite. This still may not be practical as the iterations might take too long to be useful for normal users. 68 | 69 | ## Release History 70 | * 0.2 (in Progress): 71 | * Redesign of interface to make it more consumer friendly 72 | * Addition of a new research technique (in process) - OPOM, one person / one mask 73 | * onboard training to create a class-wise mask 74 | * Thanks to all the beta testers for your feeback! 75 | * 0.1 76 | * Android code checking in (finally) 77 | * 0.0.1 78 | * Work in progress - iOS code working and checked in 79 | 80 | ## Furture work 81 | * web assembly offloading 82 | * adjustment (independently) of simplex noise and style transfer 83 | * onboard checking of facial classification 84 | (dependent on advancement of tensorflow lite, etc) 85 | * Redesign of app from PoC to actual, you know, usability 86 | 87 | ## Meta 88 | 89 | 90 | Distributed under the MIT license. See ``LICENSE`` for more information. 91 | 92 | [https://github.com/derrumbe/Ruse](https://github.com/derrumbe/Ruse) 93 | 94 | 95 | 96 | 97 | [travis-image]: https://travis-ci.com/derrumbe/Ruse.svg?branch=master 98 | [travis-url]: https://travis-ci.com/derrumbe/Ruse/ 99 | [wiki]: https://github.com/derrumbe/Ruse/wiki 100 | -------------------------------------------------------------------------------- /android/README.md: -------------------------------------------------------------------------------- 1 | # Ruse -------------------------------------------------------------------------------- /android/app/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | apply plugin: 'com.android.application' 18 | apply plugin: 'kotlin-android' 19 | apply plugin: 'kotlin-android-extensions' 20 | apply plugin: 'kotlin-kapt' 21 | apply plugin: 'de.undercouch.download' 22 | 23 | android { 24 | signingConfigs { 25 | release { 26 | storeFile file('/Users/lgusm/projects/keys/release') 27 | storePassword 'teste123' 28 | keyAlias = 'key0' 29 | keyPassword '123456' 30 | } 31 | } 32 | compileSdkVersion 29 33 | buildToolsVersion "29.0.2" 34 | defaultConfig { 35 | applicationId "org.tensorflow.lite.examples.styletransfersample" 36 | minSdkVersion 26 37 | targetSdkVersion 28 38 | versionCode 1 39 | versionName "1.0" 40 | testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" 41 | } 42 | buildTypes { 43 | 44 | } 45 | aaptOptions { 46 | noCompress "tflite" 47 | } 48 | compileOptions { 49 | sourceCompatibility JavaVersion.VERSION_1_8 50 | targetCompatibility JavaVersion.VERSION_1_8 51 | } 52 | } 53 | 54 | // import DownloadModels task 55 | project.ext.ASSET_DIR = projectDir.toString() + '/src/main/assets' 56 | 57 | // Download default models; if you wish to use your own models then 58 | // place them in the "assets" directory and comment out this line. 59 | //apply from: "download_model.gradle" 60 | 61 | apply from: 'download_model.gradle' 62 | 63 | dependencies { 64 | implementation fileTree(dir: 'libs', include: ['*.jar']) 65 | 66 | def camerax_version = "1.0.0-alpha02" 67 | def lifecycle_version = "2.1.0-alpha02" 68 | def coroutines_version = "1.1.1" 69 | def tfl_version = "0.0.0-nightly-SNAPSHOT" 70 | def glide_version = "4.9.0" 71 | 72 | implementation 'androidx.appcompat:appcompat:1.1.0-rc01' 73 | implementation "androidx.camera:camera-core:${camerax_version}" 74 | implementation "androidx.camera:camera-camera2:${camerax_version}" 75 | implementation 'androidx.core:core-ktx:1.0.2' 76 | implementation 'androidx.constraintlayout:constraintlayout:1.1.3' 77 | implementation "androidx.lifecycle:lifecycle-viewmodel-ktx:${lifecycle_version}" 78 | implementation "androidx.lifecycle:lifecycle-extensions:${lifecycle_version}" 79 | implementation 'androidx.recyclerview:recyclerview:1.0.0' 80 | kapt "androidx.lifecycle:lifecycle-compiler:${lifecycle_version}" 81 | 82 | implementation "com.github.bumptech.glide:glide:${glide_version}" 83 | kapt "com.github.bumptech.glide:compiler:${glide_version}" 84 | 85 | implementation 'com.google.android.material:material:1.1.0-alpha08' 86 | 87 | implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:${kotlin_version}" 88 | implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:${coroutines_version}" 89 | implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:${coroutines_version}" 90 | 91 | implementation("org.tensorflow:tensorflow-lite:${tfl_version}") { changing = true } 92 | implementation("org.tensorflow:tensorflow-lite-gpu:${tfl_version}") { changing = true } 93 | } 94 | -------------------------------------------------------------------------------- /android/app/download_model.gradle: -------------------------------------------------------------------------------- 1 | task downloadPredictInt8ModelFile(type: Download) { 2 | src 'https://tfhub.dev/google/lite-model/magenta/arbitrary-image-stylization-v1-256/int8/prediction/1?lite-format=tflite' 3 | dest project.ext.ASSET_DIR + '/style_predict_quantized_256.tflite' 4 | overwrite false 5 | } 6 | 7 | task downloadTransformInt8ModelFile(type: Download) { 8 | src 'https://tfhub.dev/google/lite-model/magenta/arbitrary-image-stylization-v1-256/int8/transfer/1?lite-format=tflite' 9 | dest project.ext.ASSET_DIR + '/style_transfer_quantized_384.tflite' 10 | overwrite false 11 | } 12 | 13 | task downloadPredictFloat16ModelFile(type: Download) { 14 | src 'https://tfhub.dev/google/lite-model/magenta/arbitrary-image-stylization-v1-256/fp16/prediction/1?lite-format=tflite' 15 | dest project.ext.ASSET_DIR + '/style_predict_f16_256.tflite' 16 | overwrite false 17 | } 18 | 19 | task downloadTransformFloat16ModelFile(type: Download) { 20 | src 'https://tfhub.dev/google/lite-model/magenta/arbitrary-image-stylization-v1-256/fp16/transfer/1?lite-format=tflite' 21 | dest project.ext.ASSET_DIR + '/style_transfer_f16_384.tflite' 22 | overwrite false 23 | } 24 | 25 | tasks.whenTaskAdded { task -> 26 | if ((task.name == 'assembleDebug') || (task.name == 'assembleRelease')) { 27 | task.dependsOn 'downloadPredictInt8ModelFile' 28 | task.dependsOn 'downloadTransformInt8ModelFile' 29 | task.dependsOn 'downloadPredictFloat16ModelFile' 30 | task.dependsOn 'downloadTransformFloat16ModelFile' 31 | } 32 | } 33 | 34 | 35 | -------------------------------------------------------------------------------- /android/app/proguard-rules.pro: -------------------------------------------------------------------------------- 1 | # Add project specific ProGuard rules here. 2 | # You can control the set of applied configuration files using the 3 | # proguardFiles setting in build.gradle. 4 | # 5 | # For more details, see 6 | # http://developer.android.com/guide/developing/tools/proguard.html 7 | 8 | # If your project uses WebView with JS, uncomment the following 9 | # and specify the fully qualified class name to the JavaScript interface 10 | # class: 11 | #-keepclassmembers class fqcn.of.javascript.interface.for.webview { 12 | # public *; 13 | #} 14 | 15 | # Uncomment this to preserve the line number information for 16 | # debugging stack traces. 17 | #-keepattributes SourceFile,LineNumberTable 18 | 19 | # If you keep the line number information, uncomment this to 20 | # hide the original source file name. 21 | #-renamesourcefileattribute SourceFile 22 | -------------------------------------------------------------------------------- /android/app/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style0.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style1.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style10.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style10.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style11.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style11.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style12.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style12.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style13.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style13.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style14.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style14.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style15.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style15.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style16.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style16.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style17.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style17.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style18.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style18.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style19.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style19.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style2.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style20.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style20.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style21.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style21.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style22.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style22.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style23.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style23.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style24.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style24.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style25.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style25.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style3.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style4.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style5.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style6.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style7.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style8.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style8.jpg -------------------------------------------------------------------------------- /android/app/src/main/assets/thumbnails/style9.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/assets/thumbnails/style9.jpg -------------------------------------------------------------------------------- /android/app/src/main/java/org/tensorflow/lite/examples/styletransfer/ImageUtils.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.tensorflow.lite.examples.styletransfer 18 | 19 | import android.content.Context 20 | import android.graphics.Bitmap 21 | import android.graphics.BitmapFactory 22 | import android.graphics.Color 23 | import android.graphics.Matrix 24 | import android.graphics.RectF 25 | import androidx.exifinterface.media.ExifInterface 26 | import java.io.File 27 | import java.nio.ByteBuffer 28 | import java.nio.ByteOrder 29 | 30 | /** 31 | * Collection of image reading and manipulation utilities in the form of static functions. 32 | * TODO: this class should be moved to the common code in the future 33 | */ 34 | abstract class ImageUtils { 35 | companion object { 36 | 37 | /** 38 | * Helper function used to convert an EXIF orientation enum into a transformation matrix 39 | * that can be applied to a bitmap. 40 | * 41 | * @param orientation - One of the constants from [ExifInterface] 42 | */ 43 | private fun decodeExifOrientation(orientation: Int): Matrix { 44 | val matrix = Matrix() 45 | 46 | // Apply transformation corresponding to declared EXIF orientation 47 | when (orientation) { 48 | ExifInterface.ORIENTATION_NORMAL, ExifInterface.ORIENTATION_UNDEFINED -> Unit 49 | ExifInterface.ORIENTATION_ROTATE_90 -> matrix.postRotate(90F) 50 | ExifInterface.ORIENTATION_ROTATE_180 -> matrix.postRotate(180F) 51 | ExifInterface.ORIENTATION_ROTATE_270 -> matrix.postRotate(270F) 52 | ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.postScale(-1F, 1F) 53 | ExifInterface.ORIENTATION_FLIP_VERTICAL -> matrix.postScale(1F, -1F) 54 | ExifInterface.ORIENTATION_TRANSPOSE -> { 55 | matrix.postScale(-1F, 1F) 56 | matrix.postRotate(270F) 57 | } 58 | ExifInterface.ORIENTATION_TRANSVERSE -> { 59 | matrix.postScale(-1F, 1F) 60 | matrix.postRotate(90F) 61 | } 62 | 63 | // Error out if the EXIF orientation is invalid 64 | else -> throw IllegalArgumentException("Invalid orientation: $orientation") 65 | } 66 | 67 | // Return the resulting matrix 68 | return matrix 69 | } 70 | 71 | /** 72 | * sets the Exif orientation of an image. 73 | * this method is used to fix the exit of pictures taken by the camera 74 | * 75 | * @param filePath - The image file to change 76 | * @param value - the orientation of the file 77 | */ 78 | fun setExifOrientation( 79 | filePath: String, 80 | value: String 81 | ) { 82 | val exif = ExifInterface(filePath) 83 | exif.setAttribute( 84 | ExifInterface.TAG_ORIENTATION, value 85 | ) 86 | exif.saveAttributes() 87 | } 88 | 89 | /** Transforms rotation and mirroring information into one of the [ExifInterface] constants */ 90 | fun computeExifOrientation(rotationDegrees: Int, mirrored: Boolean) = when { 91 | rotationDegrees == 0 && !mirrored -> ExifInterface.ORIENTATION_NORMAL 92 | rotationDegrees == 0 && mirrored -> ExifInterface.ORIENTATION_FLIP_HORIZONTAL 93 | rotationDegrees == 180 && !mirrored -> ExifInterface.ORIENTATION_ROTATE_180 94 | rotationDegrees == 180 && mirrored -> ExifInterface.ORIENTATION_FLIP_VERTICAL 95 | rotationDegrees == 270 && mirrored -> ExifInterface.ORIENTATION_TRANSVERSE 96 | rotationDegrees == 90 && !mirrored -> ExifInterface.ORIENTATION_ROTATE_90 97 | rotationDegrees == 90 && mirrored -> ExifInterface.ORIENTATION_TRANSPOSE 98 | rotationDegrees == 270 && mirrored -> ExifInterface.ORIENTATION_ROTATE_270 99 | rotationDegrees == 270 && !mirrored -> ExifInterface.ORIENTATION_TRANSVERSE 100 | else -> ExifInterface.ORIENTATION_UNDEFINED 101 | } 102 | 103 | /** 104 | * Decode a bitmap from a file and apply the transformations described in its EXIF data 105 | * 106 | * @param file - The image file to be read using [BitmapFactory.decodeFile] 107 | */ 108 | fun decodeBitmap(file: File): Bitmap { 109 | // First, decode EXIF data and retrieve transformation matrix 110 | val exif = ExifInterface(file.absolutePath) 111 | val transformation = 112 | decodeExifOrientation( 113 | exif.getAttributeInt( 114 | ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_ROTATE_90 115 | ) 116 | ) 117 | 118 | // Read bitmap using factory methods, and transform it using EXIF data 119 | val bitmap = BitmapFactory.decodeFile(file.absolutePath) 120 | return Bitmap.createBitmap( 121 | BitmapFactory.decodeFile(file.absolutePath), 122 | 0, 0, bitmap.width, bitmap.height, transformation, true 123 | ) 124 | } 125 | 126 | fun scaleBitmapAndKeepRatio( 127 | targetBmp: Bitmap, 128 | reqHeightInPixels: Int, 129 | reqWidthInPixels: Int 130 | ): Bitmap { 131 | if (targetBmp.height == reqHeightInPixels && targetBmp.width == reqWidthInPixels) { 132 | return targetBmp 133 | } 134 | val matrix = Matrix() 135 | matrix.setRectToRect( 136 | RectF( 137 | 0f, 0f, 138 | targetBmp.width.toFloat(), 139 | targetBmp.width.toFloat() 140 | ), 141 | RectF( 142 | 0f, 0f, 143 | reqWidthInPixels.toFloat(), 144 | reqHeightInPixels.toFloat() 145 | ), 146 | Matrix.ScaleToFit.FILL 147 | ) 148 | return Bitmap.createBitmap( 149 | targetBmp, 0, 0, 150 | targetBmp.width, 151 | targetBmp.width, matrix, true 152 | ) 153 | } 154 | 155 | fun bitmapToByteBuffer( 156 | bitmapIn: Bitmap, 157 | width: Int, 158 | height: Int, 159 | mean: Float = 0.0f, 160 | std: Float = 255.0f 161 | ): ByteBuffer { 162 | val bitmap = scaleBitmapAndKeepRatio(bitmapIn, width, height) 163 | val inputImage = ByteBuffer.allocateDirect(1 * width * height * 3 * 4) 164 | inputImage.order(ByteOrder.nativeOrder()) 165 | inputImage.rewind() 166 | 167 | val intValues = IntArray(width * height) 168 | bitmap.getPixels(intValues, 0, width, 0, 0, width, height) 169 | var pixel = 0 170 | for (y in 0 until height) { 171 | for (x in 0 until width) { 172 | val value = intValues[pixel++] 173 | 174 | // Normalize channel values to [-1.0, 1.0]. This requirement varies by 175 | // model. For example, some models might require values to be normalized 176 | // to the range [0.0, 1.0] instead. 177 | inputImage.putFloat(((value shr 16 and 0xFF) - mean) / std) 178 | inputImage.putFloat(((value shr 8 and 0xFF) - mean) / std) 179 | inputImage.putFloat(((value and 0xFF) - mean) / std) 180 | } 181 | } 182 | 183 | inputImage.rewind() 184 | return inputImage 185 | } 186 | 187 | fun createEmptyBitmap(imageWidth: Int, imageHeigth: Int, color: Int = 0): Bitmap { 188 | val ret = Bitmap.createBitmap(imageWidth, imageHeigth, Bitmap.Config.RGB_565) 189 | if (color != 0) { 190 | ret.eraseColor(color) 191 | } 192 | return ret 193 | } 194 | 195 | fun loadBitmapFromResources(context: Context, path: String): Bitmap { 196 | val inputStream = context.assets.open(path) 197 | return BitmapFactory.decodeStream(inputStream) 198 | } 199 | 200 | fun convertArrayToBitmap( 201 | imageArray: Array>>, 202 | imageWidth: Int, 203 | imageHeight: Int 204 | ): Bitmap { 205 | val conf = Bitmap.Config.ARGB_8888 // see other conf types 206 | val styledImage = Bitmap.createBitmap(imageWidth, imageHeight, conf) 207 | 208 | for (x in imageArray[0].indices) { 209 | for (y in imageArray[0][0].indices) { 210 | val color = Color.rgb( 211 | ((imageArray[0][x][y][0] * 255).toInt()), 212 | ((imageArray[0][x][y][1] * 255).toInt()), 213 | (imageArray[0][x][y][2] * 255).toInt() 214 | ) 215 | 216 | // this y, x is in the correct order!!! 217 | styledImage.setPixel(y, x, color) 218 | } 219 | } 220 | return styledImage 221 | } 222 | } 223 | } 224 | -------------------------------------------------------------------------------- /android/app/src/main/java/org/tensorflow/lite/examples/styletransfer/MLExecutionViewModel.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.tensorflow.lite.examples.styletransfer 18 | 19 | import androidx.lifecycle.ViewModel 20 | import android.content.Context 21 | import androidx.lifecycle.LiveData 22 | import androidx.lifecycle.MutableLiveData 23 | import kotlinx.coroutines.CoroutineScope 24 | import kotlinx.coroutines.ExecutorCoroutineDispatcher 25 | import kotlinx.coroutines.Job 26 | import kotlinx.coroutines.launch 27 | 28 | class MLExecutionViewModel : ViewModel() { 29 | 30 | private val _styledBitmap = MutableLiveData() 31 | 32 | val styledBitmap: LiveData 33 | get() = _styledBitmap 34 | 35 | private val viewModelJob = Job() 36 | private val viewModelScope = CoroutineScope(viewModelJob) 37 | 38 | /* 39 | Older version of onApplyStyle; static , fixed paths and a few other older approaches 40 | fun onApplyStyle( 41 | context: Context, 42 | contentFileStaticPath: String, 43 | styleFileStaticPath: String, 44 | styleTransferModelExecutor: StyleTransferModelExecutor, 45 | inferenceThread: ExecutorCoroutineDispatcher 46 | ) { 47 | viewModelScope.launch(inferenceThread) { 48 | val result = styleTransferStaticModelExecutor.execute(contentFilePath, styleFilePath, context) 49 | _styledBitmap.postValue(result) 50 | } 51 | }*/ 52 | 53 | fun onApplyStyle( 54 | context: Context, 55 | contentFilePath: String, 56 | styleFilePath: String, 57 | styleTransferModelExecutor: StyleTransferModelExecutor, 58 | inferenceThread: ExecutorCoroutineDispatcher 59 | ) { 60 | viewModelScope.launch(inferenceThread) { 61 | val result = styleTransferModelExecutor.execute(contentFilePath, styleFilePath, context) 62 | _styledBitmap.postValue(result) 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /android/app/src/main/java/org/tensorflow/lite/examples/styletransfer/MainActivity.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.tensorflow.lite.examples.styletransfer 18 | 19 | import android.Manifest 20 | import androidx.lifecycle.ViewModelProvider.AndroidViewModelFactory 21 | import android.content.pm.PackageManager 22 | import android.graphics.Bitmap 23 | import android.hardware.camera2.CameraCharacteristics 24 | import android.os.Bundle 25 | import android.os.Process 26 | import androidx.appcompat.app.AppCompatActivity 27 | import androidx.appcompat.widget.Toolbar 28 | import android.util.Log 29 | import android.view.View 30 | import android.view.animation.AnimationUtils 31 | import android.view.animation.BounceInterpolator 32 | import android.widget.Button 33 | import android.widget.FrameLayout 34 | import android.widget.HorizontalScrollView 35 | import android.widget.ImageButton 36 | import android.widget.ImageView 37 | import android.widget.ProgressBar 38 | import android.widget.Switch 39 | import android.widget.TextView 40 | import android.widget.Toast 41 | import androidx.core.app.ActivityCompat 42 | import androidx.lifecycle.Observer 43 | import com.bumptech.glide.Glide 44 | import com.bumptech.glide.load.engine.bitmap_recycle.BitmapPool 45 | import com.bumptech.glide.load.resource.bitmap.BitmapTransformation 46 | import com.bumptech.glide.request.RequestOptions 47 | import java.io.File 48 | import java.nio.charset.Charset 49 | import java.security.MessageDigest 50 | import java.util.concurrent.Executors 51 | import kotlinx.coroutines.MainScope 52 | import kotlinx.coroutines.asCoroutineDispatcher 53 | import kotlinx.coroutines.async 54 | import org.tensorflow.lite.examples.styletransfer.camera.CameraFragment 55 | 56 | // This is an arbitrary number we are using to keep tab of the permission 57 | // request. Where an app has multiple context for requesting permission, 58 | // this can help differentiate the different contexts 59 | private const val REQUEST_CODE_PERMISSIONS = 10 60 | 61 | // This is an array of all the permission specified in the manifest 62 | private val REQUIRED_PERMISSIONS = arrayOf(Manifest.permission.CAMERA) 63 | 64 | private const val TAG = "MainActivity" 65 | 66 | class MainActivity : 67 | AppCompatActivity(), 68 | StyleFragment.OnListFragmentInteractionListener, 69 | CameraFragment.OnCaptureFinished { 70 | 71 | private var isRunningModel = false 72 | private val stylesFragment: StyleFragment = StyleFragment() 73 | private var selectedStyle: String = "" 74 | 75 | private lateinit var cameraFragment: CameraFragment 76 | private lateinit var viewModel: MLExecutionViewModel 77 | private lateinit var viewFinder: FrameLayout 78 | private lateinit var resultImageView: ImageView 79 | private lateinit var originalImageView: ImageView 80 | private lateinit var styleImageView: ImageView 81 | private lateinit var rerunButton: Button 82 | private lateinit var captureButton: ImageButton 83 | private lateinit var progressBar: ProgressBar 84 | private lateinit var horizontalScrollView: HorizontalScrollView 85 | 86 | private var lastSavedFile = "" 87 | private var useGPU = false 88 | private lateinit var styleTransferModelExecutor: StyleTransferModelExecutor 89 | private val inferenceThread = Executors.newSingleThreadExecutor().asCoroutineDispatcher() 90 | private val mainScope = MainScope() 91 | 92 | private var lensFacing = CameraCharacteristics.LENS_FACING_FRONT 93 | 94 | override fun onCreate(savedInstanceState: Bundle?) { 95 | super.onCreate(savedInstanceState) 96 | setContentView(R.layout.activity_main) 97 | 98 | val toolbar: Toolbar = findViewById(R.id.toolbar) 99 | setSupportActionBar(toolbar) 100 | supportActionBar?.setDisplayShowTitleEnabled(false) 101 | 102 | viewFinder = findViewById(R.id.view_finder) 103 | resultImageView = findViewById(R.id.result_imageview) 104 | originalImageView = findViewById(R.id.original_imageview) 105 | styleImageView = findViewById(R.id.style_imageview) 106 | captureButton = findViewById(R.id.capture_button) 107 | progressBar = findViewById(R.id.progress_circular) 108 | horizontalScrollView = findViewById(R.id.horizontal_scroll_view) 109 | val useGpuSwitch: Switch = findViewById(R.id.switch_use_gpu) 110 | 111 | // Request camera permissions 112 | if (allPermissionsGranted()) { 113 | addCameraFragment() 114 | } else { 115 | ActivityCompat.requestPermissions(this, REQUIRED_PERMISSIONS, REQUEST_CODE_PERMISSIONS) 116 | } 117 | 118 | viewModel = AndroidViewModelFactory(application).create(MLExecutionViewModel::class.java) 119 | 120 | viewModel.styledBitmap.observe( 121 | this, 122 | Observer { resultImage -> 123 | if (resultImage != null) { 124 | updateUIWithResults(resultImage) 125 | } 126 | } 127 | ) 128 | 129 | mainScope.async(inferenceThread) { 130 | styleTransferModelExecutor = StyleTransferModelExecutor(this@MainActivity, useGPU) 131 | Log.d(TAG, "Executor created") 132 | } 133 | 134 | useGpuSwitch.setOnCheckedChangeListener { _, isChecked -> 135 | useGPU = isChecked 136 | // Disable control buttons to avoid running model before initialization 137 | enableControls(false) 138 | 139 | // Reinitialize TF Lite models with new GPU setting 140 | mainScope.async(inferenceThread) { 141 | styleTransferModelExecutor.close() 142 | styleTransferModelExecutor = StyleTransferModelExecutor(this@MainActivity, useGPU) 143 | 144 | // Re-enable control buttons 145 | runOnUiThread { enableControls(true) } 146 | } 147 | } 148 | 149 | rerunButton = findViewById(R.id.rerun_button) 150 | rerunButton.setOnClickListener { startRunningModel() } 151 | 152 | styleImageView.setOnClickListener { 153 | if (!isRunningModel) { 154 | stylesFragment.show(supportFragmentManager, "StylesFragment") 155 | } 156 | } 157 | 158 | progressBar.visibility = View.INVISIBLE 159 | lastSavedFile = getLastTakenPicture() 160 | setImageView(originalImageView, lastSavedFile) 161 | 162 | animateCameraButton() 163 | setupControls() 164 | enableControls(true) 165 | 166 | Log.d(TAG, "finished onCreate!!") 167 | } 168 | 169 | private fun animateCameraButton() { 170 | val animation = AnimationUtils.loadAnimation(this, R.anim.scale_anim) 171 | animation.interpolator = BounceInterpolator() 172 | captureButton.animation = animation 173 | captureButton.animation.start() 174 | } 175 | 176 | private fun setImageView(imageView: ImageView, image: Bitmap) { 177 | Glide.with(baseContext).load(image).override(512, 512).fitCenter().into(imageView) 178 | } 179 | 180 | private fun setImageView(imageView: ImageView, imagePath: String) { 181 | Glide.with(baseContext) 182 | .asBitmap() 183 | .load(imagePath) 184 | .override(512, 512) 185 | .apply(RequestOptions().transform(CropTop())) 186 | .into(imageView) 187 | } 188 | 189 | private fun updateUIWithResults(modelExecutionResult: ModelExecutionResult) { 190 | progressBar.visibility = View.INVISIBLE 191 | resultImageView.visibility = View.VISIBLE 192 | setImageView(resultImageView, modelExecutionResult.styledImage) 193 | val logText: TextView = findViewById(R.id.log_view) 194 | logText.text = modelExecutionResult.executionLog 195 | enableControls(true) 196 | horizontalScrollView.fullScroll(HorizontalScrollView.FOCUS_RIGHT) 197 | } 198 | 199 | private fun enableControls(enable: Boolean) { 200 | isRunningModel = !enable 201 | rerunButton.isEnabled = enable 202 | captureButton.isEnabled = enable 203 | } 204 | 205 | private fun setupControls() { 206 | captureButton.setOnClickListener { 207 | it.clearAnimation() 208 | cameraFragment.takePicture() 209 | } 210 | 211 | findViewById(R.id.toggle_button).setOnClickListener { 212 | lensFacing = 213 | if (lensFacing == CameraCharacteristics.LENS_FACING_BACK) { 214 | CameraCharacteristics.LENS_FACING_FRONT 215 | } else { 216 | CameraCharacteristics.LENS_FACING_BACK 217 | } 218 | cameraFragment.setFacingCamera(lensFacing) 219 | addCameraFragment() 220 | } 221 | } 222 | 223 | private fun addCameraFragment() { 224 | cameraFragment = CameraFragment.newInstance() 225 | cameraFragment.setFacingCamera(lensFacing) 226 | supportFragmentManager.popBackStack() 227 | supportFragmentManager.beginTransaction().replace(R.id.view_finder, cameraFragment).commit() 228 | } 229 | 230 | /** 231 | * Process result from permission request dialog box, has the request been granted? If yes, start 232 | * Camera. Otherwise display a toast 233 | */ 234 | override fun onRequestPermissionsResult( 235 | requestCode: Int, 236 | permissions: Array, 237 | grantResults: IntArray 238 | ) { 239 | super.onRequestPermissionsResult(requestCode, permissions, grantResults) 240 | if (requestCode == REQUEST_CODE_PERMISSIONS) { 241 | if (allPermissionsGranted()) { 242 | addCameraFragment() 243 | viewFinder.post { setupControls() } 244 | } else { 245 | Toast.makeText(this, "Permissions not granted by the user.", Toast.LENGTH_SHORT).show() 246 | finish() 247 | } 248 | } 249 | } 250 | 251 | /** Check if all permission specified in the manifest have been granted */ 252 | private fun allPermissionsGranted() = 253 | REQUIRED_PERMISSIONS.all { 254 | checkPermission(it, Process.myPid(), Process.myUid()) == PackageManager.PERMISSION_GRANTED 255 | } 256 | 257 | override fun onCaptureFinished(file: File) { 258 | val msg = "Photo capture succeeded: ${file.absolutePath}" 259 | Log.d(TAG, msg) 260 | 261 | lastSavedFile = file.absolutePath 262 | setImageView(originalImageView, lastSavedFile) 263 | } 264 | 265 | // And update once new picture is taken? 266 | // Alternatively we can provide user an ability to select any of taken photos 267 | private fun getLastTakenPicture(): String { 268 | val directory = baseContext.filesDir // externalMediaDirs.first() 269 | var files = 270 | directory.listFiles()?.filter { file -> file.absolutePath.endsWith(".jpg") }?.sorted() 271 | if (files == null || files.isEmpty()) { 272 | Log.d(TAG, "there is no previous saved file") 273 | return "" 274 | } 275 | 276 | val file = files.last() 277 | Log.d(TAG, "lastsavedfile: " + file.absolutePath) 278 | return file.absolutePath 279 | } 280 | 281 | override fun onListFragmentInteraction(item: String) { 282 | Log.d(TAG, item) 283 | selectedStyle = item 284 | stylesFragment.dismiss() 285 | 286 | startRunningModel() 287 | } 288 | 289 | private fun getUriFromAssetThumb(thumb: String): String { 290 | return "file:///android_asset/thumbnails/$thumb" 291 | } 292 | 293 | private fun startRunningModel() { 294 | if (!isRunningModel && lastSavedFile.isNotEmpty() && selectedStyle.isNotEmpty()) { 295 | val chooseStyleLabel: TextView = findViewById(R.id.choose_style_text_view) 296 | chooseStyleLabel.visibility = View.GONE 297 | enableControls(false) 298 | setImageView(styleImageView, getUriFromAssetThumb(selectedStyle)) 299 | resultImageView.visibility = View.INVISIBLE 300 | progressBar.visibility = View.VISIBLE 301 | viewModel.onApplyStyle( 302 | baseContext, 303 | lastSavedFile, 304 | selectedStyle, 305 | styleTransferModelExecutor, 306 | inferenceThread 307 | ) 308 | } else { 309 | Toast.makeText(this, "Previous Model still running", Toast.LENGTH_SHORT).show() 310 | } 311 | } 312 | 313 | // this transformation is necessary to show the top square of the image as the model 314 | // will work on this part only, making the preview and the result show the same base 315 | // NB: need to extend this model a bit to incorporate higher-res images 316 | class CropTop : BitmapTransformation() { 317 | override fun transform( 318 | pool: BitmapPool, 319 | toTransform: Bitmap, 320 | outWidth: Int, 321 | outHeight: Int 322 | ): Bitmap { 323 | return if (toTransform.width == outWidth && toTransform.height == outHeight) { 324 | toTransform 325 | } else ImageUtils.scaleBitmapAndKeepRatio(toTransform, outWidth, outHeight) 326 | } 327 | 328 | override fun equals(other: Any?): Boolean { 329 | return other is CropTop 330 | } 331 | 332 | override fun hashCode(): Int { 333 | return ID.hashCode() 334 | } 335 | 336 | override fun updateDiskCacheKey(messageDigest: MessageDigest) { 337 | messageDigest.update(ID_BYTES) 338 | } 339 | 340 | companion object { 341 | private const val ID = "org.tensorflow.lite.examples.styletransfer.CropTop" 342 | private val ID_BYTES = ID.toByteArray(Charset.forName("UTF-8")) 343 | } 344 | } 345 | } 346 | -------------------------------------------------------------------------------- /android/app/src/main/java/org/tensorflow/lite/examples/styletransfer/ModelExecutionResult.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.tensorflow.lite.examples.styletransfer 18 | 19 | import android.graphics.Bitmap 20 | 21 | @SuppressWarnings("GoodTime") 22 | data class ModelExecutionResult( 23 | val styledImage: Bitmap, 24 | val preProcessTime: Long = 0L, 25 | val stylePredictTime: Long = 0L, 26 | val styleTransferTime: Long = 0L, 27 | val postProcessTime: Long = 0L, 28 | val totalExecutionTime: Long = 0L, 29 | val executionLog: String = "", 30 | val errorMessage: String = "" 31 | ) 32 | -------------------------------------------------------------------------------- /android/app/src/main/java/org/tensorflow/lite/examples/styletransfer/StyleFragment.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.tensorflow.lite.examples.styletransfer 18 | 19 | import android.content.Context 20 | import android.os.Bundle 21 | import androidx.fragment.app.DialogFragment 22 | import androidx.recyclerview.widget.GridLayoutManager 23 | import androidx.recyclerview.widget.RecyclerView 24 | import android.view.LayoutInflater 25 | import android.view.View 26 | import android.view.ViewGroup 27 | 28 | /** 29 | * A fragment representing a list of available Styles to apply 30 | * Activities containing this fragment MUST implement the 31 | * [StyleFragment.OnListFragmentInteractionListener] interface. 32 | */ 33 | class StyleFragment : DialogFragment() { 34 | 35 | private var listener: OnListFragmentInteractionListener? = null 36 | 37 | override fun onCreateView( 38 | inflater: LayoutInflater, 39 | container: ViewGroup?, 40 | savedInstanceState: Bundle? 41 | ): View? { 42 | val view = inflater.inflate(R.layout.fragment_style_list, container, false) 43 | 44 | val styles = ArrayList() 45 | styles.addAll(activity!!.assets.list("thumbnails")!!) 46 | 47 | // Set the adapter 48 | if (view is RecyclerView) { 49 | with(view) { 50 | layoutManager = GridLayoutManager(context, 3) 51 | adapter = StyleRecyclerViewAdapter(styles, context, listener) 52 | } 53 | } 54 | return view 55 | } 56 | 57 | override fun onAttach(context: Context) { 58 | super.onAttach(context) 59 | if (context is OnListFragmentInteractionListener) { 60 | listener = context 61 | } else { 62 | throw RuntimeException("$context must implement OnListFragmentInteractionListener") 63 | } 64 | } 65 | 66 | override fun onDetach() { 67 | super.onDetach() 68 | listener = null 69 | } 70 | 71 | /** 72 | * This interface must be implemented by activities that contain this 73 | * fragment to allow an interaction in this fragment to be communicated 74 | * to the activity and potentially other fragments contained in that 75 | * activity. 76 | * 77 | * 78 | * See the Android Training lesson 79 | * [Communicating with Other Fragments](http://developer.android.com/training/basics/fragments/communicating.html) 80 | * for more information. 81 | */ 82 | interface OnListFragmentInteractionListener { 83 | fun onListFragmentInteraction(item: String) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /android/app/src/main/java/org/tensorflow/lite/examples/styletransfer/StyleRecyclerViewAdapter.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.tensorflow.lite.examples.styletransfer 18 | 19 | import android.content.Context 20 | import android.net.Uri 21 | import androidx.recyclerview.widget.RecyclerView 22 | import android.view.LayoutInflater 23 | import android.view.View 24 | import android.view.ViewGroup 25 | import android.widget.ImageView 26 | import com.bumptech.glide.Glide 27 | import org.tensorflow.lite.examples.styletransfer.StyleFragment.OnListFragmentInteractionListener 28 | 29 | /** 30 | * [StyleRecyclerViewAdapter] that can display a [StyleItem] and makes a call to the 31 | * specified [OnListFragmentInteractionListener]. 32 | */ 33 | class StyleRecyclerViewAdapter( 34 | private val styles: List, 35 | private val context: Context, 36 | private val mListener: OnListFragmentInteractionListener? 37 | ) : RecyclerView.Adapter() { 38 | 39 | private val mOnClickListener: View.OnClickListener 40 | 41 | init { 42 | mOnClickListener = View.OnClickListener { v -> 43 | val item = v.tag as String 44 | // Notify the active callbacks interface (the activity, if the fragment is attached to 45 | // one) that an item has been selected. 46 | mListener?.onListFragmentInteraction(item) 47 | } 48 | } 49 | 50 | override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): StyleItemViewHolder { 51 | val view = LayoutInflater.from(parent.context) 52 | .inflate(R.layout.image_item, parent, false) 53 | return StyleItemViewHolder(view) 54 | } 55 | 56 | override fun onBindViewHolder(holder: StyleItemViewHolder, position: Int) { 57 | val imagePath = styles[position] 58 | 59 | Glide.with(context) 60 | .load(Uri.parse("file:///android_asset/thumbnails/$imagePath")) 61 | .centerInside() 62 | .into(holder.imageView) 63 | 64 | with(holder.mView) { 65 | tag = imagePath 66 | setOnClickListener(mOnClickListener) 67 | } 68 | } 69 | 70 | override fun getItemCount(): Int = styles.size 71 | 72 | inner class StyleItemViewHolder(val mView: View) : RecyclerView.ViewHolder(mView) { 73 | var imageView: ImageView = mView.findViewById(R.id.image_view) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /android/app/src/main/java/org/tensorflow/lite/examples/styletransfer/StyleTransferModelExecutor.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.tensorflow.lite.examples.styletransfer 18 | 19 | import android.content.Context 20 | import android.os.SystemClock 21 | import android.util.Log 22 | import java.io.File 23 | import java.io.FileInputStream 24 | import java.io.IOException 25 | import java.nio.MappedByteBuffer 26 | import java.nio.channels.FileChannel 27 | import kotlin.collections.set 28 | import org.tensorflow.lite.Interpreter 29 | import org.tensorflow.lite.gpu.GpuDelegate 30 | 31 | @SuppressWarnings("GoodTime") 32 | class StyleTransferModelExecutor( 33 | context: Context, 34 | private var useGPU: Boolean = false 35 | ) { 36 | private var gpuDelegate: GpuDelegate? = null 37 | private var numberThreads = 4 38 | 39 | private val interpreterPredict: Interpreter 40 | private val interpreterTransform: Interpreter 41 | 42 | private var fullExecutionTime = 0L 43 | private var preProcessTime = 0L 44 | private var stylePredictTime = 0L 45 | private var styleTransferTime = 0L 46 | private var postProcessTime = 0L 47 | 48 | init { 49 | if (useGPU) { 50 | interpreterPredict = getInterpreter(context, STYLE_PREDICT_FLOAT16_MODEL, true) 51 | interpreterTransform = getInterpreter(context, STYLE_TRANSFER_FLOAT16_MODEL, true) 52 | } else { 53 | interpreterPredict = getInterpreter(context, STYLE_PREDICT_INT8_MODEL, false) 54 | interpreterTransform = getInterpreter(context, STYLE_TRANSFER_INT8_MODEL, false) 55 | } 56 | } 57 | 58 | companion object { 59 | private const val TAG = "StyleTransferMExec" 60 | private const val STYLE_IMAGE_SIZE = 256 61 | private const val CONTENT_IMAGE_SIZE = 384 62 | private const val BOTTLENECK_SIZE = 100 63 | private const val STYLE_PREDICT_INT8_MODEL = "style_predict_quantized_256.tflite" 64 | private const val STYLE_TRANSFER_INT8_MODEL = "style_transfer_quantized_384.tflite" 65 | private const val STYLE_PREDICT_FLOAT16_MODEL = "style_predict_f16_256.tflite" 66 | private const val STYLE_TRANSFER_FLOAT16_MODEL = "style_transfer_f16_384.tflite" 67 | } 68 | 69 | fun execute( 70 | contentImagePath: String, 71 | styleImageName: String, 72 | context: Context 73 | ): ModelExecutionResult { 74 | try { 75 | Log.i(TAG, "running models") 76 | 77 | fullExecutionTime = SystemClock.uptimeMillis() 78 | preProcessTime = SystemClock.uptimeMillis() 79 | 80 | val contentImage = ImageUtils.decodeBitmap(File(contentImagePath)) 81 | val contentArray = 82 | ImageUtils.bitmapToByteBuffer(contentImage, CONTENT_IMAGE_SIZE, CONTENT_IMAGE_SIZE) 83 | val styleBitmap = 84 | ImageUtils.loadBitmapFromResources(context, "thumbnails/$styleImageName") 85 | val input = ImageUtils.bitmapToByteBuffer(styleBitmap, STYLE_IMAGE_SIZE, STYLE_IMAGE_SIZE) 86 | 87 | val inputsForPredict = arrayOf(input) 88 | val outputsForPredict = HashMap() 89 | val styleBottleneck = Array(1) { Array(1) { Array(1) { FloatArray(BOTTLENECK_SIZE) } } } 90 | outputsForPredict[0] = styleBottleneck 91 | preProcessTime = SystemClock.uptimeMillis() - preProcessTime 92 | 93 | stylePredictTime = SystemClock.uptimeMillis() 94 | // The results of this inference could be reused given the style does not change 95 | // That would be a good practice in case this was applied to a video stream. 96 | interpreterPredict.runForMultipleInputsOutputs(inputsForPredict, outputsForPredict) 97 | stylePredictTime = SystemClock.uptimeMillis() - stylePredictTime 98 | Log.d(TAG, "Style Predict Time to run: $stylePredictTime") 99 | 100 | val inputsForStyleTransfer = arrayOf(contentArray, styleBottleneck) 101 | val outputsForStyleTransfer = HashMap() 102 | val outputImage = 103 | Array(1) { Array(CONTENT_IMAGE_SIZE) { Array(CONTENT_IMAGE_SIZE) { FloatArray(3) } } } 104 | outputsForStyleTransfer[0] = outputImage 105 | 106 | styleTransferTime = SystemClock.uptimeMillis() 107 | interpreterTransform.runForMultipleInputsOutputs( 108 | inputsForStyleTransfer, 109 | outputsForStyleTransfer 110 | ) 111 | styleTransferTime = SystemClock.uptimeMillis() - styleTransferTime 112 | Log.d(TAG, "Style apply Time to run: $styleTransferTime") 113 | 114 | postProcessTime = SystemClock.uptimeMillis() 115 | var styledImage = 116 | ImageUtils.convertArrayToBitmap(outputImage, CONTENT_IMAGE_SIZE, CONTENT_IMAGE_SIZE) 117 | postProcessTime = SystemClock.uptimeMillis() - postProcessTime 118 | 119 | fullExecutionTime = SystemClock.uptimeMillis() - fullExecutionTime 120 | Log.d(TAG, "Time to run everything: $fullExecutionTime") 121 | 122 | return ModelExecutionResult( 123 | styledImage, 124 | preProcessTime, 125 | stylePredictTime, 126 | styleTransferTime, 127 | postProcessTime, 128 | fullExecutionTime, 129 | formatExecutionLog() 130 | ) 131 | } catch (e: Exception) { 132 | val exceptionLog = "something went wrong: ${e.message}" 133 | Log.d(TAG, exceptionLog) 134 | 135 | val emptyBitmap = 136 | ImageUtils.createEmptyBitmap( 137 | CONTENT_IMAGE_SIZE, 138 | CONTENT_IMAGE_SIZE 139 | ) 140 | return ModelExecutionResult( 141 | emptyBitmap, errorMessage = e.message!! 142 | ) 143 | } 144 | } 145 | 146 | @Throws(IOException::class) 147 | private fun loadModelFile(context: Context, modelFile: String): MappedByteBuffer { 148 | val fileDescriptor = context.assets.openFd(modelFile) 149 | val inputStream = FileInputStream(fileDescriptor.fileDescriptor) 150 | val fileChannel = inputStream.channel 151 | val startOffset = fileDescriptor.startOffset 152 | val declaredLength = fileDescriptor.declaredLength 153 | val retFile = fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength) 154 | fileDescriptor.close() 155 | return retFile 156 | } 157 | 158 | @Throws(IOException::class) 159 | private fun getInterpreter( 160 | context: Context, 161 | modelName: String, 162 | useGpu: Boolean = false 163 | ): Interpreter { 164 | val tfliteOptions = Interpreter.Options() 165 | tfliteOptions.setNumThreads(numberThreads) 166 | 167 | gpuDelegate = null 168 | if (useGpu) { 169 | gpuDelegate = GpuDelegate() 170 | tfliteOptions.addDelegate(gpuDelegate) 171 | } 172 | 173 | tfliteOptions.setNumThreads(numberThreads) 174 | return Interpreter(loadModelFile(context, modelName), tfliteOptions) 175 | } 176 | 177 | private fun formatExecutionLog(): String { 178 | val sb = StringBuilder() 179 | sb.append("Input Image Size: $CONTENT_IMAGE_SIZE x $CONTENT_IMAGE_SIZE\n") 180 | sb.append("GPU enabled: $useGPU\n") 181 | sb.append("Number of threads: $numberThreads\n") 182 | sb.append("Pre-process execution time: $preProcessTime ms\n") 183 | sb.append("Predicting style execution time: $stylePredictTime ms\n") 184 | sb.append("Transferring style execution time: $styleTransferTime ms\n") 185 | sb.append("Post-process execution time: $postProcessTime ms\n") 186 | sb.append("Full execution time: $fullExecutionTime ms\n") 187 | return sb.toString() 188 | } 189 | 190 | fun close() { 191 | interpreterPredict.close() 192 | interpreterTransform.close() 193 | if (gpuDelegate != null) { 194 | gpuDelegate!!.close() 195 | } 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /android/app/src/main/java/org/tensorflow/lite/examples/styletransfer/camera/AutoFitSurfaceView.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.tensorflow.lite.examples.styletransfer.camera 18 | 19 | import android.content.Context 20 | import android.util.AttributeSet 21 | import android.util.Log 22 | import android.view.SurfaceView 23 | import kotlin.math.roundToInt 24 | 25 | /** 26 | * A [SurfaceView] that can be adjusted to a specified aspect ratio and 27 | * performs center-crop transformation of input frames. 28 | */ 29 | class AutoFitSurfaceView( 30 | context: Context, 31 | attrs: AttributeSet? = null, 32 | defStyle: Int = 0 33 | ) : SurfaceView(context, attrs, defStyle) { 34 | 35 | private var aspectRatio = 0f 36 | private var widthDiff = 0 37 | private var heightDiff = 0 38 | private var requestLayout = false 39 | 40 | /** 41 | * Sets the aspect ratio for this view. The size of the view will be 42 | * measured based on the ratio calculated from the parameters. Note that 43 | * the actual sizes of parameters don't matter, that is, calling 44 | * setAspectRatio(2, 3) and setAspectRatio(4, 6) make the same result. 45 | * 46 | * @param width Relative horizontal size 47 | * @param height Relative vertical size 48 | */ 49 | fun setAspectRatio(width: Int, height: Int) { 50 | require(width > 0 && height > 0) { "Size cannot be negative" } 51 | aspectRatio = width.toFloat() / height.toFloat() 52 | requestLayout() 53 | } 54 | 55 | override fun onMeasure(widthMeasureSpec: Int, heightMeasureSpec: Int) { 56 | val width = MeasureSpec.getSize(widthMeasureSpec) 57 | val height = MeasureSpec.getSize(heightMeasureSpec) 58 | if (aspectRatio == 0f) { 59 | setMeasuredDimension(width, height) 60 | } else { 61 | // Performs center-crop transformation of the camera frames 62 | val newWidth: Int 63 | val newHeight: Int 64 | if (width < height * aspectRatio) { 65 | newHeight = height 66 | newWidth = (height / aspectRatio).roundToInt() 67 | } else { 68 | newWidth = width 69 | newHeight = (width / aspectRatio).roundToInt() 70 | } 71 | 72 | Log.d(TAG, "Measured dimensions set: $newWidth x $newHeight") 73 | widthDiff = width - newWidth 74 | heightDiff = height - newHeight 75 | requestLayout = true 76 | setMeasuredDimension(newWidth, newHeight) 77 | } 78 | } 79 | 80 | override fun onLayout(changed: Boolean, left: Int, top: Int, right: Int, bottom: Int) { 81 | if (requestLayout) { 82 | requestLayout = false 83 | layout( 84 | widthDiff / 2, 85 | heightDiff / 2, 86 | right + (widthDiff / 2), 87 | bottom + (heightDiff / 2) 88 | ) 89 | } 90 | super.onLayout(changed, left, top, right, bottom) 91 | } 92 | 93 | companion object { 94 | private val TAG = AutoFitSurfaceView::class.java.simpleName 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /android/app/src/main/java/org/tensorflow/lite/examples/styletransfer/camera/CameraSizes.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.tensorflow.lite.examples.styletransfer.camera 18 | 19 | import android.graphics.Point 20 | import android.hardware.camera2.CameraCharacteristics 21 | import android.hardware.camera2.params.StreamConfigurationMap 22 | import android.util.Size 23 | import android.view.Display 24 | import kotlin.math.max 25 | import kotlin.math.min 26 | 27 | /** Helper class used to pre-compute shortest and longest sides of a [Size] */ 28 | class SmartSize(width: Int, height: Int) { 29 | var size = Size(width, height) 30 | var long = max(size.width, size.height) 31 | var short = min(size.width, size.height) 32 | override fun toString() = "SmartSize(${long}x$short)" 33 | } 34 | 35 | /** Standard High Definition size for pictures and video */ 36 | val SIZE_1080P: SmartSize = SmartSize(1920, 1080) 37 | 38 | /** Returns a [SmartSize] object for the given [Display] */ 39 | fun getDisplaySmartSize(display: Display): SmartSize { 40 | val outPoint = Point() 41 | display.getRealSize(outPoint) 42 | return SmartSize(outPoint.x, outPoint.y) 43 | } 44 | 45 | // verify that the given width and height are on the expected aspect ratio 46 | fun verifyAspectRatio(width: Int, height: Int, aspectRatio: Size): Boolean { 47 | return (width * aspectRatio.height) == (height * aspectRatio.width) 48 | } 49 | 50 | /** 51 | * Returns the largest available PREVIEW size. For more information, see: 52 | * https://d.android.com/reference/android/hardware/camera2/CameraDevice 53 | */ 54 | fun getPreviewOutputSize( 55 | display: Display, 56 | characteristics: CameraCharacteristics, 57 | targetClass: Class, 58 | aspectRatio: Size, 59 | format: Int? = null 60 | ): Size { 61 | // Find which is smaller: screen or 1080p 62 | val screenSize = getDisplaySmartSize(display) 63 | val hdScreen = screenSize.long >= SIZE_1080P.long || screenSize.short >= SIZE_1080P.short 64 | val maxSize = if (hdScreen) SIZE_1080P else screenSize 65 | 66 | // If image format is provided, use it to determine supported sizes; else use target class 67 | val config = characteristics.get( 68 | CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP 69 | )!! 70 | if (format == null) { 71 | assert(StreamConfigurationMap.isOutputSupportedFor(targetClass)) 72 | } else { 73 | assert(config.isOutputSupportedFor(format)) 74 | } 75 | val allSizes = if (format == null) { 76 | config.getOutputSizes(targetClass) 77 | } else { 78 | config.getOutputSizes(format) 79 | } 80 | 81 | // Get available sizes and sort them by area from largest to smallest 82 | val validSizes = allSizes 83 | .sortedWith(compareBy { it.height * it.width }) 84 | .filter { verifyAspectRatio(it.width, it.height, aspectRatio) } 85 | .map { SmartSize(it.width, it.height) }.reversed() 86 | 87 | // Then, get the largest output size that is smaller or equal than our max size 88 | return validSizes.first { it.long <= maxSize.long && it.short <= maxSize.short }.size 89 | } 90 | -------------------------------------------------------------------------------- /android/app/src/main/java/org/tensorflow/lite/examples/styletransfer/camera/OrientationLiveData.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.tensorflow.lite.examples.styletransfer.camera 18 | 19 | import android.content.Context 20 | import android.hardware.camera2.CameraCharacteristics 21 | import android.view.OrientationEventListener 22 | import android.view.Surface 23 | import androidx.lifecycle.LiveData 24 | 25 | /** 26 | * Calculates closest 90-degree orientation to compensate for the device rotation relative to sensor 27 | * orientation, i.e., allows user to see camera frames with the expected orientation. 28 | */ 29 | class OrientationLiveData(context: Context, characteristics: CameraCharacteristics) : 30 | LiveData() { 31 | 32 | private val listener = 33 | object : OrientationEventListener(context.applicationContext) { 34 | override fun onOrientationChanged(orientation: Int) { 35 | val rotation = 36 | when { 37 | orientation <= 45 -> Surface.ROTATION_0 38 | orientation <= 135 -> Surface.ROTATION_90 39 | orientation <= 225 -> Surface.ROTATION_180 40 | orientation <= 315 -> Surface.ROTATION_270 41 | else -> Surface.ROTATION_0 42 | } 43 | val relative = computeRelativeRotation(characteristics, rotation) 44 | if (relative != value) postValue(relative) 45 | } 46 | } 47 | 48 | override fun onActive() { 49 | super.onActive() 50 | listener.enable() 51 | } 52 | 53 | override fun onInactive() { 54 | super.onInactive() 55 | listener.disable() 56 | } 57 | 58 | companion object { 59 | 60 | /** 61 | * Computes rotation required to transform from the camera sensor orientation to the device's 62 | * current orientation in degrees. 63 | * 64 | * @param characteristics the [CameraCharacteristics] to query for the sensor orientation. 65 | * @param surfaceRotation the current device orientation as a Surface constant 66 | * @return the relative rotation from the camera sensor to the current device orientation. 67 | */ 68 | @JvmStatic 69 | private fun computeRelativeRotation( 70 | characteristics: CameraCharacteristics, 71 | surfaceRotation: Int 72 | ): Int { 73 | val sensorOrientationDegrees = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION)!! 74 | 75 | val deviceOrientationDegrees = 76 | when (surfaceRotation) { 77 | Surface.ROTATION_0 -> 0 78 | Surface.ROTATION_90 -> 90 79 | Surface.ROTATION_180 -> 180 80 | Surface.ROTATION_270 -> 270 81 | else -> 0 82 | } 83 | 84 | // Reverse device orientation for front-facing cameras 85 | val sign = 86 | if (characteristics.get(CameraCharacteristics.LENS_FACING) == 87 | CameraCharacteristics.LENS_FACING_FRONT 88 | ) 89 | 1 90 | else -1 91 | 92 | // Calculate desired JPEG orientation relative to camera orientation to make 93 | // the image upright relative to the device orientation 94 | return (sensorOrientationDegrees - (deviceOrientationDegrees * sign) + 360) % 360 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /android/app/src/main/res/anim/scale_anim.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 13 | -------------------------------------------------------------------------------- /android/app/src/main/res/drawable-hdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/res/drawable-hdpi/ic_launcher.png -------------------------------------------------------------------------------- /android/app/src/main/res/drawable-mdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/res/drawable-mdpi/ic_launcher.png -------------------------------------------------------------------------------- /android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml: -------------------------------------------------------------------------------- 1 | 7 | 12 | 13 | 19 | 22 | 25 | 26 | 27 | 28 | 34 | 35 | -------------------------------------------------------------------------------- /android/app/src/main/res/drawable-xxhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/res/drawable-xxhdpi/ic_launcher.png -------------------------------------------------------------------------------- /android/app/src/main/res/drawable-xxhdpi/icn_chevron_down.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/res/drawable-xxhdpi/icn_chevron_down.png -------------------------------------------------------------------------------- /android/app/src/main/res/drawable-xxhdpi/icn_chevron_up.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/res/drawable-xxhdpi/icn_chevron_up.png -------------------------------------------------------------------------------- /android/app/src/main/res/drawable-xxhdpi/styles_square_thumb.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/res/drawable-xxhdpi/styles_square_thumb.jpg -------------------------------------------------------------------------------- /android/app/src/main/res/drawable-xxhdpi/tfl2_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/res/drawable-xxhdpi/tfl2_logo.png -------------------------------------------------------------------------------- /android/app/src/main/res/drawable-xxhdpi/tfl2_logo_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrumbe/Ruse/4467e14fb15274812aed2cd949f210ba813594ba/android/app/src/main/res/drawable-xxhdpi/tfl2_logo_dark.png -------------------------------------------------------------------------------- /android/app/src/main/res/drawable/ic_launcher_background.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 10 | 15 | 20 | 25 | 30 | 35 | 40 | 45 | 50 | 55 | 60 | 65 | 70 | 75 | 80 | 85 | 90 | 95 | 100 | 105 | 110 | 115 | 120 | 125 | 130 | 135 | 140 | 145 | 150 | 155 | 160 | 165 | 170 | 171 | -------------------------------------------------------------------------------- /android/app/src/main/res/drawable/ic_switchcam.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /android/app/src/main/res/drawable/rounded_edge.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 10 | 11 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /android/app/src/main/res/layout/activity_main.xml: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | 14 | 15 | 19 | 20 | 21 | 27 | 28 | 32 | 33 | 38 | 39 | 40 | 44 | 45 | 49 | 50 | 56 | 57 | 60 | 61 | 67 | 68 | 79 | 80 | 81 | 84 | 85 | 90 | 91 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 116 | 117 | 124 | 125 | 136 | -------------------------------------------------------------------------------- /android/app/src/main/res/layout/bottom_sheet_layout.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 23 | 24 | 32 | 33 | 39 | 40 |