├── .gitignore ├── README.md ├── buildozer.spec ├── data └── pba_quantized │ ├── labels.txt │ └── model.tflite ├── main.py ├── screenshot.jpg └── tflwrapper ├── __init__.py └── tfl_android.py /.gitignore: -------------------------------------------------------------------------------- 1 | .buildozer 2 | bin 3 | *.pyc 4 | __pycache__ 5 | .vscode -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TensorFlow lite with Kivy 2 | 3 | This is an experimental project, no support will be done. 4 | 5 | Works only on Android for now. 6 | 7 | ![screenshot](https://github.com/tito/experiment-tensorflow-lite/blob/main/screenshot.jpg?raw=true) 8 | 9 | It uses: 10 | - Kivy android camera to grab RGBA pixels 11 | - Pillow to convert the image to the tensor input format 12 | - TensorFlow lite Android libraries to detect 13 | 14 | This example is mostly based on the Image classification from TensorFlow 15 | lite example repository. 16 | 17 | ## Create a model 18 | 19 | - Train your model https://teachablemachine.withgoogle.com/ 20 | - Export as a Quantized model for TensorFlow lite 21 | - Copy it into the app, ajust the paths. 22 | -------------------------------------------------------------------------------- /buildozer.spec: -------------------------------------------------------------------------------- 1 | [app] 2 | 3 | # (str) Title of your application 4 | title = TFLite Test 5 | 6 | # (str) Package name 7 | package.name = tflitetest 8 | 9 | # (str) Package domain (needed for android/ios packaging) 10 | package.domain = com.meltingrocks 11 | 12 | # (str) Source code where the main.py live 13 | source.dir = . 14 | 15 | # (list) Source files to include (let empty to include all the files) 16 | source.include_exts = py,png,jpg,kv,atlas,tflite,txt 17 | 18 | # (list) List of inclusions using pattern matching 19 | #source.include_patterns = assets/*,images/*.png 20 | 21 | # (list) Source files to exclude (let empty to not exclude anything) 22 | #source.exclude_exts = spec 23 | 24 | # (list) List of directory to exclude (let empty to not exclude anything) 25 | #source.exclude_dirs = tests, bin 26 | 27 | # (list) List of exclusions using pattern matching 28 | #source.exclude_patterns = license,images/*/*.jpg 29 | 30 | # (str) Application versioning (method 1) 31 | version = 0.1 32 | 33 | # (str) Application versioning (method 2) 34 | # version.regex = __version__ = ['"](.*)['"] 35 | # version.filename = %(source.dir)s/main.py 36 | 37 | # (list) Application requirements 38 | # comma separated e.g. requirements = sqlite3,kivy 39 | requirements = python3,kivy,pillow,pyjnius==master 40 | 41 | # (str) Custom source folders for requirements 42 | # Sets custom source for any requirements with recipes 43 | # requirements.source.kivy = ../../kivy 44 | 45 | # (list) Garden requirements 46 | #garden_requirements = 47 | 48 | # (str) Presplash of the application 49 | #presplash.filename = %(source.dir)s/data/presplash.png 50 | 51 | # (str) Icon of the application 52 | #icon.filename = %(source.dir)s/data/icon.png 53 | 54 | # (str) Supported orientation (one of landscape, sensorLandscape, portrait or all) 55 | orientation = sensorLandscape 56 | 57 | # (list) List of service to declare 58 | #services = NAME:ENTRYPOINT_TO_PY,NAME2:ENTRYPOINT2_TO_PY 59 | 60 | # 61 | # OSX Specific 62 | # 63 | 64 | # 65 | # author = © Copyright Info 66 | 67 | # change the major version of python used by the app 68 | osx.python_version = 3 69 | 70 | # Kivy version to use 71 | osx.kivy_version = 1.9.1 72 | 73 | # 74 | # Android specific 75 | # 76 | 77 | # (bool) Indicate if the application should be fullscreen or not 78 | fullscreen = 0 79 | 80 | # (string) Presplash background color (for new android toolchain) 81 | # Supported formats are: #RRGGBB #AARRGGBB or one of the following names: 82 | # red, blue, green, black, white, gray, cyan, magenta, yellow, lightgray, 83 | # darkgray, grey, lightgrey, darkgrey, aqua, fuchsia, lime, maroon, navy, 84 | # olive, purple, silver, teal. 85 | #android.presplash_color = #FFFFFF 86 | 87 | # (list) Permissions 88 | android.permissions = INTERNET,CAMERA 89 | 90 | # (int) Target Android API, should be as high as possible. 91 | #android.api = 27 92 | 93 | # (int) Minimum API your APK will support. 94 | #android.minapi = 21 95 | 96 | # (int) Android SDK version to use 97 | #android.sdk = 20 98 | 99 | # (str) Android NDK version to use 100 | #android.ndk = 19b 101 | 102 | # (int) Android NDK API to use. This is the minimum API your app will support, it should usually match android.minapi. 103 | #android.ndk_api = 21 104 | 105 | # (bool) Use --private data storage (True) or --dir public storage (False) 106 | #android.private_storage = True 107 | 108 | # (str) Android NDK directory (if empty, it will be automatically downloaded.) 109 | #android.ndk_path = 110 | 111 | # (str) Android SDK directory (if empty, it will be automatically downloaded.) 112 | #android.sdk_path = 113 | 114 | # (str) ANT directory (if empty, it will be automatically downloaded.) 115 | #android.ant_path = 116 | 117 | # (bool) If True, then skip trying to update the Android sdk 118 | # This can be useful to avoid excess Internet downloads or save time 119 | # when an update is due and you just want to test/build your package 120 | # android.skip_update = False 121 | 122 | # (bool) If True, then automatically accept SDK license 123 | # agreements. This is intended for automation only. If set to False, 124 | # the default, you will be shown the license when first running 125 | # buildozer. 126 | # android.accept_sdk_license = False 127 | 128 | # (str) Android entry point, default is ok for Kivy-based app 129 | #android.entrypoint = org.renpy.android.PythonActivity 130 | 131 | # (str) Android app theme, default is ok for Kivy-based app 132 | # android.apptheme = "@android:style/Theme.NoTitleBar" 133 | 134 | # (list) Pattern to whitelist for the whole project 135 | #android.whitelist = 136 | 137 | # (str) Path to a custom whitelist file 138 | #android.whitelist_src = 139 | 140 | # (str) Path to a custom blacklist file 141 | #android.blacklist_src = 142 | 143 | # (list) List of Java .jar files to add to the libs so that pyjnius can access 144 | # their classes. Don't add jars that you do not need, since extra jars can slow 145 | # down the build process. Allows wildcards matching, for example: 146 | # OUYA-ODK/libs/*.jar 147 | #android.add_jars = foo.jar,bar.jar,path/to/more/*.jar 148 | 149 | # (list) List of Java files to add to the android project (can be java or a 150 | # directory containing the files) 151 | #android.add_src = 152 | 153 | # (list) Android AAR archives to add (currently works only with sdl2_gradle 154 | # bootstrap) 155 | #android.add_aars = 156 | 157 | # (list) Gradle dependencies to add (currently works only with sdl2_gradle 158 | # bootstrap) 159 | android.gradle_dependencies = "org.tensorflow:tensorflow-lite:+",'org.tensorflow:tensorflow-lite-support:0.0.0-nightly' 160 | #org.tensorflow:tensorflow-android:+" 161 | 162 | # (list) add java compile options 163 | # this can for example be necessary when importing certain java libraries using the 'android.gradle_dependencies' option 164 | # see https://developer.android.com/studio/write/java8-support for further information 165 | # android.add_compile_options = "sourceCompatibility = 1.8", "targetCompatibility = 1.8" 166 | 167 | # (list) Gradle repositories to add {can be necessary for some android.gradle_dependencies} 168 | # please enclose in double quotes 169 | # e.g. android.gradle_repositories = "maven { url 'https://kotlin.bintray.com/ktor' }" 170 | #android.add_gradle_repositories = 171 | 172 | # (list) packaging options to add 173 | # see https://google.github.io/android-gradle-dsl/current/com.android.build.gradle.internal.dsl.PackagingOptions.html 174 | # can be necessary to solve conflicts in gradle_dependencies 175 | # please enclose in double quotes 176 | # e.g. android.add_packaging_options = "exclude 'META-INF/common.kotlin_module'", "exclude 'META-INF/*.kotlin_module'" 177 | #android.add_gradle_repositories = 178 | 179 | # (list) Java classes to add as activities to the manifest. 180 | #android.add_activities = com.example.ExampleActivity 181 | 182 | # (str) OUYA Console category. Should be one of GAME or APP 183 | # If you leave this blank, OUYA support will not be enabled 184 | #android.ouya.category = GAME 185 | 186 | # (str) Filename of OUYA Console icon. It must be a 732x412 png image. 187 | #android.ouya.icon.filename = %(source.dir)s/data/ouya_icon.png 188 | 189 | # (str) XML file to include as an intent filters in tag 190 | #android.manifest.intent_filters = 191 | 192 | # (str) launchMode to set for the main activity 193 | #android.manifest.launch_mode = standard 194 | 195 | # (list) Android additional libraries to copy into libs/armeabi 196 | #android.add_libs_armeabi = libs/android/*.so 197 | #android.add_libs_armeabi_v7a = libs/android-v7/*.so 198 | #android.add_libs_arm64_v8a = libs/android-v8/*.so 199 | #android.add_libs_x86 = libs/android-x86/*.so 200 | #android.add_libs_mips = libs/android-mips/*.so 201 | 202 | # (bool) Indicate whether the screen should stay on 203 | # Don't forget to add the WAKE_LOCK permission if you set this to True 204 | #android.wakelock = False 205 | 206 | # (list) Android application meta-data to set (key=value format) 207 | #android.meta_data = 208 | 209 | # (list) Android library project to add (will be added in the 210 | # project.properties automatically.) 211 | #android.library_references = 212 | 213 | # (list) Android shared libraries which will be added to AndroidManifest.xml using tag 214 | #android.uses_library = 215 | 216 | # (str) Android logcat filters to use 217 | #android.logcat_filters = *:S python:D 218 | 219 | # (bool) Copy library instead of making a libpymodules.so 220 | #android.copy_libs = 1 221 | 222 | # (str) The Android arch to build for, choices: armeabi-v7a, arm64-v8a, x86, x86_64 223 | android.arch = armeabi-v7a 224 | 225 | # (int) overrides automatic versionCode computation (used in build.gradle) 226 | # this is not the same as app version and should only be edited if you know what you're doing 227 | # android.numeric_version = 1 228 | 229 | # 230 | # Python for android (p4a) specific 231 | # 232 | 233 | # (str) python-for-android fork to use, defaults to upstream (kivy) 234 | #p4a.fork = kivy 235 | 236 | # (str) python-for-android branch to use, defaults to master 237 | #p4a.branch = master 238 | 239 | # (str) python-for-android git clone directory (if empty, it will be automatically cloned from github) 240 | p4a.source_dir = /home/tito/code/python-for-android-implementation 241 | 242 | # (str) The directory in which python-for-android should look for your own build recipes (if any) 243 | #p4a.local_recipes = 244 | 245 | # (str) Filename to the hook for p4a 246 | #p4a.hook = 247 | 248 | # (str) Bootstrap to use for android builds 249 | p4a.bootstrap = sdl2 250 | 251 | # (int) port number to specify an explicit --port= p4a argument (eg for bootstrap flask) 252 | #p4a.port = 253 | 254 | 255 | # 256 | # iOS specific 257 | # 258 | 259 | # (str) Path to a custom kivy-ios folder 260 | #ios.kivy_ios_dir = ../kivy-ios 261 | # Alternately, specify the URL and branch of a git checkout: 262 | ios.kivy_ios_url = https://github.com/kivy/kivy-ios 263 | ios.kivy_ios_branch = master 264 | 265 | # Another platform dependency: ios-deploy 266 | # Uncomment to use a custom checkout 267 | #ios.ios_deploy_dir = ../ios_deploy 268 | # Or specify URL and branch 269 | ios.ios_deploy_url = https://github.com/phonegap/ios-deploy 270 | ios.ios_deploy_branch = 1.7.0 271 | 272 | # (str) Name of the certificate to use for signing the debug version 273 | # Get a list of available identities: buildozer ios list_identities 274 | #ios.codesign.debug = "iPhone Developer: ()" 275 | 276 | # (str) Name of the certificate to use for signing the release version 277 | #ios.codesign.release = %(ios.codesign.debug)s 278 | 279 | 280 | [buildozer] 281 | 282 | # (int) Log level (0 = error only, 1 = info, 2 = debug (with command output)) 283 | log_level = 2 284 | 285 | # (int) Display warning if buildozer is run as root (0 = False, 1 = True) 286 | warn_on_root = 1 287 | 288 | # (str) Path to build artifact storage, absolute or relative to spec file 289 | # build_dir = ./.buildozer 290 | 291 | # (str) Path to build output (i.e. .apk, .ipa) storage 292 | # bin_dir = ./bin 293 | 294 | # ----------------------------------------------------------------------------- 295 | # List as sections 296 | # 297 | # You can define all the "list" as [section:key]. 298 | # Each line will be considered as a option to the list. 299 | # Let's take [app] / source.exclude_patterns. 300 | # Instead of doing: 301 | # 302 | #[app] 303 | #source.exclude_patterns = license,data/audio/*.wav,data/images/original/* 304 | # 305 | # This can be translated into: 306 | # 307 | #[app:source.exclude_patterns] 308 | #license 309 | #data/audio/*.wav 310 | #data/images/original/* 311 | # 312 | 313 | 314 | # ----------------------------------------------------------------------------- 315 | # Profiles 316 | # 317 | # You can extend section / key with a profile 318 | # For example, you want to deploy a demo version of your application without 319 | # HD content. You could first change the title to add "(demo)" in the name 320 | # and extend the excluded directories to remove the HD content. 321 | # 322 | #[app@demo] 323 | #title = My Application (demo) 324 | # 325 | #[app:source.exclude_patterns@demo] 326 | #images/hd/* 327 | # 328 | # Then, invoke the command line with the "demo" profile: 329 | # 330 | #buildozer --profile demo android debug 331 | 332 | [app@x86_64] 333 | android.arch = x86_64 334 | build_dir = .buildozer/build/x86_64/ 335 | [app@x86] 336 | android.arch = x86 337 | build_dir = .buildozer/build/x86/ 338 | [app@arm64-v8a] 339 | android.arch = arm64-v8a 340 | build_dir = .buildozer/build/arm64-v8a/ -------------------------------------------------------------------------------- /data/pba_quantized/labels.txt: -------------------------------------------------------------------------------- 1 | 0 Aquamanile en forme de lion 2 | 1 Bourreau du portement de Croix de Heinsberg 3 | 2 Buste de saint Jean-Baptiste enfant 4 | 3 Calice 5 | 4 Christ en croix 6 | 5 Christ en croix et larrons 7 | -------------------------------------------------------------------------------- /data/pba_quantized/model.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tito/experiment-tensorflow-lite/1ff35f504a0ffc24bd81626275411ef2db1c33f2/data/pba_quantized/model.tflite -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from kivy.app import App 2 | from kivy.factory import Factory as F 3 | from kivy.lang import Builder 4 | from kivy.uix.boxlayout import BoxLayout 5 | from kivy.properties import StringProperty 6 | from kivy.clock import Clock, mainthread 7 | from time import time 8 | 9 | Builder.load_string(''' 10 | : 11 | orientation: 'vertical' 12 | Camera: 13 | id: camera 14 | resolution: (1920, 1080) 15 | play: False 16 | Label: 17 | text: app.normalized_result 18 | text_size: self.width - dp(48), self.height - dp(48) 19 | y: dp(24) 20 | markup: True 21 | ''') 22 | 23 | 24 | class TestCamera(App): 25 | 26 | normalized_result = StringProperty("") 27 | 28 | def build(self): 29 | Clock.schedule_once(self.start_camera, .5) 30 | return F.CameraClassifier() 31 | 32 | def start_camera(self, *largs): 33 | self.root.ids.camera.play = True 34 | self.root.ids.camera._camera.bind(on_texture=self.on_camera_texture) 35 | from tflwrapper.tfl_android import TFLWrapperAndroid 36 | self.tflite = TFLWrapperAndroid( 37 | "data/pba_quantized/model.tflite", 38 | "data/pba_quantized/labels.txt", 39 | on_detect=self.on_tflite_detect) 40 | 41 | def on_camera_texture(self, camera): 42 | pixels = camera._fbo.pixels 43 | w, h = camera.resolution 44 | self.tflite.async_detect(pixels, w, h) 45 | 46 | @mainthread 47 | def on_tflite_detect(self, result): 48 | self.result = result 49 | labels = self.tflite.get_labels_with_value(result) 50 | textresult = [] 51 | for label, value in labels: 52 | text = f"{value:.05f}: {label}" 53 | if value > .8: 54 | text = f"[color=44FF44]{text}[/color]" 55 | else: 56 | text = f"[color=FF4444]{text}[/color]" 57 | textresult.append(text) 58 | self.normalized_result = "\n".join(textresult) 59 | 60 | def detect(self, *largs): 61 | timer = time() 62 | camera = self.root.ids.camera._camera 63 | w, h = camera.resolution 64 | pixels = camera._fbo.pixels 65 | self.result = self.tfl.detect(pixels, w, h) 66 | 67 | TestCamera().run() -------------------------------------------------------------------------------- /screenshot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tito/experiment-tensorflow-lite/1ff35f504a0ffc24bd81626275411ef2db1c33f2/screenshot.jpg -------------------------------------------------------------------------------- /tflwrapper/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tito/experiment-tensorflow-lite/1ff35f504a0ffc24bd81626275411ef2db1c33f2/tflwrapper/__init__.py -------------------------------------------------------------------------------- /tflwrapper/tfl_android.py: -------------------------------------------------------------------------------- 1 | from jnius import autoclass, cast 2 | from PIL import Image 3 | import os 4 | import threading 5 | 6 | 7 | File = autoclass("java.io.File") 8 | Interpreter = autoclass("org.tensorflow.lite.Interpreter") 9 | InterpreterOptions = autoclass("org.tensorflow.lite.Interpreter$Options") 10 | TensorImage = autoclass("org.tensorflow.lite.support.image.TensorImage") 11 | TensorBuffer = autoclass("org.tensorflow.lite.support.tensorbuffer.TensorBuffer") 12 | ByteBuffer = autoclass("java.nio.ByteBuffer") 13 | 14 | 15 | class TFLWrapper(threading.Thread): 16 | def __init__(self, model, labels, on_detect=None): 17 | super().__init__() 18 | self.event = threading.Event() 19 | self.quit = False 20 | self.on_detect = on_detect 21 | self.async_running = False 22 | self.init_from_model(model, labels) 23 | 24 | def init_from_model(self, model, labels): 25 | raise NotImplementedError() 26 | 27 | def async_start(self): 28 | if self.async_running: 29 | return 30 | self.next_frame = None 31 | self.daemon = True 32 | self.async_running = True 33 | self.start() 34 | 35 | def async_stop(self): 36 | self.quit = True 37 | 38 | def run(self): 39 | try: 40 | while not self.quit: 41 | if self.event.wait(0.5) is None: 42 | continue 43 | next_frame, self.next_frame = self.next_frame, None 44 | self.event.clear() 45 | if next_frame is None: 46 | continue 47 | result = self.detect(*next_frame) 48 | if self.on_detect: 49 | self.on_detect(result) 50 | except Exception as e: 51 | print("Exception in TFLWrapper:", e) 52 | import traceback; traceback.print_exc() 53 | 54 | 55 | class TFLWrapperAndroid(TFLWrapper): 56 | def init_from_model(self, model, labels): 57 | options = InterpreterOptions() 58 | options.setNumThreads(4) 59 | model_filename = os.path.realpath(model) 60 | label_filename = os.path.realpath(labels) 61 | model = File(model_filename) 62 | 63 | with open(label_filename, encoding="utf-8") as fd: 64 | self.labels = {} 65 | for line in fd.read().strip().splitlines(): 66 | if not line.strip(): 67 | return 68 | index, label = line.split(" ", 1) 69 | self.labels[int(index)] = label 70 | 71 | tflite = Interpreter(model, options) 72 | shape = tflite.getInputTensor(0).shape() 73 | imgwidth, imgheight = shape[1:3] 74 | 75 | # Creates the input tensor. 76 | imageDataType = tflite.getInputTensor(0).dataType() 77 | inputImageBuffer = TensorImage(imageDataType) 78 | 79 | # Create output tensor 80 | probabilityShape = tflite.getOutputTensor(0).shape() 81 | probabilityDataType = tflite.getOutputTensor(0).dataType() 82 | outputProbabilityBuffer = TensorBuffer.createFixedSize( 83 | probabilityShape, probabilityDataType) 84 | 85 | self.tflite = tflite 86 | self.inputImageBuffer = inputImageBuffer 87 | self.outputProbabilityBuffer = outputProbabilityBuffer 88 | self.imgwidth = imgwidth 89 | self.imgheight = imgheight 90 | 91 | def async_detect(self, frame, cw, ch): 92 | self.async_start() 93 | self.next_frame = (frame, cw, ch) 94 | self.event.set() 95 | 96 | def detect(self, frame, cw, ch): 97 | cropsize = min(cw, ch) 98 | 99 | # use pillow 100 | image = Image.frombytes("RGBA", (cw, ch), frame, "raw") 101 | left = (cw - cropsize) / 2 102 | top = (ch - cropsize) / 2 103 | image = image.crop((left, top, left + cropsize, top + cropsize)) 104 | image = image.resize((self.imgwidth, self.imgheight)) 105 | image = image.convert("RGB") 106 | pixels = image.tobytes() 107 | 108 | buffer = ByteBuffer.wrap(pixels) 109 | self.tflite.run(buffer, self.outputProbabilityBuffer.getBuffer().rewind()) 110 | result = self.outputProbabilityBuffer.getFloatArray() 111 | return result 112 | 113 | def get_labels_with_value(self, result): 114 | unsorted_labels = [( 115 | self.labels[index], value / 255 116 | ) for index, value in enumerate(result)] 117 | return list(sorted(unsorted_labels, key=lambda x: x[1], reverse=True)) --------------------------------------------------------------------------------