├── .gitignore ├── .metadata ├── CHANGELOG.md ├── LICENSE ├── README.md ├── analysis_options.yaml ├── android ├── .gitignore ├── build.gradle ├── gradle.properties ├── gradle │ └── wrapper │ │ └── gradle-wrapper.properties ├── settings.gradle └── src │ ├── main │ ├── AndroidManifest.xml │ └── java │ │ └── flutter │ │ └── tflite_audio │ │ ├── AudioChunk.java │ │ ├── AudioData.java │ │ ├── AudioFile.java │ │ ├── AudioProcessing.java │ │ ├── Debugging.java │ │ ├── LabelSmoothing.java │ │ ├── MediaDecoder.java │ │ ├── Recording.java │ │ ├── RecordingData.java │ │ ├── SignalProcessing.java │ │ ├── TfliteAudioPlugin.java │ │ └── lib │ │ └── jlibrosa-1.1.8-SNAPSHOT-jar-with-dependencies.jar │ └── test │ └── java │ └── flutter │ └── tflite_audio │ ├── AudioFileTest.java │ └── RecordingTest.java ├── example ├── .gitignore ├── .metadata ├── README.md ├── android │ ├── .gitignore │ ├── app │ │ ├── build.gradle │ │ └── src │ │ │ ├── androidTest │ │ │ └── java │ │ │ │ └── flutter │ │ │ │ ├── DartIntegrationTest.java │ │ │ │ └── tflite_audio_example │ │ │ │ └── FlutterActivityTest.java │ │ │ ├── debug │ │ │ └── AndroidManifest.xml │ │ │ ├── main │ │ │ ├── AndroidManifest.xml │ │ │ ├── java │ │ │ │ └── flutter │ │ │ │ │ └── tflite_audio_example │ │ │ │ │ └── MainActivity.java │ │ │ └── res │ │ │ │ ├── drawable │ │ │ │ └── launch_background.xml │ │ │ │ ├── mipmap-hdpi │ │ │ │ └── ic_launcher.png │ │ │ │ ├── mipmap-mdpi │ │ │ │ └── ic_launcher.png │ │ │ │ ├── mipmap-xhdpi │ │ │ │ └── ic_launcher.png │ │ │ │ ├── mipmap-xxhdpi │ │ │ │ └── ic_launcher.png │ │ │ │ ├── mipmap-xxxhdpi │ │ │ │ └── ic_launcher.png │ │ │ │ └── values │ │ │ │ └── styles.xml │ │ │ └── profile │ │ │ └── AndroidManifest.xml │ ├── build.gradle │ ├── gradle.properties │ ├── gradle │ │ └── wrapper │ │ │ └── gradle-wrapper.properties │ └── settings.gradle ├── assets │ ├── decoded_wav_label.txt │ ├── decoded_wav_model.tflite │ ├── google_teach_machine_label.txt │ ├── google_teach_machine_model.tflite │ ├── mfcc_label.txt │ ├── mfcc_model.tflite │ ├── sample_audio_16k_mono.wav │ ├── sample_audio_44k_mono.wav │ ├── spectrogram_label.txt │ └── spectrogram_model.tflite ├── audio_recognition_example.jpg ├── ios │ ├── .gitignore │ ├── Flutter │ │ ├── AppFrameworkInfo.plist │ │ ├── Debug.xcconfig │ │ └── Release.xcconfig │ ├── Podfile │ ├── Runner.xcodeproj │ │ ├── project.pbxproj │ │ ├── project.xcworkspace │ │ │ ├── contents.xcworkspacedata │ │ │ └── xcshareddata │ │ │ │ ├── IDEWorkspaceChecks.plist │ │ │ │ └── WorkspaceSettings.xcsettings │ │ └── xcshareddata │ │ │ └── xcschemes │ │ │ └── Runner.xcscheme │ ├── Runner.xcworkspace │ │ ├── contents.xcworkspacedata │ │ └── xcshareddata │ │ │ └── IDEWorkspaceChecks.plist │ ├── Runner │ │ ├── AppDelegate.swift │ │ ├── Assets.xcassets │ │ │ ├── AppIcon.appiconset │ │ │ │ ├── Contents.json │ │ │ │ ├── Icon-App-1024x1024@1x.png │ │ │ │ ├── Icon-App-20x20@1x.png │ │ │ │ ├── Icon-App-20x20@2x.png │ │ │ │ ├── Icon-App-20x20@3x.png │ │ │ │ ├── Icon-App-29x29@1x.png │ │ │ │ ├── Icon-App-29x29@2x.png │ │ │ │ ├── Icon-App-29x29@3x.png │ │ │ │ ├── Icon-App-40x40@1x.png │ │ │ │ ├── Icon-App-40x40@2x.png │ │ │ │ ├── Icon-App-40x40@3x.png │ │ │ │ ├── Icon-App-60x60@2x.png │ │ │ │ ├── Icon-App-60x60@3x.png │ │ │ │ ├── Icon-App-76x76@1x.png │ │ │ │ ├── Icon-App-76x76@2x.png │ │ │ │ └── Icon-App-83.5x83.5@2x.png │ │ │ └── LaunchImage.imageset │ │ │ │ ├── Contents.json │ │ │ │ ├── LaunchImage.png │ │ │ │ ├── LaunchImage@2x.png │ │ │ │ ├── LaunchImage@3x.png │ │ │ │ └── README.md │ │ ├── Base.lproj │ │ │ ├── LaunchScreen.storyboard │ │ │ └── Main.storyboard │ │ ├── Info.plist │ │ └── Runner-Bridging-Header.h │ └── RunnerTests │ │ ├── AudioFileTest.swift │ │ └── RecordingTest.swift ├── lib │ └── main.dart └── pubspec.yaml ├── ios ├── .gitignore ├── Assets │ └── .gitkeep ├── Classes │ ├── LabelSmoothing.swift │ ├── SwiftTfliteAudioPlugin.swift │ ├── TfliteAudioPlugin.h │ ├── TfliteAudioPlugin.m │ └── processing │ │ ├── AudioFile.swift │ │ ├── AudioFileData.swift │ │ ├── Recording.swift │ │ └── RecordingData.swift └── tflite_audio.podspec ├── lib └── tflite_audio.dart ├── pictures ├── deployment-target.png ├── finish.png ├── model-label-asset.png ├── start.png └── tflite-select-ops-installation.png ├── pubspec.yaml └── test ├── tflite_audio_channel_test.dart └── tflite_audio_stream_test.dart /.gitignore: -------------------------------------------------------------------------------- 1 | # Miscellaneous 2 | *.class 3 | *.lock 4 | *.log 5 | *.pyc 6 | *.swp 7 | .DS_Store 8 | .atom/ 9 | .buildlog/ 10 | .history 11 | .svn/ 12 | .classpath 13 | .project 14 | .settings 15 | 16 | # IntelliJ related 17 | *.iml 18 | *.ipr 19 | *.iws 20 | .idea/ 21 | 22 | # Visual Studio Code related 23 | .vscode/ 24 | 25 | # Flutter repo-specific 26 | /bin/cache/ 27 | /bin/mingit/ 28 | /dev/benchmarks/mega_gallery/ 29 | /dev/bots/.recipe_deps 30 | /dev/bots/android_tools/ 31 | /dev/docs/doc/ 32 | /dev/docs/flutter.docs.zip 33 | /dev/docs/lib/ 34 | /dev/docs/pubspec.yaml 35 | /dev/integration_tests/**/xcuserdata 36 | /dev/integration_tests/**/Pods 37 | /packages/flutter/coverage/ 38 | version 39 | 40 | # packages file containing multi-root paths 41 | .packages.generated 42 | 43 | # Flutter/Dart/Pub related 44 | **/doc/api/ 45 | **/ios/Flutter/.last_build_id 46 | .dart_tool/ 47 | .flutter-plugins 48 | .packages 49 | .pub-cache/ 50 | .pub/ 51 | build/ 52 | flutter_*.png 53 | linked_*.ds 54 | unlinked.ds 55 | unlinked_spec.ds 56 | 57 | # Android related 58 | **/android/**/gradle-wrapper.jar 59 | **/android/.gradle 60 | **/android/captures/ 61 | **/android/gradlew 62 | **/android/gradlew.bat 63 | **/android/local.properties 64 | **/android/**/GeneratedPluginRegistrant.java 65 | **/android/key.properties 66 | *.jks 67 | 68 | # iOS/XCode related 69 | **/ios/**/*.mode1v3 70 | **/ios/**/*.mode2v3 71 | **/ios/**/*.moved-aside 72 | **/ios/**/*.pbxuser 73 | **/ios/**/*.perspectivev3 74 | **/ios/**/*sync/ 75 | **/ios/**/.sconsign.dblite 76 | **/ios/**/.tags* 77 | **/ios/**/.vagrant/ 78 | **/ios/**/DerivedData/ 79 | **/ios/**/Icon? 80 | **/ios/**/Pods/ 81 | **/ios/**/.symlinks/ 82 | **/ios/**/profile 83 | **/ios/**/xcuserdata 84 | **/ios/.generated/ 85 | **/ios/Flutter/App.framework 86 | **/ios/Flutter/Flutter.framework 87 | **/ios/Flutter/Generated.xcconfig 88 | **/ios/Flutter/app.flx 89 | **/ios/Flutter/app.zip 90 | **/ios/Flutter/flutter_assets/ 91 | **/ios/Flutter/flutter_export_environment.sh 92 | **/ios/ServiceDefinitions.json 93 | **/ios/Runner/GeneratedPluginRegistrant.* 94 | 95 | # Coverage 96 | coverage/ 97 | 98 | # Exceptions to above rules. 99 | !**/ios/**/default.mode1v3 100 | !**/ios/**/default.mode2v3 101 | !**/ios/**/default.pbxuser 102 | !**/ios/**/default.perspectivev3 103 | !/packages/flutter_tools/test/data/dart_dependencies_test/**/.packages -------------------------------------------------------------------------------- /.metadata: -------------------------------------------------------------------------------- 1 | # This file tracks properties of this Flutter project. 2 | # Used by Flutter tool to assess capabilities and perform upgrades etc. 3 | # 4 | # This file should be version controlled and should not be manually edited. 5 | 6 | version: 7 | revision: f139b11009aeb8ed2a3a3aa8b0066e482709dde3 8 | channel: stable 9 | 10 | project_type: plugin 11 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.3.0 2 | * BREAK CHANGE: Recording bufferSize now takes in 2x the number of samples. To keep the same recording length, simply divide your previous bufferSize by 2. 3 | * Experimental: Support MFCC, melSpectrogram and spectrogram inputs 4 | * Feature: Can automatically or manually set audio length 5 | * Feature: Can automatically or manually transpose input shape 6 | * Improvement: Stability of asyncronous operations with RxJava and RxSwift 7 | * Improvement: (iOS) Removed meta info when extracting data from audio file. 8 | * Improvement: (Android) Splicing algorithm passes all test case. Audio recogntion should now be more accurate. 9 | * Fixed: (iOS) Duplicate symbol error. Set version of TensorFlowLite to 2.6.0. Problem found [here][i25]. 10 | * Fixed: (Android & iOS) Incorrect padding when splicing audio file. All test cases have passed. 11 | 12 | [i25]: https://github.com/Caldarie/flutter_tflite_audio/issues/25 13 | 14 | ## 0.2.2+4 15 | * Handled NaN exception for raw output on swift 16 | 17 | ## 0.2.2+3 18 | * Hot fixed iOS issue where it will record indefintetly. 19 | 20 | ## 0.2.2+2 21 | * Hot fixed missing AudioProcessing class. 22 | 23 | ## 0.2.2+1 24 | * Hot fixed issue with unresponsive forced stop recognition. 25 | 26 | ## 0.2.2 27 | * Feature: Added ability to recognise stored audio files 28 | * Breaking Change: RecordingLength will no longer be required as a parameter. 29 | * Fixed: NaN output for bufferRates that are non divisible to audioLength 30 | * Fixed: android permission error when granted outside app. 31 | 32 | ## 0.2.1+2 33 | * Fixed NaN raw score output for Android. 34 | 35 | ## 0.2.1+1 36 | * Fixed inaccurate numOfInference count for iOS and android. 37 | 38 | ## 0.2.1 39 | * Improved recognition accuracy for Google Teachable Machine models 40 | * Fixed memory crash on android 41 | * Improved memory performance on iOS 42 | * Added feature to output raw scores 43 | * moved inputType to loadModel() instead of startAudioRecognition() 44 | 45 | ## 0.2.0 46 | * Fixed crashed on Android when force stopping recognition 47 | * Improve recognition latency on android by reducing number of event calls. 48 | 49 | ## 0.1.9 50 | * Added support for android V2 embedding 51 | * Breaking change - no longer supports deprecated versions of Android (pre 1.12) 52 | 53 | ## 0.1.8+2 54 | * Fixed null safety incompatability with example 55 | 56 | ## 0.1.8+1 57 | * Fixed the problem with bridge NSNumber to Float 58 | * Merged rawAudioRecognize() and decodedWavRecognize() on native platforms 59 | * Set detection parameters to 0 for better performance. 60 | 61 | ## 0.1.8 62 | * Added null safety compatability 63 | 64 | ## 0.1.7+1 65 | * Hotfixed iOS crash when casting double to float for detectionThreshold 66 | 67 | ## 0.1.7 68 | * Fixed iOS bug where stream wont close when permission has been denied. 69 | * Added feature where you can adjust the detection sensitivity of the model 70 | 71 | ## 0.1.6+2 72 | * Fixed podsec error 73 | * Fixed iOS incompatability with fluter 2.0.0 74 | 75 | ## 0.1.6+1 76 | * Hotfixed missing value for recording. 77 | 78 | ## 0.1.6 79 | * bufferSize no longer needs to be divisible to recording length. 80 | 81 | ## 0.1.5+3 82 | * Fixed major android crash, where forcibly stopping the stream causes recorder.stop() to be prematurely called. 83 | * Fixed minor iOS crash, where forcibly stopping the stream during recognition returns a nil exception. 84 | * Cleaned up example for easy switch between decodedWav and Google's Teachable Machine model 85 | 86 | ## 0.1.5+2 87 | * Disabled Google's Teachable Machine by default to reduce app footprint. (This can be enabled manually) 88 | * Adjusted example's values to improve inference accuracy 89 | 90 | ## 0.1.5+1 91 | * Added documentation 92 | * Added example model from Google's Teachable Machine. 93 | * Fixed iOS crash when loading text file with empty elements. 94 | 95 | ## 0.1.5 96 | * Added support for Google Teachable Machine models. 97 | * Fixed inaccurate reading with recording 98 | * Added feature to switch between decodedwav and Google's Teachable machine model. 99 | 100 | ## 0.1.4 101 | * Added a new feature where you can run multiple inferences per recording. 102 | * Replaced future with stream when getting results from inferences 103 | * Added a button to cancel the stream / inference 104 | * Removed unnecessary code for easier reading. 105 | 106 | ## 0.1.3+1 107 | * Used reusable widgets for easier to read code. 108 | * Added some documentation 109 | 110 | ## 0.1.3 111 | * Hotfix for crash when permission has been denied. 112 | * Added the key 'hasPermission' for the future startAudioRecognitions(). 113 | * Added feature in example where it'll show inference times 114 | 115 | ## 0.1.2 116 | * Instead of returning a single string value, the future startAudioRecognition() now returns a map with the following keys: 117 | - recognitionResult 118 | - inferenceTime 119 | * Fixed issue in example where pressing the record button multiple times will crash the app. 120 | * Added feature in example where pressing the recording button changes color. 121 | 122 | ## 0.1.1 123 | * Made some fixes with making options explicit 124 | * Added alert dialog when permission is denied. 125 | 126 | ## 0.1.0 127 | * Added iOS support 128 | 129 | ## 0.0.4 130 | * Added the following arguments into the future: startAudioRecognition() 131 | - sampleRate 132 | - audioLength 133 | - bufferSize 134 | 135 | ## 0.0.3 136 | * Merged permission and audio recognition futures into one future. 137 | 138 | ## 0.0.2 139 | * Fixed image url 140 | 141 | ## 0.0.1 142 | 143 | * Initial release. 144 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) 2020 Michael Nguyen 3 | 4 | Permission is hereby granted, free of charge, to any person 5 | obtaining a copy of this software and associated documentation 6 | files (the "Software"), to deal in the Software without restriction, 7 | including without limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of the Software, 9 | and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included 13 | in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 18 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 19 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 20 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 21 | USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TFlite Audio Plugin for Flutter 2 | 3 | [![pub package](https://img.shields.io/pub/v/tflite_audio.svg?label=version&color=blue)](https://pub.dev/packages/tflite_audio) 4 | [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT) 5 | [![style: effective dart](https://img.shields.io/badge/style-effective_dart-40c4ff.svg)](https://pub.dev/packages/effective_dart) 6 | 7 |
8 | 9 | Audio classification Tflite package for flutter (iOS & Android). Can also support Google Teachable Machine models. 10 | 11 | If you are a complete newbie to audio classification, you can read the tutorial [here](https://carolinamalbuquerque.medium.com/audio-recognition-using-tensorflow-lite-in-flutter-application-8a4ad39964ae). Credit to [Carolina](https://github.com/cmalbuquerque) for writing a comprehensive article. 12 | 13 | To keep this project alive, consider giving a star or a like. Pull requests or bug reports are also welcome. 14 | 15 |
16 | 17 | Recording | Inference result 18 | :-------------------------:|:-------------------------: 19 | ![](https://github.com/Caldarie/flutter_tflite_audio/blob/master/pictures/finish.png) | ![](https://github.com/Caldarie/flutter_tflite_audio/blob/master/pictures/start.png) 20 | 21 |
22 | 23 | ## Table of Contents 24 | 25 | * [About this plugin](#about-this-plugin) 26 | * [Known Issues/Commonly asked questions](#known-issuescommonly-asked-questions) 27 | * [Please read if you are using Google's Teachable Machine. Otherwise skip.](#please-read-if-you-are-using-googles-teachable-machine-otherwise-skip) 28 | * [How to add tflite model and label to flutter](#how-to-add-tflite-model-and-label-to-flutter) 29 | * [How to use this plugin](#how-to-use-this-plugin) 30 | * [Rough guide on parameters](#rough-guide-on-the-parameters) 31 | * [Android Installation & Permissions](#android-installation--permissions) 32 | * [iOS Installation & Permissions](#ios-installation--permissions) 33 | * [References](#references) 34 | 35 |
36 | 37 | ## About This Plugin 38 | 39 | ### The plugin has several features: 40 | 41 | 1. Audio recognition for stored audio files. (Only mono wav files for now) 42 | 43 | 2. Audio recognition for recordings. 44 | 45 | 3. Tunable parameters for recording/inferences 46 | * Please look a the [parameters](#rough-guide-on-the-parameters) below for more information. 47 | 48 | 4. Automatically reshape/transpose audio inputs. 49 | 50 |
51 | 52 | ### This plugin can support several model types: 53 | 54 | 1. Google Teachable Machine (Raw audio input) 55 | 56 | * For beginners with little to no machine learning knowledge. You can read can read the tutorial [here](https://carolinamalbuquerque.medium.com/audio-recognition-using-tensorflow-lite-in-flutter-application-8a4ad39964ae) if you are a newbie. 57 | * Training can be done [here](https://teachablemachine.withgoogle.com/train/audio) 58 | 59 | 2. Raw audio input. 60 | 61 | * Can recognize the following inputs: float32[audioLength, 1] or float32[1, audioLength] 62 | * For more information on how to train your own model, take a look [here](https://github.com/tensorflow/examples/tree/master/lite/examples/speech_commands/ml). 63 | 64 | 3. Decoded wav input. 65 | 66 | * Supports two inputs: float32[audioLength, 1] and int32[1] 67 | * For more information on how to train your own model. Take a look [here](https://github.com/tensorflow/docs/blob/master/site/en/r1/tutorials/sequences/audio_recognition.md) 68 | * To train a decoded wave with MFCC, take a look [here](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/examples/speech_commands) 69 | 70 | 4. **(Experimental feature)** Spectogram, melspectrogram, and MFCC inputs. 71 | 72 | * Please note that this feature is experimental, and results may not be accurate compared to raw audio / decoded wav. 73 | * Spectrogram model can be trained here [tutorial](https://www.tensorflow.org/tutorials/audio/simple_audio). 74 | 75 | 5. **(Currently worked on feature)** Multiple input and outputs. 76 | 77 |
78 | 79 | ## Known Issues/Commonly asked questions 80 | 81 | 1. **How to adjust the recording length/time** 82 | 83 | There are two ways to reduce adjust recording length/time: 84 | 85 | * You can increase the recording time by adjusting the bufferSize to a lower value. 86 | 87 | * You can also increase recording time by lowering the sample rate. 88 | 89 | **Note:** That stretching the value too low will cause problems with model accuracy. In that case, you may want to consider lowering your sample rate as well. Likewise, a very low sample rate can also cause problems with accuracy. It is your job to find the sweetspot for both values. 90 | 91 | 2. **How to reduce false positives in my model** 92 | 93 | To reduce false positives, you may want to adjust the default values of `detectionThreshold=0.3` and `averageWindowDuration=1000` to a higher value. A good value for both respectively are `0.7` and `1500`. For more details about these parameters, please visit this [section](#rough-guide-on-the-parameters). 94 | 95 | 3. **I am getting build errors on iOS** 96 | 97 | There are several ways to fix this: 98 | 99 | * Some have reported to fix this issue by replacing the following line: 100 | 101 | ```ruby 102 | target 'Runner' do 103 | use_frameworks! 104 | use_modular_headers! 105 | #pod 'TensorFlowLiteSelectTfOps' #Old line 106 | pod'TensorFlowLiteSelectTfOps','~> 2.6.0' #New line 107 | 108 | flutter_install_all_ios_pods File.dirname(File.realpath(__FILE__)) 109 | end 110 | ``` 111 | 112 | * Others have fixed this issue building the app without the line: `pod 'TensorFlowLiteSelectTfOps`. Then rebuilding the app by re-adding the line again. 113 | 114 | * Remember to run the following below: 115 | 116 | ``` 117 | 1. cd into iOS folder 118 | 119 | 2. Run `flutter pub get` on terminal 120 | 121 | 3. Run `pod install` on terminal 122 | 123 | 4. Run `flutter clean` on terminal 124 | 125 | 5. Run `flutter run` on terminal. All done! 126 | ``` 127 | 128 | 4. **I am getting TensorFlow Lite Error on iOS. - Regular TensorFlow ops are not supported by this interpreter. Make sure you apply/link the Flex delegate before inference** 129 | 130 | * Please make sure that you have enabled ops-select on your [podfile - step 4 & Xcode - step 5](#ios-if-you-are-using-googles-teachable-machine-model-otherwise-skip) and [build gradle - step 3](#android-if-you-are-using-googles-teachable-machine-otherwise-skip) 131 | 132 | * If you tried above, please run the example on a device (not emulator). If you still recieved this error, its very likely that theres an issue with cocoapod or Xcode configuration. Please check the [issue #7](https://github.com/Caldarie/flutter_tflite_audio/issues/7) 133 | 134 | * If you recieved this error from your custom model (not GTM), its likely that you're using unsupported tensorflow operators for tflite, as found in [issue #5](https://github.com/Caldarie/flutter_tflite_audio/issues/5#issuecomment-789260402). For more details on which operators are supported, look at the official documentation [here](https://www.tensorflow.org/lite/guide/ops_compatibility) 135 | 136 | * Take a looking at issue number 3 if none of the above works. 137 | 138 | 5. **(iOS) App crashes when running Google's Teachable Machine model** 139 | 140 | Please run your simulation on actual iOS device. Running your device on M1 macs should also be ok. 141 | 142 | As of this moment, there's [limited support](https://github.com/tensorflow/tensorflow/issues/44997#issuecomment-734001671) for x86_64 architectures from the Tensorflow Lite select-ops framework. If you absolutely need to run it on an emulator, you can consider building the select ops framework yourself. Instructions can be found [here](https://www.tensorflow.org/lite/guide/ops_select#ios) 143 | 144 | 6. **(Android) Fatal signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr 0xfffffff4 in tid 5403** 145 | 146 | It seems like the latest tflite package for android is causing this issue. Until this issue is fixed, please run this package on an actual Android Device. 147 | 148 |
149 | 150 | ## Please Read If You Are Using Google's Teachable Machine. (Otherwise Skip) 151 | 152 |
153 | 154 | **BE AWARE:** Google's Teachable Machine requires [select tensorflow operators](https://www.tensorflow.org/lite/guide/ops_select#using_bazel_xcode) to work. This feature is experimental and will cause the following issues: 155 | 156 | 1. Increase the overall size of your app. If this is unnacceptable for you, it's recommended that you build your own custom model. Tutorials can be found in the [About this plugin section](#about-this-plugin) 157 | 158 | 2. Emulators for iOS do not work due to limited support for x86_64 architectures. You need to run your simulation on an actual device. Issue can be found [here](https://github.com/tensorflow/tensorflow/issues/44997) 159 | 160 | 3. You will need to manually implement ops-select on your [podfile - step 4 & Xcode - step 5](#note-skip-below-if-your-are-not-using-google-teachable-machine-ios) and [build gradle - step 3](#note-skip-below-if-your-are-not-using-google-teachable-machine-android) 161 | 162 |
163 | 164 | ## How to add tflite model and label to flutter: 165 | 166 |
167 | 168 | 1. Create an assets folder and then place your custom tflite model and labels inside. 169 | 170 | ![](https://github.com/Caldarie/flutter_tflite_audio/blob/master/pictures/model-label-asset.png) 171 | 172 | 2. In pubsec.yaml, link your tflite model and label under 'assets'. For example: 173 | 174 | ``` 175 | assets: 176 | - assets/decoded_wav_model.tflite 177 | - assets/decoded_wav_label.txt 178 | 179 | ``` 180 | 181 |
182 | 183 | ## How to use this plugin 184 | 185 |
186 | 187 | Please look at the [example](https://github.com/Caldarie/flutter_tflite_audio/tree/master/example) on how to implement these futures. 188 | 189 | 190 | 1. To add the package in pubspec.yaml, open your terminal and run this line in your flutter project: 191 | 192 | ``` 193 | flutter pub add tflite_audio 194 | ``` 195 | 196 | 2. Import the plugin. For example: 197 | 198 | ``` 199 | import 'package:tflite_audio/tflite_audio.dart'; 200 | ``` 201 | 202 | 203 | 3. To load your model: 204 | 205 | 206 | ```dart 207 | //Example for decodedWav models 208 | TfliteAudio.loadModel( 209 | model: 'assets/conv_actions_frozen.tflite', 210 | label: 'assets/conv_actions_label.txt', 211 | inputType: 'decodedWav'); 212 | 213 | 214 | //Example for Google's Teachable Machine models 215 | TfliteAudio.loadModel( 216 | model: 'assets/google_teach_machine_model.tflite', 217 | label: 'assets/google_teach_machine_label.txt', 218 | inputType: 'rawAudio'); 219 | 220 | //Example if you want to take advantage of all optional parameters from loadModel() 221 | TfliteAudio.loadModel( 222 | model: 'assets/conv_actions_frozen.tflite', 223 | label: 'assets/conv_actions_label.txt', 224 | inputType: 'decodedWav', 225 | outputRawScores: false, 226 | numThreads: 1, 227 | isAsset: true, 228 | ); 229 | ``` 230 | 231 | 232 | 4. To start and listen to the stream for inference results: 233 | 234 | * Declare stream value 235 | ```dart 236 | Stream> recognitionStream; 237 | ``` 238 | 239 | * If you want to use the recognition stream for recording: 240 | ```dart 241 | //Example values for Google's Teachable Machine models 242 | recognitionStream = TfliteAudio.startAudioRecognition( 243 | sampleRate: 44100, 244 | bufferSize: 22016, 245 | ) 246 | 247 | //Example values for decodedWav 248 | recognitionStream = TfliteAudio.startAudioRecognition( 249 | sampleRate: 16000, 250 | bufferSize: 2000, 251 | ) 252 | 253 | //Example for advanced users who want to utilise all optional parameters from this package. 254 | //Note the values are default. 255 | recognitionStream = TfliteAudio.startAudioRecognition( 256 | sampleRate: 44100, 257 | bufferSize: 22016, 258 | numOfInferences: 5, 259 | audioLength = 44032, 260 | detectionThreshold: 0.3, 261 | averageWindowDuration = 1000, 262 | minimumTimeBetweenSamples = 30, 263 | suppressionTime = 1500, 264 | ) 265 | 266 | ``` 267 | 268 | * If you want to use the recognition stream for stored audio files. 269 | 270 | ```dart 271 | //Example values for Google teachable models 272 | recognitionStream = TfliteAudio.startFileRecognition( 273 | sampleRate: 44100, 274 | audioDirectory: "assets/sampleAudio.wav", 275 | ); 276 | 277 | //Examples values for decodedWav 278 | recognitionStream = TfliteAudio.startFileRecognition( 279 | sampleRate: 16000, 280 | audioDirectory: "assets/sampleAudio.wav", 281 | ); 282 | 283 | //Example for advanced users who want to utilise all optional parameters from this package. 284 | recognitionStream = TfliteAudio.startFileRecognition( 285 | sampleRate: 44100, 286 | audioDirectory: "assets/sampleAudio.wav", 287 | audioLength: 44032, 288 | detectionThreshold: 0.3, 289 | averageWindowDuration: 1000, 290 | minimumTimeBetweenSamples: 30, 291 | suppressionTime: 1500, 292 | ); 293 | ``` 294 | 295 | * Listen for results 296 | ```dart 297 | String result = ''; 298 | int inferenceTime = 0; 299 | 300 | recognitionStream.listen((event){ 301 | result = event["inferenceTime"]; 302 | inferenceTime = event["recognitionResult"]; 303 | }) 304 | .onDone( 305 | //Do something here when stream closes 306 | ); 307 | ``` 308 | 309 | 5. To forcibly cancel recognition stream 310 | 311 | ```dart 312 | TfliteAudio.stopAudioRecognition(); 313 | ``` 314 | 315 |
316 | 317 | ## Rough guide on the parameters 318 | 319 | * outputRawScores - Will output the result as an array in string format. For example `'[0.2, 0.6, 0.1, 0.1]'` 320 | 321 | * numThreads - Higher threads will reduce inferenceTime. However, will utilise the more cpu resource. 322 | 323 | * isAsset - is your model, label or audio file in the asset file? If yes, set true. If the files are outside (such as external storage), set false. 324 | 325 | * numOfInferences - determines how many times you want to loop the recording and inference. For example: 326 | `numOfInference = 3` will repeat the recording three times, so recording length will be (1 to 2 seconds) x 3 = (3 to 6 seconds). Also the model will output the scores three times. 327 | 328 | * sampleRate - A higher sample rate may improve accuracy for recordings. Recommened values are 16000, 22050, 44100 329 | 330 | * audioLength - Default is 0 as the plugin will determine the length for you. You can manually adjust this if you wish to shorten or extend the number of audio samples. 331 | 332 | * bufferSize - A lower value will lengthen the recording. Likewise, a higehr value will shorten the recording. Make sure this value is equal or below your recording length. 333 | 334 | * detectionThreshold - Will ignore any predictions where its probability does not exceed the detection threshold. Useful for situations where you pickup unwanted/unintentional sounds. Lower the value if your model's performance isn't doing too well. 335 | 336 | * suppressionMs - If your detection triggers too early, the result may be poor or inaccurate. Adjust the values to avoid this situation. 337 | 338 | * averageWindowDurationMs - Use to remove earlier results that are too old. 339 | 340 | * minimumTimeBetweenSamples - Ignore any results that are coming in too frequently 341 | 342 |
343 | 344 | ## Android Installation & Permissions 345 | 346 | 1. Add the permissions below to your AndroidManifest. This could be found in `/android/app/src`. For example: 347 | 348 | ``` 349 | 350 | 351 | ``` 352 | 353 | 2. Edit the following below to your build.gradle. This could be found in `/app/src/`. For example: 354 | 355 | ```Gradle 356 | aaptOptions { 357 | noCompress 'tflite' 358 | ``` 359 | 360 |
361 | 362 | #### **NOTE:** Skip below if your are not using Google Teachable Machine (Android) 363 | 364 |
365 | 366 | 367 | 1. Enable select-ops under dependencies in your build gradle. 368 | 369 | ```Gradle 370 | dependencies { 371 | compile 'org.tensorflow:tensorflow-lite-select-tf-ops:+' 372 | } 373 | ``` 374 | 375 |
376 | 377 | ## iOS Installation & Permissions 378 | 379 | 1. Add the following key to Info.plist for iOS. This could be found in `/ios/Runner` 380 | ``` 381 | NSMicrophoneUsageDescription 382 | Record audio for playback 383 | ``` 384 | 385 | 2. Change the deployment target to a minumum of 12.0 or higher. This could be done by: 386 | 387 | * Open your project workspace on xcode. Project workspace can be found here: `/ios/Runner.xcworkspace` 388 | 389 | * Select the top level Runner on the left panel 390 | 391 | * Select the Runner under Project. 392 | 393 | * Under the info tab, change the iOS deployment target to a minimum of 12.0 or higher 394 | 395 | ![](https://github.com/Caldarie/flutter_tflite_audio/blob/master/pictures/deployment-target.png) 396 | 397 | 3. Open your podfile (found here: `/ios/Podfile`) and change platform ios to a minimum 12 or higher. 398 | 399 | ```ruby 400 | platform :ios, '12.0' 401 | ``` 402 | 403 |
404 | 405 | #### **NOTE:** Skip below if your are not using Google Teachable Machine (iOS) 406 | 407 |
408 | 409 | 410 | 1. In the same podfile, add `pod 'TensorFlowLiteSelectTfOps' under target. 411 | 412 | ```ruby 413 | target 'Runner' do 414 | use_frameworks! 415 | use_modular_headers! 416 | pod'TensorFlowLiteSelectTfOps','~> 2.6.0' #Add this line here 417 | 418 | flutter_install_all_ios_pods File.dirname(File.realpath(__FILE__)) 419 | end 420 | ``` 421 | 422 | 2. Force load Select Ops for Tensorflow. To do that: 423 | 424 | * Open your project on xcode 425 | 426 | * click on runner under "Targets" 427 | 428 | * Click on "Build settings" tab 429 | 430 | * Click on "All" tab 431 | 432 | * Click on the empty space which is on the right side of "Other Links Flag" 433 | 434 | * Add the following line: `-force_load $(SRCROOT)/Pods/TensorFlowLiteSelectTfOps/Frameworks/TensorFlowLiteSelectTfOps.framework/TensorFlowLiteSelectTfOps` 435 | 436 | ![](https://github.com/Caldarie/flutter_tflite_audio/blob/master/pictures/tflite-select-ops-installation.png) 437 | 438 | 439 | 3. Install the ops-select package to pod. To do this: 440 | 441 | * cd into iOS folder 442 | 443 | * Run `flutter pub get` on terminal 444 | 445 | * Run `pod install` on terminal 446 | 447 | * Run `flutter clean` on terminal 448 | 449 | * Run `flutter run` on terminal. All done! 450 | 451 |
452 | 453 | ## References 454 | 455 | This project wouldn't of been possible if it wasn't for the following: 456 | 457 | 1. Project is based on: 458 | * https://github.com/tensorflow/examples/tree/master/lite/examples/speech_commands 459 | 2. Tflite & select ops: 460 | * https://www.tensorflow.org/lite/guide/ops_select 461 | * https://libraries.io/cocoapods/TensorFlowLiteSelectTfOps 462 | 3. Spectogram libraries: 463 | * https://github.com/Subtitle-Synchronizer/jlibrosa 464 | * https://github.com/dhrebeniuk/RosaKit 465 | 4. RxJava and RxSwift 466 | * https://github.com/ReactiveX/RxAndroid 467 | * https://github.com/ReactiveX/RxJava 468 | * https://github.com/ReactiveX/RxSwift 469 | -------------------------------------------------------------------------------- /analysis_options.yaml: -------------------------------------------------------------------------------- 1 | include: package:effective_dart/analysis_options.yaml 2 | 3 | linter: 4 | rules: 5 | await_only_futures: true 6 | prefer_const_constructors: true 7 | prefer_const_literals_to_create_immutables: true 8 | prefer_relative_imports: false 9 | prefer_single_quotes: true 10 | public_member_api_docs: true 11 | sort_child_properties_last: true 12 | sort_constructors_first: true 13 | unawaited_futures: true 14 | analyzer: 15 | exclude: 16 | - lib/**/*.freezed.dart 17 | - lib/**/*.g.dart 18 | - lib/intl/arb/* 19 | - lib/intl/messages/* 20 | - 'example/**' 21 | strong-mode: 22 | implicit-casts: false 23 | errors: 24 | close_sinks: ignore 25 | missing_return: error 26 | dead_code: warning 27 | -------------------------------------------------------------------------------- /android/.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | .gradle 3 | /local.properties 4 | /.idea/workspace.xml 5 | /.idea/libraries 6 | .DS_Store 7 | /build 8 | /captures 9 | -------------------------------------------------------------------------------- /android/build.gradle: -------------------------------------------------------------------------------- 1 | /* Point of reference 2 | https://github.com/flutter/plugins/blob/main/packages/camera/camera/android/build.gradle 3 | */ 4 | 5 | group 'flutter.tflite_audio' //do not edit 6 | version '1.0' 7 | 8 | buildscript { 9 | repositories { 10 | google() 11 | mavenCentral() 12 | maven { url "https://oss.jfrog.org/libs-snapshot" } //rxjava 13 | } 14 | 15 | dependencies { 16 | classpath 'com.android.tools.build:gradle:7.0.4' 17 | } 18 | } 19 | 20 | rootProject.allprojects { 21 | repositories { 22 | google() 23 | mavenLocal() 24 | } 25 | } 26 | 27 | apply plugin: 'com.android.library' 28 | 29 | android { 30 | compileSdkVersion 31 31 | 32 | defaultConfig { 33 | ndk { 34 | abiFilters 'armeabi-v7a', 'arm64-v8a' 35 | } 36 | targetSdkVersion 31 37 | minSdkVersion 24 38 | testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" 39 | 40 | } 41 | 42 | aaptOptions { 43 | noCompress "tflite" 44 | } 45 | 46 | lintOptions { 47 | disable 'InvalidPackage' 48 | } 49 | 50 | compileOptions { 51 | sourceCompatibility JavaVersion.VERSION_1_8 52 | targetCompatibility JavaVersion.VERSION_1_8 53 | } 54 | 55 | // https://stackoverflow.com/questions/57958239/how-to-mock-android-util-log 56 | testOptions { 57 | unitTests.includeAndroidResources = true 58 | unitTests.returnDefaultValues = true 59 | unitTests.all { 60 | testLogging { 61 | events "passed", "skipped", "failed", "standardOut", "standardError" 62 | outputs.upToDateWhen {false} 63 | showStandardStreams = true 64 | } 65 | } 66 | } 67 | 68 | dependencies{ 69 | implementation "androidx.appcompat:appcompat:1.4.1" 70 | implementation 'androidx.annotation:annotation:1.3.0' 71 | implementation files('../android/src/main/java/flutter/tflite_audio/lib/jlibrosa-1.1.8-SNAPSHOT-jar-with-dependencies.jar') 72 | implementation 'org.tensorflow:tensorflow-lite:+' 73 | implementation 'io.reactivex.rxjava3:rxandroid:3.0.0' 74 | implementation 'io.reactivex.rxjava3:rxjava:3.1.3' 75 | // compile 'org.apache.commons:commons-math3:3.6.1' 76 | // compile 'org.tensorflow:tensorflow-lite-select-tf-ops:+' 77 | // implementation 'org.tensorflow:tensorflow-lite:0.1.100' 78 | // implementation 'org.tensorflow:tensorflow-lite-select-tf-ops:0.1.100' 79 | // implementation "org.tensorflow:tensorflow-lite-support:0.1.0-rc1" 80 | // implementation "org.tensorflow:tensorflow-lite-metadata:0.1.0-rc2" 81 | 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /android/gradle.properties: -------------------------------------------------------------------------------- 1 | ## For more details on how to configure your build environment visit 2 | # http://www.gradle.org/docs/current/userguide/build_environment.html 3 | # 4 | # Specifies the JVM arguments used for the daemon process. 5 | # The setting is particularly useful for tweaking memory settings. 6 | # Default value: -Xmx1024m -XX:MaxPermSize=256m 7 | # org.gradle.jvmargs=-Xmx2048m -XX:MaxPermSize=512m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 8 | # 9 | # When configured, Gradle will run in incubating parallel mode. 10 | # This option should only be used with decoupled projects. More details, visit 11 | # http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects 12 | # org.gradle.parallel=true 13 | #Sat May 16 22:49:36 JST 2020 14 | android.enableJetifier=true 15 | android.enableR8=true 16 | org.gradle.jvmargs=-Xmx2048M -Dkotlin.daemon.jvm.options\="-Xmx2048M" 17 | android.useAndroidX=true 18 | -------------------------------------------------------------------------------- /android/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | zipStoreBase=GRADLE_USER_HOME 4 | zipStorePath=wrapper/dists 5 | distributionUrl = https\://services.gradle.org/distributions/gradle-7.2-all.zip 6 | -------------------------------------------------------------------------------- /android/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'tflite_audio' 2 | -------------------------------------------------------------------------------- /android/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | -------------------------------------------------------------------------------- /android/src/main/java/flutter/tflite_audio/AudioChunk.java: -------------------------------------------------------------------------------- 1 | package flutter.tflite_audio; 2 | 3 | 4 | // https://stackoverflow.com/questions/54566753/escaping-closure-in-swift-and-how-to-perform-it-in-java 5 | public interface AudioChunk { 6 | void get(short [] data); 7 | } -------------------------------------------------------------------------------- /android/src/main/java/flutter/tflite_audio/AudioData.java: -------------------------------------------------------------------------------- 1 | package flutter.tflite_audio; 2 | 3 | import android.util.Log; 4 | 5 | import java.util.Arrays; 6 | 7 | public class AudioData { 8 | 9 | private static final String LOG_TAG = "AudioData"; 10 | 11 | private final int audioLength; 12 | private final int fileSize; 13 | 14 | private final boolean requirePadding; 15 | private final int numOfInferences; 16 | 17 | private int indexCount = 0; 18 | private int inferenceCount = 1; 19 | 20 | private short[] audioChunk; 21 | 22 | public AudioData(int audioLength, int fileSize){ 23 | this.audioLength = audioLength; 24 | this.fileSize = fileSize; 25 | this.audioChunk = new short [audioLength]; 26 | 27 | int excessSample = fileSize % audioLength; 28 | int missingSamples = audioLength - excessSample; 29 | requirePadding = getPaddingRequirement(excessSample, missingSamples); 30 | 31 | int totalWithPad = fileSize + missingSamples; 32 | int totalWithoutPad = fileSize - excessSample; 33 | numOfInferences = getNumOfInferences(totalWithoutPad, totalWithPad); 34 | } 35 | 36 | 37 | private boolean getPaddingRequirement(int excessSample, int missingSamples) { 38 | 39 | boolean hasMissingSamples = missingSamples != 0 || excessSample != audioLength; 40 | double missingSampleThreshold = 0.40; 41 | double missingSamplesRatio = (double) missingSamples / (double) audioLength; 42 | boolean shouldPad = missingSamplesRatio < missingSampleThreshold; 43 | 44 | if (hasMissingSamples && shouldPad) return true; 45 | else if (hasMissingSamples && !shouldPad) return false; 46 | else if (!hasMissingSamples && shouldPad) return false; 47 | else return false; 48 | 49 | } 50 | 51 | private int getNumOfInferences(int totalWithoutPad, int totalWithPad) { 52 | return requirePadding ? (int) totalWithPad / audioLength : (int) totalWithoutPad / audioLength; 53 | } 54 | 55 | public String getState(int i) { 56 | boolean reachInputSize = (i + 1) % audioLength == 0; 57 | boolean reachFileSize = (i + 1) == fileSize; 58 | boolean reachInferenceLimit = inferenceCount == numOfInferences; 59 | 60 | if (reachInputSize && !reachInferenceLimit) { 61 | return "recognise"; 62 | } // inferences > 1 && < not final 63 | else if (!reachInputSize && reachInferenceLimit && !reachFileSize) { 64 | return "append"; 65 | } // Inferences = 1 66 | else if (!reachInputSize && !reachInferenceLimit) { 67 | return "append"; 68 | } // Inferences > 1 69 | else if (!reachInputSize && reachInferenceLimit && reachFileSize) { 70 | return "padAndFinalise"; 71 | } // for padding last infernce 72 | else if (reachInputSize && reachInferenceLimit) { 73 | return "finalise"; 74 | } // inference is final 75 | else { 76 | return "Error"; 77 | } 78 | } 79 | 80 | public AudioData append(short dataPoint) { 81 | audioChunk[indexCount] = dataPoint; 82 | indexCount += 1; 83 | return this; 84 | } 85 | 86 | public AudioData displayInference() { 87 | Log.d(LOG_TAG, "Inference count: " + (inferenceCount) + "/" + numOfInferences); 88 | return this; 89 | } 90 | 91 | /* 92 | https://www.javatpoint.com/java-closure 93 | https://www.geeksforgeeks.org/method-within-method-in-java/ 94 | https://stackoverflow.com/questions/54566753/escaping-closure-in-swift-and-how-to-perform-it-in-java */ 95 | public AudioData emit(AudioChunk audioChunk) { 96 | audioChunk.get(this.audioChunk); 97 | return this; 98 | } 99 | 100 | public AudioData reset() { 101 | indexCount = 0; 102 | inferenceCount += 1; 103 | audioChunk = new short[audioLength]; 104 | return this; 105 | } 106 | 107 | public AudioData padSilence(int i) { 108 | AudioProcessing audioData = new AudioProcessing(); 109 | int missingSamples = audioLength - indexCount; 110 | 111 | if (requirePadding) { 112 | Log.d(LOG_TAG, "Missing samples found in short audio chunk.."); 113 | audioChunk = audioData.addSilence(missingSamples, audioChunk, indexCount); 114 | } else { 115 | Log.d(LOG_TAG, "Under threshold. Padding not required"); 116 | } 117 | 118 | return this; 119 | } 120 | 121 | 122 | } 123 | 124 | 125 | -------------------------------------------------------------------------------- /android/src/main/java/flutter/tflite_audio/AudioFile.java: -------------------------------------------------------------------------------- 1 | package flutter.tflite_audio; 2 | 3 | import android.util.Log; 4 | 5 | import io.reactivex.rxjava3.core.Observable; 6 | import io.reactivex.rxjava3.core.*; 7 | import io.reactivex.rxjava3.disposables.Disposable; 8 | import io.reactivex.rxjava3.subjects.PublishSubject; 9 | 10 | import java.nio.ShortBuffer; 11 | import java.util.Arrays; 12 | import java.nio.ByteBuffer; 13 | import java.nio.ByteOrder; 14 | 15 | /* 16 | !References 17 | https://www.javatpoint.com/java-closure 18 | https://www.geeksforgeeks.org/method-within-method-in-java/ 19 | https://stackoverflow.com/questions/54566753/escaping-closure-in-swift-and-how-to-perform-it-in-java 20 | 21 | ``` 22 | .emit(new AudioChunk(){ 23 | @Override 24 | public void get(short [] data) { 25 | subject.onNext(data); 26 | } 27 | }) 28 | ``` 29 | !Code above same as lambda below: 30 | 31 | `.emit(data -> subject.onNext(data))` 32 | 33 | */ 34 | 35 | public class AudioFile { 36 | 37 | private static final String LOG_TAG = "AudioFile"; 38 | 39 | private final ShortBuffer shortBuffer; 40 | private final PublishSubject subject; 41 | private final AudioData audioData; 42 | 43 | private boolean isSplicing = false; 44 | 45 | public AudioFile(byte[] byteData, int audioLength) { 46 | 47 | shortBuffer = ByteBuffer.wrap(byteData).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer(); 48 | subject = PublishSubject.create(); 49 | audioData = new AudioData(audioLength, shortBuffer.limit()); 50 | 51 | } 52 | 53 | public Observable getObservable() { 54 | return (Observable) this.subject; 55 | } 56 | 57 | public void stop() { 58 | isSplicing = false; 59 | subject.onComplete(); 60 | } 61 | 62 | public void splice() { 63 | isSplicing = true; 64 | 65 | for (int i = 0; i < shortBuffer.limit(); i++) { 66 | 67 | short dataPoint = shortBuffer.get(i); 68 | 69 | if (!isSplicing) { 70 | subject.onComplete(); 71 | break; 72 | } 73 | 74 | switch (audioData.getState(i)) { 75 | case "append": 76 | audioData 77 | .append(dataPoint); 78 | break; 79 | case "recognise": 80 | Log.d(LOG_TAG, "Recognising"); 81 | audioData 82 | .append(dataPoint) 83 | .displayInference() 84 | .emit(data -> subject.onNext(data)) 85 | .reset(); 86 | break; 87 | case "finalise": 88 | Log.d(LOG_TAG, "Finalising"); 89 | audioData 90 | .append(dataPoint) 91 | .displayInference() 92 | .emit(data -> subject.onNext(data)); 93 | stop(); 94 | break; 95 | case "padAndFinalise": 96 | Log.d(LOG_TAG, "Padding and finalising"); 97 | audioData 98 | .append(dataPoint) 99 | .padSilence(i) 100 | .displayInference() 101 | .emit(data -> subject.onNext(data)); 102 | stop(); 103 | break; 104 | 105 | default: 106 | throw new AssertionError("Incorrect state when preprocessing"); 107 | } 108 | } 109 | } 110 | 111 | } 112 | -------------------------------------------------------------------------------- /android/src/main/java/flutter/tflite_audio/AudioProcessing.java: -------------------------------------------------------------------------------- 1 | package flutter.tflite_audio; 2 | 3 | import android.util.Log; 4 | 5 | import java.util.ArrayList; 6 | import java.util.Arrays; 7 | import java.util.Random; 8 | 9 | import org.apache.commons.math3.complex.Complex; 10 | 11 | 12 | /* Used for debugging. Implement this?? 13 | Log.v(LOG_TAG, "smax: " + audioData.getMaxAbsoluteValue(audioBuffer)); 14 | Log.v(LOG_TAG, "smin: " + audioData.getMinAbsoluteValue(audioBuffer)); 15 | Log.v(LOG_TAG, "audio data: " + Arrays.toString(audioBuffer)); 16 | */ 17 | 18 | public class AudioProcessing { 19 | 20 | private static final String LOG_TAG = "Audio_Data"; 21 | 22 | //TODO - user controlled ranges? 23 | public short[] addSilence(int missingSamples, short[] audioChunk, int indexCount) { 24 | short[] padding = new short[missingSamples]; 25 | Random random = new Random(); 26 | Log.d("Preprocess:", "Padding " + missingSamples + " samples to audioChunk.."); 27 | for (int x = 0; x < missingSamples; x++) { 28 | int rand = random.nextInt(10 + 10) - 10; //range from negative to positive 29 | // int rand = 0; 30 | short value = (short) rand; 31 | padding[x] = value; 32 | } 33 | System.arraycopy(padding, 0, audioChunk, indexCount, missingSamples); 34 | return audioChunk; 35 | } 36 | 37 | public float[][] transpose2D(float[][] matrix){ 38 | int m = matrix.length; 39 | int n = matrix[0].length; 40 | 41 | float[][] transposedMatrix = new float[n][m]; 42 | 43 | for(int x = 0; x < n; x++) { 44 | for(int y = 0; y < m; y++) { 45 | transposedMatrix[x][y] = matrix[y][x]; 46 | } 47 | } 48 | return transposedMatrix; 49 | } 50 | 51 | 52 | public float[][] normalise(short [] inputBuffer16){ 53 | final float maxRes16 = (float) Math.pow(2, 15) -1; //outputs 32767.0f 54 | float[][] floatInputBuffer = new float [1][inputBuffer16.length]; 55 | 56 | for (int i = 0; i < inputBuffer16.length; ++i) { 57 | floatInputBuffer[0][i] = inputBuffer16[i] / maxRes16; 58 | } 59 | 60 | return floatInputBuffer; 61 | } 62 | 63 | 64 | public float[][] normaliseAndTranspose(short [] inputBuffer16){ 65 | final float maxRes16 = (float) Math.pow(2, 15) -1; //outputs 32767.0f 66 | float[][] floatInputBuffer = new float [inputBuffer16.length][1]; 67 | 68 | for (int i = 0; i < inputBuffer16.length; ++i) { 69 | floatInputBuffer[i][0] = inputBuffer16[i] / maxRes16; 70 | } 71 | 72 | return floatInputBuffer; 73 | } 74 | 75 | 76 | public float [] normalizeBySigned16(short [] inputBuffer16){ 77 | final float maxRes16 = (float) Math.pow(2, 15) -1; //outputs 32767.0f 78 | float inputBuffer32[] = new float[inputBuffer16.length]; 79 | 80 | //normalise audio to -1 to 1 values 81 | for (int i = 0; i < inputBuffer16.length; ++i) 82 | inputBuffer32[i] = inputBuffer16[i] / maxRes16; 83 | // inputBuffer32[i] = inputBuffer16[i] / 32767.0f; 84 | 85 | return inputBuffer32; 86 | } 87 | 88 | public float [] normalizeByMaxAmplitude(short[] inputBuffer16){ 89 | //normalise what?? 90 | float [] result = new float [inputBuffer16.length]; 91 | short wmax = getMaxAbsoluteValue(inputBuffer16); 92 | for (int i = 0; i < inputBuffer16.length; ++i) 93 | result[i] = (float) inputBuffer16[i] / wmax; 94 | 95 | return result; 96 | } 97 | 98 | 99 | //https://github.com/mkvenkit/simple_audio_pi/blob/main/simple_audio.py 100 | public float [] scaleAndCentre(float [] inputBuffer32){ 101 | 102 | float [] result = new float [inputBuffer32.length]; 103 | final float ptp = getMaxMinRange(inputBuffer32); 104 | final float min = getMinValue(inputBuffer32); 105 | 106 | // Log.d(LOG_TAG, "audio chunk: " + Arrays.toString(inputBuffer32)); 107 | Log.d(LOG_TAG, "ptp: " + ptp); 108 | Log.d(LOG_TAG, "min: " + min); 109 | 110 | //scale to center waveform 111 | for (int i = 0; i < inputBuffer32.length; ++i) 112 | result[i] = 2.0f*(inputBuffer32[i] - min) / (float)(ptp - 1); 113 | 114 | return result; 115 | } 116 | 117 | // public float[][] doubleTo2dFloat(double[][] doubleInput) { 118 | 119 | // int height = doubleInput.length; 120 | // int width = doubleInput[0].length; 121 | 122 | // float[][] floatOutput = new float[height][width]; 123 | // for (int h = 0; h < height; h++) { 124 | // for (int w = 0; w < width; w++) { 125 | // floatOutput[h][w] = (float) doubleInput[h][w]; 126 | // } 127 | // } 128 | // return floatOutput; 129 | // } 130 | 131 | 132 | // preprocessing 133 | public byte[] appendByteData(byte[] src, byte[] dst) { 134 | byte[] result = new byte[src.length + dst.length]; 135 | System.arraycopy(src, 0, result, 0, src.length); 136 | System.arraycopy(dst, 0, result, src.length, dst.length); 137 | return result; 138 | } 139 | 140 | public short[] appendShortData(short[] src, short[] dst) { 141 | short[] result = new short[src.length + dst.length]; 142 | System.arraycopy(src, 0, result, 0, src.length); 143 | System.arraycopy(dst, 0, result, src.length, dst.length); 144 | return result; 145 | } 146 | 147 | //https://stackoverflow.com/questions/40361324/maximum-minus-minimum-in-an-array 148 | private float getMaxMinRange(float [] input){ 149 | float min = Float.MAX_VALUE; 150 | float max = Float.MIN_VALUE; 151 | for (float elem : input) { 152 | if (elem < min) min = elem; 153 | if (elem > max) max = elem; 154 | } 155 | return (float) (max - min); 156 | } 157 | 158 | //http://www.java2s.com/example/java/collection-framework/calculates-max-absolute-value-in-short-type-array.html 159 | // private float getMaxAbsoluteValue(float[] input) { 160 | // float max = Float.MIN_VALUE; 161 | // for (int i = 0; i < input.length; i++) { 162 | // if (Math.abs(input[i]) > max) { 163 | // max = (float) Math.abs(input[i]); 164 | // } 165 | // } 166 | // return max; 167 | // } 168 | 169 | private short getMaxAbsoluteValue(short[] input) { 170 | short max = Short.MIN_VALUE; 171 | for (int i = 0; i < input.length; i++) { 172 | if (Math.abs(input[i]) > max) { 173 | max = (short) Math.abs(input[i]); 174 | } 175 | } 176 | return max; 177 | } 178 | 179 | private float getMinValue(float[] input) { 180 | float min = Float.MAX_VALUE; 181 | for (int i = 0; i < input.length; i++) { 182 | if (Math.abs(input[i]) < min) { 183 | min = (float) input[i]; 184 | } 185 | } 186 | return min; 187 | } 188 | 189 | // private float getMinAbsoluteValue(float[] input) { 190 | // float min = Float.MAX_VALUE; 191 | // for (int i = 0; i < input.length; i++) { 192 | // if (Math.abs(input[i]) < min) { 193 | // min = (float) Math.abs(input[i]); 194 | // } 195 | // } 196 | // return min; 197 | // } 198 | 199 | // public short getMinAbsoluteValue(short[] input) { 200 | // short min = Short.MAX_VALUE; 201 | // for (int i = 0; i < input.length; i++) { 202 | // if (Math.abs(input[i]) < min) { 203 | // min = (short) Math.abs(input[i]); 204 | // } 205 | // } 206 | // return min; 207 | // } 208 | } 209 | -------------------------------------------------------------------------------- /android/src/main/java/flutter/tflite_audio/Debugging.java: -------------------------------------------------------------------------------- 1 | package flutter.tflite_audio; 2 | 3 | import android.util.Log; 4 | import java.util.Arrays; 5 | 6 | class DisplayLogs { 7 | 8 | public void matrix(float [][] matrix) { 9 | for (int i = 0; i < matrix.length; i++) { 10 | Log.d("matrix", Arrays.toString(matrix[i])); 11 | } 12 | } 13 | 14 | 15 | // Preprocessing - silence padding on final inference 16 | public void logs(String LOG_TAG, short[] audioChunk, int indexCount, int inputSize) { 17 | Log.d(LOG_TAG, "audioChunk first element: " + audioChunk[0]); 18 | Log.d(LOG_TAG, "audioChunk second last element: " + audioChunk[indexCount - 2]); 19 | Log.d(LOG_TAG, "audioChunk last element: " + audioChunk[indexCount - 1]); 20 | Log.d(LOG_TAG, "audioChunk first missing element: " + audioChunk[indexCount]); 21 | Log.d(LOG_TAG, "audioChunk second missing element: " + audioChunk[indexCount + 1]); 22 | Log.d(LOG_TAG, "audioChunk second last missing element: " + audioChunk[inputSize - 2]); 23 | Log.d(LOG_TAG, "audioChunk last missing element: " + audioChunk[inputSize - 1]); 24 | } 25 | 26 | // Preprocessing - keep track of index & inference count 27 | public void logs(String LOG_TAG, int i, int indexCount, int inferenceCount, short[] audioChunk) { 28 | Log.d(LOG_TAG, "Index: " + i); 29 | Log.d(LOG_TAG, "IndexCount: " + indexCount); 30 | Log.d(LOG_TAG, "InferenceCount " + inferenceCount); 31 | Log.d(LOG_TAG, "audioChunk: " + Arrays.toString(audioChunk)); 32 | } 33 | 34 | // Preprocesing - show basic info about raw audio data 35 | public void logs(String LOG_TAG, int byteDataLength, int shortDataLength, int numOfInferences) { 36 | Log.d(LOG_TAG, "byte length: " + byteDataLength); 37 | Log.d(LOG_TAG, "short length: " + shortDataLength); 38 | Log.d(LOG_TAG, "numOfInference " + numOfInferences); 39 | } 40 | 41 | // recording - before and after trim 42 | public void logs(String LOG_TAG, short[] recordingBuffer, int recordingOffset) { 43 | Log.v("RECORDING_BUFFER", "2nd last index before added date: " + recordingBuffer[recordingOffset - 2]); 44 | Log.v("RECORDING_BUFFER", "last buffer index before added data: " + recordingBuffer[recordingOffset - 1]); 45 | Log.v("RECORDING_BUFFER", "1st index after added data: " + recordingBuffer[recordingOffset]); 46 | Log.v("RECORDING_BUFFER", "2nd index after added data: " + recordingBuffer[recordingOffset + 1]); 47 | } 48 | 49 | // recording - final inference with padding 50 | public void logs(String LOG_TAG, short[] remainingRecordingFrame, int remainingRecordingLength, 51 | short[] excessRecordingFrame, int excessRecordingLength) { 52 | Log.v(LOG_TAG, "First remain index: " + remainingRecordingFrame[0]); 53 | Log.v(LOG_TAG, "last remain index: " + 54 | remainingRecordingFrame[remainingRecordingLength - 1]); 55 | Log.v(LOG_TAG, "First excess index: " + excessRecordingFrame[0]); 56 | Log.v(LOG_TAG, "last excess index: " + 57 | excessRecordingFrame[excessRecordingLength - 1]); 58 | } 59 | 60 | //recording - determine which inference is causing the strange behaviour 61 | public void logs(String LOG_TAG, int inferenceCount, int numOfInferences, 62 | int recordingOffset, int recordingOffsetCount, int inputSize) { 63 | Log.v(LOG_TAG, "countNumOfInference: " + inferenceCount); 64 | Log.v(LOG_TAG, "numOfInference: " + numOfInferences); 65 | Log.v(LOG_TAG, "recordingOffset: " + recordingOffset); 66 | Log.v(LOG_TAG, "recordingOffsetCount " + recordingOffsetCount); 67 | Log.v(LOG_TAG, "inputSize " + inputSize); 68 | } 69 | 70 | 71 | } 72 | -------------------------------------------------------------------------------- /android/src/main/java/flutter/tflite_audio/LabelSmoothing.java: -------------------------------------------------------------------------------- 1 | package flutter.tflite_audio; 2 | 3 | import android.util.Log; 4 | import android.util.Pair; 5 | import java.util.ArrayDeque; 6 | import java.util.ArrayList; 7 | import java.util.Arrays; 8 | import java.util.Deque; 9 | import java.util.List; 10 | 11 | /** Reads in results from an instantaneous audio recognition model and smoothes them over time. */ 12 | public class LabelSmoothing { 13 | // Configuration settings. 14 | private List labels = new ArrayList(); 15 | private long averageWindowDurationMs; 16 | private float detectionThreshold; 17 | private int suppressionMs; 18 | private long minimumTimeBetweenSamplesMs; 19 | 20 | // Working variables. 21 | private Deque> previousResults = new ArrayDeque>(); 22 | private String previousTopLabel; 23 | private int labelsCount; 24 | private long previousTopLabelTime; 25 | private float previousTopLabelScore; 26 | 27 | private static final String SILENCE_LABEL = "_silence_"; 28 | private static final long MINIMUM_TIME_FRACTION = 4; 29 | 30 | public LabelSmoothing( 31 | List inLabels, 32 | long inAverageWindowDurationMs, 33 | float inDetectionThreshold, 34 | int inSuppressionMS, 35 | long inMinimumTimeBetweenSamplesMS) { 36 | labels = inLabels; 37 | averageWindowDurationMs = inAverageWindowDurationMs; 38 | detectionThreshold = inDetectionThreshold; 39 | suppressionMs = inSuppressionMS; 40 | labelsCount = inLabels.size(); 41 | previousTopLabel = SILENCE_LABEL; 42 | previousTopLabelTime = Long.MIN_VALUE; 43 | previousTopLabelScore = 0.0f; 44 | minimumTimeBetweenSamplesMs = inMinimumTimeBetweenSamplesMS; 45 | } 46 | 47 | /** Holds information about what's been recognized. */ 48 | public static class RecognitionResult { 49 | public final String foundCommand; 50 | public final float score; 51 | public final boolean isNewCommand; 52 | 53 | public RecognitionResult(String inFoundCommand, float inScore, boolean inIsNewCommand) { 54 | foundCommand = inFoundCommand; 55 | score = inScore; 56 | isNewCommand = inIsNewCommand; 57 | } 58 | } 59 | 60 | private static class ScoreForSorting implements Comparable { 61 | public final float score; 62 | public final int index; 63 | 64 | public ScoreForSorting(float inScore, int inIndex) { 65 | score = inScore; 66 | index = inIndex; 67 | } 68 | 69 | @Override 70 | public int compareTo(ScoreForSorting other) { 71 | if (this.score > other.score) { 72 | return -1; 73 | } else if (this.score < other.score) { 74 | return 1; 75 | } else { 76 | return 0; 77 | } 78 | } 79 | } 80 | 81 | public RecognitionResult processLatestResults(float[] currentResults, long currentTimeMS) { 82 | if (currentResults.length != labelsCount) { 83 | throw new RuntimeException( 84 | "The results for recognition should contain " 85 | + labelsCount 86 | + " elements, but there are " 87 | + currentResults.length); 88 | } 89 | 90 | if ((!previousResults.isEmpty()) && (currentTimeMS < previousResults.getFirst().first)) { 91 | throw new RuntimeException( 92 | "You must feed results in increasing time order, but received a timestamp of " 93 | + currentTimeMS 94 | + " that was earlier than the previous one of " 95 | + previousResults.getFirst().first); 96 | } 97 | 98 | int howManyResults = previousResults.size(); 99 | // Ignore any results that are coming in too frequently. 100 | if (howManyResults > 1) { 101 | final long timeSinceMostRecent = currentTimeMS - previousResults.getLast().first; 102 | if (timeSinceMostRecent < minimumTimeBetweenSamplesMs) { 103 | return new RecognitionResult(previousTopLabel, previousTopLabelScore, false); 104 | } 105 | } 106 | 107 | // Add the latest results to the head of the queue. 108 | previousResults.addLast(new Pair(currentTimeMS, currentResults)); 109 | 110 | // Prune any earlier results that are too old for the averaging window. 111 | final long timeLimit = currentTimeMS - averageWindowDurationMs; 112 | while (previousResults.getFirst().first < timeLimit) { 113 | previousResults.removeFirst(); 114 | } 115 | 116 | howManyResults = previousResults.size(); 117 | 118 | // Calculate the average score across all the results in the window. 119 | float[] averageScores = new float[labelsCount]; 120 | for (Pair previousResult : previousResults) { 121 | final float[] scoresTensor = previousResult.second; 122 | int i = 0; 123 | while (i < scoresTensor.length) { 124 | averageScores[i] += scoresTensor[i] / howManyResults; 125 | ++i; 126 | } 127 | } 128 | 129 | // Sort the averaged results in descending score order. 130 | ScoreForSorting[] sortedAverageScores = new ScoreForSorting[labelsCount]; 131 | for (int i = 0; i < labelsCount; ++i) { 132 | sortedAverageScores[i] = new ScoreForSorting(averageScores[i], i); 133 | } 134 | Arrays.sort(sortedAverageScores); 135 | 136 | // See if the latest top score is enough to trigger a detection. 137 | final int currentTopIndex = sortedAverageScores[0].index; 138 | final String currentTopLabel = labels.get(currentTopIndex); 139 | final float currentTopScore = sortedAverageScores[0].score; 140 | // If we've recently had another label trigger, assume one that occurs too 141 | // soon afterwards is a bad result. 142 | long timeSinceLastTop; 143 | if (previousTopLabel.equals(SILENCE_LABEL) || (previousTopLabelTime == Long.MIN_VALUE)) { 144 | timeSinceLastTop = Long.MAX_VALUE; 145 | } else { 146 | timeSinceLastTop = currentTimeMS - previousTopLabelTime; 147 | } 148 | boolean isNewCommand; 149 | if ((currentTopScore > detectionThreshold) && (timeSinceLastTop > suppressionMs)) { 150 | previousTopLabel = currentTopLabel; 151 | previousTopLabelTime = currentTimeMS; 152 | previousTopLabelScore = currentTopScore; 153 | isNewCommand = true; 154 | } else { 155 | isNewCommand = false; 156 | } 157 | return new RecognitionResult(currentTopLabel, currentTopScore, isNewCommand); 158 | } 159 | } -------------------------------------------------------------------------------- /android/src/main/java/flutter/tflite_audio/MediaDecoder.java: -------------------------------------------------------------------------------- 1 | /* Referemces 2 | 1. https://gist.github.com/a-m-s/1991ab18fbcb0fcc2cf9 3 | 2. https://github.com/tuntorius/mightier_amp/blob/7256c1cb120cc0c4fa1da7fd08ef6464964cadb4/android/app/src/main/java/com/tuntori/mightieramp/MediaDecoder.java 4 | */ 5 | 6 | package flutter.tflite_audio; 7 | 8 | import android.media.MediaCodec; 9 | import android.media.MediaCodec.BufferInfo; 10 | import android.media.MediaExtractor; 11 | import android.media.MediaFormat; 12 | import android.content.res.AssetFileDescriptor; 13 | 14 | import java.nio.ByteBuffer; 15 | import java.nio.ByteOrder; 16 | 17 | public class MediaDecoder { 18 | private static final String LOG_TAG = "Media_Decoder"; 19 | 20 | private MediaExtractor extractor = new MediaExtractor(); 21 | private MediaCodec decoder; 22 | 23 | private MediaFormat inputFormat; 24 | 25 | private ByteBuffer[] inputBuffers; 26 | private boolean end_of_input_file; 27 | 28 | private ByteBuffer[] outputBuffers; 29 | private int outputBufferIndex = -1; 30 | 31 | public MediaDecoder(AssetFileDescriptor fileDescriptor, long startOffset, long declaredLength) { 32 | 33 | try { 34 | extractor.setDataSource(fileDescriptor.getFileDescriptor(), startOffset, declaredLength); 35 | } catch(Exception e) { 36 | throw new RuntimeException("Failed to load audio file: ", e); 37 | } 38 | 39 | // Select the first audio track we find. 40 | int numTracks = extractor.getTrackCount(); 41 | for (int i = 0; i < numTracks; ++i) { 42 | MediaFormat format = extractor.getTrackFormat(i); 43 | String mime = format.getString(MediaFormat.KEY_MIME); 44 | if (mime.startsWith("audio/")) { 45 | extractor.selectTrack(i); 46 | try { 47 | decoder = MediaCodec.createDecoderByType(mime); 48 | } catch(Exception e) { 49 | throw new RuntimeException("Extractor.selectTrack error: ", e); 50 | } 51 | decoder.configure(format, null, null, 0); 52 | 53 | /* when adding encoder, use these settings 54 | format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1); 55 | format.setInteger(MediaFormat.KEY_SAMPLE_RATE, 8000); 56 | format.setInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_8BIT); 57 | decoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);*/ 58 | 59 | inputFormat = format; 60 | break; 61 | } 62 | } 63 | 64 | if (decoder == null) { 65 | throw new IllegalArgumentException("No decoder for file format"); 66 | } 67 | 68 | decoder.start(); 69 | inputBuffers = decoder.getInputBuffers(); 70 | outputBuffers = decoder.getOutputBuffers(); 71 | end_of_input_file = false; 72 | } 73 | 74 | public void release() 75 | { 76 | extractor.release(); 77 | } 78 | 79 | // Read the raw data from MediaCodec. 80 | // The caller should copy the data out of the ByteBuffer before calling this again 81 | // or else it may get overwritten. 82 | // private ByteBuffer readData(BufferInfo info) { 83 | private BufferInfo readData() { 84 | if (decoder == null) 85 | return null; 86 | 87 | BufferInfo info = new BufferInfo(); 88 | 89 | for (;;) { 90 | // Read data from the file into the codec. 91 | if (!end_of_input_file) { 92 | int inputBufferIndex = decoder.dequeueInputBuffer(10000); 93 | if (inputBufferIndex >= 0) { 94 | int size = extractor.readSampleData(inputBuffers[inputBufferIndex], 0); 95 | if (size < 0) { 96 | // End Of File 97 | decoder.queueInputBuffer(inputBufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); 98 | end_of_input_file = true; 99 | } else { 100 | decoder.queueInputBuffer(inputBufferIndex, 0, size, extractor.getSampleTime(), 0); 101 | extractor.advance(); 102 | } 103 | } 104 | } 105 | 106 | // Read the output from the codec. 107 | if (outputBufferIndex >= 0) 108 | // Ensure that the data is placed at the start of the buffer 109 | outputBuffers[outputBufferIndex].position(0); 110 | 111 | outputBufferIndex = decoder.dequeueOutputBuffer(info, 10000); 112 | if (outputBufferIndex >= 0) { 113 | // Handle EOF 114 | if (info.flags != 0) { 115 | decoder.stop(); 116 | decoder.release(); 117 | decoder = null; 118 | return null; 119 | } 120 | 121 | return info; 122 | 123 | } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 124 | // This usually happens once at the start of the file. 125 | outputBuffers = decoder.getOutputBuffers(); 126 | } 127 | } 128 | } 129 | 130 | // Return the Audio sample rate, in samples/sec. 131 | public int getSampleRate() { 132 | return inputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); 133 | } 134 | 135 | // Read the raw audio data in 16-bit format 136 | // Returns null on EOF 137 | public byte[] readByteData() { 138 | BufferInfo info = readData(); 139 | 140 | if (info==null) 141 | return null; 142 | 143 | ByteBuffer data = currentBuffer(); 144 | 145 | if (data == null) 146 | return null; 147 | 148 | byte[] returnData = new byte[info.size]; 149 | 150 | if (info.size>0) 151 | data.get(returnData); 152 | 153 | releaseBuffer(); 154 | return returnData; 155 | } 156 | 157 | 158 | private ByteBuffer currentBuffer() 159 | { 160 | return outputBuffers[outputBufferIndex]; 161 | } 162 | 163 | private void releaseBuffer() 164 | { 165 | // Release the buffer so MediaCodec can use it again. 166 | // The data should stay there until the next time we are called. 167 | decoder.releaseOutputBuffer(outputBufferIndex, false); 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /android/src/main/java/flutter/tflite_audio/Recording.java: -------------------------------------------------------------------------------- 1 | package flutter.tflite_audio; 2 | 3 | import android.annotation.SuppressLint; 4 | import android.media.MediaRecorder; 5 | import android.media.AudioRecord; 6 | import android.media.AudioFormat; 7 | import android.util.Log; 8 | 9 | import java.util.concurrent.locks.ReentrantLock; 10 | 11 | import io.reactivex.rxjava3.core.Observable; 12 | import io.reactivex.rxjava3.core.*; 13 | import io.reactivex.rxjava3.disposables.Disposable; 14 | import io.reactivex.rxjava3.subjects.PublishSubject; 15 | 16 | /* 17 | !References 18 | https://www.javatpoint.com/java-closure 19 | https://www.geeksforgeeks.org/method-within-method-in-java/ 20 | https://stackoverflow.com/questions/54566753/escaping-closure-in-swift-and-how-to-perform-it-in-java 21 | 22 | ``` 23 | .emit(new AudioChunk(){ 24 | @Override 25 | public void get(short [] data) { 26 | subject.onNext(data); 27 | } 28 | }) 29 | ``` 30 | !Code above same as lambda below: 31 | 32 | `.emit(data -> subject.onNext(data))` 33 | 34 | */ 35 | 36 | 37 | public class Recording{ 38 | 39 | private static final String LOG_TAG = "Recording"; 40 | 41 | private int bufferSize; 42 | private int audioLength; 43 | private int sampleRate; 44 | private int numOfInferences; 45 | 46 | private AudioRecord record; 47 | private boolean shouldContinue; 48 | 49 | private RecordingData recordingData; 50 | private PublishSubject subject; 51 | private ReentrantLock recordingBufferLock; 52 | 53 | @SuppressLint("MissingPermission") 54 | public Recording(int bufferSize, int audioLength, int sampleRate, int numOfInferences){ 55 | this.bufferSize = bufferSize; 56 | this.audioLength = audioLength; 57 | this.sampleRate = sampleRate; 58 | this.numOfInferences = numOfInferences; 59 | 60 | this.subject = PublishSubject.create(); 61 | this.recordingData = new RecordingData(audioLength, bufferSize, numOfInferences); 62 | this.record = new AudioRecord( 63 | MediaRecorder.AudioSource.DEFAULT, 64 | sampleRate, 65 | AudioFormat.CHANNEL_IN_MONO, 66 | AudioFormat.ENCODING_PCM_16BIT, 67 | bufferSize); 68 | 69 | } 70 | 71 | public void setReentrantLock(ReentrantLock recordingBufferLock){ 72 | this.recordingBufferLock = recordingBufferLock; 73 | } 74 | 75 | 76 | public Observable getObservable() { 77 | return (Observable) this.subject; 78 | } 79 | 80 | 81 | public void stop(){ 82 | shouldContinue = false; 83 | record.stop(); 84 | record.release(); 85 | subject.onComplete(); 86 | } 87 | 88 | public void start(){ 89 | Log.v(LOG_TAG, "Recording started"); 90 | shouldContinue = true; 91 | record.startRecording(); 92 | splice(); 93 | 94 | } 95 | 96 | public void splice(){ 97 | 98 | if (record.getState() != AudioRecord.STATE_INITIALIZED) { 99 | Log.e(LOG_TAG, "Audio Record can't initialize!"); 100 | return; 101 | } 102 | 103 | while (shouldContinue) { 104 | 105 | short[] shortData = new short [bufferSize]; 106 | record.read(shortData, 0, shortData.length); 107 | recordingBufferLock.lock(); 108 | 109 | try { 110 | switch (recordingData.getState()) { 111 | case "append": 112 | recordingData 113 | .append(shortData); 114 | break; 115 | 116 | case "recognise": 117 | recordingData 118 | .append(shortData) 119 | .emit(data -> subject.onNext(data)) 120 | .updateInferenceCount() 121 | .clear(); 122 | break; 123 | 124 | case "finalise": 125 | recordingData 126 | .append(shortData) 127 | .emit(data -> subject.onNext(data)); 128 | stop(); 129 | break; 130 | 131 | case "trimAndRecognise": 132 | recordingData 133 | .updateRemain() 134 | .trimToRemain(shortData) 135 | .emit(data -> subject.onNext(data)) 136 | .updateInferenceCount() 137 | .clear() 138 | .updateExcess() 139 | .addExcessToNew(shortData); 140 | break; 141 | 142 | case "trimAndFinalise": 143 | recordingData 144 | .updateRemain() 145 | .trimToRemain(shortData) 146 | .emit(data -> subject.onNext(data)); 147 | stop(); 148 | break; 149 | 150 | default: 151 | recordingData.displayErrorLog(); 152 | throw new AssertionError("Incorrect state when preprocessing"); 153 | } 154 | } finally { 155 | recordingBufferLock.unlock(); 156 | } 157 | 158 | 159 | } 160 | } 161 | } -------------------------------------------------------------------------------- /android/src/main/java/flutter/tflite_audio/RecordingData.java: -------------------------------------------------------------------------------- 1 | package flutter.tflite_audio; 2 | 3 | import android.util.Log; 4 | 5 | import java.util.Arrays; 6 | 7 | public class RecordingData { 8 | 9 | 10 | private static final String LOG_TAG = "RecordingData"; 11 | 12 | private int audioLength; 13 | private int numOfInferences; 14 | private int bufferSize; 15 | 16 | //Count 17 | private int inferenceCount = 1; 18 | private int recordingOffset = 0; 19 | private int readCount; //keep track of state 20 | 21 | //Excess count 22 | private int remainingLength = 0; 23 | private int excessLength = 0; 24 | private short[] remainingFrame; 25 | private short[] excessFrame; 26 | 27 | //result 28 | private short[] recordingBuffer; 29 | 30 | public RecordingData(int audioLength, int bufferSize, int numOfInferences){ 31 | this.audioLength = audioLength; 32 | this.bufferSize = bufferSize; 33 | this.numOfInferences = numOfInferences; 34 | 35 | this.recordingBuffer = new short[audioLength]; 36 | this.readCount = bufferSize; //readcount should always be recordingOffset + bufferSize; 37 | } 38 | 39 | public void displayErrorLog(){ 40 | System.out.println("inferenceCount " + inferenceCount); 41 | System.out.println("numOfInference " + numOfInferences); 42 | System.out.println("readCount " + readCount); 43 | System.out.println("audioLength " + audioLength); 44 | 45 | Log.v(LOG_TAG, "State error log:"); 46 | Log.v(LOG_TAG, "inferenceCount " + inferenceCount); 47 | Log.v(LOG_TAG, "numOfInference " + numOfInferences); 48 | Log.v(LOG_TAG, "readCount " + readCount); 49 | Log.v(LOG_TAG, "audioLength " + audioLength); 50 | } 51 | 52 | public String getState(){ 53 | if (inferenceCount <= numOfInferences && readCount < audioLength) { return "append"; } 54 | else if(inferenceCount < numOfInferences && readCount == audioLength){return "recognise"; } 55 | else if(inferenceCount == numOfInferences && readCount == audioLength){return "finalise";} 56 | else if(inferenceCount < numOfInferences && readCount > audioLength){ return "trimAndRecognise";} 57 | else if(inferenceCount == numOfInferences && readCount > audioLength){return "trimAndFinalise";} 58 | else{ return "error"; } 59 | } 60 | 61 | 62 | public RecordingData append(short[] shortData){ 63 | 64 | //TODO - break when excess or remainframe is full 65 | 66 | System.arraycopy(shortData, 0, recordingBuffer, recordingOffset, shortData.length); 67 | recordingOffset += shortData.length; 68 | readCount = recordingOffset + shortData.length; 69 | 70 | Log.v(LOG_TAG, "recordingOffset: " + recordingOffset + "/" + audioLength + " | inferenceCount: " 71 | + inferenceCount + "/" + numOfInferences); 72 | return this; 73 | } 74 | 75 | public RecordingData emit(AudioChunk audioChunk){ 76 | audioChunk.get(recordingBuffer); 77 | return this; 78 | } 79 | 80 | public RecordingData updateInferenceCount(){ 81 | inferenceCount += 1; 82 | return this; 83 | } 84 | 85 | public RecordingData updateRemain(){ 86 | remainingLength = audioLength - recordingOffset; 87 | remainingFrame = new short[remainingLength]; 88 | return this; 89 | } 90 | 91 | 92 | public RecordingData trimToRemain(short[] shortData){ 93 | System.arraycopy(shortData, 0, remainingFrame, 0, remainingLength); 94 | System.arraycopy(remainingFrame, 0, recordingBuffer, recordingOffset, remainingLength); 95 | 96 | //!Dont need [readCount] as you are trimming the data. (Already readcount in append) 97 | recordingOffset += remainingLength; 98 | Log.v(LOG_TAG, "recordingOffset: " + recordingOffset + "/" + audioLength + " | inferenceCount: " 99 | + inferenceCount + "/" + numOfInferences + " (" + remainingLength + " samples trimmed to remaining buffer)"); 100 | return this; 101 | } 102 | 103 | public RecordingData updateExcess(){ 104 | excessLength = bufferSize - remainingLength; 105 | excessFrame = new short[excessLength]; 106 | return this; 107 | } 108 | 109 | public RecordingData addExcessToNew(short[] shortData){ 110 | System.arraycopy(shortData, remainingLength, excessFrame, 0, excessLength); 111 | System.arraycopy(excessFrame, 0, recordingBuffer, 0, excessLength); 112 | 113 | //!need [readCount] as your add excess data in appeneded to new data 114 | recordingOffset += excessLength; 115 | readCount = recordingOffset + shortData.length; 116 | Log.v(LOG_TAG, "recordingOffset: " + recordingOffset + "/" + audioLength + " | inferenceCount: " 117 | + inferenceCount + "/" + numOfInferences + " (" + excessLength + " excess samples added to new buffer)"); 118 | return this; 119 | } 120 | 121 | public RecordingData clear(){ 122 | recordingBuffer = new short[audioLength]; 123 | recordingOffset = 0; 124 | readCount = bufferSize; //readcount should always be recordingOffset + bufferSize; 125 | return this; 126 | } 127 | 128 | public RecordingData resetCount(){ 129 | recordingOffset = 0; 130 | inferenceCount = 1; 131 | return this; 132 | } 133 | 134 | public RecordingData resetExcessCount(){ 135 | remainingLength = 0; 136 | excessLength = 0; 137 | return this; 138 | } 139 | } 140 | 141 | -------------------------------------------------------------------------------- /android/src/main/java/flutter/tflite_audio/SignalProcessing.java: -------------------------------------------------------------------------------- 1 | package flutter.tflite_audio; 2 | 3 | import org.apache.commons.math3.complex.Complex; 4 | 5 | import java.util.Arrays; 6 | 7 | import com.jlibrosa.audio.JLibrosa; 8 | import com.jlibrosa.audio.process.AudioFeatureExtraction; 9 | 10 | import android.util.Log; 11 | 12 | /* Spectogram - shape guidelines 13 | First dimension: [Frequency Bins] = 1+nFFT/2 (note: frequency bins = mel bins) 14 | Second dimension: [Frame Rate] = sampleRate/hopLength (note: by default hopLength = winLength) 15 | 16 | For example for input shape [129, 124] 17 | nFFT = 256 18 | hopLength = 129 19 | 20 | First dimenstion: 129 = 1+nFTT/2 21 | Second dimension: 124 = 16000/hopLength 22 | 23 | https://kinwaicheuk.github.io/nnAudio/v0.1.5/_autosummary/nnAudio.Spectrogram.STFT.html 24 | https://stackoverflow.com/questions/62584184/understanding-the-shape-of-spectrograms-and-n-mels 25 | nFFT = number of samples per frame (needs to be power to 2/ if nFFT > hop length - need to pad) 26 | hopLength = sample_rate/frame_rate = 512 = 22050 Hz/43 Hz 27 | */ 28 | 29 | /* MFCC - shape guidelines 30 | https://stackoverflow.com/questions/56911774/mfcc-feature-extraction-librosa 31 | first dimension: n_mfcc 32 | second dimension: (sr * time) / hop length 33 | 34 | example, given a samplerate of 16000/sec, inputshape is [1,40] 35 | first dimension = 40 36 | second dimension is (16000 * 1 / 16384) = [1] 37 | 38 | 16,384 is hoplength 39 | */ 40 | 41 | /* Mel spectrogram 42 | number of mel bins 43 | time 44 | 45 | */ 46 | 47 | 48 | public class SignalProcessing{ 49 | 50 | private static final String LOG_TAG = "Signal_Processing"; 51 | private JLibrosa jLibrosa = new JLibrosa(); 52 | private AudioFeatureExtraction featureExtractor = new AudioFeatureExtraction(); 53 | private boolean showPreprocessLogs = true; 54 | 55 | private int sampleRate; 56 | private int nMFCC; 57 | private int nFFT; 58 | private int nMels; 59 | private int hopLength; 60 | 61 | public SignalProcessing(int sampleRate, int nMFCC, int nFFT, int nMels, int hopLength){ 62 | this.sampleRate = sampleRate; 63 | this.nMFCC = nMFCC; 64 | this.nFFT = nFFT; 65 | this.nMels = nMels; 66 | this.hopLength = hopLength; 67 | }; 68 | 69 | 70 | public float [][] getMFCC(float [] inputBuffer32){ 71 | float [][] MFCC = jLibrosa.generateMFCCFeatures(inputBuffer32, sampleRate, nMFCC, nFFT, nMels, hopLength); 72 | if (showPreprocessLogs) displayShape(MFCC); 73 | return MFCC; 74 | } 75 | 76 | public float [][] getMelSpectrogram(float [] inputBuffer32){ 77 | float [][] melSpectrogram = jLibrosa.generateMelSpectroGram(inputBuffer32, sampleRate, nFFT, nMels, hopLength); 78 | if (showPreprocessLogs) displayShape(melSpectrogram); 79 | return melSpectrogram; 80 | } 81 | 82 | public float[][] getSpectrogram(float [] inputBuffer32){ 83 | featureExtractor.setSampleRate(sampleRate); 84 | featureExtractor.setN_mfcc(nMFCC); 85 | featureExtractor.setN_fft(nFFT); 86 | featureExtractor.setN_mels(nMels); 87 | featureExtractor.setHop_length(hopLength); 88 | 89 | Complex [][] stft = featureExtractor.extractSTFTFeaturesAsComplexValues(inputBuffer32, true); 90 | //float [][] spectrogram = getSpectroAbsVal(stft); 91 | float [][] spectrogram = getFloatABSValue(stft); 92 | if (showPreprocessLogs) displayShape(spectrogram); 93 | return spectrogram; 94 | } 95 | 96 | private float [][] getFloatABSValue(Complex [][] spectro){ 97 | 98 | float[][] spectroAbsVal = new float[spectro.length][spectro[0].length]; 99 | 100 | for(int i=0;i result = splice(audioData, audioLength, audioData.length); 27 | assertArrayEquals(convertToArray(result), expectedData); 28 | } 29 | 30 | @Test 31 | public void testSingleSplice_lackData_noPad(){ 32 | short [] audioData = {1}; 33 | int audioLength = 3; 34 | short [] expectedData = new short [] {}; 35 | 36 | List result = splice(audioData, audioLength, audioData.length); 37 | assertArrayEquals(convertToArray(result), expectedData); 38 | } 39 | 40 | @Test 41 | public void testSingleSplice_lackData_withPad(){ 42 | short [] audioData = {1, 2}; 43 | int audioLength = 3; 44 | int expectedWithPadLength = 3; 45 | 46 | List resultList = splice(audioData, audioLength, audioData.length); 47 | short [] result = convertToArray(resultList); 48 | short [] resultNoPad = Arrays.copyOfRange(result, 0, audioData.length); 49 | 50 | System.out.println("singleSplice_lackData_withPad: " + resultList.toString()); 51 | assertArrayEquals(resultNoPad, audioData); 52 | assertEquals(result.length, expectedWithPadLength); 53 | } 54 | 55 | @Test 56 | public void testMultiSplice() { 57 | 58 | short [] audioData = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; 59 | int audioLength = 5; 60 | short [] expectedData = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; 61 | 62 | List result = splice(audioData, audioLength, audioData.length); 63 | assertArrayEquals(convertToArray(result), expectedData); 64 | } 65 | 66 | @Test 67 | public void testMultiSplice_withExcess_noPadding() { 68 | 69 | short [] audioData = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}; 70 | int audioLength = 5; 71 | short [] expectedData = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; 72 | 73 | List result = splice(audioData, audioLength, audioData.length); 74 | assertArrayEquals(convertToArray(result), expectedData); 75 | } 76 | 77 | @Test 78 | public void testMultiSplice_withExcess_withPadding() { 79 | 80 | short [] audioData = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; 81 | int audioLength = 6; 82 | int expectedWithPadLength = 12; 83 | 84 | List resultList = splice(audioData, audioLength, audioData.length); 85 | short [] result = convertToArray(resultList); 86 | short [] resultNoPad = Arrays.copyOfRange(result, 0, audioData.length); 87 | 88 | System.out.println("multiSplice_lackData_withPad: " + resultList.toString()); 89 | assertArrayEquals(resultNoPad, audioData); 90 | assertEquals(result.length, expectedWithPadLength); 91 | } 92 | 93 | 94 | //https://stackoverflow.com/questions/60072435/how-to-convert-short-into-listshort-in-java-with-streams 95 | public List convertToList(short [] shortArray){ 96 | return IntStream.range(0, shortArray.length) 97 | .mapToObj(s -> shortArray[s]) 98 | .collect(Collectors.toList()); 99 | } 100 | 101 | //https://stackoverflow.com/questions/718554/how-to-convert-an-arraylist-containing-integers-to-primitive-int-array 102 | public short [] convertToArray(List shortList){ 103 | short [] result = new short[shortList.size()]; 104 | 105 | for (int i=0; i < result.length; i++) { 106 | result[i] = shortList.get(i); 107 | } 108 | 109 | return result; 110 | } 111 | 112 | public List splice(short [] shortBuffer, int audioLength, int bufferSize){ 113 | 114 | List result = new ArrayList(); 115 | AudioProcessing audioProcessing = new AudioProcessing(); 116 | AudioData audioData = new AudioData(audioLength, bufferSize); 117 | boolean isSplicing = true; 118 | 119 | for (int i = 0; i < shortBuffer.length; i++) { 120 | 121 | if (!isSplicing) { 122 | break; 123 | } 124 | 125 | short dataPoint = shortBuffer[i]; 126 | 127 | switch (audioData.getState(i)) { 128 | case "append": 129 | audioData 130 | .append(dataPoint); 131 | break; 132 | case "recognise": 133 | System.out.println("recognising"); 134 | audioData 135 | .append(dataPoint) 136 | .displayInference() 137 | .emit(data -> { 138 | result.addAll(convertToList(data)); 139 | }) 140 | .reset(); 141 | break; 142 | case "finalise": 143 | System.out.println("finalising"); 144 | audioData 145 | .append(dataPoint) 146 | .displayInference() 147 | .emit(data -> { 148 | result.addAll(convertToList(data)); 149 | }); 150 | isSplicing = false; 151 | break; 152 | case "padAndFinalise": 153 | System.out.println("padding and finalising"); 154 | audioData 155 | .append(dataPoint) 156 | .padSilence(i) 157 | .displayInference() 158 | .emit(data -> { 159 | result.addAll(convertToList(data)); 160 | }); 161 | isSplicing = false; 162 | break; 163 | 164 | default: 165 | throw new AssertionError("Incorrect state when preprocessing"); 166 | } 167 | } 168 | return result; 169 | } 170 | 171 | } 172 | -------------------------------------------------------------------------------- /android/src/test/java/flutter/tflite_audio/RecordingTest.java: -------------------------------------------------------------------------------- 1 | package flutter.tflite_audio; 2 | 3 | import static org.junit.Assert.assertArrayEquals; 4 | 5 | import org.junit.Test; 6 | 7 | import java.util.ArrayList; 8 | import java.util.Arrays; 9 | import java.util.List; 10 | import java.util.stream.Collectors; 11 | import java.util.stream.IntStream; 12 | 13 | 14 | public class RecordingTest { 15 | 16 | @Test 17 | public void testSingleSplice() { 18 | 19 | short [] recordingData = {1, 2, 3, 4}; 20 | int audioLength = 4; 21 | int numOfInferences = 1; 22 | short [] expectedData = {1, 2, 3, 4}; 23 | 24 | List result = mockRecording(recordingData, audioLength, numOfInferences); 25 | assertArrayEquals(convertToArray(result), expectedData); 26 | } 27 | 28 | @Test 29 | public void testSingleSplice_lackDataEven() { 30 | 31 | short [] recordingData = {1, 2}; 32 | int audioLength = 4; 33 | int numOfInferences = 1; 34 | short [] expectedData = {1, 2, 1, 2}; 35 | 36 | List result = mockRecording(recordingData, audioLength, numOfInferences); 37 | assertArrayEquals(convertToArray(result), expectedData); 38 | } 39 | 40 | @Test 41 | public void testSingleSplice_lackDataOdd() { 42 | 43 | short [] recordingData = {1, 2, 3}; 44 | int audioLength = 4; 45 | int numOfInferences = 1; 46 | short [] expectedData = {1, 2, 3, 1}; 47 | 48 | List result = mockRecording(recordingData, audioLength, numOfInferences); 49 | assertArrayEquals(convertToArray(result), expectedData); 50 | } 51 | 52 | @Test 53 | public void testSingleSplice_withExcess() { 54 | 55 | short [] recordingData = {1, 2, 3, 4, 5}; 56 | int audioLength = 4; 57 | int numOfInferences = 1; 58 | short [] expectedData = {1, 2, 3, 4}; 59 | 60 | List result = mockRecording(recordingData, audioLength, numOfInferences); 61 | assertArrayEquals(convertToArray(result), expectedData); 62 | } 63 | 64 | @Test 65 | public void testMultiSplice() { 66 | 67 | short [] recordingData = {1, 2, 3, 4}; 68 | int audioLength = 4; 69 | int numOfInferences = 3; 70 | short [] expectedData = {1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4}; 71 | 72 | List result = mockRecording(recordingData, audioLength, numOfInferences); 73 | assertArrayEquals(convertToArray(result), expectedData); 74 | } 75 | 76 | @Test 77 | public void testMultiSplice_lackDataEven() { 78 | 79 | short [] recordingData = {1, 2}; 80 | int audioLength = 4; 81 | int numOfInferences = 3; 82 | short [] expectedData = {1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2}; 83 | 84 | List result = mockRecording(recordingData, audioLength, numOfInferences); 85 | assertArrayEquals(convertToArray(result), expectedData); 86 | } 87 | 88 | @Test 89 | public void testMultiSplice_lackDataOdd() { 90 | 91 | short [] recordingData = {1, 2, 3}; 92 | int audioLength = 4; 93 | int numOfInferences = 3; 94 | short [] expectedData = {1, 2, 3, 1, 2, 3, 1 ,2, 3, 1, 2, 3}; 95 | 96 | List result = mockRecording(recordingData, audioLength, numOfInferences); 97 | assertArrayEquals(convertToArray(result), expectedData); 98 | } 99 | 100 | @Test 101 | public void testMultiSplice_withExcess() { 102 | 103 | short [] recordingData = {1, 2, 3, 4, 5, 6}; 104 | int audioLength = 4; 105 | int numOfInferences = 3; 106 | short [] expectedData = {1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6}; 107 | 108 | List result = mockRecording(recordingData, audioLength, numOfInferences); 109 | System.out.println(result.toString()); 110 | assertArrayEquals(convertToArray(result), expectedData); 111 | } 112 | 113 | //https://stackoverflow.com/questions/60072435/how-to-convert-short-into-listshort-in-java-with-streams 114 | public List convertToList(short [] shortArray){ 115 | return IntStream.range(0, shortArray.length) 116 | .mapToObj(s -> shortArray[s]) 117 | .collect(Collectors.toList()); 118 | } 119 | 120 | //https://stackoverflow.com/questions/718554/how-to-convert-an-arraylist-containing-integers-to-primitive-int-array 121 | public short [] convertToArray(List shortList){ 122 | short [] result = new short[shortList.size()]; 123 | 124 | for (int i=0; i < result.length; i++) { 125 | result[i] = shortList.get(i); 126 | } 127 | 128 | return result; 129 | } 130 | 131 | public List mockRecording(short [] shortBuffer, int audioLength, int numOfInferences){ 132 | 133 | boolean isRecording = true; 134 | List result = new ArrayList<>(); 135 | 136 | RecordingData recordingData = new RecordingData(audioLength, shortBuffer.length ,numOfInferences); 137 | 138 | while (isRecording) { 139 | 140 | switch (recordingData.getState()) { 141 | case "append": 142 | System.out.println("appending"); 143 | recordingData 144 | .append(shortBuffer); 145 | break; 146 | 147 | case "recognise": 148 | System.out.println("recognising"); 149 | recordingData 150 | .append(shortBuffer) 151 | .emit(data -> { 152 | System.out.println(Arrays.toString(data)); 153 | result.addAll(convertToList(data)); 154 | }) 155 | .updateInferenceCount() 156 | .clear(); 157 | break; 158 | 159 | case "finalise": 160 | // Log.v(LOG_TAG, "finalising"); 161 | System.out.println("finalising"); 162 | recordingData 163 | .append(shortBuffer) 164 | .emit(data -> { 165 | System.out.println(Arrays.toString(data)); 166 | result.addAll(convertToList(data)); 167 | }); 168 | isRecording = false; 169 | break; 170 | 171 | case "trimAndRecognise": 172 | System.out.println("Trimming and recognising"); 173 | recordingData 174 | .updateRemain() 175 | .trimToRemain(shortBuffer) 176 | .emit(data -> { 177 | System.out.println(Arrays.toString(data)); 178 | result.addAll(convertToList(data)); 179 | }) 180 | .updateInferenceCount() 181 | .clear() 182 | .updateExcess() 183 | .addExcessToNew(shortBuffer); 184 | break; 185 | 186 | case "trimAndFinalise": 187 | System.out.println("Trimming and finalising"); 188 | recordingData 189 | .updateRemain() 190 | .trimToRemain(shortBuffer) 191 | .emit(data -> { 192 | System.out.println(Arrays.toString(data)); 193 | result.addAll(convertToList(data)); 194 | }); 195 | isRecording = false; 196 | break; 197 | 198 | default: 199 | recordingData.displayErrorLog(); 200 | throw new AssertionError("Incorrect state when preprocessing"); 201 | } 202 | } 203 | return result; 204 | } 205 | 206 | } 207 | -------------------------------------------------------------------------------- /example/.gitignore: -------------------------------------------------------------------------------- 1 | # Miscellaneous 2 | *.class 3 | *.log 4 | *.pyc 5 | *.swp 6 | .DS_Store 7 | .atom/ 8 | .buildlog/ 9 | .history 10 | .svn/ 11 | 12 | # IntelliJ related 13 | *.iml 14 | *.ipr 15 | *.iws 16 | .idea/ 17 | 18 | # The .vscode folder contains launch configuration and tasks you configure in 19 | # VS Code which you may wish to be included in version control, so this line 20 | # is commented out by default. 21 | #.vscode/ 22 | 23 | # Flutter/Dart/Pub related 24 | **/doc/api/ 25 | .dart_tool/ 26 | .flutter-plugins 27 | .flutter-plugins-dependencies 28 | .packages 29 | .pub-cache/ 30 | .pub/ 31 | /build/ 32 | 33 | # Web related 34 | lib/generated_plugin_registrant.dart 35 | 36 | # Exceptions to above rules. 37 | !/packages/flutter_tools/test/data/dart_dependencies_test/**/.packages 38 | -------------------------------------------------------------------------------- /example/.metadata: -------------------------------------------------------------------------------- 1 | # This file tracks properties of this Flutter project. 2 | # Used by Flutter tool to assess capabilities and perform upgrades etc. 3 | # 4 | # This file should be version controlled and should not be manually edited. 5 | 6 | version: 7 | revision: f139b11009aeb8ed2a3a3aa8b0066e482709dde3 8 | channel: stable 9 | 10 | project_type: app 11 | -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- 1 | # tflite_audio_example 2 | 3 | Demonstrates how to use the tflite_audio plugin. 4 | 5 | ![](audio_recognition_example.jpg) -------------------------------------------------------------------------------- /example/android/.gitignore: -------------------------------------------------------------------------------- 1 | gradle-wrapper.jar 2 | /.gradle 3 | /captures/ 4 | /gradlew 5 | /gradlew.bat 6 | /local.properties 7 | GeneratedPluginRegistrant.java 8 | -------------------------------------------------------------------------------- /example/android/app/build.gradle: -------------------------------------------------------------------------------- 1 | /* Point of reference 2 | https://github.com/flutter/plugins/blob/main/packages/camera/camera/example/android/app/build.gradle 3 | */ 4 | 5 | 6 | def localProperties = new Properties() 7 | def localPropertiesFile = rootProject.file('local.properties') 8 | if (localPropertiesFile.exists()) { 9 | localPropertiesFile.withReader('UTF-8') { reader -> 10 | localProperties.load(reader) 11 | } 12 | } 13 | 14 | def flutterRoot = localProperties.getProperty('flutter.sdk') 15 | if (flutterRoot == null) { 16 | throw new GradleException("Flutter SDK not found. Define location with flutter.sdk in the local.properties file.") 17 | } 18 | 19 | def flutterVersionCode = localProperties.getProperty('flutter.versionCode') 20 | if (flutterVersionCode == null) { 21 | flutterVersionCode = '1' 22 | } 23 | 24 | def flutterVersionName = localProperties.getProperty('flutter.versionName') 25 | if (flutterVersionName == null) { 26 | flutterVersionName = '1.0' 27 | } 28 | 29 | apply plugin: 'com.android.application' 30 | apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle" 31 | 32 | android { 33 | compileSdkVersion 31 34 | 35 | lintOptions { 36 | disable 'InvalidPackage' 37 | } 38 | 39 | aaptOptions { 40 | noCompress 'tflite' 41 | } 42 | 43 | defaultConfig { 44 | applicationId "flutter.tflite_audio_example" 45 | minSdkVersion 24 46 | targetSdkVersion 31 47 | versionCode flutterVersionCode.toInteger() 48 | versionName flutterVersionName 49 | testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" 50 | } 51 | 52 | //Support java.util.Array.streams 53 | //https://stackoverflow.com/questions/47042805/androidstudio-3-0-java-8-stream-not-compile-call-requires-api-level-24 54 | //https://developer.android.com/studio/releases/gradle-plugin#kts 55 | compileOptions { 56 | // Flag to enable support for the new language APIs 57 | coreLibraryDesugaringEnabled true 58 | // Sets Java compatibility to Java 8 59 | sourceCompatibility JavaVersion.VERSION_1_8 60 | targetCompatibility JavaVersion.VERSION_1_8 61 | } 62 | 63 | buildTypes { 64 | release { 65 | signingConfig signingConfigs.debug 66 | } 67 | } 68 | } 69 | 70 | flutter { 71 | source '../..' 72 | } 73 | 74 | dependencies { 75 | coreLibraryDesugaring 'com.android.tools:desugar_jdk_libs:1.0.9' 76 | testImplementation 'junit:junit:4.12' 77 | androidTestImplementation 'androidx.test:runner:1.2.0' 78 | androidTestImplementation 'androidx.test:rules:1.2.0' 79 | androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' 80 | implementation 'org.tensorflow:tensorflow-lite-select-tf-ops:+' 81 | } 82 | -------------------------------------------------------------------------------- /example/android/app/src/androidTest/java/flutter/DartIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package flutter.tflite_audio_example; 2 | 3 | import java.lang.annotation.ElementType; 4 | import java.lang.annotation.Retention; 5 | import java.lang.annotation.RetentionPolicy; 6 | import java.lang.annotation.Target; 7 | 8 | @Retention(RetentionPolicy.RUNTIME) 9 | @Target(ElementType.TYPE) 10 | public @interface DartIntegrationTest {} 11 | -------------------------------------------------------------------------------- /example/android/app/src/androidTest/java/flutter/tflite_audio_example/FlutterActivityTest.java: -------------------------------------------------------------------------------- 1 | package flutter.tflite_audio_example; 2 | 3 | import androidx.test.rule.ActivityTestRule; 4 | import dev.flutter.plugins.integration_test.FlutterTestRunner; 5 | import io.flutter.embedding.android.FlutterActivity; 6 | import io.flutter.plugins.DartIntegrationTest; 7 | import org.junit.Rule; 8 | import org.junit.runner.RunWith; 9 | 10 | @DartIntegrationTest 11 | @RunWith(FlutterTestRunner.class) 12 | public class FlutterActivityTest { 13 | @Rule 14 | public ActivityTestRule rule = new ActivityTestRule<>(FlutterActivity.class); 15 | } -------------------------------------------------------------------------------- /example/android/app/src/debug/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 3 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /example/android/app/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 3 | 8 | 9 | 10 | 11 | 12 | 15 | 23 | 24 | 25 | 26 | 27 | 28 | 30 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /example/android/app/src/main/java/flutter/tflite_audio_example/MainActivity.java: -------------------------------------------------------------------------------- 1 | package flutter.tflite_audio_example; 2 | 3 | import androidx.annotation.NonNull; 4 | import io.flutter.embedding.engine.FlutterEngine; 5 | import io.flutter.embedding.android.FlutterActivity; 6 | 7 | public class MainActivity extends FlutterActivity { 8 | } 9 | -------------------------------------------------------------------------------- /example/android/app/src/main/res/drawable/launch_background.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 12 | 13 | -------------------------------------------------------------------------------- /example/android/app/src/main/res/mipmap-hdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/android/app/src/main/res/mipmap-hdpi/ic_launcher.png -------------------------------------------------------------------------------- /example/android/app/src/main/res/mipmap-mdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/android/app/src/main/res/mipmap-mdpi/ic_launcher.png -------------------------------------------------------------------------------- /example/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png -------------------------------------------------------------------------------- /example/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png -------------------------------------------------------------------------------- /example/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png -------------------------------------------------------------------------------- /example/android/app/src/main/res/values/styles.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | -------------------------------------------------------------------------------- /example/android/app/src/profile/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 3 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /example/android/build.gradle: -------------------------------------------------------------------------------- 1 | /* Point of reference 2 | https://github.com/flutter/plugins/blob/main/packages/camera/camera/example/android/build.gradle 3 | */ 4 | 5 | 6 | buildscript { 7 | repositories { 8 | google() 9 | mavenCentral() 10 | } 11 | 12 | dependencies { 13 | classpath 'com.android.tools.build:gradle:7.0.4' 14 | } 15 | } 16 | 17 | allprojects { 18 | repositories { 19 | google() 20 | jcenter() 21 | } 22 | } 23 | 24 | rootProject.buildDir = '../build' 25 | subprojects { 26 | project.buildDir = "${rootProject.buildDir}/${project.name}" 27 | } 28 | subprojects { 29 | project.evaluationDependsOn(':app') 30 | } 31 | 32 | task clean(type: Delete) { 33 | delete rootProject.buildDir 34 | } 35 | -------------------------------------------------------------------------------- /example/android/gradle.properties: -------------------------------------------------------------------------------- 1 | org.gradle.jvmargs=-Xmx1536M 2 | android.enableR8=true 3 | android.useAndroidX=true 4 | android.enableJetifier=true 5 | -------------------------------------------------------------------------------- /example/android/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Fri Jun 23 08:50:38 CEST 2017 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl = https\://services.gradle.org/distributions/gradle-7.2-all.zip 7 | -------------------------------------------------------------------------------- /example/android/settings.gradle: -------------------------------------------------------------------------------- 1 | include ':app' 2 | 3 | def flutterProjectRoot = rootProject.projectDir.parentFile.toPath() 4 | 5 | def plugins = new Properties() 6 | def pluginsFile = new File(flutterProjectRoot.toFile(), '.flutter-plugins') 7 | if (pluginsFile.exists()) { 8 | pluginsFile.withReader('UTF-8') { reader -> plugins.load(reader) } 9 | } 10 | 11 | plugins.each { name, path -> 12 | def pluginDirectory = flutterProjectRoot.resolve(path).resolve('android').toFile() 13 | include ":$name" 14 | project(":$name").projectDir = pluginDirectory 15 | } 16 | -------------------------------------------------------------------------------- /example/assets/decoded_wav_label.txt: -------------------------------------------------------------------------------- 1 | _silence_ 2 | _unknown_ 3 | yes 4 | no 5 | up 6 | down 7 | left 8 | right 9 | on 10 | off 11 | stop 12 | go -------------------------------------------------------------------------------- /example/assets/decoded_wav_model.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/assets/decoded_wav_model.tflite -------------------------------------------------------------------------------- /example/assets/google_teach_machine_label.txt: -------------------------------------------------------------------------------- 1 | 0 Background Noise 2 | 1 no 3 | 2 yes 4 | -------------------------------------------------------------------------------- /example/assets/google_teach_machine_model.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/assets/google_teach_machine_model.tflite -------------------------------------------------------------------------------- /example/assets/mfcc_label.txt: -------------------------------------------------------------------------------- 1 | cough 2 | hiss -------------------------------------------------------------------------------- /example/assets/mfcc_model.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/assets/mfcc_model.tflite -------------------------------------------------------------------------------- /example/assets/sample_audio_16k_mono.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/assets/sample_audio_16k_mono.wav -------------------------------------------------------------------------------- /example/assets/sample_audio_44k_mono.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/assets/sample_audio_44k_mono.wav -------------------------------------------------------------------------------- /example/assets/spectrogram_label.txt: -------------------------------------------------------------------------------- 1 | go 2 | down 3 | up 4 | stop 5 | yes 6 | left 7 | right 8 | no -------------------------------------------------------------------------------- /example/assets/spectrogram_model.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/assets/spectrogram_model.tflite -------------------------------------------------------------------------------- /example/audio_recognition_example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/audio_recognition_example.jpg -------------------------------------------------------------------------------- /example/ios/.gitignore: -------------------------------------------------------------------------------- 1 | *.mode1v3 2 | *.mode2v3 3 | *.moved-aside 4 | *.pbxuser 5 | *.perspectivev3 6 | **/*sync/ 7 | .sconsign.dblite 8 | .tags* 9 | **/.vagrant/ 10 | **/DerivedData/ 11 | Icon? 12 | **/Pods/ 13 | **/.symlinks/ 14 | profile 15 | xcuserdata 16 | **/.generated/ 17 | Flutter/App.framework 18 | Flutter/Flutter.framework 19 | Flutter/Flutter.podspec 20 | Flutter/Generated.xcconfig 21 | Flutter/app.flx 22 | Flutter/app.zip 23 | Flutter/flutter_assets/ 24 | Flutter/flutter_export_environment.sh 25 | ServiceDefinitions.json 26 | Runner/GeneratedPluginRegistrant.* 27 | 28 | # Exceptions to above rules. 29 | !default.mode1v3 30 | !default.mode2v3 31 | !default.pbxuser 32 | !default.perspectivev3 33 | -------------------------------------------------------------------------------- /example/ios/Flutter/AppFrameworkInfo.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | $(DEVELOPMENT_LANGUAGE) 7 | CFBundleExecutable 8 | App 9 | CFBundleIdentifier 10 | io.flutter.flutter.app 11 | CFBundleInfoDictionaryVersion 12 | 6.0 13 | CFBundleName 14 | App 15 | CFBundlePackageType 16 | FMWK 17 | CFBundleShortVersionString 18 | 1.0 19 | CFBundleSignature 20 | ???? 21 | CFBundleVersion 22 | 1.0 23 | MinimumOSVersion 24 | 9.0 25 | 26 | 27 | -------------------------------------------------------------------------------- /example/ios/Flutter/Debug.xcconfig: -------------------------------------------------------------------------------- 1 | #include "Pods/Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig" 2 | #include "Generated.xcconfig" 3 | -------------------------------------------------------------------------------- /example/ios/Flutter/Release.xcconfig: -------------------------------------------------------------------------------- 1 | #include "Pods/Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig" 2 | #include "Generated.xcconfig" 3 | -------------------------------------------------------------------------------- /example/ios/Podfile: -------------------------------------------------------------------------------- 1 | # Uncomment this line to define a global platform for your project 2 | platform :ios, '12.0' 3 | 4 | # CocoaPods analytics sends network stats synchronously affecting flutter build latency. 5 | ENV['COCOAPODS_DISABLE_STATS'] = 'true' 6 | 7 | project 'Runner', { 8 | 'Debug' => :debug, 9 | 'Profile' => :release, 10 | 'Release' => :release, 11 | } 12 | 13 | def flutter_root 14 | generated_xcode_build_settings_path = File.expand_path(File.join('..', 'Flutter', 'Generated.xcconfig'), __FILE__) 15 | unless File.exist?(generated_xcode_build_settings_path) 16 | raise "#{generated_xcode_build_settings_path} must exist. If you're running pod install manually, make sure flutter pub get is executed first" 17 | end 18 | 19 | File.foreach(generated_xcode_build_settings_path) do |line| 20 | matches = line.match(/FLUTTER_ROOT\=(.*)/) 21 | return matches[1].strip if matches 22 | end 23 | raise "FLUTTER_ROOT not found in #{generated_xcode_build_settings_path}. Try deleting Generated.xcconfig, then run flutter pub get" 24 | end 25 | 26 | require File.expand_path(File.join('packages', 'flutter_tools', 'bin', 'podhelper'), flutter_root) 27 | 28 | flutter_ios_podfile_setup 29 | 30 | target 'Runner' do 31 | use_frameworks! 32 | use_modular_headers! 33 | pod 'TensorFlowLiteSelectTfOps', '~> 2.6.0' 34 | 35 | flutter_install_all_ios_pods File.dirname(File.realpath(__FILE__)) 36 | 37 | target 'RunnerTests' do 38 | platform :ios, '9.0' 39 | inherit! :search_paths 40 | # Pods for testing 41 | pod 'OCMock', '~> 3.8.1' 42 | end 43 | end 44 | 45 | post_install do |installer| 46 | installer.pods_project.targets.each do |target| 47 | flutter_additional_ios_build_settings(target) 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /example/ios/Runner.xcodeproj/project.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /example/ios/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | IDEDidComputeMac32BitWarning 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /example/ios/Runner.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | PreviewsEnabled 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme: -------------------------------------------------------------------------------- 1 | 2 | 5 | 8 | 9 | 15 | 21 | 22 | 23 | 24 | 25 | 30 | 31 | 37 | 38 | 39 | 40 | 42 | 48 | 49 | 50 | 52 | 58 | 59 | 60 | 62 | 68 | 69 | 70 | 71 | 72 | 82 | 84 | 90 | 91 | 92 | 93 | 97 | 98 | 99 | 100 | 106 | 108 | 114 | 115 | 116 | 117 | 119 | 120 | 123 | 124 | 125 | -------------------------------------------------------------------------------- /example/ios/Runner.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /example/ios/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | IDEDidComputeMac32BitWarning 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /example/ios/Runner/AppDelegate.swift: -------------------------------------------------------------------------------- 1 | import UIKit 2 | import Flutter 3 | 4 | @UIApplicationMain 5 | @objc class AppDelegate: FlutterAppDelegate { 6 | override func application( 7 | _ application: UIApplication, 8 | didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? 9 | ) -> Bool { 10 | GeneratedPluginRegistrant.register(with: self) 11 | return super.application(application, didFinishLaunchingWithOptions: launchOptions) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "size" : "20x20", 5 | "idiom" : "iphone", 6 | "filename" : "Icon-App-20x20@2x.png", 7 | "scale" : "2x" 8 | }, 9 | { 10 | "size" : "20x20", 11 | "idiom" : "iphone", 12 | "filename" : "Icon-App-20x20@3x.png", 13 | "scale" : "3x" 14 | }, 15 | { 16 | "size" : "29x29", 17 | "idiom" : "iphone", 18 | "filename" : "Icon-App-29x29@1x.png", 19 | "scale" : "1x" 20 | }, 21 | { 22 | "size" : "29x29", 23 | "idiom" : "iphone", 24 | "filename" : "Icon-App-29x29@2x.png", 25 | "scale" : "2x" 26 | }, 27 | { 28 | "size" : "29x29", 29 | "idiom" : "iphone", 30 | "filename" : "Icon-App-29x29@3x.png", 31 | "scale" : "3x" 32 | }, 33 | { 34 | "size" : "40x40", 35 | "idiom" : "iphone", 36 | "filename" : "Icon-App-40x40@2x.png", 37 | "scale" : "2x" 38 | }, 39 | { 40 | "size" : "40x40", 41 | "idiom" : "iphone", 42 | "filename" : "Icon-App-40x40@3x.png", 43 | "scale" : "3x" 44 | }, 45 | { 46 | "size" : "60x60", 47 | "idiom" : "iphone", 48 | "filename" : "Icon-App-60x60@2x.png", 49 | "scale" : "2x" 50 | }, 51 | { 52 | "size" : "60x60", 53 | "idiom" : "iphone", 54 | "filename" : "Icon-App-60x60@3x.png", 55 | "scale" : "3x" 56 | }, 57 | { 58 | "size" : "20x20", 59 | "idiom" : "ipad", 60 | "filename" : "Icon-App-20x20@1x.png", 61 | "scale" : "1x" 62 | }, 63 | { 64 | "size" : "20x20", 65 | "idiom" : "ipad", 66 | "filename" : "Icon-App-20x20@2x.png", 67 | "scale" : "2x" 68 | }, 69 | { 70 | "size" : "29x29", 71 | "idiom" : "ipad", 72 | "filename" : "Icon-App-29x29@1x.png", 73 | "scale" : "1x" 74 | }, 75 | { 76 | "size" : "29x29", 77 | "idiom" : "ipad", 78 | "filename" : "Icon-App-29x29@2x.png", 79 | "scale" : "2x" 80 | }, 81 | { 82 | "size" : "40x40", 83 | "idiom" : "ipad", 84 | "filename" : "Icon-App-40x40@1x.png", 85 | "scale" : "1x" 86 | }, 87 | { 88 | "size" : "40x40", 89 | "idiom" : "ipad", 90 | "filename" : "Icon-App-40x40@2x.png", 91 | "scale" : "2x" 92 | }, 93 | { 94 | "size" : "76x76", 95 | "idiom" : "ipad", 96 | "filename" : "Icon-App-76x76@1x.png", 97 | "scale" : "1x" 98 | }, 99 | { 100 | "size" : "76x76", 101 | "idiom" : "ipad", 102 | "filename" : "Icon-App-76x76@2x.png", 103 | "scale" : "2x" 104 | }, 105 | { 106 | "size" : "83.5x83.5", 107 | "idiom" : "ipad", 108 | "filename" : "Icon-App-83.5x83.5@2x.png", 109 | "scale" : "2x" 110 | }, 111 | { 112 | "size" : "1024x1024", 113 | "idiom" : "ios-marketing", 114 | "filename" : "Icon-App-1024x1024@1x.png", 115 | "scale" : "1x" 116 | } 117 | ], 118 | "info" : { 119 | "version" : 1, 120 | "author" : "xcode" 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-1024x1024@1x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-1024x1024@1x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@1x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@1x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@2x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@3x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@3x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@1x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@1x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@2x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@3x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@3x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@1x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@1x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@2x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@3x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@3x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-60x60@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-60x60@2x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-60x60@3x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-60x60@3x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-76x76@1x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-76x76@1x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-76x76@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-76x76@2x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-83.5x83.5@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-83.5x83.5@2x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/LaunchImage.imageset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "idiom" : "universal", 5 | "filename" : "LaunchImage.png", 6 | "scale" : "1x" 7 | }, 8 | { 9 | "idiom" : "universal", 10 | "filename" : "LaunchImage@2x.png", 11 | "scale" : "2x" 12 | }, 13 | { 14 | "idiom" : "universal", 15 | "filename" : "LaunchImage@3x.png", 16 | "scale" : "3x" 17 | } 18 | ], 19 | "info" : { 20 | "version" : 1, 21 | "author" : "xcode" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage@2x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage@3x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/example/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage@3x.png -------------------------------------------------------------------------------- /example/ios/Runner/Assets.xcassets/LaunchImage.imageset/README.md: -------------------------------------------------------------------------------- 1 | # Launch Screen Assets 2 | 3 | You can customize the launch screen with your own desired assets by replacing the image files in this directory. 4 | 5 | You can also do it by opening your Flutter project's Xcode project with `open ios/Runner.xcworkspace`, selecting `Runner/Assets.xcassets` in the Project Navigator and dropping in the desired images. -------------------------------------------------------------------------------- /example/ios/Runner/Base.lproj/LaunchScreen.storyboard: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /example/ios/Runner/Base.lproj/Main.storyboard: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /example/ios/Runner/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | NSMicrophoneUsageDescription 6 | Record audio for playback 7 | CFBundleDevelopmentRegion 8 | $(DEVELOPMENT_LANGUAGE) 9 | CFBundleExecutable 10 | $(EXECUTABLE_NAME) 11 | CFBundleIdentifier 12 | $(PRODUCT_BUNDLE_IDENTIFIER) 13 | CFBundleInfoDictionaryVersion 14 | 6.0 15 | CFBundleName 16 | tflite_audio_example 17 | CFBundlePackageType 18 | APPL 19 | CFBundleShortVersionString 20 | $(MARKETING_VERSION) 21 | CFBundleSignature 22 | ???? 23 | CFBundleVersion 24 | $(FLUTTER_BUILD_NUMBER) 25 | LSRequiresIPhoneOS 26 | 27 | UILaunchStoryboardName 28 | LaunchScreen 29 | UIMainStoryboardFile 30 | Main 31 | UISupportedInterfaceOrientations 32 | 33 | UIInterfaceOrientationPortrait 34 | UIInterfaceOrientationLandscapeLeft 35 | UIInterfaceOrientationLandscapeRight 36 | 37 | UISupportedInterfaceOrientations~ipad 38 | 39 | UIInterfaceOrientationPortrait 40 | UIInterfaceOrientationPortraitUpsideDown 41 | UIInterfaceOrientationLandscapeLeft 42 | UIInterfaceOrientationLandscapeRight 43 | 44 | UIViewControllerBasedStatusBarAppearance 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /example/ios/Runner/Runner-Bridging-Header.h: -------------------------------------------------------------------------------- 1 | #import "GeneratedPluginRegistrant.h" 2 | -------------------------------------------------------------------------------- /example/ios/RunnerTests/AudioFileTest.swift: -------------------------------------------------------------------------------- 1 | // 2 | // RunnerTests.swift 3 | // RunnerTests 4 | // 5 | // Created by Michael Nguyen on 2022/03/08. 6 | // 7 | 8 | import XCTest 9 | @testable import tflite_audio 10 | 11 | class AudioFileTest: XCTestCase { 12 | 13 | 14 | func testSingleSplice(){ 15 | 16 | let audioData: [Int16] = [1, 2, 3] 17 | let audioLength = 3 18 | let expectedData: [Int16] = [1, 2, 3] 19 | let result = splice(audioData: audioData, audioLength: audioLength, bufferSize: audioData.count) 20 | 21 | 22 | XCTAssertEqual(result, expectedData) 23 | } 24 | 25 | func testSingleSplice_LackData_NoPad(){ 26 | //Assumes that threshold is set to 0.4 27 | //Mostly for situations where file sampleRate is below model input size 28 | //Todo - maybe force pad this situation??? 29 | 30 | let audioData: [Int16] = [1] 31 | let audioLength = 3 32 | let expectedData: [Int16] = [] 33 | let result = splice(audioData: audioData, audioLength: audioLength, bufferSize: audioData.count) 34 | 35 | 36 | XCTAssertEqual(result, expectedData) 37 | } 38 | 39 | func testSingleSplice_LackData_WithPadding(){ 40 | //Assumes that threshold is set to 0.4 41 | 42 | let audioData: [Int16] = [1, 2] 43 | let audioLength = 3 44 | let expectedLength = 3 45 | let result = splice(audioData: audioData, audioLength: audioLength, bufferSize: audioData.count) 46 | let resultWithNoPad: [Int16] = Array(result[0.. [Int16]{ 95 | 96 | let audioFileData: AudioFileData = AudioFileData(audioLength: audioLength, bufferSize: bufferSize) 97 | var result: [Int16] = [] 98 | var isSplicing = true 99 | 100 | for (index, data) in audioData.enumerated(){ 101 | 102 | if(isSplicing == false){ break } 103 | let state = audioFileData.getState(i: index) 104 | 105 | switch state{ 106 | case "recognise": 107 | print("recognising") 108 | audioFileData 109 | .append(data: data) 110 | .displayCount() 111 | .emit{ (audioChunk) in result.append(contentsOf: audioChunk) } 112 | .reset() 113 | break 114 | case "append": 115 | audioFileData 116 | .append(data: data) 117 | .displayCount() //only for testing 118 | break 119 | case "finalise": 120 | print("finalising") 121 | audioFileData 122 | .append(data: data) 123 | .displayCount() 124 | .emit{ (audioChunk) in result.append(contentsOf: audioChunk) } 125 | isSplicing = false //BREAK HERE - excess data ignored 126 | break 127 | case "padAndFinalise": 128 | print("trimming and finalising") 129 | audioFileData 130 | .append(data: data) 131 | .padSilence(i: index) 132 | .displayCount() 133 | .emit{ (audioChunk) in result.append(contentsOf: audioChunk)} 134 | break 135 | default: 136 | print("Error") 137 | 138 | } 139 | } 140 | 141 | return result 142 | 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /example/ios/RunnerTests/RecordingTest.swift: -------------------------------------------------------------------------------- 1 | // 2 | // RecordingTest.swift 3 | // RecordingTest 4 | // 5 | // Created by Michael Nguyen on 2022/03/10. 6 | // 7 | 8 | import XCTest 9 | @testable import tflite_audio 10 | 11 | class RecordingTest: XCTestCase { 12 | 13 | func test_singleSplice(){ 14 | let recordingData: [Int16] = [1, 2, 3, 4] 15 | let audioLength = 4 16 | let numOfInferences = 1 17 | let expectedResult: [Int16] = [1, 2, 3, 4] 18 | let result = mockRecording(data: recordingData, length: audioLength, num: numOfInferences) 19 | XCTAssertEqual(result, expectedResult) 20 | } 21 | 22 | func test_singleSplice_lackData(){ 23 | let recordingData: [Int16] = [1, 2, 3] 24 | let audioLength = 4 25 | let numOfInferences = 1 26 | let expectedResult: [Int16] = [1, 2, 3, 1] 27 | let result = mockRecording(data: recordingData, length: audioLength, num: numOfInferences) 28 | XCTAssertEqual(result, expectedResult) 29 | } 30 | 31 | 32 | func test_singleSplice_withExcess(){ 33 | let recordingData: [Int16] = [1, 2, 3, 4, 5] 34 | let audioLength = 4 35 | let numOfInferences = 1 36 | let expectedResult: [Int16] = [1, 2, 3, 4] 37 | let result = mockRecording(data: recordingData, length: audioLength, num: numOfInferences) 38 | XCTAssertEqual(result, expectedResult) 39 | } 40 | 41 | 42 | 43 | func test_multiSplice(){ 44 | let recordingData: [Int16] = [1, 2, 3, 4] 45 | let audioLength = 4 46 | let numOfInferences = 3 47 | let expectedResult: [Int16] = [1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4] 48 | let result = mockRecording(data: recordingData, length: audioLength, num: numOfInferences) 49 | XCTAssertEqual(result, expectedResult) 50 | } 51 | 52 | func test_multiSplice_lackData(){ 53 | let recordingData: [Int16] = [1, 2, 3] 54 | let audioLength = 4 55 | let numOfInferences = 3 56 | let expectedResult: [Int16] = [1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3] 57 | let result = mockRecording(data: recordingData, length: audioLength, num: numOfInferences) 58 | XCTAssertEqual(result, expectedResult) 59 | } 60 | 61 | 62 | func test_multiSplice_withExcess(){ 63 | 64 | let recordingData: [Int16] = [1, 2, 3, 4, 5] 65 | let audioLength = 4 66 | let numOfInferences = 3 67 | let expectedResult: [Int16] = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2] 68 | let result = mockRecording(data: recordingData, length: audioLength, num: numOfInferences) 69 | XCTAssertEqual(result, expectedResult) 70 | } 71 | 72 | func mockRecording(data: [Int16], length: Int, num: Int) -> [Int16]{ 73 | 74 | 75 | var result: [Int16] = [] 76 | var isRecording = true 77 | 78 | let recordingData: RecordingData = RecordingData() 79 | recordingData.setAudioLength(audioLength: length) 80 | recordingData.setNumOfInferences(numOfInferences: num) 81 | 82 | while(isRecording){ 83 | let state = recordingData.getState() 84 | 85 | switch state{ 86 | case "append": 87 | recordingData.append(data: data) 88 | break 89 | case "recognise": 90 | recordingData 91 | .emit{ (audioChunk) in result.append(contentsOf: audioChunk)} 92 | .updateCount() 93 | .clear() 94 | .append(data: data) 95 | break 96 | case "trimAndRecognise": 97 | recordingData 98 | .emit{ (audioChunk) in result.append(contentsOf: audioChunk)} 99 | .updateCount() 100 | .trimExcessToNewBuffer() 101 | break 102 | case "finalise": 103 | recordingData 104 | .emit{ (audioChunk) in result.append(contentsOf: audioChunk)} 105 | .updateCount() 106 | .resetCount() 107 | .clear() 108 | isRecording = false 109 | break 110 | default: 111 | print("Error: \(state)") 112 | 113 | } 114 | } 115 | 116 | return result 117 | } 118 | 119 | } 120 | -------------------------------------------------------------------------------- /example/lib/main.dart: -------------------------------------------------------------------------------- 1 | import 'package:flutter/material.dart'; 2 | import 'dart:async'; 3 | import 'dart:developer'; 4 | import 'package:tflite_audio/tflite_audio.dart'; 5 | import 'package:flutter/services.dart'; 6 | import 'dart:convert'; 7 | 8 | void main() => runApp(const MyApp()); 9 | 10 | ///This example showcases how to take advantage of all the futures and streams 11 | ///from the plugin. 12 | class MyApp extends StatefulWidget { 13 | const MyApp({Key? key}) : super(key: key); 14 | 15 | @override 16 | _MyAppState createState() => _MyAppState(); 17 | } 18 | 19 | class _MyAppState extends State { 20 | final GlobalKey _scaffoldKey = GlobalKey(); 21 | final isRecording = ValueNotifier(false); 22 | Stream>? result; 23 | 24 | ///example values for decodedwav models 25 | // final String model = 'assets/decoded_wav_model.tflite'; 26 | // final String label = 'assets/decoded_wav_label.txt'; 27 | // final String audioDirectory = 'assets/sample_audio_16k_mono.wav'; 28 | // final String inputType = 'decodedWav'; 29 | // final int sampleRate = 16000; 30 | // final int bufferSize = 2000; 31 | // // final int audioLength = 16000; 32 | 33 | ///example values for google's teachable machine model 34 | final String model = 'assets/google_teach_machine_model.tflite'; 35 | final String label = 'assets/google_teach_machine_label.txt'; 36 | final String inputType = 'rawAudio'; 37 | final String audioDirectory = 'assets/sample_audio_44k_mono.wav'; 38 | final int sampleRate = 44100; 39 | final int bufferSize = 11016; 40 | // final int audioLength = 44032; 41 | 42 | ///example values for MFCC, melspectrogram, spectrogram models 43 | // final String model = 'assets/spectrogram_model.tflite'; 44 | // final String label = 'assets/spectrogram_label.txt'; 45 | // final String inputType = 'spectrogram'; 46 | 47 | // final String model = 'assets/melspectrogram_model.tflite'; 48 | // final String label = 'assets/melspectrogram_label.txt'; 49 | // final String inputType = 'melSpectrogram'; 50 | 51 | // final String model = 'assets/mfcc_model.tflite'; 52 | // final String label = 'assets/mfcc_label.txt'; 53 | // final String inputType = 'mfcc'; 54 | 55 | // final String audioDirectory = 'assets/sample_audio_16k_mono.wav'; 56 | // final int sampleRate = 16000; 57 | // final int bufferSize = 2000; 58 | // // final int audioLength = 16000; 59 | 60 | ///Optional parameters you can adjust to modify your input and output 61 | final bool outputRawScores = false; 62 | final int numOfInferences = 5; 63 | final int numThreads = 1; 64 | final bool isAsset = true; 65 | 66 | ///Adjust the values below when tuning model detection. 67 | final double detectionThreshold = 0.3; 68 | final int averageWindowDuration = 1000; 69 | final int minimumTimeBetweenSamples = 30; 70 | final int suppressionTime = 1500; 71 | 72 | @override 73 | void initState() { 74 | super.initState(); 75 | TfliteAudio.loadModel( 76 | // numThreads: this.numThreads, 77 | // isAsset: this.isAsset, 78 | // outputRawScores: outputRawScores, 79 | inputType: inputType, 80 | model: model, 81 | label: label, 82 | ); 83 | 84 | //spectrogram parameters 85 | // TfliteAudio.setSpectrogramParameters(nFFT: 256, hopLength: 129); 86 | 87 | // mfcc parameters 88 | TfliteAudio.setSpectrogramParameters(nMFCC: 40, hopLength: 16384); 89 | } 90 | 91 | void getResult() { 92 | ///example for stored audio file recognition 93 | // result = TfliteAudio.startFileRecognition( 94 | // audioDirectory: audioDirectory, 95 | // sampleRate: sampleRate, 96 | // // audioLength: audioLength, 97 | // // detectionThreshold: detectionThreshold, 98 | // // averageWindowDuration: averageWindowDuration, 99 | // // minimumTimeBetweenSamples: minimumTimeBetweenSamples, 100 | // // suppressionTime: suppressionTime, 101 | // ); 102 | 103 | ///example for recording recognition 104 | result = TfliteAudio.startAudioRecognition( 105 | sampleRate: sampleRate, 106 | bufferSize: bufferSize, 107 | numOfInferences: numOfInferences, 108 | // audioLength: audioLength, 109 | // detectionThreshold: detectionThreshold, 110 | // averageWindowDuration: averageWindowDuration, 111 | // minimumTimeBetweenSamples: minimumTimeBetweenSamples, 112 | // suppressionTime: suppressionTime, 113 | ); 114 | 115 | ///Below returns a map of values. The keys are: 116 | ///"recognitionResult", "hasPermission", "inferenceTime" 117 | result 118 | ?.listen((event) => 119 | log("Recognition Result: " + event["recognitionResult"].toString())) 120 | .onDone(() => isRecording.value = false); 121 | } 122 | 123 | ///fetches the labels from the text file in assets 124 | Future> fetchLabelList() async { 125 | List _labelList = []; 126 | await rootBundle.loadString(this.label).then((q) { 127 | for (String i in const LineSplitter().convert(q)) { 128 | _labelList.add(i); 129 | } 130 | }); 131 | return _labelList; 132 | } 133 | 134 | ///handles null exception if snapshot is null. 135 | String showResult(AsyncSnapshot snapshot, String key) => 136 | snapshot.hasData ? snapshot.data[key].toString() : '0 '; 137 | 138 | @override 139 | Widget build(BuildContext context) { 140 | return MaterialApp( 141 | home: Scaffold( 142 | key: _scaffoldKey, 143 | appBar: AppBar( 144 | title: const Text('Tflite-audio/speech'), 145 | ), 146 | 147 | ///Streambuilder for inference results 148 | body: StreamBuilder>( 149 | stream: result, 150 | builder: (BuildContext context, 151 | AsyncSnapshot> inferenceSnapshot) { 152 | ///futurebuilder for getting the label list 153 | return FutureBuilder( 154 | future: fetchLabelList(), 155 | builder: (BuildContext context, 156 | AsyncSnapshot> labelSnapshot) { 157 | switch (inferenceSnapshot.connectionState) { 158 | case ConnectionState.none: 159 | //Loads the asset file. 160 | if (labelSnapshot.hasData) { 161 | return labelListWidget(labelSnapshot.data); 162 | } else { 163 | return const CircularProgressIndicator(); 164 | } 165 | case ConnectionState.waiting: 166 | 167 | ///Widets will let the user know that its loading when waiting for results 168 | return Stack(children: [ 169 | Align( 170 | alignment: Alignment.bottomRight, 171 | child: inferenceTimeWidget('calculating..')), 172 | labelListWidget(labelSnapshot.data), 173 | ]); 174 | 175 | ///Widgets will display the final results. 176 | default: 177 | return Stack(children: [ 178 | Align( 179 | alignment: Alignment.bottomRight, 180 | child: inferenceTimeWidget(showResult( 181 | inferenceSnapshot, 'inferenceTime') + 182 | 'ms')), 183 | labelListWidget( 184 | labelSnapshot.data, 185 | showResult( 186 | inferenceSnapshot, 'recognitionResult')) 187 | ]); 188 | } 189 | }); 190 | }), 191 | floatingActionButtonLocation: 192 | FloatingActionButtonLocation.centerFloat, 193 | floatingActionButton: ValueListenableBuilder( 194 | valueListenable: isRecording, 195 | builder: (context, value, widget) { 196 | if (value == false) { 197 | return FloatingActionButton( 198 | onPressed: () { 199 | isRecording.value = true; 200 | setState(() { 201 | getResult(); 202 | }); 203 | }, 204 | backgroundColor: Colors.blue, 205 | child: const Icon(Icons.mic), 206 | ); 207 | } else { 208 | return FloatingActionButton( 209 | onPressed: () { 210 | log('Audio Recognition Stopped'); 211 | TfliteAudio.stopAudioRecognition(); 212 | }, 213 | backgroundColor: Colors.red, 214 | child: const Icon(Icons.adjust), 215 | ); 216 | } 217 | }))); 218 | } 219 | 220 | ///If snapshot data matches the label, it will change colour 221 | Widget labelListWidget(List? labelList, [String? result]) { 222 | return Center( 223 | child: Column( 224 | mainAxisAlignment: MainAxisAlignment.center, 225 | crossAxisAlignment: CrossAxisAlignment.center, 226 | children: labelList!.map((labels) { 227 | if (labels == result) { 228 | return Padding( 229 | padding: const EdgeInsets.all(5.0), 230 | child: Text(labels.toString(), 231 | textAlign: TextAlign.center, 232 | style: const TextStyle( 233 | fontWeight: FontWeight.bold, 234 | fontSize: 25, 235 | color: Colors.green, 236 | ))); 237 | } else { 238 | return Padding( 239 | padding: const EdgeInsets.all(5.0), 240 | child: Text(labels.toString(), 241 | textAlign: TextAlign.center, 242 | style: const TextStyle( 243 | fontWeight: FontWeight.bold, 244 | color: Colors.black, 245 | ))); 246 | } 247 | }).toList())); 248 | } 249 | 250 | ///If the future isn't completed, shows 'calculating'. Else shows inference time. 251 | Widget inferenceTimeWidget(String result) { 252 | return Padding( 253 | padding: const EdgeInsets.all(20.0), 254 | child: Text(result, 255 | textAlign: TextAlign.center, 256 | style: const TextStyle( 257 | fontWeight: FontWeight.bold, 258 | fontSize: 20, 259 | color: Colors.black, 260 | ))); 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /example/pubspec.yaml: -------------------------------------------------------------------------------- 1 | name: tflite_audio_example 2 | description: Demonstrates how to use the tflite_audio plugin. 3 | publish_to: 'none' 4 | 5 | environment: 6 | sdk: ">=2.12.0 <3.0.0" 7 | 8 | dependencies: 9 | flutter: 10 | sdk: flutter 11 | 12 | # The following adds the Cupertino Icons font to your application. 13 | # Use with the CupertinoIcons class for iOS style icons. 14 | cupertino_icons: ^0.1.2 15 | 16 | dev_dependencies: 17 | flutter_test: 18 | sdk: flutter 19 | 20 | tflite_audio: 21 | path: ../ 22 | 23 | # For information on the generic Dart part of this file, see the 24 | # following page: https://dart.dev/tools/pub/pubspec 25 | 26 | # The following section is specific to Flutter. 27 | flutter: 28 | 29 | # The following line ensures that the Material Icons font is 30 | # included with your application, so that you can use the icons in 31 | # the material Icons class. 32 | uses-material-design: true 33 | 34 | # To add assets to your application, add an assets section, like this: 35 | assets: 36 | - assets/ 37 | 38 | # An image asset can refer to one or more resolution-specific "variants", see 39 | # https://flutter.dev/assets-and-images/#resolution-aware. 40 | 41 | # For details regarding adding assets from package dependencies, see 42 | # https://flutter.dev/assets-and-images/#from-packages 43 | 44 | # To add custom fonts to your application, add a fonts section here, 45 | # in this "flutter" section. Each entry in this list should have a 46 | # "family" key with the font family name, and a "fonts" key with a 47 | # list giving the asset and other descriptors for the font. For 48 | # example: 49 | # fonts: 50 | # - family: Schyler 51 | # fonts: 52 | # - asset: fonts/Schyler-Regular.ttf 53 | # - asset: fonts/Schyler-Italic.ttf 54 | # style: italic 55 | # - family: Trajan Pro 56 | # fonts: 57 | # - asset: fonts/TrajanPro.ttf 58 | # - asset: fonts/TrajanPro_Bold.ttf 59 | # weight: 700 60 | # 61 | # For details regarding fonts from package dependencies, 62 | # see https://flutter.dev/custom-fonts/#from-packages 63 | -------------------------------------------------------------------------------- /ios/.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .vagrant/ 3 | .sconsign.dblite 4 | .svn/ 5 | 6 | .DS_Store 7 | *.swp 8 | profile 9 | 10 | DerivedData/ 11 | build/ 12 | GeneratedPluginRegistrant.h 13 | GeneratedPluginRegistrant.m 14 | 15 | .generated/ 16 | 17 | *.pbxuser 18 | *.mode1v3 19 | *.mode2v3 20 | *.perspectivev3 21 | 22 | !default.pbxuser 23 | !default.mode1v3 24 | !default.mode2v3 25 | !default.perspectivev3 26 | 27 | xcuserdata 28 | 29 | *.moved-aside 30 | 31 | *.pyc 32 | *sync/ 33 | Icon? 34 | .tags* 35 | 36 | /Flutter/Generated.xcconfig 37 | /Flutter/flutter_export_environment.sh -------------------------------------------------------------------------------- /ios/Assets/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/ios/Assets/.gitkeep -------------------------------------------------------------------------------- /ios/Classes/LabelSmoothing.swift: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | import Foundation 16 | 17 | struct RecognitionResult { 18 | var score: Float 19 | var name: String 20 | var isNew: Bool 21 | } 22 | 23 | /** 24 | This class smoothes out the results by averaging them over a window duration and making sure the 25 | commands are not duplicated for display. 26 | */ 27 | class LabelSmoothing { 28 | // MARK: Structures that handles results. 29 | private struct Command { 30 | var score: Float 31 | let name: String 32 | } 33 | 34 | private struct ResultsAtTime { 35 | let time: TimeInterval 36 | let scores: [Float] 37 | } 38 | 39 | // MARK: Constants 40 | private let averageWindowDuration: Double 41 | private let suppressionTime: Double 42 | private let minimumTimeBetweenSamples: Double 43 | private let detectionThreshold: Float 44 | private let classLabels: [String] 45 | private let silenceLabel = "_silence_" 46 | private var previousTopLabel = "_silence_" 47 | 48 | 49 | private var previousTopScore: Float = 0.0 50 | private var previousTopLabelTime: TimeInterval = Date.distantPast.timeIntervalSince1970 * 1000 51 | private var previousResults: [ResultsAtTime] = [] 52 | 53 | /** 54 | Initializes LabelSmoothing with specified parameters. 55 | */ 56 | init(averageWindowDuration: Double, detectionThreshold: Float, minimumTimeBetweenSamples: Double, suppressionTime: Double, classLabels: [String]) { 57 | self.averageWindowDuration = averageWindowDuration 58 | self.detectionThreshold = detectionThreshold 59 | self.minimumTimeBetweenSamples = minimumTimeBetweenSamples 60 | self.suppressionTime = suppressionTime 61 | self.classLabels = classLabels 62 | } 63 | 64 | /** 65 | This function averages the results obtained over an average window duration and prunes out any 66 | old results. 67 | */ 68 | func process(latestResults: [Float], currentTime: TimeInterval) -> RecognitionResult? { 69 | 70 | guard latestResults.count == classLabels.count else { 71 | fatalError("There should be \(classLabels.count) in results. But there are \(latestResults.count) results") 72 | } 73 | 74 | // Checks if the new results were identified at a later time than the currently identified 75 | // results. 76 | if let first = previousResults.first, first.time > currentTime { 77 | fatalError("Results should be provided in increasing time order") 78 | } 79 | 80 | // if let lastResult = previousResults.last { 81 | 82 | // let timeSinceMostRecent = currentTime - previousResults[previousResults.count - 1].time 83 | 84 | // // If not enough time has passed after the last inference, we return the previously identified 85 | // // result as legitimate one. 86 | // if timeSinceMostRecent < minimumTimeBetweenSamples { 87 | // return RecognitionResult(score: previousTopScore, name: previousTopLabel, isNew: false) 88 | // } 89 | // } 90 | 91 | // Appends the new results to the identified results 92 | let results: ResultsAtTime = ResultsAtTime(time: currentTime, scores: latestResults) 93 | 94 | previousResults.append(results) 95 | 96 | let timeLimit = currentTime - averageWindowDuration 97 | 98 | // Flushes out all the results currently held that less than the average window duration since 99 | // they are considered too old for averaging. 100 | while previousResults[0].time < timeLimit { 101 | previousResults.removeFirst() 102 | 103 | guard previousResults.count > 0 else { 104 | break 105 | } 106 | } 107 | 108 | // If number of results currently held to average is less than a minimum count, return the score 109 | // as zero so that no command is identified. 110 | // if previousResults.count < minimumCount { 111 | // return RecognitionResult(score: 0.0, name: previousTopLabel, isNew: false) 112 | // } 113 | 114 | // Creates an average of the scores of each classes currently held by this class. 115 | var averageScores:[Command] = [] 116 | for i in 0...classLabels.count - 1 { 117 | 118 | let command = Command(score: 0.0, name: classLabels[i]) 119 | averageScores.append(command) 120 | 121 | } 122 | 123 | for result in previousResults { 124 | 125 | let scores = result.scores 126 | for i in 0...scores.count - 1 { 127 | averageScores[i].score = averageScores[i].score + scores[i] / Float(previousResults.count) 128 | 129 | } 130 | } 131 | 132 | // Sorts scores in descending order of confidence. 133 | averageScores.sort { (first, second) -> Bool in 134 | return first.score > second.score 135 | } 136 | 137 | var timeSinceLastTop: Double = 0.0 138 | 139 | // If silence was detected previously, consider the current result with the best average as a 140 | // new command to be displayed. 141 | if (previousTopLabel == silenceLabel || 142 | previousTopLabelTime == (Date.distantPast.timeIntervalSince1970 * 1000)) { 143 | 144 | timeSinceLastTop = Date.distantFuture.timeIntervalSince1970 * 1000 145 | } 146 | else { 147 | timeSinceLastTop = currentTime - previousTopLabelTime 148 | } 149 | 150 | // Return the results 151 | var isNew = false 152 | if (averageScores[0].score > detectionThreshold && timeSinceLastTop > suppressionTime) { 153 | 154 | previousTopScore = averageScores[0].score 155 | previousTopLabel = averageScores[0].name 156 | previousTopLabelTime = currentTime 157 | isNew = true 158 | } 159 | else { 160 | isNew = false 161 | } 162 | 163 | return RecognitionResult( 164 | score: previousTopScore, name: previousTopLabel, isNew: isNew) 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /ios/Classes/TfliteAudioPlugin.h: -------------------------------------------------------------------------------- 1 | #import 2 | 3 | @interface TfliteAudioPlugin : NSObject 4 | @end 5 | -------------------------------------------------------------------------------- /ios/Classes/TfliteAudioPlugin.m: -------------------------------------------------------------------------------- 1 | #import "TfliteAudioPlugin.h" 2 | #if __has_include() 3 | #import 4 | #else 5 | // Support project import fallback if the generated compatibility header 6 | // is not copied when this plugin is created as a library. 7 | // https://forums.swift.org/t/swift-static-libraries-dont-copy-generated-objective-c-header/19816 8 | #import "tflite_audio-Swift.h" 9 | #endif 10 | 11 | @implementation TfliteAudioPlugin 12 | + (void)registerWithRegistrar:(NSObject*)registrar { 13 | [SwiftTfliteAudioPlugin registerWithRegistrar:registrar]; 14 | } 15 | @end 16 | -------------------------------------------------------------------------------- /ios/Classes/processing/AudioFile.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import AVFoundation 3 | import RxCocoa 4 | import RxSwift 5 | 6 | /* why nil is assigned to pcmBuffer abd subject 7 | https://stackoverflow.com/questions/34474545/self-used-before-all-stored-properties-are-initialized 8 | */ 9 | 10 | class AudioFile{ 11 | 12 | private var pcmBuffer: AVAudioPCMBuffer? = nil 13 | private var subject: PublishSubject<[Int16]>? = nil 14 | private var audioFileData: AudioFileData? = nil 15 | 16 | private var isSplicing = false 17 | 18 | init(fileURL: URL, audioLength: Int){ 19 | self.pcmBuffer = getBuffer(fileURL: fileURL) 20 | self.subject = PublishSubject() 21 | self.audioFileData = AudioFileData(audioLength: audioLength, bufferSize: Int(pcmBuffer!.frameCapacity)) 22 | print(Int(pcmBuffer!.frameCapacity)) 23 | } 24 | 25 | func getObservable() -> Observable<[Int16]>{ 26 | return self.subject!.asObservable() 27 | } 28 | 29 | func getBuffer(fileURL : URL) -> AVAudioPCMBuffer{ 30 | 31 | let pcmBuffer: AVAudioPCMBuffer 32 | 33 | do { 34 | let file = try! AVAudioFile(forReading: fileURL, commonFormat: .pcmFormatInt16, interleaved: false) 35 | let format = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: file.fileFormat.sampleRate, channels: 1, interleaved: false) 36 | pcmBuffer = AVAudioPCMBuffer(pcmFormat: format!, frameCapacity: AVAudioFrameCount(file.length))! 37 | try file.read(into: pcmBuffer) 38 | } catch let error as NSError { 39 | print("Buffer loading error: ", error.localizedDescription) 40 | } 41 | 42 | return pcmBuffer 43 | 44 | } 45 | 46 | func stop(){ 47 | isSplicing = false 48 | self.subject!.onCompleted() 49 | audioFileData = nil 50 | } 51 | 52 | func splice(){ 53 | 54 | isSplicing = true 55 | let buffer = UnsafeBufferPointer(start: pcmBuffer?.int16ChannelData![0], count:Int(pcmBuffer!.frameCapacity)) 56 | 57 | for (index, data) in buffer.enumerated(){ 58 | 59 | if(isSplicing == false){ break } 60 | 61 | let state = audioFileData!.getState(i: index) 62 | 63 | switch state{ 64 | case "recognise": 65 | audioFileData! 66 | .append(data: data) 67 | .displayCount() 68 | .emit{ (audioChunk) in self.subject!.onNext(audioChunk) } 69 | .reset() 70 | break 71 | case "append": 72 | audioFileData! 73 | .append(data: data) 74 | break 75 | case "finalise": 76 | audioFileData! 77 | .append(data: data) 78 | .displayCount() 79 | .emit{ (audioChunk) in self.subject!.onNext(audioChunk) } 80 | self.stop() //Force break loop - excess data will be ignored 81 | break 82 | case "padAndFinalise": 83 | audioFileData! 84 | .append(data: data) 85 | .padSilence(i: index) 86 | .displayCount() 87 | .emit{ (audioChunk) in self.subject!.onNext(audioChunk) } 88 | self.stop() 89 | break 90 | default: 91 | print("Error") 92 | } 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /ios/Classes/processing/AudioFileData.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | class AudioFileData{ 4 | 5 | private var audioLength: Int 6 | private var int16DataSize: Int 7 | 8 | private var requirePadding: Bool? = nil 9 | private var numOfInferences: Int? = nil 10 | 11 | private var indexCount = 0 12 | private var inferenceCount = 1 13 | 14 | private var audioChunk: [Int16] = [] 15 | private var result = [Int16]() 16 | 17 | init(audioLength: Int, bufferSize: Int){ 18 | self.audioLength = audioLength 19 | self.int16DataSize = bufferSize 20 | 21 | let excessSamples = int16DataSize % audioLength; 22 | let missingSamples = audioLength - excessSamples; 23 | requirePadding = getPaddingRequirement(excessSamples: excessSamples, missingSamples: missingSamples); 24 | 25 | let totalWithPad = int16DataSize + missingSamples; 26 | let totalWithoutPad = int16DataSize - excessSamples; 27 | numOfInferences = getNumOfInferences(totalWithoutPad: totalWithoutPad, totalWithPad: totalWithPad); 28 | 29 | // //Force pad data that is lacking 30 | // let isLackingData = numOfInferences == 0 && requirePadding == false 31 | // if(isLackingData){ 32 | // print("Insufficient Data. Audio Chunk set to be padded regardless of threshold") 33 | // self.requirePadding == true 34 | // self.numOfInferences == 1 35 | // } 36 | } 37 | 38 | 39 | func getPaddingRequirement(excessSamples: Int, missingSamples: Int) -> Bool{ 40 | let hasMissingSamples = missingSamples != 0 || excessSamples != audioLength 41 | let missingSampleThreshold: Double = 0.40 42 | let missingSamplesRatio: Double = Double(missingSamples) / Double(audioLength) 43 | let shouldPad = missingSamplesRatio < missingSampleThreshold 44 | // print(missingSampleThreshold) //testing 45 | // print(missingSamplesRatio) //testing 46 | // print(shouldPad) //testing 47 | 48 | if (hasMissingSamples && shouldPad) {return true} 49 | else if (hasMissingSamples && !shouldPad) {return false} 50 | else if (!hasMissingSamples && shouldPad) {return false} 51 | else {return false} 52 | 53 | } 54 | 55 | func getNumOfInferences(totalWithoutPad: Int, totalWithPad: Int) -> Int{ 56 | return requirePadding! ? Int(totalWithPad/audioLength) : Int(totalWithoutPad/audioLength) 57 | } 58 | 59 | func getState(i: Int) -> String{ 60 | let reachInputSize: Bool = (i + 1) % audioLength == 0 61 | let reachFileSize: Bool = (i + 1) == int16DataSize 62 | let reachInferenceLimit: Bool = inferenceCount == numOfInferences 63 | 64 | if (reachInputSize && !reachInferenceLimit) { 65 | return "recognise"; 66 | } // inferences > 1 && < not final 67 | else if (!reachInputSize && reachInferenceLimit && !reachFileSize) { 68 | return "append"; 69 | } // Inferences = 1 70 | else if (!reachInputSize && !reachInferenceLimit) { 71 | return "append"; 72 | } // Inferences > 1 73 | else if (!reachInputSize && reachInferenceLimit && reachFileSize) { 74 | return "padAndFinalise"; 75 | } // for padding last infernce 76 | else if (reachInputSize && reachInferenceLimit) { 77 | return "finalise"; 78 | } // inference is final 79 | else { 80 | return "Error"; 81 | } 82 | 83 | } 84 | 85 | //TODO - add after rest 86 | @discardableResult 87 | func append(data: Int16) -> AudioFileData{ 88 | audioChunk.append(data) 89 | indexCount += 1 90 | return self 91 | } 92 | 93 | @discardableResult 94 | func displayCount() -> AudioFileData{ 95 | // print("\(inferenceCount)/\(numOfInferences!) | \(audioChunk.count)/\(audioLength)") //TESTING 96 | print("inference count: \(inferenceCount)/\(numOfInferences!)") 97 | return self 98 | } 99 | 100 | @discardableResult 101 | func emit(result: @escaping ([Int16]) -> Void) -> AudioFileData{ 102 | result(Array(audioChunk[0.. AudioFileData{ 109 | indexCount = 0; 110 | inferenceCount += 1 111 | audioChunk.removeAll() 112 | return self 113 | } 114 | 115 | @discardableResult 116 | func padSilence(i: Int) -> AudioFileData{ 117 | let missingSamples = audioLength - indexCount 118 | if(requirePadding!){ 119 | let paddedArray: [Int16] = (-10...10).randomElements(missingSamples) 120 | audioChunk.append(contentsOf: paddedArray) 121 | print( "(\(missingSamples)) samples have been padded to audio chunk") 122 | }else{ 123 | print( "Under threshold. Padding not required") 124 | } 125 | return self 126 | } 127 | 128 | 129 | } 130 | -------------------------------------------------------------------------------- /ios/Classes/processing/Recording.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import AVFoundation 3 | import RxCocoa 4 | import RxSwift 5 | 6 | /* RxSwift 7 | https://github.com/chat-sdk/firestream-swift/blob/7cdcb2bfe18a50d982609d972e1fadc5d3d5655a/Sources/RX/MultiQueueSubject.swift 8 | */ 9 | 10 | /* observable schedulers/threads 11 | https://stackoverflow.com/questions/37973445/does-the-order-of-subscribeon-and-observeon-matter 12 | https://stackoverflow.com/questions/52931989/schedulers-for-network-requests-in-rxswift 13 | */ 14 | 15 | /* serial vs concurrent 16 | let main = MainScheduler.instance 17 | let concurrentMain = ConcurrentMainScheduler.instance 18 | 19 | let serialBackground = SerialDispatchQueueScheduler.init(qos: .background) 20 | let concurrentBackground = ConcurrentDispatchQueueScheduler.init(qos: .background) 21 | https://www.avanderlee.com/swift/concurrent-serial-dispatchqueue/ 22 | */ 23 | 24 | class Recording{ 25 | 26 | private var bufferSize: Int 27 | private var audioLength: Int 28 | private var sampleRate: Int 29 | private var numOfInferences: Int 30 | 31 | private var recordingData: RecordingData 32 | private var audioEngine: AVAudioEngine 33 | private let subject: PublishSubject<[Int16]> 34 | 35 | init(bufferSize: Int, audioLength: Int, sampleRate: Int, numOfInferences: Int){ 36 | 37 | recordingData = RecordingData() 38 | audioEngine = AVAudioEngine() 39 | subject = PublishSubject() 40 | 41 | self.bufferSize = bufferSize 42 | self.audioLength = audioLength 43 | self.sampleRate = sampleRate 44 | self.numOfInferences = numOfInferences 45 | 46 | recordingData.setAudioLength(audioLength: audioLength) 47 | recordingData.setNumOfInferences(numOfInferences: numOfInferences) 48 | } 49 | 50 | func getObservable() -> Observable<[Int16]>{ 51 | return self.subject.asObservable() 52 | } 53 | 54 | func start(){ 55 | 56 | let recordingFrame = AVAudioFrameCount(bufferSize) 57 | let inputNode = audioEngine.inputNode 58 | let inputFormat = inputNode.outputFormat(forBus: 0) 59 | let recordingFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: Double(sampleRate), channels: 1, interleaved: true) 60 | guard let formatConverter = AVAudioConverter(from:inputFormat, to: recordingFormat!) else { 61 | return 62 | } 63 | 64 | audioEngine.inputNode.installTap(onBus: 0, bufferSize: AVAudioFrameCount(bufferSize), format: inputFormat) { (buffer, time) in 65 | 66 | let pcmBuffer = AVAudioPCMBuffer(pcmFormat: recordingFormat!, frameCapacity: recordingFrame) 67 | var error: NSError? = nil 68 | 69 | let inputBlock: AVAudioConverterInputBlock = {inNumPackets, outStatus in 70 | outStatus.pointee = AVAudioConverterInputStatus.haveData 71 | return buffer 72 | } 73 | 74 | formatConverter.convert(to: pcmBuffer!, error: &error, withInputFrom: inputBlock) 75 | 76 | if error != nil { 77 | print(error!.localizedDescription) 78 | } 79 | else if let channelData = pcmBuffer!.int16ChannelData { 80 | 81 | let channelDataValue = channelData.pointee 82 | let channelDataValueArray = stride(from: 0, 83 | to: Int(pcmBuffer!.frameLength), 84 | by: buffer.stride).map{ channelDataValue[$0] } 85 | self.splice(data: channelDataValueArray) 86 | } 87 | } 88 | 89 | audioEngine.prepare() 90 | do { 91 | try audioEngine.start() 92 | } 93 | catch { 94 | print(error.localizedDescription) 95 | } 96 | } 97 | 98 | func stop(){ 99 | audioEngine.stop() 100 | audioEngine.inputNode.removeTap(onBus: 0) 101 | self.subject.onCompleted() 102 | } 103 | 104 | func splice(data: [Int16]){ 105 | 106 | let state = recordingData.getState() 107 | 108 | switch state{ 109 | case "append": 110 | recordingData.append(data: data) 111 | break 112 | case "recognise": 113 | recordingData 114 | .emit{ (result) in self.subject.onNext(result) } 115 | .updateCount() 116 | .clear() 117 | .append(data: data) 118 | break 119 | case "trimAndRecognise": 120 | recordingData 121 | .emit{ (result) in self.subject.onNext(result) } 122 | .updateCount() 123 | .trimExcessToNewBuffer() 124 | break 125 | case "finalise": 126 | recordingData 127 | .emit{ (result) in self.subject.onNext(result) } 128 | .updateCount() 129 | .resetCount() 130 | .clear() 131 | self.stop() 132 | break 133 | default: 134 | print("Error: \(state)") 135 | 136 | } 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /ios/Classes/processing/RecordingData.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /* emit function 4 | allow chaining to continue after callback clouse (notice two "->") 5 | @escaping callback closure allows parameter to return in arguments 6 | https://abhimuralidharan.medium.com/chaining-methods-in-swift-not-optional-chaining-3007d1714985 7 | http://www.albertopasca.it/whiletrue/swift-chaining-methods/ 8 | */ 9 | 10 | /* @discardableResult 11 | supress call warning 12 | */ 13 | 14 | /* @escaping 15 | https://www.donnywals.com/what-is-escaping-in-swift/ 16 | */ 17 | 18 | /* optionals "?" on class variables 19 | avoid need to call init() since we are using setters 20 | */ 21 | 22 | class RecordingData{ 23 | 24 | private var audioLength: Int? 25 | private var numOfInferences: Int? 26 | 27 | private var inferenceCount = 1 28 | private var recordingBuffer: [Int16] = [] 29 | private var result = [Int16]() 30 | 31 | func setAudioLength(audioLength: Int){ 32 | self.audioLength = audioLength 33 | } 34 | 35 | func setNumOfInferences(numOfInferences: Int){ 36 | self.numOfInferences = numOfInferences 37 | } 38 | 39 | func getState() -> String{ 40 | var state: String = "" 41 | 42 | if(inferenceCount <= numOfInferences! && recordingBuffer.count < audioLength!){ 43 | state = "append" 44 | } else if (inferenceCount < numOfInferences! && recordingBuffer.count == audioLength!){ 45 | state = "recognise" 46 | } else if (inferenceCount < numOfInferences! && recordingBuffer.count > audioLength!) { 47 | state = "trimAndRecognise" 48 | } else if (inferenceCount == numOfInferences! && recordingBuffer.count >= audioLength!) { 49 | state = "finalise" 50 | } else { 51 | state = "Incorrect state: \(displayLogCount())" 52 | } 53 | 54 | return state 55 | } 56 | 57 | func displayLogCount() -> String{ 58 | return "\(inferenceCount)/\(numOfInferences!) | \(recordingBuffer.count)/\(audioLength!)" 59 | } 60 | 61 | @discardableResult 62 | func append(data: [Int16]) -> RecordingData{ 63 | recordingBuffer.append(contentsOf: data) 64 | print("recordingBuffer length: \(displayLogCount())") 65 | return self 66 | } 67 | 68 | func emit(result: @escaping ([Int16]) -> Void) -> RecordingData{ 69 | // Swift.print(recordingBuffer[0.. RecordingData{ 77 | let excessRecordingBuffer: [Int16] = Array(recordingBuffer[audioLength!.. RecordingData{ 85 | inferenceCount += 1 86 | return self 87 | } 88 | 89 | @discardableResult 90 | func clear() -> RecordingData{ 91 | recordingBuffer = [] 92 | return self 93 | } 94 | 95 | @discardableResult 96 | func resetCount() -> RecordingData{ 97 | inferenceCount = 1 98 | return self 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /ios/tflite_audio.podspec: -------------------------------------------------------------------------------- 1 | # 2 | # To learn more about a Podspec see http://guides.cocoapods.org/syntax/podspec.html. 3 | # Run `pod lib lint tflite_audio.podspec' to validate before publishing. 4 | # 5 | Pod::Spec.new do |s| 6 | s.name = 'tflite_audio' 7 | s.version = '0.3.0' 8 | s.summary = 'A new flutter plugin project.' 9 | s.description = <<-DESC 10 | A new flutter plugin project. 11 | DESC 12 | s.homepage = 'http://example.com' 13 | s.license = { :file => '../LICENSE' } 14 | s.author = { 'Your Company' => 'email@example.com' } 15 | s.source = { :path => '.' } 16 | s.source_files = 'Classes/**/*' 17 | s.dependency 'Flutter' 18 | s.dependency 'TensorFlowLiteSwift', '~> 2.6.0' 19 | s.dependency 'RxSwift', '6.5.0' 20 | s.dependency 'RxCocoa', '6.5.0' 21 | s.dependency 'RosaKit' 22 | # s.dependency 'TensorFlowLiteSwift' 23 | # s.dependency 'TensorFlowLiteSelectTfOps' 24 | # s.dependency 'TensorFlowLiteSwift', '~> 2.4.0' 25 | # s.dependency 'TensorFlowLiteSelectTfOps', '~> 2.4.0' 26 | # s.dependency 'TensorFlowLiteSwift', '~> 0.0.1-nightly' 27 | # s.dependency 'TensorFlowLiteSelectTfOps', '~> 0.0.1-nightly' 28 | s.platform = :ios, '12.0' 29 | s.static_framework = true 30 | 31 | # Flutter.framework does not contain a i386 slice. Only x86_64 simulators are supported. 32 | s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', 'VALID_ARCHS[sdk=iphonesimulator*]' => 'x86_64' } 33 | s.swift_version = '5.0' 34 | end 35 | -------------------------------------------------------------------------------- /lib/tflite_audio.dart: -------------------------------------------------------------------------------- 1 | import 'dart:async'; 2 | 3 | import 'package:flutter/services.dart'; 4 | 5 | // ignore: avoid_classes_with_only_static_members 6 | /// Class which manages the future and stream for the plugins 7 | class TfliteAudio { 8 | static const MethodChannel _channel = MethodChannel('tflite_audio'); 9 | static const EventChannel audioRecongitionChannel = 10 | EventChannel('AudioRecognitionStream'); 11 | static const EventChannel fileRecognitionChannel = 12 | EventChannel('FileRecognitionStream'); 13 | 14 | /// [startAudioRecognition] returns map objects with the following values: 15 | /// String recognitionResult, int inferenceTime, bool hasPermission 16 | /// Do not change the parameter 'method' 17 | static Stream> startAudioRecognition( 18 | {required int sampleRate, 19 | required int bufferSize, 20 | int audioLength = 0, 21 | double detectionThreshold = 0.3, 22 | int numOfInferences = 1, 23 | int averageWindowDuration = 0, 24 | int minimumTimeBetweenSamples = 0, 25 | int suppressionTime = 0, 26 | String method = 'setAudioRecognitionStream'}) { 27 | final audioRecognitionStream = 28 | audioRecongitionChannel.receiveBroadcastStream({ 29 | 'sampleRate': sampleRate, 30 | 'bufferSize': bufferSize, 31 | 'audioLength': audioLength, 32 | 'numOfInferences': numOfInferences, 33 | 'averageWindowDuration': averageWindowDuration, 34 | 'detectionThreshold': detectionThreshold, 35 | 'minimumTimeBetweenSamples': minimumTimeBetweenSamples, 36 | 'suppressionTime': suppressionTime, 37 | 'method': method 38 | }); 39 | 40 | ///cast the result of the stream a map object. 41 | return audioRecognitionStream 42 | .cast>() 43 | .map((event) => Map.from(event)); 44 | } 45 | 46 | ///Load stored audio file, preprocess and then fed into model. 47 | static Stream> startFileRecognition( 48 | {required String audioDirectory, 49 | required int sampleRate, 50 | int audioLength = 0, 51 | double detectionThreshold = 0.3, 52 | int averageWindowDuration = 0, 53 | int minimumTimeBetweenSamples = 0, 54 | int suppressionTime = 0, 55 | final String method = 'setFileRecognitionStream'}) { 56 | final fileRecognitionStream = 57 | fileRecognitionChannel.receiveBroadcastStream({ 58 | 'audioDirectory': audioDirectory, 59 | 'sampleRate': sampleRate, 60 | 'audioLength': audioLength, 61 | 'averageWindowDuration': averageWindowDuration, 62 | 'detectionThreshold': detectionThreshold, 63 | 'minimumTimeBetweenSamples': minimumTimeBetweenSamples, 64 | 'suppressionTime': suppressionTime, 65 | 'method': method 66 | }); 67 | 68 | ///cast the result of the stream a map object. 69 | return fileRecognitionStream 70 | .cast>() 71 | .map((event) => Map.from(event)); 72 | } 73 | 74 | ///call [stopAudioRecognition] to forcibly stop recording, recognition and 75 | ///stream. 76 | static Future stopAudioRecognition() async { 77 | return _channel.invokeMethod('stopAudioRecognition'); 78 | } 79 | 80 | /// Call [setSpectrogramParameters] to adjust the default spectro parameters 81 | static Future setSpectrogramParameters({ 82 | bool shouldTranspose = false, 83 | int nMFCC = 20, 84 | int nFFT = 2048, 85 | int nMels = 128, 86 | int hopLength = 512, 87 | }) async { 88 | return _channel.invokeMethod( 89 | 'setSpectrogramParameters', 90 | { 91 | 'shouldTranspose': shouldTranspose, 92 | 'nMFCC': nMFCC, 93 | 'nFFT': nFFT, 94 | 'nMels': nMels, 95 | 'hopLength': hopLength, 96 | }, 97 | ); 98 | } 99 | 100 | ///initialize [loadModel] before calling any other streams and futures. 101 | static Future loadModel( 102 | {required String model, 103 | required String label, 104 | required String inputType, 105 | bool outputRawScores = false, 106 | int numThreads = 1, 107 | bool isAsset = true}) async { 108 | return _channel.invokeMethod( 109 | 'loadModel', 110 | { 111 | 'model': model, 112 | 'label': label, 113 | 'inputType': inputType, 114 | 'outputRawScores': outputRawScores, 115 | 'numThreads': numThreads, 116 | 'isAsset': isAsset, 117 | }, 118 | ); 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /pictures/deployment-target.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/pictures/deployment-target.png -------------------------------------------------------------------------------- /pictures/finish.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/pictures/finish.png -------------------------------------------------------------------------------- /pictures/model-label-asset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/pictures/model-label-asset.png -------------------------------------------------------------------------------- /pictures/start.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/pictures/start.png -------------------------------------------------------------------------------- /pictures/tflite-select-ops-installation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Caldarie/flutter_tflite_audio/15ae04ce29493dbb85763cc4b154486e3844c9c5/pictures/tflite-select-ops-installation.png -------------------------------------------------------------------------------- /pubspec.yaml: -------------------------------------------------------------------------------- 1 | name: tflite_audio 2 | maintainer: Michael Nguyen 3 | description: Audio classification Tflite package for flutter (iOS & Android). Can support Google Teachable Machine models. 4 | version: 0.3.0 5 | homepage: https://github.com/Caldarie/flutter_tflite_audio 6 | 7 | environment: 8 | sdk: '>=2.12.0 <3.0.0' 9 | flutter: ">=1.10.0" 10 | 11 | dependencies: 12 | flutter: 13 | sdk: flutter 14 | 15 | dev_dependencies: 16 | effective_dart: ^1.3.1 17 | flutter_test: 18 | sdk: flutter 19 | 20 | flutter: 21 | plugin: 22 | platforms: 23 | android: 24 | package: flutter.tflite_audio 25 | pluginClass: TfliteAudioPlugin 26 | ios: 27 | pluginClass: TfliteAudioPlugin 28 | -------------------------------------------------------------------------------- /test/tflite_audio_channel_test.dart: -------------------------------------------------------------------------------- 1 | /* 2 | check if argument passes 3 | https://github.com/flutter/plugins/blob/f93314bb3779ebb0151bc326a0e515ca5f46533c/packages/image_picker/image_picker_platform_interface/test/new_method_channel_image_picker_test.dart 4 | */ 5 | 6 | import 'package:flutter/material.dart'; 7 | import 'package:flutter/services.dart'; 8 | import 'package:flutter_test/flutter_test.dart'; 9 | import 'package:tflite_audio/tflite_audio.dart'; 10 | 11 | final List log = []; 12 | const MethodChannel channel = MethodChannel('tflite_audio'); 13 | // const MethodChannel eventChannel = MethodChannel('startAudioRecognition'); 14 | 15 | void main() { 16 | WidgetsFlutterBinding.ensureInitialized(); 17 | 18 | group('loadModel() test', () { 19 | setUp(() async { 20 | channel.setMockMethodCallHandler((methodCall) async { 21 | log.add(methodCall); 22 | return ''; 23 | }); 24 | log.clear(); 25 | }); 26 | 27 | tearDown(() { 28 | channel.setMockMethodCallHandler(null); 29 | }); 30 | 31 | test('passes optional and required arguments correctly', () async { 32 | await TfliteAudio.loadModel( 33 | inputType: 'decodedWav', 34 | model: 'assets/decoded_wav_model.tflite', 35 | label: 'assets/decoded_wav_label.txt', 36 | ); 37 | 38 | await TfliteAudio.loadModel( 39 | inputType: 'decodedWav', 40 | model: 'assets/google_teach_machine_model.tflite', 41 | label: 'assets/google_teach_machine_label.txt', 42 | numThreads: 3, 43 | isAsset: false, 44 | ); 45 | 46 | expect( 47 | log, 48 | [ 49 | isMethodCall( 50 | 'loadModel', 51 | arguments: { 52 | 'model': 'assets/decoded_wav_model.tflite', 53 | 'label': 'assets/decoded_wav_label.txt', 54 | 'numThreads': 1, 55 | 'isAsset': true, 56 | }, 57 | ), 58 | isMethodCall( 59 | 'loadModel', 60 | arguments: { 61 | 'model': 'assets/google_teach_machine_model.tflite', 62 | 'label': 'assets/google_teach_machine_label.txt', 63 | 'numThreads': 3, 64 | 'isAsset': false, 65 | }, 66 | ), 67 | ], 68 | ); 69 | }); 70 | }); 71 | 72 | // group('startAudioRecognition() test', () { 73 | // setUp(() async { 74 | // eventChannel.setMockMethodCallHandler((methodCall) async { 75 | // log.add(methodCall); 76 | // return ''; 77 | // }); 78 | // log.clear(); 79 | // }); 80 | 81 | // tearDown(() { 82 | // eventChannel.setMockMethodCallHandler(null); 83 | // }); 84 | 85 | // test('passes optional and required arguments correctly', () async { 86 | // TfliteAudio.startAudioRecognition( 87 | // inputType: 'decodedWav', 88 | // sampleRate: 16000, 89 | // audioLength: 16000, 90 | // bufferSize: 2000); 91 | 92 | // expect( 93 | // log, 94 | // [ 95 | // isMethodCall( 96 | // 'startAudioRecognition', 97 | // arguments: { 98 | // 'inputType': 'decodedWav', 99 | // 'sampleRate': 16000, 100 | // 'audioLength': 16000, 101 | // 'bufferSize': 2000, 102 | // }, 103 | // ), 104 | // ], 105 | // ); 106 | // }); 107 | // }); 108 | } 109 | -------------------------------------------------------------------------------- /test/tflite_audio_stream_test.dart: -------------------------------------------------------------------------------- 1 | /* 2 | event streams 3 | https://github.com/befovy/fijkplayer/blob/58503f3f2591d41437bef61478de57b7527dff98/test/fijkplayer_test.dart 4 | https://github.com/BugsBunnyBR/plugin_gen.flutter/blob/master/examples/counter_stream_plugin/test/counter_stream_plugin_test.dart 5 | 6 | https://github.com/ZaraclaJ/audio_recorder/blob/master/test/audio_recorder_test.dart 7 | */ 8 | 9 | import 'package:flutter/material.dart'; 10 | import 'package:flutter/services.dart'; 11 | import 'package:flutter_test/flutter_test.dart'; 12 | import 'package:tflite_audio/tflite_audio.dart'; 13 | 14 | const MethodChannel eventChannel = MethodChannel('startAudioRecognition'); 15 | 16 | void main() { 17 | WidgetsFlutterBinding.ensureInitialized(); 18 | 19 | setUp(() { 20 | eventChannel.setMockMethodCallHandler((methodCall) async { 21 | await ServicesBinding.instance!.defaultBinaryMessenger 22 | .handlePlatformMessage( 23 | 'startAudioRecognition', 24 | const StandardMethodCodec() 25 | .encodeSuccessEnvelope({ 26 | 'hasPermission': 'true', 27 | 'inferenceTime': '189', 28 | 'recognitionResult': 'no' 29 | }), 30 | (data) {}); 31 | }); 32 | }); 33 | 34 | tearDown(() { 35 | eventChannel.setMockMethodCallHandler(null); 36 | }); 37 | 38 | test('returns a map stream correctly', () async { 39 | var value = TfliteAudio.startAudioRecognition( 40 | sampleRate: 16000, audioLength: 16000, bufferSize: 2000); 41 | 42 | value.listen(expectAsync1((event) { 43 | expect(event, { 44 | 'hasPermission': 'true', 45 | 'inferenceTime': '189', 46 | 'recognitionResult': 'no' 47 | }); 48 | })); 49 | }); 50 | } 51 | --------------------------------------------------------------------------------