├── .gitignore ├── LICENSE ├── README.md ├── Train model.ipynb ├── coreml_ios ├── .gitignore ├── Core ML Demo.xcodeproj │ ├── project.pbxproj │ └── project.xcworkspace │ │ ├── contents.xcworkspacedata │ │ └── xcshareddata │ │ └── IDEWorkspaceChecks.plist ├── Core ML Demo.xcworkspace │ ├── contents.xcworkspacedata │ └── xcshareddata │ │ └── IDEWorkspaceChecks.plist ├── Core ML Demo │ ├── AppDelegate.swift │ ├── Assets.xcassets │ │ ├── AppIcon.appiconset │ │ │ └── Contents.json │ │ └── Contents.json │ ├── Base.lproj │ │ ├── LaunchScreen.storyboard │ │ └── Main.storyboard │ ├── CoreMLHelpers │ │ ├── Array.swift │ │ ├── CVPixelBuffer+Helpers.swift │ │ ├── CoreMLHelpers.h │ │ ├── Info.plist │ │ ├── MLMultiArray+Image.swift │ │ ├── Math.swift │ │ ├── MultiArray.swift │ │ ├── NonMaxSuppression.swift │ │ ├── Predictions.swift │ │ └── UIImage+CVPixelBuffer.swift │ ├── Info.plist │ ├── ViewController.swift │ └── VisionObjectRecognitionViewController.swift ├── CoreMLHelpers │ ├── Array+Extensions.swift │ ├── CGImage+CVPixelBuffer.swift │ ├── CGImage+RawBytes.swift │ ├── CGImagePropertyOrientation.swift │ ├── CVPixelBuffer+Helpers.swift │ ├── MLMultiArray+Helpers.swift │ ├── MLMultiArray+Image.swift │ ├── Math.swift │ ├── NonMaxSuppression.swift │ ├── Predictions.swift │ ├── UIImage+CVPixelBuffer.swift │ ├── UIImage+Extensions.swift │ └── UIImage+RawBytes.swift ├── Podfile └── Podfile.lock ├── images └── header.jpg ├── tf_lite_android └── demo │ ├── .gitignore │ ├── README.md │ ├── app │ ├── build.gradle │ └── src │ │ └── main │ │ ├── AndroidManifest.xml │ │ ├── BUILD │ │ ├── assets │ │ ├── BUILD │ │ ├── converted_model.tflite │ │ └── labels_emotion.txt │ │ ├── java │ │ └── me │ │ │ └── ndres │ │ │ └── tflitedemo │ │ │ ├── AutoFitTextureView.java │ │ │ ├── BorderedText.java │ │ │ ├── Camera2BasicFragment.java │ │ │ ├── CameraActivity.java │ │ │ ├── GpuDelegateHelper.java │ │ │ ├── ImageClassifier.java │ │ │ ├── ImageClassifierFloatInception.java │ │ │ ├── ImageClassifierFloatMobileNet.java │ │ │ ├── ImageClassifierQuantizedMobileNet.java │ │ │ └── OverlayView.java │ │ └── res │ │ ├── drawable-hdpi │ │ ├── ic_action_info.png │ │ ├── ic_launcher.png │ │ └── tile.9.png │ │ ├── drawable-mdpi │ │ ├── ic_action_info.png │ │ └── ic_launcher.png │ │ ├── drawable-xhdpi │ │ ├── ic_action_info.png │ │ └── ic_launcher.png │ │ ├── drawable-xxhdpi │ │ ├── ic_action_info.png │ │ ├── ic_launcher.png │ │ └── logo.png │ │ ├── drawable │ │ └── item_selector.xml │ │ ├── layout-land │ │ └── fragment_camera2_basic.xml │ │ ├── layout-v26 │ │ └── fragment_camera2_basic.xml │ │ ├── layout │ │ ├── activity_camera.xml │ │ ├── fragment_camera2_basic.xml │ │ └── listview_row.xml │ │ ├── values-sw600dp │ │ ├── template-dimens.xml │ │ └── template-styles.xml │ │ ├── values-v11 │ │ └── template-styles.xml │ │ ├── values-v21 │ │ ├── base-colors.xml │ │ └── base-template-styles.xml │ │ └── values │ │ ├── base-strings.xml │ │ ├── colors.xml │ │ ├── strings.xml │ │ ├── styles.xml │ │ ├── template-dimens.xml │ │ └── template-styles.xml │ ├── build.gradle │ ├── gradle.properties │ ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties │ ├── gradlew │ ├── gradlew.bat │ └── settings.gradle └── tfjs ├── .babelrc ├── .gitignore ├── cat.jpg ├── index.html ├── index.js ├── package-lock.json ├── package.json ├── serve.sh ├── static ├── emotion_detection │ ├── group1-shard1of4.bin │ ├── group1-shard2of4.bin │ ├── group1-shard3of4.bin │ ├── group1-shard4of4.bin │ └── model.json ├── emotion_detection_old │ ├── group1-shard1of4 │ ├── group1-shard2of4 │ ├── group1-shard3of4 │ ├── group1-shard4of4 │ └── model.json └── face_detection │ ├── tiny_face_detector_model-shard1 │ ├── tiny_face_detector_model-shard1.weight │ └── tiny_face_detector_model-weights_manifest.json ├── yarn-error.log └── yarn.lock /.gitignore: -------------------------------------------------------------------------------- 1 | *.pb 2 | *.mlmodel 3 | logs 4 | Pods 5 | .ipynb_checkpoints 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Eliot ANDRES 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Run TensorFlow 2 on any device 2 | 3 | ![header](./images/header.jpg) 4 | 5 | This project walks you through the process of creating an emotion recognition app on Android, iOS and in the browser: 6 | - Load data and train model 7 | - Export as SavedModel 8 | - Run on Android using TensorFlow Lite 9 | - Run on iOS using Core ML 10 | - Run in the browser using TensorFlow.js 11 | 12 | ## Demo 13 | 14 | ![header](./images/demo.jpg) 15 | 16 | A demo for the browser version is [available here](https://ndres.me/face-emotion/) 17 | 18 | 19 | ## Running in the browser 20 | Folder: `tfjs` 21 | 22 | Install [yarn](https://yarnpkg.com/en/) and run: 23 | 24 | cd tfjs 25 | yarn 26 | yarn watch 27 | 28 | ## Running on iOS 29 | Folder: coreml_ios 30 | You must have Xcode and [Cocoa Pods](https://cocoapods.org/) installed as well as a mac computer. 31 | 32 | cd coreml_ios/app 33 | pod install 34 | open Core\ ML\ Demo.xcworkspace 35 | 36 | ## Running on Android 37 | It is recommended to have [Android Studio](https://developer.android.com/studio) installed. 38 | Import the project and run the app. 39 | -------------------------------------------------------------------------------- /coreml_ios/.gitignore: -------------------------------------------------------------------------------- 1 | xcuserdata 2 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo.xcodeproj/project.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | IDEDidComputeMac32BitWarning 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | IDEDidComputeMac32BitWarning 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/AppDelegate.swift: -------------------------------------------------------------------------------- 1 | // 2 | // AppDelegate.swift 3 | // Core ML Demo 4 | // 5 | // Created by Eliot ANDRES on 23/12/2018. 6 | // Copyright © 2018 Eliot ANDRES. All rights reserved. 7 | // 8 | 9 | import UIKit 10 | 11 | @UIApplicationMain 12 | class AppDelegate: UIResponder, UIApplicationDelegate { 13 | 14 | var window: UIWindow? 15 | 16 | 17 | func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool { 18 | // Override point for customization after application launch. 19 | return true 20 | } 21 | 22 | func applicationWillResignActive(_ application: UIApplication) { 23 | // Sent when the application is about to move from active to inactive state. This can occur for certain types of temporary interruptions (such as an incoming phone call or SMS message) or when the user quits the application and it begins the transition to the background state. 24 | // Use this method to pause ongoing tasks, disable timers, and invalidate graphics rendering callbacks. Games should use this method to pause the game. 25 | } 26 | 27 | func applicationDidEnterBackground(_ application: UIApplication) { 28 | // Use this method to release shared resources, save user data, invalidate timers, and store enough application state information to restore your application to its current state in case it is terminated later. 29 | // If your application supports background execution, this method is called instead of applicationWillTerminate: when the user quits. 30 | } 31 | 32 | func applicationWillEnterForeground(_ application: UIApplication) { 33 | // Called as part of the transition from the background to the active state; here you can undo many of the changes made on entering the background. 34 | } 35 | 36 | func applicationDidBecomeActive(_ application: UIApplication) { 37 | // Restart any tasks that were paused (or not yet started) while the application was inactive. If the application was previously in the background, optionally refresh the user interface. 38 | } 39 | 40 | func applicationWillTerminate(_ application: UIApplication) { 41 | // Called when the application is about to terminate. Save data if appropriate. See also applicationDidEnterBackground:. 42 | } 43 | 44 | 45 | } 46 | 47 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/Assets.xcassets/AppIcon.appiconset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "idiom" : "iphone", 5 | "size" : "20x20", 6 | "scale" : "2x" 7 | }, 8 | { 9 | "idiom" : "iphone", 10 | "size" : "20x20", 11 | "scale" : "3x" 12 | }, 13 | { 14 | "idiom" : "iphone", 15 | "size" : "29x29", 16 | "scale" : "2x" 17 | }, 18 | { 19 | "idiom" : "iphone", 20 | "size" : "29x29", 21 | "scale" : "3x" 22 | }, 23 | { 24 | "idiom" : "iphone", 25 | "size" : "40x40", 26 | "scale" : "2x" 27 | }, 28 | { 29 | "idiom" : "iphone", 30 | "size" : "40x40", 31 | "scale" : "3x" 32 | }, 33 | { 34 | "idiom" : "iphone", 35 | "size" : "60x60", 36 | "scale" : "2x" 37 | }, 38 | { 39 | "idiom" : "iphone", 40 | "size" : "60x60", 41 | "scale" : "3x" 42 | }, 43 | { 44 | "idiom" : "ipad", 45 | "size" : "20x20", 46 | "scale" : "1x" 47 | }, 48 | { 49 | "idiom" : "ipad", 50 | "size" : "20x20", 51 | "scale" : "2x" 52 | }, 53 | { 54 | "idiom" : "ipad", 55 | "size" : "29x29", 56 | "scale" : "1x" 57 | }, 58 | { 59 | "idiom" : "ipad", 60 | "size" : "29x29", 61 | "scale" : "2x" 62 | }, 63 | { 64 | "idiom" : "ipad", 65 | "size" : "40x40", 66 | "scale" : "1x" 67 | }, 68 | { 69 | "idiom" : "ipad", 70 | "size" : "40x40", 71 | "scale" : "2x" 72 | }, 73 | { 74 | "idiom" : "ipad", 75 | "size" : "76x76", 76 | "scale" : "1x" 77 | }, 78 | { 79 | "idiom" : "ipad", 80 | "size" : "76x76", 81 | "scale" : "2x" 82 | }, 83 | { 84 | "idiom" : "ipad", 85 | "size" : "83.5x83.5", 86 | "scale" : "2x" 87 | }, 88 | { 89 | "idiom" : "ios-marketing", 90 | "size" : "1024x1024", 91 | "scale" : "1x" 92 | } 93 | ], 94 | "info" : { 95 | "version" : 1, 96 | "author" : "xcode" 97 | } 98 | } -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/Assets.xcassets/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "info" : { 3 | "version" : 1, 4 | "author" : "xcode" 5 | } 6 | } -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/Base.lproj/LaunchScreen.storyboard: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/Base.lproj/Main.storyboard: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/CoreMLHelpers/Array.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | import Swift 24 | 25 | extension Array where Element: Comparable { 26 | /** 27 | Returns the index and value of the largest element in the array. 28 | */ 29 | public func argmax() -> (Int, Element) { 30 | precondition(self.count > 0) 31 | var maxIndex = 0 32 | var maxValue = self[0] 33 | for i in 1.. maxValue { 35 | maxValue = self[i] 36 | maxIndex = i 37 | } 38 | } 39 | return (maxIndex, maxValue) 40 | } 41 | 42 | /** 43 | Returns the indices of the array's elements in sorted order. 44 | */ 45 | public func argsort(by areInIncreasingOrder: (Element, Element) -> Bool) -> [Array.Index] { 46 | return self.indices.sorted { areInIncreasingOrder(self[$0], self[$1]) } 47 | } 48 | 49 | /** 50 | Returns a new array containing the elements at the specified indices. 51 | */ 52 | public func gather(indices: [Array.Index]) -> [Element] { 53 | var a = [Element]() 54 | for i in indices { a.append(self[i]) } 55 | return a 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/CoreMLHelpers/CVPixelBuffer+Helpers.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | import Foundation 24 | import Accelerate 25 | import CoreImage 26 | 27 | /** 28 | Creates a RGB pixel buffer of the specified width and height. 29 | */ 30 | public func createPixelBuffer(width: Int, height: Int) -> CVPixelBuffer? { 31 | var pixelBuffer: CVPixelBuffer? 32 | let status = CVPixelBufferCreate(nil, width, height, 33 | kCVPixelFormatType_32BGRA, nil, 34 | &pixelBuffer) 35 | if status != kCVReturnSuccess { 36 | print("Error: could not create resized pixel buffer", status) 37 | return nil 38 | } 39 | return pixelBuffer 40 | } 41 | 42 | /** 43 | First crops the pixel buffer, then resizes it. 44 | */ 45 | public func resizePixelBuffer(_ srcPixelBuffer: CVPixelBuffer, 46 | cropX: Int, 47 | cropY: Int, 48 | cropWidth: Int, 49 | cropHeight: Int, 50 | scaleWidth: Int, 51 | scaleHeight: Int) -> CVPixelBuffer? { 52 | 53 | CVPixelBufferLockBaseAddress(srcPixelBuffer, CVPixelBufferLockFlags(rawValue: 0)) 54 | guard let srcData = CVPixelBufferGetBaseAddress(srcPixelBuffer) else { 55 | print("Error: could not get pixel buffer base address") 56 | return nil 57 | } 58 | let srcBytesPerRow = CVPixelBufferGetBytesPerRow(srcPixelBuffer) 59 | let offset = cropY*srcBytesPerRow + cropX*4 60 | var srcBuffer = vImage_Buffer(data: srcData.advanced(by: offset), 61 | height: vImagePixelCount(cropHeight), 62 | width: vImagePixelCount(cropWidth), 63 | rowBytes: srcBytesPerRow) 64 | 65 | let destBytesPerRow = scaleWidth*4 66 | guard let destData = malloc(scaleHeight*destBytesPerRow) else { 67 | print("Error: out of memory") 68 | return nil 69 | } 70 | var destBuffer = vImage_Buffer(data: destData, 71 | height: vImagePixelCount(scaleHeight), 72 | width: vImagePixelCount(scaleWidth), 73 | rowBytes: destBytesPerRow) 74 | 75 | let error = vImageScale_ARGB8888(&srcBuffer, &destBuffer, nil, vImage_Flags(0)) 76 | CVPixelBufferUnlockBaseAddress(srcPixelBuffer, CVPixelBufferLockFlags(rawValue: 0)) 77 | if error != kvImageNoError { 78 | print("Error:", error) 79 | free(destData) 80 | return nil 81 | } 82 | 83 | let releaseCallback: CVPixelBufferReleaseBytesCallback = { _, ptr in 84 | if let ptr = ptr { 85 | free(UnsafeMutableRawPointer(mutating: ptr)) 86 | } 87 | } 88 | 89 | let pixelFormat = CVPixelBufferGetPixelFormatType(srcPixelBuffer) 90 | var dstPixelBuffer: CVPixelBuffer? 91 | let status = CVPixelBufferCreateWithBytes(nil, scaleWidth, scaleHeight, 92 | pixelFormat, destData, 93 | destBytesPerRow, releaseCallback, 94 | nil, nil, &dstPixelBuffer) 95 | if status != kCVReturnSuccess { 96 | print("Error: could not create new pixel buffer") 97 | free(destData) 98 | return nil 99 | } 100 | return dstPixelBuffer 101 | } 102 | 103 | /** 104 | Resizes a CVPixelBuffer to a new width and height. 105 | */ 106 | public func resizePixelBuffer(_ pixelBuffer: CVPixelBuffer, 107 | width: Int, height: Int) -> CVPixelBuffer? { 108 | return resizePixelBuffer(pixelBuffer, cropX: 0, cropY: 0, 109 | cropWidth: CVPixelBufferGetWidth(pixelBuffer), 110 | cropHeight: CVPixelBufferGetHeight(pixelBuffer), 111 | scaleWidth: width, scaleHeight: height) 112 | } 113 | 114 | /** 115 | Resizes a CVPixelBuffer to a new width and height. 116 | */ 117 | public func resizePixelBuffer(_ pixelBuffer: CVPixelBuffer, 118 | width: Int, height: Int, 119 | output: CVPixelBuffer, context: CIContext) { 120 | let ciImage = CIImage(cvPixelBuffer: pixelBuffer) 121 | let sx = CGFloat(width) / CGFloat(CVPixelBufferGetWidth(pixelBuffer)) 122 | let sy = CGFloat(height) / CGFloat(CVPixelBufferGetHeight(pixelBuffer)) 123 | let scaleTransform = CGAffineTransform(scaleX: sx, y: sy) 124 | let scaledImage = ciImage.transformed(by: scaleTransform) 125 | context.render(scaledImage, to: output) 126 | } 127 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/CoreMLHelpers/CoreMLHelpers.h: -------------------------------------------------------------------------------- 1 | // 2 | // CoreMLHelpers.h 3 | // CoreMLHelpers 4 | // 5 | // Created by Francescu Santoni on 19/02/2018. 6 | // Copyright © 2018 Archery Inc. All rights reserved. 7 | // 8 | 9 | #import 10 | 11 | //! Project version number for CoreMLHelpers. 12 | FOUNDATION_EXPORT double CoreMLHelpersVersionNumber; 13 | 14 | //! Project version string for CoreMLHelpers. 15 | FOUNDATION_EXPORT const unsigned char CoreMLHelpersVersionString[]; 16 | 17 | // In this header, you should import all the public headers of your framework using statements like #import 18 | 19 | 20 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/CoreMLHelpers/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | $(DEVELOPMENT_LANGUAGE) 7 | CFBundleExecutable 8 | $(EXECUTABLE_NAME) 9 | CFBundleIdentifier 10 | $(PRODUCT_BUNDLE_IDENTIFIER) 11 | CFBundleInfoDictionaryVersion 12 | 6.0 13 | CFBundleName 14 | $(PRODUCT_NAME) 15 | CFBundlePackageType 16 | FMWK 17 | CFBundleShortVersionString 18 | 1.0 19 | CFBundleVersion 20 | $(CURRENT_PROJECT_VERSION) 21 | NSPrincipalClass 22 | 23 | UIRequiredDeviceCapabilities 24 | 25 | arm64 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/CoreMLHelpers/MLMultiArray+Image.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | import Foundation 24 | import CoreML 25 | 26 | extension MLMultiArray { 27 | /** 28 | Converts the multi-array to a UIImage. 29 | */ 30 | public func image(offset: T, scale: T) -> UIImage? { 31 | return MultiArray(self).image(offset: offset, scale: scale) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/CoreMLHelpers/Math.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | import Foundation 24 | 25 | public func clamp(_ x: T, min: T, max: T) -> T { 26 | if x < min { return min } 27 | if x > max { return max } 28 | return x 29 | } 30 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/CoreMLHelpers/MultiArray.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | import Foundation 24 | import CoreML 25 | import Swift 26 | 27 | public protocol MultiArrayType: Comparable { 28 | static var multiArrayDataType: MLMultiArrayDataType { get } 29 | static func +(lhs: Self, rhs: Self) -> Self 30 | static func *(lhs: Self, rhs: Self) -> Self 31 | init(_: Int) 32 | var toUInt8: UInt8 { get } 33 | } 34 | 35 | extension Double: MultiArrayType { 36 | public static var multiArrayDataType: MLMultiArrayDataType { return .double } 37 | public var toUInt8: UInt8 { return UInt8(self) } 38 | } 39 | 40 | extension Float: MultiArrayType { 41 | public static var multiArrayDataType: MLMultiArrayDataType { return .float32 } 42 | public var toUInt8: UInt8 { return UInt8(self) } 43 | } 44 | 45 | extension Int32: MultiArrayType { 46 | public static var multiArrayDataType: MLMultiArrayDataType { return .int32 } 47 | public var toUInt8: UInt8 { return UInt8(self) } 48 | } 49 | 50 | /** 51 | Wrapper around MLMultiArray to make it more Swifty. 52 | */ 53 | public struct MultiArray { 54 | public let array: MLMultiArray 55 | public let pointer: UnsafeMutablePointer 56 | 57 | private(set) public var strides: [Int] 58 | private(set) public var shape: [Int] 59 | 60 | /** 61 | Creates a new multi-array filled with all zeros. 62 | */ 63 | public init(shape: [Int]) { 64 | let m = try! MLMultiArray(shape: shape as [NSNumber], dataType: T.multiArrayDataType) 65 | self.init(m) 66 | memset(pointer, 0, MemoryLayout.stride * count) 67 | } 68 | 69 | /** 70 | Creates a new multi-array initialized with the specified value. 71 | */ 72 | public init(shape: [Int], initial: T) { 73 | self.init(shape: shape) 74 | for i in 0..(OpaquePointer(array.dataPointer)) 91 | } 92 | 93 | /** 94 | Returns the number of elements in the entire array. 95 | */ 96 | public var count: Int { 97 | return shape.reduce(1, *) 98 | } 99 | 100 | public subscript(a: Int) -> T { 101 | get { return pointer[a] } 102 | set { pointer[a] = newValue } 103 | } 104 | 105 | public subscript(a: Int, b: Int) -> T { 106 | get { return pointer[a*strides[0] + b*strides[1]] } 107 | set { pointer[a*strides[0] + b*strides[1]] = newValue } 108 | } 109 | 110 | public subscript(a: Int, b: Int, c: Int) -> T { 111 | get { return pointer[a*strides[0] + b*strides[1] + c*strides[2]] } 112 | set { pointer[a*strides[0] + b*strides[1] + c*strides[2]] = newValue } 113 | } 114 | 115 | public subscript(a: Int, b: Int, c: Int, d: Int) -> T { 116 | get { return pointer[a*strides[0] + b*strides[1] + c*strides[2] + d*strides[3]] } 117 | set { pointer[a*strides[0] + b*strides[1] + c*strides[2] + d*strides[3]] = newValue } 118 | } 119 | 120 | public subscript(a: Int, b: Int, c: Int, d: Int, e: Int) -> T { 121 | get { return pointer[a*strides[0] + b*strides[1] + c*strides[2] + d*strides[3] + e*strides[4]] } 122 | set { pointer[a*strides[0] + b*strides[1] + c*strides[2] + d*strides[3] + e*strides[4]] = newValue } 123 | } 124 | 125 | public subscript(indices: [Int]) -> T { 126 | get { return pointer[offset(for: indices)] } 127 | set { pointer[offset(for: indices)] = newValue } 128 | } 129 | 130 | func offset(for indices: [Int]) -> Int { 131 | var offset = 0 132 | for i in 0.. MultiArray { 143 | precondition(order.count == strides.count) 144 | var newShape = shape 145 | var newStrides = strides 146 | for i in 0.. MultiArray { 157 | let newCount = dimensions.reduce(1, *) 158 | precondition(newCount == count, "Cannot reshape \(shape) to \(dimensions)") 159 | 160 | var newStrides = [Int](repeating: 0, count: dimensions.count) 161 | newStrides[dimensions.count - 1] = 1 162 | for i in stride(from: dimensions.count - 1, to: 0, by: -1) { 163 | newStrides[i - 1] = newStrides[i] * dimensions[i] 164 | } 165 | 166 | return MultiArray(array, dimensions, newStrides) 167 | } 168 | } 169 | 170 | extension MultiArray: CustomStringConvertible { 171 | public var description: String { 172 | return description([]) 173 | } 174 | 175 | func description(_ indices: [Int]) -> String { 176 | func indent(_ x: Int) -> String { 177 | return String(repeating: " ", count: x) 178 | } 179 | 180 | // This function is called recursively for every dimension. 181 | // Add an entry for this dimension to the end of the array. 182 | var indices = indices + [0] 183 | 184 | let d = indices.count - 1 // the current dimension 185 | let N = shape[d] // how many elements in this dimension 186 | 187 | var s = "[" 188 | if indices.count < shape.count { // not last dimension yet? 189 | for i in 0.. UIImage? { 226 | if shape.count == 3, let (b, w, h) = toRawBytesRGBA(offset: offset, scale: scale) { 227 | return UIImage.fromByteArrayRGBA(b, width: w, height: h) 228 | } else if shape.count == 2, let (b, w, h) = toRawBytesGray(offset: offset, scale: scale) { 229 | return UIImage.fromByteArrayGray(b, width: w, height: h) 230 | } else { 231 | return nil 232 | } 233 | } 234 | 235 | /** 236 | Converts the multi-array into an array of RGBA pixels. 237 | 238 | - Note: The multi-array must have shape (3, height, width). If your array 239 | has a different shape, use `reshape()` or `transpose()` first. 240 | */ 241 | public func toRawBytesRGBA(offset: T, scale: T) 242 | -> (bytes: [UInt8], width: Int, height: Int)? { 243 | guard shape.count == 3 else { 244 | print("Expected a multi-array with 3 dimensions, got \(shape)") 245 | return nil 246 | } 247 | guard shape[0] == 3 else { 248 | print("Expected first dimension to have 3 channels, got \(shape[0])") 249 | return nil 250 | } 251 | 252 | let height = shape[1] 253 | let width = shape[2] 254 | var bytes = [UInt8](repeating: 0, count: height * width * 4) 255 | 256 | for h in 0.. (bytes: [UInt8], width: Int, height: Int)? { 280 | guard shape.count == 2 else { 281 | print("Expected a multi-array with 2 dimensions, got \(shape)") 282 | return nil 283 | } 284 | 285 | let height = shape[0] 286 | let width = shape[1] 287 | var bytes = [UInt8](repeating: 0, count: height * width) 288 | 289 | for h in 0.. UIImage? { 306 | guard shape.count == 3 else { 307 | print("Expected a multi-array with 3 dimensions, got \(shape)") 308 | return nil 309 | } 310 | guard channel >= 0 && channel < shape[0] else { 311 | print("Channel must be between 0 and \(shape[0] - 1)") 312 | return nil 313 | } 314 | 315 | let height = shape[1] 316 | let width = shape[2] 317 | var a = MultiArray(shape: [height, width]) 318 | for y in 0.. Float { 32 | let areaA = a.width * a.height 33 | if areaA <= 0 { return 0 } 34 | 35 | let areaB = b.width * b.height 36 | if areaB <= 0 { return 0 } 37 | 38 | let intersectionMinX = max(a.minX, b.minX) 39 | let intersectionMinY = max(a.minY, b.minY) 40 | let intersectionMaxX = min(a.maxX, b.maxX) 41 | let intersectionMaxY = min(a.maxY, b.maxY) 42 | let intersectionArea = max(intersectionMaxY - intersectionMinY, 0) * 43 | max(intersectionMaxX - intersectionMinX, 0) 44 | return Float(intersectionArea / (areaA + areaB - intersectionArea)) 45 | } 46 | 47 | public typealias NMSPrediction = (classIndex: Int, score: Float, rect: CGRect) 48 | 49 | /** 50 | Removes bounding boxes that overlap too much with other boxes that have 51 | a higher score. 52 | */ 53 | public func nonMaxSuppression(predictions: [NMSPrediction], iouThreshold: Float, maxBoxes: Int) -> [Int] { 54 | return nonMaxSuppression(predictions: predictions, 55 | indices: Array(predictions.indices), 56 | iouThreshold: iouThreshold, 57 | maxBoxes: maxBoxes) 58 | } 59 | 60 | /** 61 | Removes bounding boxes that overlap too much with other boxes that have 62 | a higher score. 63 | 64 | Based on code from https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/non_max_suppression_op.cc 65 | 66 | - Note: This version of NMS ignores the class of the bounding boxes. Since it 67 | selects the bounding boxes in a greedy fashion, if a certain class has many 68 | boxes that are selected, then it is possible none of the boxes of the other 69 | classes get selected. 70 | 71 | - Parameters: 72 | - predictions: an array of bounding boxes and their scores 73 | - indices: which predictions to look at 74 | - iouThreshold: used to decide whether boxes overlap too much 75 | - maxBoxes: the maximum number of boxes that will be selected 76 | 77 | - Returns: the array indices of the selected bounding boxes 78 | */ 79 | public func nonMaxSuppression(predictions: [NMSPrediction], 80 | indices: [Int], 81 | iouThreshold: Float, 82 | maxBoxes: Int) -> [Int] { 83 | 84 | // Sort the boxes based on their confidence scores, from high to low. 85 | let sortedIndices = indices.sorted { predictions[$0].score > predictions[$1].score } 86 | 87 | var selected: [Int] = [] 88 | 89 | // Loop through the bounding boxes, from highest score to lowest score, 90 | // and determine whether or not to keep each box. 91 | for i in 0..= maxBoxes { break } 93 | 94 | var shouldSelect = true 95 | let boxA = predictions[sortedIndices[i]] 96 | 97 | // Does the current box overlap one of the selected boxes more than the 98 | // given threshold amount? Then it's too similar, so don't keep it. 99 | for j in 0.. iouThreshold { 102 | shouldSelect = false 103 | break 104 | } 105 | } 106 | 107 | // This bounding box did not overlap too much with any previously selected 108 | // bounding box, so we'll keep it. 109 | if shouldSelect { 110 | selected.append(sortedIndices[i]) 111 | } 112 | } 113 | 114 | return selected 115 | } 116 | 117 | /** 118 | Multi-class version of non maximum suppression. 119 | 120 | Where `nonMaxSuppression()` does not look at the class of the predictions at 121 | all, the multi-class version first selects the best bounding boxes for each 122 | class, and then keeps the best ones of those. 123 | 124 | With this method you can usually expect to see at least one bounding box for 125 | each class (unless all the scores for a given class are really low). 126 | 127 | Based on code from: https://github.com/tensorflow/models/blob/master/object_detection/core/post_processing.py 128 | 129 | - Parameters: 130 | - numClasses: the number of classes 131 | - predictions: an array of bounding boxes and their scores 132 | - scoreThreshold: used to only keep bounding boxes with a high enough score 133 | - iouThreshold: used to decide whether boxes overlap too much 134 | - maxPerClass: the maximum number of boxes that will be selected per class 135 | - maxTotal: maximum number of boxes that will be selected over all classes 136 | 137 | - Returns: the array indices of the selected bounding boxes 138 | */ 139 | public func nonMaxSuppressionMultiClass(numClasses: Int, 140 | predictions: [NMSPrediction], 141 | scoreThreshold: Float, 142 | iouThreshold: Float, 143 | maxPerClass: Int, 144 | maxTotal: Int) -> [Int] { 145 | var selectedBoxes: [Int] = [] 146 | 147 | // Look at all the classes one-by-one. 148 | for c in 0.. scoreThreshold { 158 | filteredBoxes.append(p) 159 | } 160 | } 161 | } 162 | 163 | // Only keep the best bounding boxes for this class. 164 | let nmsBoxes = nonMaxSuppression(predictions: predictions, 165 | indices: filteredBoxes, 166 | iouThreshold: iouThreshold, 167 | maxBoxes: maxPerClass) 168 | 169 | // Add the indices of the surviving boxes to the big list. 170 | selectedBoxes.append(contentsOf: nmsBoxes) 171 | } 172 | 173 | // Sort all the surviving boxes by score and only keep the best ones. 174 | let sortedBoxes = selectedBoxes.sorted { predictions[$0].score > predictions[$1].score } 175 | return Array(sortedBoxes.prefix(maxTotal)) 176 | } 177 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/CoreMLHelpers/Predictions.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | import Vision 24 | 25 | /** 26 | Returns the top `k` predictions from Core ML classification results as an 27 | array of `(String, Double)` pairs. 28 | */ 29 | public func top(_ k: Int, _ prob: [String: Double]) -> [(String, Double)] { 30 | return Array(prob.map { x in (x.key, x.value) } 31 | .sorted(by: { a, b -> Bool in a.1 > b.1 }) 32 | .prefix(through: min(k, prob.count) - 1)) 33 | } 34 | 35 | /** 36 | Returns the top `k` predictions from Vision classification results as an 37 | array of `(String, Double)` pairs. 38 | */ 39 | public func top(_ k: Int, _ observations: [VNClassificationObservation]) -> [(String, Double)] { 40 | // The Vision observations are sorted by confidence already. 41 | return observations.prefix(through: min(k, observations.count) - 1) 42 | .map { ($0.identifier, Double($0.confidence)) } 43 | } 44 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/CoreMLHelpers/UIImage+CVPixelBuffer.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | import UIKit 24 | import VideoToolbox 25 | 26 | extension UIImage { 27 | /** 28 | Resizes the image to width x height and converts it to an RGB CVPixelBuffer. 29 | */ 30 | public func pixelBuffer(width: Int, height: Int) -> CVPixelBuffer? { 31 | return pixelBuffer(width: width, height: height, 32 | pixelFormatType: kCVPixelFormatType_32ARGB, 33 | colorSpace: CGColorSpaceCreateDeviceRGB(), 34 | alphaInfo: .noneSkipFirst) 35 | } 36 | 37 | /** 38 | Resizes the image to width x height and converts it to a grayscale CVPixelBuffer. 39 | */ 40 | public func pixelBufferGray(width: Int, height: Int) -> CVPixelBuffer? { 41 | return pixelBuffer(width: width, height: height, 42 | pixelFormatType: kCVPixelFormatType_OneComponent8, 43 | colorSpace: CGColorSpaceCreateDeviceGray(), 44 | alphaInfo: .none) 45 | } 46 | 47 | func pixelBuffer(width: Int, height: Int, pixelFormatType: OSType, 48 | colorSpace: CGColorSpace, alphaInfo: CGImageAlphaInfo) -> CVPixelBuffer? { 49 | var maybePixelBuffer: CVPixelBuffer? 50 | let attrs = [kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue, 51 | kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue] 52 | let status = CVPixelBufferCreate(kCFAllocatorDefault, 53 | width, 54 | height, 55 | pixelFormatType, 56 | attrs as CFDictionary, 57 | &maybePixelBuffer) 58 | 59 | guard status == kCVReturnSuccess, let pixelBuffer = maybePixelBuffer else { 60 | return nil 61 | } 62 | 63 | CVPixelBufferLockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0)) 64 | let pixelData = CVPixelBufferGetBaseAddress(pixelBuffer) 65 | 66 | guard let context = CGContext(data: pixelData, 67 | width: width, 68 | height: height, 69 | bitsPerComponent: 8, 70 | bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer), 71 | space: colorSpace, 72 | bitmapInfo: alphaInfo.rawValue) 73 | else { 74 | return nil 75 | } 76 | 77 | UIGraphicsPushContext(context) 78 | context.translateBy(x: 0, y: CGFloat(height)) 79 | context.scaleBy(x: 1, y: -1) 80 | self.draw(in: CGRect(x: 0, y: 0, width: width, height: height)) 81 | UIGraphicsPopContext() 82 | 83 | CVPixelBufferUnlockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0)) 84 | return pixelBuffer 85 | } 86 | } 87 | 88 | extension UIImage { 89 | /** 90 | Creates a new UIImage from a CVPixelBuffer. 91 | NOTE: This only works for RGB pixel buffers, not for grayscale. 92 | */ 93 | public convenience init?(pixelBuffer: CVPixelBuffer) { 94 | var cgImage: CGImage? 95 | VTCreateCGImageFromCVPixelBuffer(pixelBuffer, nil, &cgImage) 96 | 97 | if let cgImage = cgImage { 98 | self.init(cgImage: cgImage) 99 | } else { 100 | return nil 101 | } 102 | } 103 | 104 | /** 105 | Creates a new UIImage from a CVPixelBuffer, using Core Image. 106 | */ 107 | public convenience init?(pixelBuffer: CVPixelBuffer, context: CIContext) { 108 | let ciImage = CIImage(cvPixelBuffer: pixelBuffer) 109 | let rect = CGRect(x: 0, y: 0, width: CVPixelBufferGetWidth(pixelBuffer), 110 | height: CVPixelBufferGetHeight(pixelBuffer)) 111 | if let cgImage = context.createCGImage(ciImage, from: rect) { 112 | self.init(cgImage: cgImage) 113 | } else { 114 | return nil 115 | } 116 | } 117 | } 118 | 119 | extension UIImage { 120 | /** 121 | Creates a new UIImage from an array of RGBA bytes. 122 | */ 123 | @nonobjc public class func fromByteArrayRGBA(_ bytes: [UInt8], 124 | width: Int, 125 | height: Int, 126 | scale: CGFloat = 0, 127 | orientation: UIImageOrientation = .up) -> UIImage? { 128 | return fromByteArray(bytes, width: width, height: height, 129 | scale: scale, orientation: orientation, 130 | bytesPerRow: width * 4, 131 | colorSpace: CGColorSpaceCreateDeviceRGB(), 132 | alphaInfo: .premultipliedLast) 133 | } 134 | 135 | /** 136 | Creates a new UIImage from an array of grayscale bytes. 137 | */ 138 | @nonobjc public class func fromByteArrayGray(_ bytes: [UInt8], 139 | width: Int, 140 | height: Int, 141 | scale: CGFloat = 0, 142 | orientation: UIImageOrientation = .up) -> UIImage? { 143 | return fromByteArray(bytes, width: width, height: height, 144 | scale: scale, orientation: orientation, 145 | bytesPerRow: width, 146 | colorSpace: CGColorSpaceCreateDeviceGray(), 147 | alphaInfo: .none) 148 | } 149 | 150 | @nonobjc class func fromByteArray(_ bytes: [UInt8], 151 | width: Int, 152 | height: Int, 153 | scale: CGFloat, 154 | orientation: UIImageOrientation, 155 | bytesPerRow: Int, 156 | colorSpace: CGColorSpace, 157 | alphaInfo: CGImageAlphaInfo) -> UIImage? { 158 | var image: UIImage? 159 | bytes.withUnsafeBytes { ptr in 160 | if let context = CGContext(data: UnsafeMutableRawPointer(mutating: ptr.baseAddress!), 161 | width: width, 162 | height: height, 163 | bitsPerComponent: 8, 164 | bytesPerRow: bytesPerRow, 165 | space: colorSpace, 166 | bitmapInfo: alphaInfo.rawValue), 167 | let cgImage = context.makeImage() { 168 | image = UIImage(cgImage: cgImage, scale: scale, orientation: orientation) 169 | } 170 | } 171 | return image 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | $(DEVELOPMENT_LANGUAGE) 7 | CFBundleExecutable 8 | $(EXECUTABLE_NAME) 9 | CFBundleIdentifier 10 | $(PRODUCT_BUNDLE_IDENTIFIER) 11 | CFBundleInfoDictionaryVersion 12 | 6.0 13 | CFBundleName 14 | $(PRODUCT_NAME) 15 | CFBundlePackageType 16 | APPL 17 | CFBundleShortVersionString 18 | 1.0 19 | CFBundleVersion 20 | 1 21 | LSRequiresIPhoneOS 22 | 23 | NSCameraUsageDescription 24 | This sample uses the camera to capture images of breakfast items for object recognition. 25 | UILaunchStoryboardName 26 | LaunchScreen 27 | UIMainStoryboardFile 28 | Main 29 | UIRequiredDeviceCapabilities 30 | 31 | armv7 32 | 33 | UISupportedInterfaceOrientations 34 | 35 | UIInterfaceOrientationPortrait 36 | 37 | UISupportedInterfaceOrientations~ipad 38 | 39 | UIInterfaceOrientationPortrait 40 | UIInterfaceOrientationPortraitUpsideDown 41 | UIInterfaceOrientationLandscapeLeft 42 | UIInterfaceOrientationLandscapeRight 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/ViewController.swift: -------------------------------------------------------------------------------- 1 | /* 2 | See LICENSE folder for this sample’s licensing information. 3 | 4 | Abstract: 5 | Contains the view controller for the Breakfast Finder. 6 | */ 7 | 8 | import UIKit 9 | import AVFoundation 10 | import Vision 11 | 12 | class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate { 13 | 14 | var bufferSize: CGSize = .zero 15 | var rootLayer: CALayer! = nil 16 | 17 | private var detectionOverlay: CALayer! = nil 18 | 19 | @IBOutlet weak private var previewView: UIView! 20 | 21 | private let session = AVCaptureSession() 22 | private var previewLayer: AVCaptureVideoPreviewLayer! = nil 23 | private let videoDataOutput = AVCaptureVideoDataOutput() 24 | 25 | private let videoDataOutputQueue = DispatchQueue(label: "VideoDataOutput", qos: .userInitiated, attributes: [], autoreleaseFrequency: .workItem) 26 | 27 | func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { 28 | // to be implemented in the subclass 29 | } 30 | 31 | override func viewDidLoad() { 32 | super.viewDidLoad() 33 | setupAVCapture() 34 | setupLayers() 35 | 36 | 37 | } 38 | 39 | override func didReceiveMemoryWarning() { 40 | super.didReceiveMemoryWarning() 41 | // Dispose of any resources that can be recreated. 42 | } 43 | 44 | func setupLayers() { 45 | detectionOverlay = CALayer() // container layer that has all the renderings of the observations 46 | detectionOverlay.name = "DetectionOverlay" 47 | detectionOverlay.bounds = CGRect(x: 0.0, 48 | y: 0.0, 49 | width: bufferSize.width, 50 | height: bufferSize.height) 51 | detectionOverlay.position = CGPoint(x: rootLayer.bounds.midX, y: rootLayer.bounds.midY) 52 | rootLayer.addSublayer(detectionOverlay) 53 | } 54 | 55 | func setupAVCapture() { 56 | var deviceInput: AVCaptureDeviceInput! 57 | 58 | // Select a video device, make an input 59 | let videoDevice = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInWideAngleCamera], mediaType: .video, position: .front).devices.first 60 | do { 61 | deviceInput = try AVCaptureDeviceInput(device: videoDevice!) 62 | } catch { 63 | print("Could not create video device input: \(error)") 64 | return 65 | } 66 | 67 | session.beginConfiguration() 68 | session.sessionPreset = .vga640x480 // Model image size is smaller. 69 | 70 | // Add a video input 71 | guard session.canAddInput(deviceInput) else { 72 | print("Could not add video device input to the session") 73 | session.commitConfiguration() 74 | return 75 | } 76 | session.addInput(deviceInput) 77 | if session.canAddOutput(videoDataOutput) { 78 | session.addOutput(videoDataOutput) 79 | // Add a video data output 80 | videoDataOutput.alwaysDiscardsLateVideoFrames = true 81 | videoDataOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)] 82 | videoDataOutput.setSampleBufferDelegate(self, queue: videoDataOutputQueue) 83 | } else { 84 | print("Could not add video data output to the session") 85 | session.commitConfiguration() 86 | return 87 | } 88 | let captureConnection = videoDataOutput.connection(with: .video) 89 | // Always process the frames 90 | captureConnection?.isEnabled = true 91 | do { 92 | try videoDevice!.lockForConfiguration() 93 | let dimensions = CMVideoFormatDescriptionGetDimensions((videoDevice?.activeFormat.formatDescription)!) 94 | bufferSize.width = CGFloat(dimensions.width) 95 | bufferSize.height = CGFloat(dimensions.height) 96 | videoDevice!.unlockForConfiguration() 97 | } catch { 98 | print(error) 99 | } 100 | session.commitConfiguration() 101 | previewLayer = AVCaptureVideoPreviewLayer(session: session) 102 | previewLayer.videoGravity = AVLayerVideoGravity.resizeAspectFill 103 | rootLayer = previewView.layer 104 | previewLayer.frame = rootLayer.bounds 105 | rootLayer.addSublayer(previewLayer) 106 | } 107 | 108 | func startCaptureSession() { 109 | session.startRunning() 110 | } 111 | 112 | // Clean up capture setup 113 | func teardownAVCapture() { 114 | previewLayer.removeFromSuperlayer() 115 | previewLayer = nil 116 | } 117 | 118 | func captureOutput(_ captureOutput: AVCaptureOutput, didDrop didDropSampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { 119 | // print("frame dropped") 120 | } 121 | 122 | public func exifOrientationFromDeviceOrientation() -> CGImagePropertyOrientation { 123 | let curDeviceOrientation = UIDevice.current.orientation 124 | let exifOrientation: CGImagePropertyOrientation 125 | 126 | switch curDeviceOrientation { 127 | case UIDeviceOrientation.portraitUpsideDown: // Device oriented vertically, home button on the top 128 | exifOrientation = .left 129 | case UIDeviceOrientation.landscapeLeft: // Device oriented horizontally, home button on the right 130 | exifOrientation = .upMirrored 131 | case UIDeviceOrientation.landscapeRight: // Device oriented horizontally, home button on the left 132 | exifOrientation = .down 133 | case UIDeviceOrientation.portrait: // Device oriented vertically, home button on the bottom 134 | exifOrientation = .up 135 | default: 136 | exifOrientation = .up 137 | } 138 | return exifOrientation 139 | } 140 | } 141 | 142 | -------------------------------------------------------------------------------- /coreml_ios/Core ML Demo/VisionObjectRecognitionViewController.swift: -------------------------------------------------------------------------------- 1 | /* 2 | See LICENSE folder for this sample’s licensing information. 3 | 4 | Abstract: 5 | Contains the object recognition view controller for the Breakfast Finder. 6 | */ 7 | 8 | import UIKit 9 | import AVFoundation 10 | import Vision 11 | //import CoreMLHelpers 12 | 13 | class VisionObjectRecognitionViewController: ViewController { 14 | 15 | var detectionOverlayLayer: CALayer? 16 | var detectedFaceRectangleShapeLayer: CAShapeLayer? 17 | @IBOutlet weak var classificationLabel: UILabel! 18 | 19 | private let faceDetectionRequest = VNDetectFaceRectanglesRequest() 20 | private let faceDetectionHandler = VNSequenceRequestHandler() 21 | 22 | /// - Tag: MLModelSetup 23 | lazy var classificationRequest: VNCoreMLRequest = { 24 | do { 25 | /* 26 | Use the Swift class `MobileNet` Core ML generates from the model. 27 | To use a different Core ML classifier model, add it to the project 28 | and replace `MobileNet` with that model's generated Swift class. 29 | */ 30 | let model = try VNCoreMLModel(for: mobilenet().model) 31 | let request = VNCoreMLRequest(model: model, completionHandler: { [weak self] request, error in 32 | self?.processClassifications(for: request, error: error) 33 | }) 34 | request.imageCropAndScaleOption = .centerCrop 35 | return request 36 | } catch { 37 | fatalError("Failed to load Vision ML model: \(error)") 38 | } 39 | }() 40 | 41 | private lazy var dumbModel: VNCoreMLModel = try! VNCoreMLModel(for: DumbDetection().model) 42 | private lazy var dumbRequest: VNCoreMLRequest = { 43 | let request = VNCoreMLRequest(model: dumbModel) 44 | request.imageCropAndScaleOption = .scaleFill 45 | return request 46 | }() 47 | 48 | /// Updates the UI with the results of the classification. 49 | /// - Tag: ProcessClassifications 50 | func processClassifications(for request: VNRequest, error: Error?) { 51 | DispatchQueue.main.async { 52 | guard let results = request.results else { 53 | self.classificationLabel.text = "Unable to classify image.\n\(error!.localizedDescription)" 54 | return 55 | } 56 | // The `results` will always be `VNClassificationObservation`s, as specified by the Core ML model in this project. 57 | let classifications = results as! [VNClassificationObservation] 58 | if classifications.isEmpty { 59 | self.classificationLabel.text = "Nothing recognized." 60 | } else { 61 | // Display top classifications ranked by confidence in the UI. 62 | let topClassifications = classifications.prefix(3) 63 | 64 | let descriptions = topClassifications.map { classification in 65 | // Formats the classification for display; e.g. "(0.37) cliff, drop, drop-off". 66 | return String(format: " (%.2f) %@", classification.confidence, classification.identifier) 67 | } 68 | self.classificationLabel.text = "Classification:\n" + descriptions.joined(separator: "\n") 69 | } 70 | } 71 | } 72 | 73 | 74 | override func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { 75 | guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { 76 | return 77 | } 78 | 79 | let exifOrientation = exifOrientationFromDeviceOrientation() 80 | 81 | do { 82 | try faceDetectionHandler.perform([faceDetectionRequest], on: pixelBuffer, orientation: exifOrientation) 83 | 84 | guard let faceObservations = faceDetectionRequest.results as? [VNFaceObservation], faceObservations.isEmpty == false else { 85 | return 86 | } 87 | 88 | drawFaceObservations(faceObservations) 89 | 90 | for faceObservation in faceObservations { 91 | 92 | let classificationHandler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, orientation: .right, options: [:]) 93 | 94 | let box = faceObservation.boundingBox 95 | let region = CGRect(x: box.minY, y: 1 - box.maxX, width: box.height, height:box.width) 96 | // let percentage: CGFloat = 0.1 97 | // let largeRegion = region.insetBy(dx: region.width * -percentage, dy: region.height * -percentage) 98 | self.classificationRequest.regionOfInterest = region 99 | 100 | do { 101 | try classificationHandler.perform([self.classificationRequest]) 102 | } catch { 103 | print(error) 104 | } 105 | // dumbRequest.regionOfInterest = largeRegion 106 | // let requestHandlerOptions: [VNImageOption: AnyObject] = [:] 107 | 108 | // let dumbHandler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, orientation: .right, options: requestHandlerOptions) 109 | // 110 | // 111 | // try dumbHandler.perform([dumbRequest]) 112 | 113 | // guard let observations = dumbRequest.results as? [VNCoreMLFeatureValueObservation] 114 | // else { fatalError("unexpected result type from VNCoreMLRequest1") } 115 | // let predictions = observations[0].featureValue.multiArrayValue as? MLMultiArray 116 | // 117 | // let image: UIImage = (predictions?.image(min: 0, max: 1)!)! 118 | 119 | 120 | 121 | } 122 | 123 | 124 | } catch { 125 | print("Failed to perform classification.\n\(error.localizedDescription)") 126 | } 127 | 128 | } 129 | 130 | func updateLayerGeometry() { 131 | 132 | guard let overlayLayer = self.detectionOverlayLayer 133 | else { 134 | return 135 | } 136 | 137 | let bounds = rootLayer.bounds 138 | var scale: CGFloat 139 | 140 | let xScale: CGFloat = bounds.size.width / bufferSize.height 141 | let yScale: CGFloat = bounds.size.height / bufferSize.width 142 | 143 | scale = fmax(xScale, yScale) 144 | if scale.isInfinite { 145 | scale = 1.0 146 | } 147 | CATransaction.begin() 148 | CATransaction.setValue(kCFBooleanTrue, forKey: kCATransactionDisableActions) 149 | 150 | // rotate the layer into screen orientation and scale and mirror 151 | overlayLayer.setAffineTransform(CGAffineTransform(rotationAngle: CGFloat(.pi / 2.0)).scaledBy(x: scale, y: scale)) 152 | // center the layer 153 | overlayLayer.position = CGPoint (x: bounds.midX, y: bounds.midY) 154 | 155 | CATransaction.commit() 156 | 157 | } 158 | 159 | override func setupAVCapture() { 160 | super.setupAVCapture() 161 | 162 | // setup Vision parts 163 | self.setupVisionDrawingLayers() 164 | updateLayerGeometry() 165 | classificationLabel.layer.zPosition = 2 166 | 167 | // start the capture 168 | startCaptureSession() 169 | } 170 | 171 | fileprivate func setupVisionDrawingLayers() { 172 | let captureDeviceBounds = CGRect(x: 0, 173 | y: 0, 174 | width: bufferSize.width, 175 | height: bufferSize.height) 176 | 177 | let captureDeviceBoundsCenterPoint = CGPoint(x: captureDeviceBounds.midX, 178 | y: captureDeviceBounds.midY) 179 | 180 | let normalizedCenterPoint = CGPoint(x: 0.5, y: 0.5) 181 | 182 | guard let rootLayer = self.rootLayer else { 183 | print("ERROR: view was not property initialized") 184 | return 185 | } 186 | 187 | let overlayLayer = CALayer() 188 | overlayLayer.name = "DetectionOverlay" 189 | overlayLayer.masksToBounds = true 190 | overlayLayer.anchorPoint = normalizedCenterPoint 191 | overlayLayer.bounds = captureDeviceBounds 192 | overlayLayer.position = CGPoint(x: rootLayer.bounds.midX, y: rootLayer.bounds.midY) 193 | 194 | let faceRectangleShapeLayer = CAShapeLayer() 195 | faceRectangleShapeLayer.name = "RectangleOutlineLayer" 196 | faceRectangleShapeLayer.bounds = captureDeviceBounds 197 | faceRectangleShapeLayer.anchorPoint = normalizedCenterPoint 198 | faceRectangleShapeLayer.position = captureDeviceBoundsCenterPoint 199 | faceRectangleShapeLayer.fillColor = nil 200 | faceRectangleShapeLayer.strokeColor = UIColor.green.withAlphaComponent(0.7).cgColor 201 | faceRectangleShapeLayer.lineWidth = 5 202 | faceRectangleShapeLayer.shadowOpacity = 0.7 203 | faceRectangleShapeLayer.shadowRadius = 5 204 | 205 | let particleEmitter = CAEmitterLayer() 206 | 207 | particleEmitter.emitterPosition = CGPoint(x: view.frame.width / 2.0, y: -50) 208 | particleEmitter.emitterShape = .line 209 | particleEmitter.emitterSize = CGSize(width: view.frame.width, height: 1) 210 | particleEmitter.renderMode = .additive 211 | 212 | let cell = CAEmitterCell() 213 | cell.birthRate = 2 214 | cell.lifetime = 5.0 215 | cell.velocity = 100 216 | cell.velocityRange = 50 217 | cell.emissionLongitude = .pi 218 | cell.spinRange = 5 219 | cell.scale = 0.5 220 | cell.scaleRange = 0.25 221 | cell.color = UIColor(white: 1, alpha: 0.1).cgColor 222 | cell.alphaSpeed = -0.025 223 | cell.contents = UIImage(named: "particle")?.cgImage 224 | particleEmitter.emitterCells = [cell] 225 | 226 | 227 | 228 | overlayLayer.addSublayer(faceRectangleShapeLayer) 229 | rootLayer.addSublayer(overlayLayer) 230 | 231 | self.detectionOverlayLayer = overlayLayer 232 | self.detectedFaceRectangleShapeLayer = faceRectangleShapeLayer 233 | 234 | self.updateLayerGeometry() 235 | } 236 | 237 | fileprivate func drawFaceObservations(_ faceObservations: [VNFaceObservation]) { 238 | guard let faceRectangleShapeLayer = self.detectedFaceRectangleShapeLayer 239 | else { 240 | return 241 | } 242 | 243 | CATransaction.begin() 244 | 245 | CATransaction.setValue(NSNumber(value: true), forKey: kCATransactionDisableActions) 246 | 247 | let faceRectanglePath = CGMutablePath() 248 | 249 | for faceObservation in faceObservations { 250 | self.addIndicators(to: faceRectanglePath, 251 | for: faceObservation) 252 | } 253 | 254 | faceRectangleShapeLayer.path = faceRectanglePath 255 | 256 | self.updateLayerGeometry() 257 | 258 | CATransaction.commit() 259 | } 260 | 261 | fileprivate func addIndicators(to faceRectanglePath: CGMutablePath, for faceObservation: VNFaceObservation) { 262 | let displaySize = bufferSize 263 | 264 | let faceBounds = VNImageRectForNormalizedRect(faceObservation.boundingBox, Int(displaySize.width), Int(displaySize.height)) 265 | faceRectanglePath.addRect(faceBounds) 266 | } 267 | 268 | } 269 | 270 | -------------------------------------------------------------------------------- /coreml_ios/CoreMLHelpers/Array+Extensions.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | import Swift 24 | 25 | extension Array where Element: Comparable { 26 | /** 27 | Returns the index and value of the largest element in the array. 28 | 29 | - Note: This method is slow. For faster results, use the standalone 30 | version of argmax() instead. 31 | */ 32 | public func argmax() -> (Int, Element) { 33 | precondition(self.count > 0) 34 | var maxIndex = 0 35 | var maxValue = self[0] 36 | for i in 1.. maxValue { 37 | maxValue = self[i] 38 | maxIndex = i 39 | } 40 | return (maxIndex, maxValue) 41 | } 42 | 43 | /** 44 | Returns the indices of the array's elements in sorted order. 45 | */ 46 | public func argsort(by areInIncreasingOrder: (Element, Element) -> Bool) -> [Array.Index] { 47 | return self.indices.sorted { areInIncreasingOrder(self[$0], self[$1]) } 48 | } 49 | 50 | /** 51 | Returns a new array containing the elements at the specified indices. 52 | */ 53 | public func gather(indices: [Array.Index]) -> [Element] { 54 | return indices.map { self[$0] } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /coreml_ios/CoreMLHelpers/CGImage+CVPixelBuffer.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017-2018 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | import CoreGraphics 24 | import CoreImage 25 | import VideoToolbox 26 | 27 | extension CGImage { 28 | /** 29 | Resizes the image to width x height and converts it to an RGB CVPixelBuffer. 30 | */ 31 | public func pixelBuffer(width: Int, height: Int, 32 | orientation: CGImagePropertyOrientation) -> CVPixelBuffer? { 33 | return pixelBuffer(width: width, height: height, 34 | pixelFormatType: kCVPixelFormatType_32ARGB, 35 | colorSpace: CGColorSpaceCreateDeviceRGB(), 36 | alphaInfo: .noneSkipFirst, 37 | orientation: orientation) 38 | } 39 | 40 | /** 41 | Resizes the image to width x height and converts it to a grayscale CVPixelBuffer. 42 | */ 43 | public func pixelBufferGray(width: Int, height: Int, 44 | orientation: CGImagePropertyOrientation) -> CVPixelBuffer? { 45 | return pixelBuffer(width: width, height: height, 46 | pixelFormatType: kCVPixelFormatType_OneComponent8, 47 | colorSpace: CGColorSpaceCreateDeviceGray(), 48 | alphaInfo: .none, 49 | orientation: orientation) 50 | } 51 | 52 | func pixelBuffer(width: Int, height: Int, pixelFormatType: OSType, 53 | colorSpace: CGColorSpace, alphaInfo: CGImageAlphaInfo, 54 | orientation: CGImagePropertyOrientation) -> CVPixelBuffer? { 55 | 56 | // TODO: If the orientation is not .up, then rotate the CGImage. 57 | // See also: https://stackoverflow.com/a/40438893/ 58 | assert(orientation == .up) 59 | 60 | var maybePixelBuffer: CVPixelBuffer? 61 | let attrs = [kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue, 62 | kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue] 63 | let status = CVPixelBufferCreate(kCFAllocatorDefault, 64 | width, 65 | height, 66 | pixelFormatType, 67 | attrs as CFDictionary, 68 | &maybePixelBuffer) 69 | 70 | guard status == kCVReturnSuccess, let pixelBuffer = maybePixelBuffer else { 71 | return nil 72 | } 73 | 74 | let flags = CVPixelBufferLockFlags(rawValue: 0) 75 | guard kCVReturnSuccess == CVPixelBufferLockBaseAddress(pixelBuffer, flags) else { 76 | return nil 77 | } 78 | defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, flags) } 79 | 80 | guard let context = CGContext(data: CVPixelBufferGetBaseAddress(pixelBuffer), 81 | width: width, 82 | height: height, 83 | bitsPerComponent: 8, 84 | bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer), 85 | space: colorSpace, 86 | bitmapInfo: alphaInfo.rawValue) 87 | else { 88 | return nil 89 | } 90 | 91 | context.draw(self, in: CGRect(x: 0, y: 0, width: width, height: height)) 92 | return pixelBuffer 93 | } 94 | } 95 | 96 | extension CGImage { 97 | /** 98 | Creates a new CGImage from a CVPixelBuffer. 99 | 100 | - Note: Not all CVPixelBuffer pixel formats support conversion into a 101 | CGImage-compatible pixel format. 102 | */ 103 | public static func create(pixelBuffer: CVPixelBuffer) -> CGImage? { 104 | var cgImage: CGImage? 105 | VTCreateCGImageFromCVPixelBuffer(pixelBuffer, options: nil, imageOut: &cgImage) 106 | return cgImage 107 | } 108 | 109 | /* 110 | // Alternative implementation: 111 | public static func create(pixelBuffer: CVPixelBuffer) -> CGImage? { 112 | // This method creates a bitmap CGContext using the pixel buffer's memory. 113 | // It currently only handles kCVPixelFormatType_32ARGB images. To support 114 | // other pixel formats too, you'll have to change the bitmapInfo and maybe 115 | // the color space for the CGContext. 116 | 117 | guard kCVReturnSuccess == CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly) else { 118 | return nil 119 | } 120 | defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly) } 121 | 122 | if let context = CGContext(data: CVPixelBufferGetBaseAddress(pixelBuffer), 123 | width: CVPixelBufferGetWidth(pixelBuffer), 124 | height: CVPixelBufferGetHeight(pixelBuffer), 125 | bitsPerComponent: 8, 126 | bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer), 127 | space: CGColorSpaceCreateDeviceRGB(), 128 | bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue), 129 | let cgImage = context.makeImage() { 130 | return cgImage 131 | } else { 132 | return nil 133 | } 134 | } 135 | */ 136 | 137 | /** 138 | Creates a new CGImage from a CVPixelBuffer, using Core Image. 139 | */ 140 | public static func create(pixelBuffer: CVPixelBuffer, context: CIContext) -> CGImage? { 141 | let ciImage = CIImage(cvPixelBuffer: pixelBuffer) 142 | let rect = CGRect(x: 0, y: 0, width: CVPixelBufferGetWidth(pixelBuffer), 143 | height: CVPixelBufferGetHeight(pixelBuffer)) 144 | return context.createCGImage(ciImage, from: rect) 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /coreml_ios/CoreMLHelpers/CGImage+RawBytes.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017-2018 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | import CoreGraphics 24 | 25 | extension CGImage { 26 | /** 27 | Converts the image into an array of RGBA bytes. 28 | */ 29 | @nonobjc public func toByteArrayRGBA() -> [UInt8] { 30 | var bytes = [UInt8](repeating: 0, count: width * height * 4) 31 | bytes.withUnsafeMutableBytes { ptr in 32 | if let colorSpace = colorSpace, 33 | let context = CGContext( 34 | data: ptr.baseAddress, 35 | width: width, 36 | height: height, 37 | bitsPerComponent: bitsPerComponent, 38 | bytesPerRow: bytesPerRow, 39 | space: colorSpace, 40 | bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) { 41 | let rect = CGRect(x: 0, y: 0, width: width, height: height) 42 | context.draw(self, in: rect) 43 | } 44 | } 45 | return bytes 46 | } 47 | 48 | /** 49 | Creates a new CGImage from an array of RGBA bytes. 50 | */ 51 | @nonobjc public class func fromByteArrayRGBA(_ bytes: [UInt8], 52 | width: Int, 53 | height: Int) -> CGImage? { 54 | return fromByteArray(bytes, width: width, height: height, 55 | bytesPerRow: width * 4, 56 | colorSpace: CGColorSpaceCreateDeviceRGB(), 57 | alphaInfo: .premultipliedLast) 58 | } 59 | 60 | /** 61 | Creates a new CGImage from an array of grayscale bytes. 62 | */ 63 | @nonobjc public class func fromByteArrayGray(_ bytes: [UInt8], 64 | width: Int, 65 | height: Int) -> CGImage? { 66 | return fromByteArray(bytes, width: width, height: height, 67 | bytesPerRow: width, 68 | colorSpace: CGColorSpaceCreateDeviceGray(), 69 | alphaInfo: .none) 70 | } 71 | 72 | @nonobjc class func fromByteArray(_ bytes: [UInt8], 73 | width: Int, 74 | height: Int, 75 | bytesPerRow: Int, 76 | colorSpace: CGColorSpace, 77 | alphaInfo: CGImageAlphaInfo) -> CGImage? { 78 | return bytes.withUnsafeBytes { ptr in 79 | let context = CGContext(data: UnsafeMutableRawPointer(mutating: ptr.baseAddress!), 80 | width: width, 81 | height: height, 82 | bitsPerComponent: 8, 83 | bytesPerRow: bytesPerRow, 84 | space: colorSpace, 85 | bitmapInfo: alphaInfo.rawValue) 86 | return context?.makeImage() 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /coreml_ios/CoreMLHelpers/CGImagePropertyOrientation.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017-2018 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | #if canImport(UIKit) 24 | 25 | import UIKit 26 | 27 | public extension CGImagePropertyOrientation { 28 | init(_ orientation: UIImage.Orientation) { 29 | switch orientation { 30 | case .up: self = .up 31 | case .upMirrored: self = .upMirrored 32 | case .down: self = .down 33 | case .downMirrored: self = .downMirrored 34 | case .left: self = .left 35 | case .leftMirrored: self = .leftMirrored 36 | case .right: self = .right 37 | case .rightMirrored: self = .rightMirrored 38 | } 39 | } 40 | 41 | init(_ orientation: UIDeviceOrientation) { 42 | switch orientation { 43 | case .portraitUpsideDown: self = .left 44 | case .landscapeLeft: self = .up 45 | case .landscapeRight: self = .down 46 | default: self = .right 47 | } 48 | } 49 | } 50 | 51 | extension UIImage.Orientation { 52 | init(_ cgOrientation: UIImage.Orientation) { 53 | switch cgOrientation { 54 | case .up: self = .up 55 | case .upMirrored: self = .upMirrored 56 | case .down: self = .down 57 | case .downMirrored: self = .downMirrored 58 | case .left: self = .left 59 | case .leftMirrored: self = .leftMirrored 60 | case .right: self = .right 61 | case .rightMirrored: self = .rightMirrored 62 | } 63 | } 64 | } 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /coreml_ios/CoreMLHelpers/CVPixelBuffer+Helpers.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | import Foundation 24 | import Accelerate 25 | import CoreImage 26 | 27 | /** 28 | Creates a RGB pixel buffer of the specified width and height. 29 | */ 30 | public func createPixelBuffer(width: Int, height: Int) -> CVPixelBuffer? { 31 | var pixelBuffer: CVPixelBuffer? 32 | let status = CVPixelBufferCreate(nil, width, height, 33 | kCVPixelFormatType_32BGRA, nil, 34 | &pixelBuffer) 35 | if status != kCVReturnSuccess { 36 | print("Error: could not create pixel buffer", status) 37 | return nil 38 | } 39 | return pixelBuffer 40 | } 41 | 42 | /** 43 | First crops the pixel buffer, then resizes it. 44 | */ 45 | public func resizePixelBuffer(_ srcPixelBuffer: CVPixelBuffer, 46 | cropX: Int, 47 | cropY: Int, 48 | cropWidth: Int, 49 | cropHeight: Int, 50 | scaleWidth: Int, 51 | scaleHeight: Int) -> CVPixelBuffer? { 52 | let flags = CVPixelBufferLockFlags(rawValue: 0) 53 | guard kCVReturnSuccess == CVPixelBufferLockBaseAddress(srcPixelBuffer, flags) else { 54 | return nil 55 | } 56 | defer { CVPixelBufferUnlockBaseAddress(srcPixelBuffer, flags) } 57 | 58 | guard let srcData = CVPixelBufferGetBaseAddress(srcPixelBuffer) else { 59 | print("Error: could not get pixel buffer base address") 60 | return nil 61 | } 62 | let srcBytesPerRow = CVPixelBufferGetBytesPerRow(srcPixelBuffer) 63 | let offset = cropY*srcBytesPerRow + cropX*4 64 | var srcBuffer = vImage_Buffer(data: srcData.advanced(by: offset), 65 | height: vImagePixelCount(cropHeight), 66 | width: vImagePixelCount(cropWidth), 67 | rowBytes: srcBytesPerRow) 68 | 69 | let destBytesPerRow = scaleWidth*4 70 | guard let destData = malloc(scaleHeight*destBytesPerRow) else { 71 | print("Error: out of memory") 72 | return nil 73 | } 74 | var destBuffer = vImage_Buffer(data: destData, 75 | height: vImagePixelCount(scaleHeight), 76 | width: vImagePixelCount(scaleWidth), 77 | rowBytes: destBytesPerRow) 78 | 79 | let error = vImageScale_ARGB8888(&srcBuffer, &destBuffer, nil, vImage_Flags(0)) 80 | if error != kvImageNoError { 81 | print("Error:", error) 82 | free(destData) 83 | return nil 84 | } 85 | 86 | let releaseCallback: CVPixelBufferReleaseBytesCallback = { _, ptr in 87 | if let ptr = ptr { 88 | free(UnsafeMutableRawPointer(mutating: ptr)) 89 | } 90 | } 91 | 92 | let pixelFormat = CVPixelBufferGetPixelFormatType(srcPixelBuffer) 93 | var dstPixelBuffer: CVPixelBuffer? 94 | let status = CVPixelBufferCreateWithBytes(nil, scaleWidth, scaleHeight, 95 | pixelFormat, destData, 96 | destBytesPerRow, releaseCallback, 97 | nil, nil, &dstPixelBuffer) 98 | if status != kCVReturnSuccess { 99 | print("Error: could not create new pixel buffer") 100 | free(destData) 101 | return nil 102 | } 103 | return dstPixelBuffer 104 | } 105 | 106 | /** 107 | Resizes a CVPixelBuffer to a new width and height. 108 | */ 109 | public func resizePixelBuffer(_ pixelBuffer: CVPixelBuffer, 110 | width: Int, height: Int) -> CVPixelBuffer? { 111 | return resizePixelBuffer(pixelBuffer, cropX: 0, cropY: 0, 112 | cropWidth: CVPixelBufferGetWidth(pixelBuffer), 113 | cropHeight: CVPixelBufferGetHeight(pixelBuffer), 114 | scaleWidth: width, scaleHeight: height) 115 | } 116 | 117 | /** 118 | Resizes a CVPixelBuffer to a new width and height. 119 | */ 120 | public func resizePixelBuffer(_ pixelBuffer: CVPixelBuffer, 121 | width: Int, height: Int, 122 | output: CVPixelBuffer, context: CIContext) { 123 | let ciImage = CIImage(cvPixelBuffer: pixelBuffer) 124 | let sx = CGFloat(width) / CGFloat(CVPixelBufferGetWidth(pixelBuffer)) 125 | let sy = CGFloat(height) / CGFloat(CVPixelBufferGetHeight(pixelBuffer)) 126 | let scaleTransform = CGAffineTransform(scaleX: sx, y: sy) 127 | let scaledImage = ciImage.transformed(by: scaleTransform) 128 | context.render(scaledImage, to: output) 129 | } 130 | 131 | /** 132 | Rotates CVPixelBuffer by the provided factor of 90 counterclock-wise. 133 | */ 134 | public func rotate90PixelBuffer(_ srcPixelBuffer: CVPixelBuffer, factor: UInt8) -> CVPixelBuffer? { 135 | let flags = CVPixelBufferLockFlags(rawValue: 0) 136 | guard kCVReturnSuccess == CVPixelBufferLockBaseAddress(srcPixelBuffer, flags) else { 137 | return nil 138 | } 139 | defer { CVPixelBufferUnlockBaseAddress(srcPixelBuffer, flags) } 140 | 141 | guard let srcData = CVPixelBufferGetBaseAddress(srcPixelBuffer) else { 142 | print("Error: could not get pixel buffer base address") 143 | return nil 144 | } 145 | let sourceWidth = CVPixelBufferGetWidth(srcPixelBuffer) 146 | let sourceHeight = CVPixelBufferGetHeight(srcPixelBuffer) 147 | var destWidth = sourceHeight 148 | var destHeight = sourceWidth 149 | var color = UInt8(0) 150 | 151 | if factor % 2 == 0 { 152 | destWidth = sourceWidth 153 | destHeight = sourceHeight 154 | } 155 | 156 | let srcBytesPerRow = CVPixelBufferGetBytesPerRow(srcPixelBuffer) 157 | var srcBuffer = vImage_Buffer(data: srcData, 158 | height: vImagePixelCount(sourceHeight), 159 | width: vImagePixelCount(sourceWidth), 160 | rowBytes: srcBytesPerRow) 161 | 162 | let destBytesPerRow = destWidth*4 163 | guard let destData = malloc(destHeight*destBytesPerRow) else { 164 | print("Error: out of memory") 165 | return nil 166 | } 167 | var destBuffer = vImage_Buffer(data: destData, 168 | height: vImagePixelCount(destHeight), 169 | width: vImagePixelCount(destWidth), 170 | rowBytes: destBytesPerRow) 171 | 172 | let error = vImageRotate90_ARGB8888(&srcBuffer, &destBuffer, factor, &color, vImage_Flags(0)) 173 | if error != kvImageNoError { 174 | print("Error:", error) 175 | free(destData) 176 | return nil 177 | } 178 | 179 | let releaseCallback: CVPixelBufferReleaseBytesCallback = { _, ptr in 180 | if let ptr = ptr { 181 | free(UnsafeMutableRawPointer(mutating: ptr)) 182 | } 183 | } 184 | 185 | let pixelFormat = CVPixelBufferGetPixelFormatType(srcPixelBuffer) 186 | var dstPixelBuffer: CVPixelBuffer? 187 | let status = CVPixelBufferCreateWithBytes(nil, destWidth, destHeight, 188 | pixelFormat, destData, 189 | destBytesPerRow, releaseCallback, 190 | nil, nil, &dstPixelBuffer) 191 | if status != kCVReturnSuccess { 192 | print("Error: could not create new pixel buffer") 193 | free(destData) 194 | return nil 195 | } 196 | return dstPixelBuffer 197 | } 198 | -------------------------------------------------------------------------------- /coreml_ios/CoreMLHelpers/MLMultiArray+Helpers.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017-2018 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | import CoreML 24 | 25 | extension MLMultiArray { 26 | /** 27 | Returns a new MLMultiArray with the specified dimensions. 28 | 29 | - Note: This does not copy the data but uses a pointer into the original 30 | multi-array's memory. The caller is responsible for keeping the original 31 | object alive, for example using `withExtendedLifetime(originalArray) {...}` 32 | */ 33 | @nonobjc public func reshaped(to dimensions: [Int]) throws -> MLMultiArray { 34 | let newCount = dimensions.reduce(1, *) 35 | precondition(newCount == count, "Cannot reshape \(shape) to \(dimensions)") 36 | 37 | var newStrides = [Int](repeating: 0, count: dimensions.count) 38 | newStrides[dimensions.count - 1] = 1 39 | for i in stride(from: dimensions.count - 1, to: 0, by: -1) { 40 | newStrides[i - 1] = newStrides[i] * dimensions[i] 41 | } 42 | 43 | let newShape_ = dimensions.map { NSNumber(value: $0) } 44 | let newStrides_ = newStrides.map { NSNumber(value: $0) } 45 | 46 | return try MLMultiArray(dataPointer: self.dataPointer, 47 | shape: newShape_, 48 | dataType: self.dataType, 49 | strides: newStrides_) 50 | } 51 | 52 | /** 53 | Returns a transposed version of this MLMultiArray. 54 | 55 | - Note: This copies the data. 56 | 57 | - TODO: Support .float32 and .int32 types too. 58 | */ 59 | @nonobjc public func transposed(to order: [Int]) throws -> MLMultiArray { 60 | let ndim = order.count 61 | 62 | precondition(dataType == .double) 63 | precondition(ndim == strides.count) 64 | 65 | let newShape = shape.indices.map { shape[order[$0]] } 66 | let newArray = try MLMultiArray(shape: newShape, dataType: self.dataType) 67 | 68 | let srcPtr = UnsafeMutablePointer(OpaquePointer(dataPointer)) 69 | let dstPtr = UnsafeMutablePointer(OpaquePointer(newArray.dataPointer)) 70 | 71 | let srcShape = shape.map { $0.intValue } 72 | let dstStride = newArray.strides.map { $0.intValue } 73 | var idx = [Int](repeating: 0, count: ndim) 74 | 75 | for j in 0.. 0 && idx[i] >= srcShape[i] { 89 | idx[i] = 0 90 | idx[i - 1] += 1 91 | i -= 1 92 | } 93 | } 94 | return newArray 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /coreml_ios/CoreMLHelpers/Math.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017-2018 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | import Foundation 24 | import Accelerate 25 | 26 | /** 27 | Returns the index and value of the largest element in the array. 28 | 29 | - Parameters: 30 | - count: If provided, only look at the first `count` elements of the array, 31 | otherwise look at the entire array. 32 | */ 33 | public func argmax(_ array: [Float], count: Int?) -> (Int, Float) { 34 | var maxValue: Float = 0 35 | var maxIndex: vDSP_Length = 0 36 | vDSP_maxvi(array, 1, &maxValue, &maxIndex, vDSP_Length(count ?? array.count)) 37 | return (Int(maxIndex), maxValue) 38 | } 39 | 40 | /** 41 | Returns the index and value of the largest element in the array. 42 | 43 | - Parameters: 44 | - ptr: Pointer to the first element in memory. 45 | - count: How many elements to look at. 46 | - stride: The distance between two elements in memory. 47 | */ 48 | public func argmax(_ ptr: UnsafePointer, count: Int, stride: Int = 1) -> (Int, Float) { 49 | var maxValue: Float = 0 50 | var maxIndex: vDSP_Length = 0 51 | vDSP_maxvi(ptr, vDSP_Stride(stride), &maxValue, &maxIndex, vDSP_Length(count)) 52 | return (Int(maxIndex), maxValue) 53 | } 54 | 55 | /** 56 | Returns the index and value of the largest element in the array. 57 | 58 | - Parameters: 59 | - count: If provided, only look at the first `count` elements of the array, 60 | otherwise look at the entire array. 61 | */ 62 | public func argmax(_ array: [Double], count: Int?) -> (Int, Double) { 63 | var maxValue: Double = 0 64 | var maxIndex: vDSP_Length = 0 65 | vDSP_maxviD(array, 1, &maxValue, &maxIndex, vDSP_Length(count ?? array.count)) 66 | return (Int(maxIndex), maxValue) 67 | } 68 | 69 | /** 70 | Returns the index and value of the largest element in the array. 71 | 72 | - Parameters: 73 | - ptr: Pointer to the first element in memory. 74 | - count: How many elements to look at. 75 | - stride: The distance between two elements in memory. 76 | */ 77 | public func argmax(_ ptr: UnsafePointer, count: Int, stride: Int = 1) -> (Int, Double) { 78 | var maxValue: Double = 0 79 | var maxIndex: vDSP_Length = 0 80 | vDSP_maxviD(ptr, vDSP_Stride(stride), &maxValue, &maxIndex, vDSP_Length(count)) 81 | return (Int(maxIndex), maxValue) 82 | } 83 | 84 | /** Ensures that `x` is in the range `[min, max]`. */ 85 | public func clamp(_ x: T, min: T, max: T) -> T { 86 | if x < min { return min } 87 | if x > max { return max } 88 | return x 89 | } 90 | 91 | /** Logistic sigmoid. */ 92 | public func sigmoid(_ x: Float) -> Float { 93 | return 1 / (1 + exp(-x)) 94 | } 95 | 96 | /** Logistic sigmoid. */ 97 | public func sigmoid(_ x: Double) -> Double { 98 | return 1 / (1 + exp(-x)) 99 | } 100 | 101 | /* In-place logistic sigmoid: x = 1 / (1 + exp(-x)) */ 102 | public func sigmoid(_ x: UnsafeMutablePointer, count: Int) { 103 | vDSP_vneg(x, 1, x, 1, vDSP_Length(count)) 104 | var cnt = Int32(count) 105 | vvexpf(x, x, &cnt) 106 | var y: Float = 1 107 | vDSP_vsadd(x, 1, &y, x, 1, vDSP_Length(count)) 108 | vvrecf(x, x, &cnt) 109 | } 110 | 111 | /* In-place logistic sigmoid: x = 1 / (1 + exp(-x)) */ 112 | public func sigmoid(_ x: UnsafeMutablePointer, count: Int) { 113 | vDSP_vnegD(x, 1, x, 1, vDSP_Length(count)) 114 | var cnt = Int32(count) 115 | vvexp(x, x, &cnt) 116 | var y: Double = 1 117 | vDSP_vsaddD(x, 1, &y, x, 1, vDSP_Length(count)) 118 | vvrec(x, x, &cnt) 119 | } 120 | 121 | /** 122 | Computes the "softmax" function over an array. 123 | 124 | Based on code from https://github.com/nikolaypavlov/MLPNeuralNet/ 125 | 126 | This is what softmax looks like in "pseudocode" (actually using Python 127 | and numpy): 128 | 129 | x -= np.max(x) 130 | exp_scores = np.exp(x) 131 | softmax = exp_scores / np.sum(exp_scores) 132 | 133 | First we shift the values of x so that the highest value in the array is 0. 134 | This ensures numerical stability with the exponents, so they don't blow up. 135 | */ 136 | public func softmax(_ x: [Float]) -> [Float] { 137 | var x = x 138 | let len = vDSP_Length(x.count) 139 | 140 | // Find the maximum value in the input array. 141 | var max: Float = 0 142 | vDSP_maxv(x, 1, &max, len) 143 | 144 | // Subtract the maximum from all the elements in the array. 145 | // Now the highest value in the array is 0. 146 | max = -max 147 | vDSP_vsadd(x, 1, &max, &x, 1, len) 148 | 149 | // Exponentiate all the elements in the array. 150 | var count = Int32(x.count) 151 | vvexpf(&x, x, &count) 152 | 153 | // Compute the sum of all exponentiated values. 154 | var sum: Float = 0 155 | vDSP_sve(x, 1, &sum, len) 156 | 157 | // Divide each element by the sum. This normalizes the array contents 158 | // so that they all add up to 1. 159 | vDSP_vsdiv(x, 1, &sum, &x, 1, len) 160 | 161 | return x 162 | } 163 | -------------------------------------------------------------------------------- /coreml_ios/CoreMLHelpers/NonMaxSuppression.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017-2018 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | import Foundation 24 | import Accelerate 25 | 26 | public struct BoundingBox { 27 | /** Index of the predicted class. */ 28 | public let classIndex: Int 29 | 30 | /** Confidence score. */ 31 | public let score: Float 32 | 33 | /** Normalized coordinates between 0 and 1. */ 34 | public let rect: CGRect 35 | 36 | public init(classIndex: Int, score: Float, rect: CGRect) { 37 | self.classIndex = classIndex 38 | self.score = score 39 | self.rect = rect 40 | } 41 | } 42 | 43 | /** 44 | Computes intersection-over-union overlap between two bounding boxes. 45 | */ 46 | public func IOU(_ a: CGRect, _ b: CGRect) -> Float { 47 | let areaA = a.width * a.height 48 | if areaA <= 0 { return 0 } 49 | 50 | let areaB = b.width * b.height 51 | if areaB <= 0 { return 0 } 52 | 53 | let intersectionMinX = max(a.minX, b.minX) 54 | let intersectionMinY = max(a.minY, b.minY) 55 | let intersectionMaxX = min(a.maxX, b.maxX) 56 | let intersectionMaxY = min(a.maxY, b.maxY) 57 | let intersectionArea = max(intersectionMaxY - intersectionMinY, 0) * 58 | max(intersectionMaxX - intersectionMinX, 0) 59 | return Float(intersectionArea / (areaA + areaB - intersectionArea)) 60 | } 61 | 62 | /** 63 | Removes bounding boxes that overlap too much with other boxes that have 64 | a higher score. 65 | */ 66 | public func nonMaxSuppression(boundingBoxes: [BoundingBox], 67 | iouThreshold: Float, 68 | maxBoxes: Int) -> [Int] { 69 | return nonMaxSuppression(boundingBoxes: boundingBoxes, 70 | indices: Array(boundingBoxes.indices), 71 | iouThreshold: iouThreshold, 72 | maxBoxes: maxBoxes) 73 | } 74 | 75 | /** 76 | Removes bounding boxes that overlap too much with other boxes that have 77 | a higher score. 78 | 79 | Based on code from https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/non_max_suppression_op.cc 80 | 81 | - Note: This version of NMS ignores the class of the bounding boxes. Since it 82 | selects the bounding boxes in a greedy fashion, if a certain class has many 83 | boxes that are selected, then it is possible none of the boxes of the other 84 | classes get selected. 85 | 86 | - Parameters: 87 | - boundingBoxes: an array of bounding boxes and their scores 88 | - indices: which predictions to look at 89 | - iouThreshold: used to decide whether boxes overlap too much 90 | - maxBoxes: the maximum number of boxes that will be selected 91 | 92 | - Returns: the array indices of the selected bounding boxes 93 | */ 94 | public func nonMaxSuppression(boundingBoxes: [BoundingBox], 95 | indices: [Int], 96 | iouThreshold: Float, 97 | maxBoxes: Int) -> [Int] { 98 | 99 | // Sort the boxes based on their confidence scores, from high to low. 100 | let sortedIndices = indices.sorted { boundingBoxes[$0].score > boundingBoxes[$1].score } 101 | 102 | var selected: [Int] = [] 103 | 104 | // Loop through the bounding boxes, from highest score to lowest score, 105 | // and determine whether or not to keep each box. 106 | for i in 0..= maxBoxes { break } 108 | 109 | var shouldSelect = true 110 | let boxA = boundingBoxes[sortedIndices[i]] 111 | 112 | // Does the current box overlap one of the selected boxes more than the 113 | // given threshold amount? Then it's too similar, so don't keep it. 114 | for j in 0.. iouThreshold { 117 | shouldSelect = false 118 | break 119 | } 120 | } 121 | 122 | // This bounding box did not overlap too much with any previously selected 123 | // bounding box, so we'll keep it. 124 | if shouldSelect { 125 | selected.append(sortedIndices[i]) 126 | } 127 | } 128 | 129 | return selected 130 | } 131 | 132 | /** 133 | Multi-class version of non maximum suppression. 134 | 135 | Where `nonMaxSuppression()` does not look at the class of the predictions at 136 | all, the multi-class version first selects the best bounding boxes for each 137 | class, and then keeps the best ones of those. 138 | 139 | With this method you can usually expect to see at least one bounding box for 140 | each class (unless all the scores for a given class are really low). 141 | 142 | Based on code from: https://github.com/tensorflow/models/blob/master/object_detection/core/post_processing.py 143 | 144 | - Parameters: 145 | - numClasses: the number of classes 146 | - boundingBoxes: an array of bounding boxes and their scores 147 | - scoreThreshold: used to only keep bounding boxes with a high enough score 148 | - iouThreshold: used to decide whether boxes overlap too much 149 | - maxPerClass: the maximum number of boxes that will be selected per class 150 | - maxTotal: maximum number of boxes that will be selected over all classes 151 | 152 | - Returns: the array indices of the selected bounding boxes 153 | */ 154 | public func nonMaxSuppressionMultiClass(numClasses: Int, 155 | boundingBoxes: [BoundingBox], 156 | scoreThreshold: Float, 157 | iouThreshold: Float, 158 | maxPerClass: Int, 159 | maxTotal: Int) -> [Int] { 160 | var selectedBoxes: [Int] = [] 161 | 162 | // Look at all the classes one-by-one. 163 | for c in 0.. scoreThreshold { 173 | filteredBoxes.append(p) 174 | } 175 | } 176 | } 177 | 178 | // Only keep the best bounding boxes for this class. 179 | let nmsBoxes = nonMaxSuppression(boundingBoxes: boundingBoxes, 180 | indices: filteredBoxes, 181 | iouThreshold: iouThreshold, 182 | maxBoxes: maxPerClass) 183 | 184 | // Add the indices of the surviving boxes to the big list. 185 | selectedBoxes.append(contentsOf: nmsBoxes) 186 | } 187 | 188 | // Sort all the surviving boxes by score and only keep the best ones. 189 | let sortedBoxes = selectedBoxes.sorted { boundingBoxes[$0].score > boundingBoxes[$1].score } 190 | return Array(sortedBoxes.prefix(maxTotal)) 191 | } 192 | -------------------------------------------------------------------------------- /coreml_ios/CoreMLHelpers/Predictions.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | import Vision 24 | 25 | /** 26 | Returns the top `k` predictions from Core ML classification results as an 27 | array of `(String, Double)` pairs. 28 | */ 29 | public func top(_ k: Int, _ prob: [String: Double]) -> [(String, Double)] { 30 | return Array(prob.map { x in (x.key, x.value) } 31 | .sorted(by: { a, b -> Bool in a.1 > b.1 }) 32 | .prefix(through: min(k, prob.count) - 1)) 33 | } 34 | 35 | /** 36 | Returns the top `k` predictions from Vision classification results as an 37 | array of `(String, Double)` pairs. 38 | */ 39 | public func top(_ k: Int, _ observations: [VNClassificationObservation]) -> [(String, Double)] { 40 | // The Vision observations are sorted by confidence already. 41 | return observations.prefix(through: min(k, observations.count) - 1) 42 | .map { ($0.identifier, Double($0.confidence)) } 43 | } 44 | -------------------------------------------------------------------------------- /coreml_ios/CoreMLHelpers/UIImage+CVPixelBuffer.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017-2018 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | #if canImport(UIKit) 24 | 25 | import UIKit 26 | import VideoToolbox 27 | 28 | extension UIImage { 29 | /** 30 | Resizes the image to width x height and converts it to an RGB CVPixelBuffer. 31 | */ 32 | public func pixelBuffer(width: Int, height: Int) -> CVPixelBuffer? { 33 | return pixelBuffer(width: width, height: height, 34 | pixelFormatType: kCVPixelFormatType_32ARGB, 35 | colorSpace: CGColorSpaceCreateDeviceRGB(), 36 | alphaInfo: .noneSkipFirst) 37 | } 38 | 39 | /** 40 | Resizes the image to width x height and converts it to a grayscale CVPixelBuffer. 41 | */ 42 | public func pixelBufferGray(width: Int, height: Int) -> CVPixelBuffer? { 43 | return pixelBuffer(width: width, height: height, 44 | pixelFormatType: kCVPixelFormatType_OneComponent8, 45 | colorSpace: CGColorSpaceCreateDeviceGray(), 46 | alphaInfo: .none) 47 | } 48 | 49 | func pixelBuffer(width: Int, height: Int, pixelFormatType: OSType, 50 | colorSpace: CGColorSpace, alphaInfo: CGImageAlphaInfo) -> CVPixelBuffer? { 51 | var maybePixelBuffer: CVPixelBuffer? 52 | let attrs = [kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue, 53 | kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue] 54 | let status = CVPixelBufferCreate(kCFAllocatorDefault, 55 | width, 56 | height, 57 | pixelFormatType, 58 | attrs as CFDictionary, 59 | &maybePixelBuffer) 60 | 61 | guard status == kCVReturnSuccess, let pixelBuffer = maybePixelBuffer else { 62 | return nil 63 | } 64 | 65 | let flags = CVPixelBufferLockFlags(rawValue: 0) 66 | guard kCVReturnSuccess == CVPixelBufferLockBaseAddress(pixelBuffer, flags) else { 67 | return nil 68 | } 69 | defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, flags) } 70 | 71 | guard let context = CGContext(data: CVPixelBufferGetBaseAddress(pixelBuffer), 72 | width: width, 73 | height: height, 74 | bitsPerComponent: 8, 75 | bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer), 76 | space: colorSpace, 77 | bitmapInfo: alphaInfo.rawValue) 78 | else { 79 | return nil 80 | } 81 | 82 | UIGraphicsPushContext(context) 83 | context.translateBy(x: 0, y: CGFloat(height)) 84 | context.scaleBy(x: 1, y: -1) 85 | self.draw(in: CGRect(x: 0, y: 0, width: width, height: height)) 86 | UIGraphicsPopContext() 87 | 88 | return pixelBuffer 89 | } 90 | } 91 | 92 | extension UIImage { 93 | /** 94 | Creates a new UIImage from a CVPixelBuffer. 95 | 96 | - Note: Not all CVPixelBuffer pixel formats support conversion into a 97 | CGImage-compatible pixel format. 98 | */ 99 | public convenience init?(pixelBuffer: CVPixelBuffer) { 100 | if let cgImage = CGImage.create(pixelBuffer: pixelBuffer) { 101 | self.init(cgImage: cgImage) 102 | } else { 103 | return nil 104 | } 105 | } 106 | 107 | /* 108 | // Alternative implementation: 109 | public convenience init?(pixelBuffer: CVPixelBuffer) { 110 | // This converts the image to a CIImage first and then to a UIImage. 111 | // Does not appear to work on the simulator but is OK on the device. 112 | self.init(ciImage: CIImage(cvPixelBuffer: pixelBuffer)) 113 | } 114 | */ 115 | 116 | /** 117 | Creates a new UIImage from a CVPixelBuffer, using a Core Image context. 118 | */ 119 | public convenience init?(pixelBuffer: CVPixelBuffer, context: CIContext) { 120 | if let cgImage = CGImage.create(pixelBuffer: pixelBuffer, context: context) { 121 | self.init(cgImage: cgImage) 122 | } else { 123 | return nil 124 | } 125 | } 126 | } 127 | 128 | #endif 129 | -------------------------------------------------------------------------------- /coreml_ios/CoreMLHelpers/UIImage+Extensions.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017-2018 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | #if canImport(UIKit) 24 | 25 | import UIKit 26 | 27 | extension UIImage { 28 | /** 29 | Resizes the image. 30 | 31 | - Parameters: 32 | - scale: If this is 1, `newSize` is the size in pixels. 33 | */ 34 | @nonobjc public func resized(to newSize: CGSize, scale: CGFloat = 1) -> UIImage { 35 | let format = UIGraphicsImageRendererFormat.default() 36 | format.scale = scale 37 | let renderer = UIGraphicsImageRenderer(size: newSize, format: format) 38 | let image = renderer.image { _ in 39 | draw(in: CGRect(origin: .zero, size: newSize)) 40 | } 41 | return image 42 | } 43 | } 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /coreml_ios/CoreMLHelpers/UIImage+RawBytes.swift: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2017-2018 M.I. Hollemans 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | */ 22 | 23 | #if canImport(UIKit) 24 | 25 | import UIKit 26 | 27 | extension UIImage { 28 | /** 29 | Converts the image into an array of RGBA bytes. 30 | */ 31 | @nonobjc public func toByteArrayRGBA() -> [UInt8]? { 32 | return cgImage?.toByteArrayRGBA() 33 | } 34 | 35 | /** 36 | Creates a new UIImage from an array of RGBA bytes. 37 | */ 38 | @nonobjc public class func fromByteArrayRGBA(_ bytes: [UInt8], 39 | width: Int, 40 | height: Int, 41 | scale: CGFloat = 0, 42 | orientation: UIImage.Orientation = .up) -> UIImage? { 43 | if let cgImage = CGImage.fromByteArrayRGBA(bytes, width: width, height: height) { 44 | return UIImage(cgImage: cgImage, scale: scale, orientation: orientation) 45 | } else { 46 | return nil 47 | } 48 | } 49 | 50 | /** 51 | Creates a new UIImage from an array of grayscale bytes. 52 | */ 53 | @nonobjc public class func fromByteArrayGray(_ bytes: [UInt8], 54 | width: Int, 55 | height: Int, 56 | scale: CGFloat = 0, 57 | orientation: UIImage.Orientation = .up) -> UIImage? { 58 | if let cgImage = CGImage.fromByteArrayGray(bytes, width: width, height: height) { 59 | return UIImage(cgImage: cgImage, scale: scale, orientation: orientation) 60 | } else { 61 | return nil 62 | } 63 | } 64 | } 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /coreml_ios/Podfile: -------------------------------------------------------------------------------- 1 | # Uncomment the next line to define a global platform for your project 2 | # platform :ios, '9.0' 3 | 4 | target 'Core ML Demo' do 5 | # Comment the next line if you're not using Swift and don't want to use dynamic frameworks 6 | use_frameworks! 7 | 8 | # Pods for Core ML Demo 9 | pod 'CoreMLHelpers', git: 'https://github.com/hollance/CoreMLHelpers.git', branch: 'master' 10 | end 11 | -------------------------------------------------------------------------------- /coreml_ios/Podfile.lock: -------------------------------------------------------------------------------- 1 | PODS: 2 | - CoreMLHelpers (0.2.0) 3 | 4 | DEPENDENCIES: 5 | - CoreMLHelpers (from `https://github.com/hollance/CoreMLHelpers.git`, branch `master`) 6 | 7 | EXTERNAL SOURCES: 8 | CoreMLHelpers: 9 | :branch: master 10 | :git: https://github.com/hollance/CoreMLHelpers.git 11 | 12 | CHECKOUT OPTIONS: 13 | CoreMLHelpers: 14 | :commit: 6df452776c94cd55f576ad6d1ae04b7d49b3d50e 15 | :git: https://github.com/hollance/CoreMLHelpers.git 16 | 17 | SPEC CHECKSUMS: 18 | CoreMLHelpers: 6d0b6e69d950ed1abce6866c44e9441587b8f9dc 19 | 20 | PODFILE CHECKSUM: ede493c2b97920637d34e03bb3f5c6573f1919e4 21 | 22 | COCOAPODS: 1.4.0 23 | -------------------------------------------------------------------------------- /images/header.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/images/header.jpg -------------------------------------------------------------------------------- /tf_lite_android/demo/.gitignore: -------------------------------------------------------------------------------- 1 | # This file is based on https://github.com/github/gitignore/blob/master/Android.gitignore 2 | *.iml 3 | .idea/compiler.xml 4 | .idea/copyright 5 | .idea/dictionaries 6 | .idea/gradle.xml 7 | .idea/libraries 8 | .idea/inspectionProfiles 9 | .idea/misc.xml 10 | .idea/modules.xml 11 | .idea/runConfigurations.xml 12 | .idea/tasks.xml 13 | .idea/workspace.xml 14 | .gradle 15 | local.properties 16 | .DS_Store 17 | build/ 18 | gradleBuild/ 19 | *.apk 20 | *.ap_ 21 | *.dex 22 | *.class 23 | bin/ 24 | gen/ 25 | out/ 26 | *.log 27 | .navigation/ 28 | /captures 29 | .externalNativeBuild 30 | -------------------------------------------------------------------------------- /tf_lite_android/demo/README.md: -------------------------------------------------------------------------------- 1 | # TF Lite Android Image Classifier App Example 2 | 3 | A simple Android example that demonstrates image classification using the camera. 4 | 5 | ## Building in Android Studio with TensorFlow Lite AAR from JCenter. 6 | The build.gradle is configured to use TensorFlow Lite's nightly build. 7 | 8 | If you see a build error related to compatibility with Tensorflow Lite's Java API (example: method X is 9 | undefined for type Interpreter), there has likely been a backwards compatible 10 | change to the API. You will need to pull new app code that's compatible with the 11 | nightly build and may need to first wait a few days for our external and internal 12 | code to merge. 13 | 14 | ## Building from Source with Bazel 15 | 16 | 1. Follow the [Bazel steps for the TF Demo App](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#bazel): 17 | 18 | 1. [Install Bazel and Android Prerequisites](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#install-bazel-and-android-prerequisites). 19 | It's easiest with Android Studio. 20 | 21 | - You'll need at least SDK version 23. 22 | - Make sure to install the latest version of Bazel. Some distributions 23 | ship with Bazel 0.5.4, which is too old. 24 | - Bazel requires Android Build Tools `26.0.1` or higher. 25 | - You also need to install the Android Support Repository, available 26 | through Android Studio under `Android SDK Manager -> SDK Tools -> 27 | Android Support Repository`. 28 | 29 | 2. [Edit your `WORKSPACE`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#edit-workspace) 30 | to add SDK and NDK targets. 31 | 32 | NOTE: As long as you have the SDK and NDK installed, the `./configure` 33 | script will create these rules for you. Answer "Yes" when the script asks 34 | to automatically configure the `./WORKSPACE`. 35 | 36 | - Make sure the `api_level` in `WORKSPACE` is set to an SDK version that 37 | you have installed. 38 | - By default, Android Studio will install the SDK to `~/Android/Sdk` and 39 | the NDK to `~/Android/Sdk/ndk-bundle`. 40 | 41 | 2. Build the app with Bazel. The demo needs C++11: 42 | 43 | ```shell 44 | bazel build -c opt --cxxopt='--std=c++11' \ 45 | //tensorflow/lite/java/demo/app/src/main:TfLiteCameraDemo 46 | ``` 47 | 48 | 3. Install the demo on a 49 | [debug-enabled device](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#install): 50 | 51 | ```shell 52 | adb install bazel-bin/tensorflow/lite/java/demo/app/src/main/TfLiteCameraDemo.apk 53 | ``` 54 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'com.android.application' 2 | 3 | android { 4 | compileSdkVersion 26 5 | defaultConfig { 6 | applicationId "me.ndres.tflitedemo" 7 | // Required by Camera2 API. 8 | minSdkVersion 21 9 | targetSdkVersion 26 10 | versionCode 1 11 | versionName "1.0" 12 | } 13 | lintOptions { 14 | abortOnError false 15 | } 16 | buildTypes { 17 | release { 18 | minifyEnabled false 19 | proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' 20 | } 21 | } 22 | aaptOptions { 23 | noCompress "tflite" 24 | } 25 | 26 | compileOptions { 27 | sourceCompatibility JavaVersion.VERSION_1_8 28 | targetCompatibility JavaVersion.VERSION_1_8 29 | } 30 | } 31 | 32 | repositories { 33 | maven { 34 | url 'https://google.bintray.com/tensorflow' 35 | google() 36 | } 37 | } 38 | 39 | allprojects { 40 | repositories { 41 | // Uncomment if you want to use a local repo. 42 | // mavenLocal() 43 | jcenter() 44 | } 45 | } 46 | 47 | dependencies { 48 | implementation fileTree(dir: 'libs', include: ['*.jar']) 49 | implementation 'com.android.support:appcompat-v7:25.2.0' 50 | implementation 'com.android.support.constraint:constraint-layout:1.0.2' 51 | implementation 'com.android.support:design:25.2.0' 52 | implementation 'com.android.support:support-annotations:25.3.1' 53 | implementation 'com.android.support:support-v13:25.2.0' 54 | 55 | // Build off of nightly TensorFlow Lite 56 | implementation 'org.tensorflow:tensorflow-lite:0.0.0-nightly' 57 | // Use local TensorFlow library 58 | // implementation 'org.tensorflow:tensorflow-lite-local:0.0.0' 59 | implementation 'com.google.android.gms:play-services-vision:9.4.0' 60 | 61 | } 62 | 63 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 32 | 33 | 36 | 39 | 40 | 41 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/BUILD: -------------------------------------------------------------------------------- 1 | load("@build_bazel_rules_android//android:rules.bzl", "android_binary") 2 | 3 | package(default_visibility = ["//visibility:private"]) 4 | 5 | licenses(["notice"]) # Apache 2.0 6 | 7 | android_binary( 8 | name = "TfLiteCameraDemo", 9 | srcs = glob(["java/**/*.java"]), 10 | aapt_version = "aapt", 11 | assets = [ 12 | "//tensorflow/lite/java/demo/app/src/main/assets:labels_mobilenet_quant_v1_224.txt", 13 | "@tflite_mobilenet_quant//:mobilenet_v1_1.0_224_quant.tflite", 14 | "@tflite_mobilenet_float//:mobilenet_v1_1.0_224.tflite", 15 | ], 16 | assets_dir = "", 17 | custom_package = "com.example.android.tflitecamerademo", 18 | manifest = "AndroidManifest.xml", 19 | nocompress_extensions = [ 20 | ".tflite", 21 | ], 22 | resource_files = glob(["res/**"]), 23 | # In some platforms we don't have an Android SDK/NDK and this target 24 | # can't be built. We need to prevent the build system from trying to 25 | # use the target in that case. 26 | tags = ["manual"], 27 | deps = [ 28 | "//tensorflow/lite/java:tensorflowlite", 29 | "//tensorflow/lite/java/src/testhelper/java/org/tensorflow/lite:testhelper", 30 | "@androidsdk//com.android.support:support-v13-25.2.0", 31 | "@androidsdk//com.android.support:support-v4-25.2.0", 32 | ], 33 | ) 34 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/assets/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:private"]) 2 | 3 | licenses(["notice"]) # Apache 2.0 4 | 5 | exports_files( 6 | glob( 7 | ["**/*"], 8 | exclude = [ 9 | "BUILD", 10 | ], 11 | ), 12 | ) 13 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/assets/converted_model.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tf_lite_android/demo/app/src/main/assets/converted_model.tflite -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/assets/labels_emotion.txt: -------------------------------------------------------------------------------- 1 | angry 2 | disgust 3 | scared 4 | happy 5 | sad 6 | surprised 7 | neutral -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/java/me/ndres/tflitedemo/AutoFitTextureView.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | package me.ndres.tflitedemo; 17 | 18 | import android.content.Context; 19 | import android.util.AttributeSet; 20 | import android.view.TextureView; 21 | 22 | /** A {@link TextureView} that can be adjusted to a specified aspect ratio. */ 23 | public class AutoFitTextureView extends TextureView { 24 | 25 | private int mRatioWidth = 0; 26 | private int mRatioHeight = 0; 27 | 28 | public AutoFitTextureView(Context context) { 29 | this(context, null); 30 | } 31 | 32 | public AutoFitTextureView(Context context, AttributeSet attrs) { 33 | this(context, attrs, 0); 34 | } 35 | 36 | public AutoFitTextureView(Context context, AttributeSet attrs, int defStyle) { 37 | super(context, attrs, defStyle); 38 | } 39 | 40 | /** 41 | * Sets the aspect ratio for this view. The size of the view will be measured based on the ratio 42 | * calculated from the parameters. Note that the actual sizes of parameters don't matter, that is, 43 | * calling setAspectRatio(2, 3) and setAspectRatio(4, 6) make the same result. 44 | * 45 | * @param width Relative horizontal size 46 | * @param height Relative vertical size 47 | */ 48 | public void setAspectRatio(int width, int height) { 49 | if (width < 0 || height < 0) { 50 | throw new IllegalArgumentException("Size cannot be negative."); 51 | } 52 | mRatioWidth = width; 53 | mRatioHeight = height; 54 | requestLayout(); 55 | } 56 | 57 | @Override 58 | protected void onMeasure(int widthMeasureSpec, int heightMeasureSpec) { 59 | super.onMeasure(widthMeasureSpec, heightMeasureSpec); 60 | int width = MeasureSpec.getSize(widthMeasureSpec); 61 | int height = MeasureSpec.getSize(heightMeasureSpec); 62 | if (0 == mRatioWidth || 0 == mRatioHeight) { 63 | setMeasuredDimension(width, height); 64 | } else { 65 | if (width < height * mRatioWidth / mRatioHeight) { 66 | setMeasuredDimension(width, width * mRatioHeight / mRatioWidth); 67 | } else { 68 | setMeasuredDimension(height * mRatioWidth / mRatioHeight, height); 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/java/me/ndres/tflitedemo/BorderedText.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | package me.ndres.tflitedemo; 17 | 18 | import android.graphics.Canvas; 19 | import android.graphics.Color; 20 | import android.graphics.Paint; 21 | import android.graphics.Paint.Align; 22 | import android.graphics.Paint.Style; 23 | import android.graphics.Rect; 24 | import android.graphics.Typeface; 25 | 26 | import java.util.Vector; 27 | 28 | /** 29 | * A class that encapsulates the tedious bits of rendering legible, bordered text onto a canvas. 30 | */ 31 | public class BorderedText { 32 | private final Paint interiorPaint; 33 | private final Paint exteriorPaint; 34 | 35 | private final float textSize; 36 | 37 | /** 38 | * Creates a left-aligned bordered text object with a white interior, and a black exterior with 39 | * the specified text size. 40 | * 41 | * @param textSize text size in pixels 42 | */ 43 | public BorderedText(final float textSize) { 44 | this(Color.WHITE, Color.BLACK, textSize); 45 | } 46 | 47 | /** 48 | * Create a bordered text object with the specified interior and exterior colors, text size and 49 | * alignment. 50 | * 51 | * @param interiorColor the interior text color 52 | * @param exteriorColor the exterior text color 53 | * @param textSize text size in pixels 54 | */ 55 | public BorderedText(final int interiorColor, final int exteriorColor, final float textSize) { 56 | interiorPaint = new Paint(); 57 | interiorPaint.setTextSize(textSize); 58 | interiorPaint.setColor(interiorColor); 59 | interiorPaint.setStyle(Style.FILL); 60 | interiorPaint.setAntiAlias(false); 61 | interiorPaint.setAlpha(255); 62 | 63 | exteriorPaint = new Paint(); 64 | exteriorPaint.setTextSize(textSize); 65 | exteriorPaint.setColor(exteriorColor); 66 | exteriorPaint.setStyle(Style.FILL_AND_STROKE); 67 | exteriorPaint.setStrokeWidth(textSize / 8); 68 | exteriorPaint.setAntiAlias(false); 69 | exteriorPaint.setAlpha(255); 70 | 71 | this.textSize = textSize; 72 | } 73 | 74 | public void setTypeface(Typeface typeface) { 75 | interiorPaint.setTypeface(typeface); 76 | exteriorPaint.setTypeface(typeface); 77 | } 78 | 79 | public void drawText(final Canvas canvas, final float posX, final float posY, final String text) { 80 | canvas.drawText(text, posX, posY, exteriorPaint); 81 | canvas.drawText(text, posX, posY, interiorPaint); 82 | } 83 | 84 | public void drawLines(Canvas canvas, final float posX, final float posY, Vector lines) { 85 | int lineNum = 0; 86 | for (final String line : lines) { 87 | drawText(canvas, posX, posY - getTextSize() * (lines.size() - lineNum - 1), line); 88 | // drawText(canvas, posX, posY+ getTextSize() * (lines.size() - lineNum - 1), line); 89 | 90 | ++lineNum; 91 | } 92 | } 93 | 94 | public void setInteriorColor(final int color) { 95 | interiorPaint.setColor(color); 96 | } 97 | 98 | public void setExteriorColor(final int color) { 99 | exteriorPaint.setColor(color); 100 | } 101 | 102 | public float getTextSize() { 103 | return textSize; 104 | } 105 | 106 | public void setAlpha(final int alpha) { 107 | interiorPaint.setAlpha(alpha); 108 | exteriorPaint.setAlpha(alpha); 109 | } 110 | 111 | public void getTextBounds( 112 | final String line, final int index, final int count, final Rect lineBounds) { 113 | interiorPaint.getTextBounds(line, index, count, lineBounds); 114 | } 115 | 116 | public void setTextAlign(final Align align) { 117 | interiorPaint.setTextAlign(align); 118 | exteriorPaint.setTextAlign(align); 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/java/me/ndres/tflitedemo/CameraActivity.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | package me.ndres.tflitedemo; 17 | 18 | import android.app.Activity; 19 | import android.os.Bundle; 20 | 21 | /** Main {@code Activity} class for the Camera app. */ 22 | public class CameraActivity extends Activity { 23 | 24 | @Override 25 | protected void onCreate(Bundle savedInstanceState) { 26 | super.onCreate(savedInstanceState); 27 | setContentView(R.layout.activity_camera); 28 | if (null == savedInstanceState) { 29 | getFragmentManager() 30 | .beginTransaction() 31 | .replace(R.id.container, Camera2BasicFragment.newInstance()) 32 | .commit(); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/java/me/ndres/tflitedemo/GpuDelegateHelper.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | package me.ndres.tflitedemo; 17 | 18 | import org.tensorflow.lite.Delegate; 19 | 20 | /** 21 | * Helper class for {@code GpuDelegate}. 22 | * 23 | *

WARNING: This is an experimental API and subject to change. 24 | */ 25 | public class GpuDelegateHelper { 26 | private GpuDelegateHelper() {} 27 | 28 | /** Checks whether {@code GpuDelegate} is available. */ 29 | public static boolean isGpuDelegateAvailable() { 30 | try { 31 | Class.forName("org.tensorflow.lite.experimental.GpuDelegate"); 32 | return true; 33 | } catch (Exception e) { 34 | return false; 35 | } 36 | } 37 | 38 | /** Returns an instance of {@code GpuDelegate} if available. */ 39 | public static Delegate createGpuDelegate() { 40 | try { 41 | return Class.forName("org.tensorflow.lite.experimental.GpuDelegate") 42 | .asSubclass(Delegate.class) 43 | .getDeclaredConstructor() 44 | .newInstance(); 45 | } catch (Exception e) { 46 | throw new IllegalStateException(e); 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/java/me/ndres/tflitedemo/ImageClassifierFloatInception.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | package me.ndres.tflitedemo; 17 | 18 | import android.app.Activity; 19 | 20 | import java.io.IOException; 21 | 22 | /** 23 | * This classifier works with the Inception-v3 slim model. 24 | * It applies floating point inference rather than using a quantized model. 25 | */ 26 | public class ImageClassifierFloatInception extends ImageClassifier { 27 | 28 | /** 29 | * The inception net requires additional normalization of the used input. 30 | */ 31 | private static final int IMAGE_MEAN = 128; 32 | private static final float IMAGE_STD = 128.0f; 33 | 34 | /** 35 | * An array to hold inference results, to be feed into Tensorflow Lite as outputs. 36 | * This isn't part of the super class, because we need a primitive array here. 37 | */ 38 | private float[][] labelProbArray = null; 39 | 40 | /** 41 | * Initializes an {@code ImageClassifier}. 42 | * 43 | * @param activity 44 | */ 45 | ImageClassifierFloatInception(Activity activity) throws IOException { 46 | super(activity); 47 | labelProbArray = new float[1][getNumLabels()]; 48 | } 49 | 50 | @Override 51 | protected String getModelPath() { 52 | // you can download this file from 53 | // https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip 54 | return "inceptionv3_slim_2016.tflite"; 55 | } 56 | 57 | @Override 58 | protected String getLabelPath() { 59 | return "labels_imagenet_slim.txt"; 60 | } 61 | 62 | @Override 63 | protected int getImageSizeX() { 64 | return 299; 65 | } 66 | 67 | @Override 68 | protected int getImageSizeY() { 69 | return 299; 70 | } 71 | 72 | @Override 73 | protected int getNumBytesPerChannel() { 74 | // a 32bit float value requires 4 bytes 75 | return 4; 76 | } 77 | 78 | @Override 79 | protected void addPixelValue(int pixelValue) { 80 | imgData.putFloat((((pixelValue >> 16) & 0xFF) - IMAGE_MEAN) / IMAGE_STD); 81 | imgData.putFloat((((pixelValue >> 8) & 0xFF) - IMAGE_MEAN) / IMAGE_STD); 82 | imgData.putFloat(((pixelValue & 0xFF) - IMAGE_MEAN) / IMAGE_STD); 83 | } 84 | 85 | @Override 86 | protected float getProbability(int labelIndex) { 87 | return labelProbArray[0][labelIndex]; 88 | } 89 | 90 | @Override 91 | protected void setProbability(int labelIndex, Number value) { 92 | labelProbArray[0][labelIndex] = value.floatValue(); 93 | } 94 | 95 | @Override 96 | protected float getNormalizedProbability(int labelIndex) { 97 | // TODO the following value isn't in [0,1] yet, but may be greater. Why? 98 | return getProbability(labelIndex); 99 | } 100 | 101 | @Override 102 | protected void runInference() { 103 | tflite.run(imgData, labelProbArray); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/java/me/ndres/tflitedemo/ImageClassifierFloatMobileNet.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | package me.ndres.tflitedemo; 17 | 18 | import android.app.Activity; 19 | import java.io.IOException; 20 | 21 | /** This classifier works with the float MobileNet model. */ 22 | public class ImageClassifierFloatMobileNet extends ImageClassifier { 23 | 24 | /** 25 | * An array to hold inference results, to be feed into Tensorflow Lite as outputs. This isn't part 26 | * of the super class, because we need a primitive array here. 27 | */ 28 | private float[][] labelProbArray = null; 29 | 30 | /** 31 | * Initializes an {@code ImageClassifierFloatMobileNet}. 32 | * 33 | * @param activity 34 | */ 35 | ImageClassifierFloatMobileNet(Activity activity) throws IOException { 36 | super(activity); 37 | labelProbArray = new float[1][getNumLabels()]; 38 | } 39 | 40 | @Override 41 | protected String getModelPath() { 42 | // you can download this file from 43 | // see build.gradle for where to obtain this file. It should be auto 44 | // downloaded into assets. 45 | return "converted_model.tflite"; 46 | } 47 | 48 | @Override 49 | protected String getLabelPath() { 50 | return "labels_emotion.txt"; 51 | } 52 | 53 | @Override 54 | protected int getImageSizeX() { 55 | return 48; 56 | } 57 | 58 | @Override 59 | protected int getImageSizeY() { 60 | return 48; 61 | } 62 | 63 | @Override 64 | protected int getNumBytesPerChannel() { 65 | return 4; // Float.SIZE / Byte.SIZE; 66 | } 67 | 68 | @Override 69 | protected void addPixelValue(int pixelValue) { 70 | float ret = (((pixelValue >> 16) & 0xFF) + ((pixelValue >> 8) & 0xFF) + (pixelValue & 0xFF)) / 3.0f / 127.0f; 71 | imgData.putFloat(ret - 1.0f); 72 | } 73 | 74 | @Override 75 | protected float getProbability(int labelIndex) { 76 | return labelProbArray[0][labelIndex]; 77 | } 78 | 79 | @Override 80 | protected void setProbability(int labelIndex, Number value) { 81 | labelProbArray[0][labelIndex] = value.floatValue(); 82 | } 83 | 84 | @Override 85 | protected float getNormalizedProbability(int labelIndex) { 86 | return labelProbArray[0][labelIndex]; 87 | } 88 | 89 | @Override 90 | protected void runInference() { 91 | tflite.run(imgData, labelProbArray); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/java/me/ndres/tflitedemo/ImageClassifierQuantizedMobileNet.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | package me.ndres.tflitedemo; 17 | 18 | import android.app.Activity; 19 | import java.io.IOException; 20 | 21 | /** 22 | * This classifier works with the quantized MobileNet model. 23 | */ 24 | public class ImageClassifierQuantizedMobileNet extends ImageClassifier { 25 | 26 | /** 27 | * An array to hold inference results, to be feed into Tensorflow Lite as outputs. 28 | * This isn't part of the super class, because we need a primitive array here. 29 | */ 30 | private byte[][] labelProbArray = null; 31 | 32 | /** 33 | * Initializes an {@code ImageClassifier}. 34 | * 35 | * @param activity 36 | */ 37 | ImageClassifierQuantizedMobileNet(Activity activity) throws IOException { 38 | super(activity); 39 | labelProbArray = new byte[1][getNumLabels()]; 40 | } 41 | 42 | @Override 43 | protected String getModelPath() { 44 | // you can download this file from 45 | // see build.gradle for where to obtain this file. It should be auto 46 | // downloaded into assets. 47 | return "mobilenet_v1_1.0_224_quant.tflite"; 48 | } 49 | 50 | @Override 51 | protected String getLabelPath() { 52 | return "labels_emotion.txt"; 53 | } 54 | 55 | @Override 56 | protected int getImageSizeX() { 57 | return 224; 58 | } 59 | 60 | @Override 61 | protected int getImageSizeY() { 62 | return 224; 63 | } 64 | 65 | @Override 66 | protected int getNumBytesPerChannel() { 67 | // the quantized model uses a single byte only 68 | return 1; 69 | } 70 | 71 | @Override 72 | protected void addPixelValue(int pixelValue) { 73 | imgData.put((byte) ((pixelValue >> 16) & 0xFF)); 74 | imgData.put((byte) ((pixelValue >> 8) & 0xFF)); 75 | imgData.put((byte) (pixelValue & 0xFF)); 76 | } 77 | 78 | @Override 79 | protected float getProbability(int labelIndex) { 80 | return labelProbArray[0][labelIndex]; 81 | } 82 | 83 | @Override 84 | protected void setProbability(int labelIndex, Number value) { 85 | labelProbArray[0][labelIndex] = value.byteValue(); 86 | } 87 | 88 | @Override 89 | protected float getNormalizedProbability(int labelIndex) { 90 | return (labelProbArray[0][labelIndex] & 0xff) / 255.0f; 91 | } 92 | 93 | @Override 94 | protected void runInference() { 95 | tflite.run(imgData, labelProbArray); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/java/me/ndres/tflitedemo/OverlayView.java: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | package me.ndres.tflitedemo; 17 | 18 | import android.content.Context; 19 | import android.graphics.Canvas; 20 | import android.util.AttributeSet; 21 | import android.view.View; 22 | 23 | import java.util.LinkedList; 24 | import java.util.List; 25 | 26 | /** 27 | * A simple View providing a render callback to other classes. 28 | */ 29 | public class OverlayView extends View { 30 | private final List callbacks = new LinkedList(); 31 | 32 | public OverlayView(final Context context, final AttributeSet attrs) { 33 | super(context, attrs); 34 | } 35 | 36 | /** 37 | * Interface defining the callback for client classes. 38 | */ 39 | public interface DrawCallback { 40 | public void drawCallback(final Canvas canvas); 41 | } 42 | 43 | public void addCallback(final DrawCallback callback) { 44 | callbacks.add(callback); 45 | } 46 | 47 | @Override 48 | public synchronized void draw(final Canvas canvas) { 49 | for (final DrawCallback callback : callbacks) { 50 | callback.drawCallback(canvas); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/drawable-hdpi/ic_action_info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tf_lite_android/demo/app/src/main/res/drawable-hdpi/ic_action_info.png -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/drawable-hdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tf_lite_android/demo/app/src/main/res/drawable-hdpi/ic_launcher.png -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/drawable-hdpi/tile.9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tf_lite_android/demo/app/src/main/res/drawable-hdpi/tile.9.png -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/drawable-mdpi/ic_action_info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tf_lite_android/demo/app/src/main/res/drawable-mdpi/ic_action_info.png -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/drawable-mdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tf_lite_android/demo/app/src/main/res/drawable-mdpi/ic_launcher.png -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/drawable-xhdpi/ic_action_info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tf_lite_android/demo/app/src/main/res/drawable-xhdpi/ic_action_info.png -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/drawable-xhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tf_lite_android/demo/app/src/main/res/drawable-xhdpi/ic_launcher.png -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/drawable-xxhdpi/ic_action_info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tf_lite_android/demo/app/src/main/res/drawable-xxhdpi/ic_action_info.png -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/drawable-xxhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tf_lite_android/demo/app/src/main/res/drawable-xxhdpi/ic_launcher.png -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/drawable-xxhdpi/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tf_lite_android/demo/app/src/main/res/drawable-xxhdpi/logo.png -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/drawable/item_selector.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/layout-land/fragment_camera2_basic.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 23 | 24 | 29 | 30 | 35 | 36 | 42 | 43 | 51 | 56 | 57 | 64 | 65 | 69 | 70 | 71 | 72 | 77 | 78 | 85 | 86 | 90 | 91 | 92 | 93 | 97 | 98 | 105 | 106 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/layout-v26/fragment_camera2_basic.xml: -------------------------------------------------------------------------------- 1 | 16 | 21 | 22 | 27 | 28 | 38 | 39 | 46 | 47 | 58 | 59 | 60 | 61 | 70 | 71 | 75 | 76 | 83 | 84 | 91 | 92 | 93 | 94 | 99 | 100 | 107 | 108 | 112 | 113 | 114 | 115 | 116 | 121 | 122 | 129 | 130 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/layout/activity_camera.xml: -------------------------------------------------------------------------------- 1 | 16 | 23 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/layout/fragment_camera2_basic.xml: -------------------------------------------------------------------------------- 1 | 16 | 21 | 22 | 23 | 24 | 28 | 29 | 34 | 35 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 57 | 58 | 69 | 70 | 71 | 72 | 82 | 83 | 87 | 88 | 95 | 96 | 103 | 104 | 105 | 106 | 111 | 112 | 119 | 120 | 124 | 125 | 126 | 127 | 128 | 133 | 134 | 141 | 142 | 146 | 147 | 148 | 149 | 150 | 151 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/layout/listview_row.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 16 | 17 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/values-sw600dp/template-dimens.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | 20 | 21 | @dimen/margin_huge 22 | @dimen/margin_medium 23 | 24 | 25 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/values-sw600dp/template-styles.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/values-v11/template-styles.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/values/base-strings.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | TfLite Camera Demo 20 | 21 | 29 | 30 | Threads: 31 | 32 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/values/colors.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | #cc4285f4 19 | #aaaaaa 20 | #eeaa55 21 | #eeeeee 22 | 23 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/values/strings.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | Picture 18 | Info 19 | This sample needs camera permission. 20 | This device doesn\'t support Camera2 API. 21 | NN:On 22 | NN:Off 23 | Use NNAPI 24 | tflite 25 | NNAPI 26 | GPU 27 | CPU 28 | Model 29 | Device 30 | mobilenet v1 quant; 31 | mobilenet v1 float;; 32 | 33 | 34 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/values/styles.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/values/template-dimens.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | 20 | 21 | 4dp 22 | 8dp 23 | 16dp 24 | 32dp 25 | 64dp 26 | 27 | 28 | 29 | @dimen/margin_medium 30 | @dimen/margin_medium 31 | 32 | 33 | -------------------------------------------------------------------------------- /tf_lite_android/demo/app/src/main/res/values/template-styles.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | 20 | 21 | 34 | 35 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /tf_lite_android/demo/build.gradle: -------------------------------------------------------------------------------- 1 | // Top-level build file where you can add configuration options common to all sub-projects/modules. 2 | 3 | buildscript { 4 | repositories { 5 | google() 6 | jcenter() 7 | } 8 | dependencies { 9 | classpath 'com.android.tools.build:gradle:3.3.0' 10 | 11 | // NOTE: Do not place your application dependencies here; they belong 12 | // in the individual module build.gradle files 13 | } 14 | } 15 | 16 | allprojects { 17 | repositories { 18 | jcenter() 19 | } 20 | } 21 | 22 | task clean(type: Delete) { 23 | delete rootProject.buildDir 24 | } 25 | -------------------------------------------------------------------------------- /tf_lite_android/demo/gradle.properties: -------------------------------------------------------------------------------- 1 | # Project-wide Gradle settings. 2 | 3 | # IDE (e.g. Android Studio) users: 4 | # Gradle settings configured through the IDE *will override* 5 | # any settings specified in this file. 6 | 7 | # For more details on how to configure your build environment visit 8 | # http://www.gradle.org/docs/current/userguide/build_environment.html 9 | 10 | # Specifies the JVM arguments used for the daemon process. 11 | # The setting is particularly useful for tweaking memory settings. 12 | org.gradle.jvmargs=-Xmx1536m 13 | 14 | # When configured, Gradle will run in incubating parallel mode. 15 | # This option should only be used with decoupled projects. More details, visit 16 | # http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects 17 | # org.gradle.parallel=true 18 | -------------------------------------------------------------------------------- /tf_lite_android/demo/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tf_lite_android/demo/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /tf_lite_android/demo/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Tue Feb 05 16:26:59 COT 2019 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.1-all.zip 7 | -------------------------------------------------------------------------------- /tf_lite_android/demo/gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 10 | DEFAULT_JVM_OPTS="" 11 | 12 | APP_NAME="Gradle" 13 | APP_BASE_NAME=`basename "$0"` 14 | 15 | # Use the maximum available, or set MAX_FD != -1 to use that value. 16 | MAX_FD="maximum" 17 | 18 | warn ( ) { 19 | echo "$*" 20 | } 21 | 22 | die ( ) { 23 | echo 24 | echo "$*" 25 | echo 26 | exit 1 27 | } 28 | 29 | # OS specific support (must be 'true' or 'false'). 30 | cygwin=false 31 | msys=false 32 | darwin=false 33 | case "`uname`" in 34 | CYGWIN* ) 35 | cygwin=true 36 | ;; 37 | Darwin* ) 38 | darwin=true 39 | ;; 40 | MINGW* ) 41 | msys=true 42 | ;; 43 | esac 44 | 45 | # Attempt to set APP_HOME 46 | # Resolve links: $0 may be a link 47 | PRG="$0" 48 | # Need this for relative symlinks. 49 | while [ -h "$PRG" ] ; do 50 | ls=`ls -ld "$PRG"` 51 | link=`expr "$ls" : '.*-> \(.*\)$'` 52 | if expr "$link" : '/.*' > /dev/null; then 53 | PRG="$link" 54 | else 55 | PRG=`dirname "$PRG"`"/$link" 56 | fi 57 | done 58 | SAVED="`pwd`" 59 | cd "`dirname \"$PRG\"`/" >/dev/null 60 | APP_HOME="`pwd -P`" 61 | cd "$SAVED" >/dev/null 62 | 63 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 64 | 65 | # Determine the Java command to use to start the JVM. 66 | if [ -n "$JAVA_HOME" ] ; then 67 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 68 | # IBM's JDK on AIX uses strange locations for the executables 69 | JAVACMD="$JAVA_HOME/jre/sh/java" 70 | else 71 | JAVACMD="$JAVA_HOME/bin/java" 72 | fi 73 | if [ ! -x "$JAVACMD" ] ; then 74 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 75 | 76 | Please set the JAVA_HOME variable in your environment to match the 77 | location of your Java installation." 78 | fi 79 | else 80 | JAVACMD="java" 81 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 82 | 83 | Please set the JAVA_HOME variable in your environment to match the 84 | location of your Java installation." 85 | fi 86 | 87 | # Increase the maximum file descriptors if we can. 88 | if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then 89 | MAX_FD_LIMIT=`ulimit -H -n` 90 | if [ $? -eq 0 ] ; then 91 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 92 | MAX_FD="$MAX_FD_LIMIT" 93 | fi 94 | ulimit -n $MAX_FD 95 | if [ $? -ne 0 ] ; then 96 | warn "Could not set maximum file descriptor limit: $MAX_FD" 97 | fi 98 | else 99 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 100 | fi 101 | fi 102 | 103 | # For Darwin, add options to specify how the application appears in the dock 104 | if $darwin; then 105 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 106 | fi 107 | 108 | # For Cygwin, switch paths to Windows format before running java 109 | if $cygwin ; then 110 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 111 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 112 | JAVACMD=`cygpath --unix "$JAVACMD"` 113 | 114 | # We build the pattern for arguments to be converted via cygpath 115 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 116 | SEP="" 117 | for dir in $ROOTDIRSRAW ; do 118 | ROOTDIRS="$ROOTDIRS$SEP$dir" 119 | SEP="|" 120 | done 121 | OURCYGPATTERN="(^($ROOTDIRS))" 122 | # Add a user-defined pattern to the cygpath arguments 123 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 124 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 125 | fi 126 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 127 | i=0 128 | for arg in "$@" ; do 129 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 130 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 131 | 132 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 133 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 134 | else 135 | eval `echo args$i`="\"$arg\"" 136 | fi 137 | i=$((i+1)) 138 | done 139 | case $i in 140 | (0) set -- ;; 141 | (1) set -- "$args0" ;; 142 | (2) set -- "$args0" "$args1" ;; 143 | (3) set -- "$args0" "$args1" "$args2" ;; 144 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 145 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 146 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 147 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 148 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 149 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 150 | esac 151 | fi 152 | 153 | # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules 154 | function splitJvmOpts() { 155 | JVM_OPTS=("$@") 156 | } 157 | eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS 158 | JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" 159 | 160 | exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" 161 | -------------------------------------------------------------------------------- /tf_lite_android/demo/gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 12 | set DEFAULT_JVM_OPTS= 13 | 14 | set DIRNAME=%~dp0 15 | if "%DIRNAME%" == "" set DIRNAME=. 16 | set APP_BASE_NAME=%~n0 17 | set APP_HOME=%DIRNAME% 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windowz variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | if "%@eval[2+2]" == "4" goto 4NT_args 53 | 54 | :win9xME_args 55 | @rem Slurp the command line arguments. 56 | set CMD_LINE_ARGS= 57 | set _SKIP=2 58 | 59 | :win9xME_args_slurp 60 | if "x%~1" == "x" goto execute 61 | 62 | set CMD_LINE_ARGS=%* 63 | goto execute 64 | 65 | :4NT_args 66 | @rem Get arguments from the 4NT Shell from JP Software 67 | set CMD_LINE_ARGS=%$ 68 | 69 | :execute 70 | @rem Setup the command line 71 | 72 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if "%ERRORLEVEL%"=="0" goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 85 | exit /b 1 86 | 87 | :mainEnd 88 | if "%OS%"=="Windows_NT" endlocal 89 | 90 | :omega 91 | -------------------------------------------------------------------------------- /tf_lite_android/demo/settings.gradle: -------------------------------------------------------------------------------- 1 | include ':app' 2 | -------------------------------------------------------------------------------- /tfjs/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "env", 5 | { 6 | "esmodules": false, 7 | "targets": { 8 | "browsers": [ 9 | "> 3%" 10 | ] 11 | } 12 | } 13 | ] 14 | ], 15 | "plugins": [ 16 | "transform-runtime" 17 | ] 18 | } -------------------------------------------------------------------------------- /tfjs/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .cache 3 | dist 4 | -------------------------------------------------------------------------------- /tfjs/cat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tfjs/cat.jpg -------------------------------------------------------------------------------- /tfjs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | TensorFlow.js: Using a pretrained MobileNet 5 | 6 | 7 | 14 | 15 | 16 | 17 | 45 | 46 | 47 |

48 |
49 |

TensorFlow.js: Emotion detection

50 |
51 | 52 |
53 |

Description

54 |

55 | 56 |

57 |
58 | 59 |
60 |

Status

61 |
62 |
63 | 64 |
65 | 66 |
67 | 68 | 69 |
70 |
71 | 72 | 73 |
74 | 75 | 76 |
77 | 78 | -------------------------------------------------------------------------------- /tfjs/index.js: -------------------------------------------------------------------------------- 1 | import * as tf from '@tensorflow/tfjs'; 2 | import * as faceapi from 'face-api.js'; 3 | 4 | const isDev = process.env.NODE_ENV === 'development'; 5 | 6 | const MOBILENET_MODEL_PATH = isDev ? 'http://localhost:1235/emotion_detection/model.json' : './emotion_detection/model.json'; 7 | const DETECTION_MODEL_PATH = isDev ? 'http://localhost:1235/face_detection': './face_detection'; 8 | const FACE_EXPRESSIONS = ["angry","disgust","scared", "happy", "sad", "surprised","neutral"] 9 | 10 | 11 | const IMAGE_SIZE = 48; 12 | 13 | let mobilenet; 14 | const mobilenetDemo = async () => { 15 | status('Loading model...'); 16 | 17 | mobilenet = await tf.loadGraphModel(MOBILENET_MODEL_PATH); 18 | 19 | // Warmup the model. This isn't necessary, but makes the first prediction 20 | // faster. Call `dispose` to release the WebGL memory allocated for the return 21 | // value of `predict`. 22 | mobilenet.predict(tf.zeros([1, IMAGE_SIZE, IMAGE_SIZE, 1])).dispose(); 23 | 24 | status(''); 25 | }; 26 | 27 | /** 28 | * Given an image element, makes a prediction through mobilenet returning the 29 | * probabilities of the top K classes. 30 | */ 31 | async function predict(imgElement) { 32 | let img = await tf.browser.fromPixels(imgElement, 3).toFloat(); 33 | 34 | const logits = tf.tidy(() => { 35 | // tf.fromPixels() returns a Tensor from an image element. 36 | img = tf.image.resizeBilinear(img, [IMAGE_SIZE, IMAGE_SIZE]); 37 | img = img.mean(2); 38 | const offset = tf.scalar(127.5); 39 | // Normalize the image from [0, 255] to [-1, 1]. 40 | const normalized = img.sub(offset).div(offset); 41 | 42 | // Reshape to a single-element batch so we can pass it to predict. 43 | const batched = normalized.reshape([1, IMAGE_SIZE, IMAGE_SIZE, 1]); 44 | 45 | // Make a prediction through mobilenet. 46 | return mobilenet.predict(batched); 47 | }); 48 | 49 | return logits 50 | } 51 | 52 | /** 53 | * Computes the probabilities of the topK classes given logits by computing 54 | * softmax to get probabilities and then sorting the probabilities. 55 | * @param logits Tensor representing the logits from MobileNet. 56 | * @param topK The number of top predictions to show. 57 | */ 58 | export async function getTopClass(values) { 59 | 60 | const valuesAndIndices = []; 61 | for (let i = 0; i < values.length; i++) { 62 | valuesAndIndices.push({value: values[i], index: i}); 63 | } 64 | valuesAndIndices.sort((a, b) => { 65 | return b.value - a.value; 66 | }); 67 | 68 | return valuesAndIndices[0] 69 | } 70 | 71 | // 72 | // UI 73 | // 74 | 75 | 76 | const demoStatusElement = document.getElementById('status'); 77 | const status = msg => demoStatusElement.innerText = msg; 78 | const predictionsElement = document.getElementById('predictions'); 79 | 80 | 81 | 82 | window.onPlay = async function onPlay() { 83 | const video = document.getElementById('video'); 84 | const overlay = document.getElementById('overlay'); 85 | 86 | const detection = await faceapi.detectSingleFace(video, new faceapi.TinyFaceDetectorOptions()) 87 | 88 | if (detection) { 89 | 90 | const faceCanvases = await faceapi.extractFaces(video, [detection]) 91 | 92 | const prediction = await predict(faceCanvases[0]); 93 | 94 | const values = await prediction.data(); 95 | const topClass = await getTopClass(values) 96 | 97 | // TODO(eliot): fix this hack. we should not use private properties 98 | detection._className = FACE_EXPRESSIONS[topClass.index] 99 | detection._classScore = topClass.value 100 | drawDetections(video, overlay, detection) 101 | 102 | } 103 | 104 | setTimeout(window.onPlay, 100) 105 | }; 106 | 107 | function resizeCanvasAndResults(dimensions, canvas, results) { 108 | const { width, height } = dimensions instanceof HTMLVideoElement 109 | ? faceapi.getMediaDimensions(dimensions) 110 | : dimensions 111 | canvas.width = width 112 | canvas.height = height 113 | 114 | // resize detections (and landmarks) in case displayed image is smaller than 115 | // original size 116 | return faceapi.resizeResults(results, { width, height }) 117 | } 118 | 119 | function drawDetections(dimensions, canvas, detections) { 120 | const resizedDetections = resizeCanvasAndResults(dimensions, canvas, detections) 121 | faceapi.drawDetection(canvas, resizedDetections) 122 | } 123 | 124 | async function init() { 125 | var video = document.getElementById('video'); 126 | 127 | await faceapi.loadTinyFaceDetectorModel(DETECTION_MODEL_PATH) 128 | const stream = await navigator.mediaDevices.getUserMedia({ video: {} }) 129 | video.srcObject = stream 130 | }; 131 | 132 | window.onload = async function () { 133 | await mobilenetDemo() 134 | init() 135 | } 136 | -------------------------------------------------------------------------------- /tfjs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tfjs-examples-mobilenet", 3 | "version": "0.1.0", 4 | "description": "", 5 | "main": "index.js", 6 | "license": "Apache-2.0", 7 | "private": true, 8 | "engines": { 9 | "node": ">=8.9.0" 10 | }, 11 | "dependencies": { 12 | "@tensorflow/tfjs": "1.0.4", 13 | "face-api.js": "0.19.0", 14 | "parcel-plugin-static-files-copy": "^2.0.0" 15 | }, 16 | "scripts": { 17 | "watch": "./serve.sh", 18 | "build": "cross-env NODE_ENV=production parcel build index.html --no-minify --public-url ./", 19 | "link-local": "yalc link", 20 | "postinstall": "yarn upgrade --pattern @tensorflow" 21 | }, 22 | "devDependencies": { 23 | "babel-core": "^6.26.3", 24 | "babel-plugin-transform-runtime": "^6.23.0", 25 | "babel-preset-env": "^1.7.0", 26 | "clang-format": "~1.2.2", 27 | "cross-env": "^5.1.6", 28 | "http-server": "~0.10.0", 29 | "parcel-bundler": "~1.10.3", 30 | "yalc": "~1.0.0-pre.22" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /tfjs/serve.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2018 Google LLC. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ============================================================================= 17 | 18 | # This script starts two HTTP servers on different ports: 19 | # * Port 1234 (using parcel) serves HTML and JavaScript. 20 | # * Port 1235 (using http-server) serves pretrained model resources. 21 | # 22 | # The reason for this arrangement is that Parcel currently has a limitation that 23 | # prevents it from serving the pretrained models; see 24 | # https://github.com/parcel-bundler/parcel/issues/1098. Once that issue is 25 | # resolved, a single Parcel server will be sufficient. 26 | 27 | NODE_ENV=development 28 | RESOURCE_PORT=1235 29 | 30 | 31 | echo Starting the pretrained model server... 32 | node_modules/http-server/bin/http-server dist --cors -p "${RESOURCE_PORT}" > /dev/null & HTTP_SERVER_PID=$! 33 | 34 | echo Starting the example html/js server... 35 | # This uses port 1234 by default. 36 | NODE_ENV=development node_modules/parcel-bundler/bin/cli.js serve -d dist index.html --open --no-hmr --public-url / 37 | 38 | # When the Parcel server exits, kill the http-server too. 39 | kill $HTTP_SERVER_PID 40 | 41 | -------------------------------------------------------------------------------- /tfjs/static/emotion_detection/group1-shard1of4.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tfjs/static/emotion_detection/group1-shard1of4.bin -------------------------------------------------------------------------------- /tfjs/static/emotion_detection/group1-shard2of4.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tfjs/static/emotion_detection/group1-shard2of4.bin -------------------------------------------------------------------------------- /tfjs/static/emotion_detection/group1-shard3of4.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tfjs/static/emotion_detection/group1-shard3of4.bin -------------------------------------------------------------------------------- /tfjs/static/emotion_detection/group1-shard4of4.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tfjs/static/emotion_detection/group1-shard4of4.bin -------------------------------------------------------------------------------- /tfjs/static/emotion_detection_old/group1-shard1of4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tfjs/static/emotion_detection_old/group1-shard1of4 -------------------------------------------------------------------------------- /tfjs/static/emotion_detection_old/group1-shard2of4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tfjs/static/emotion_detection_old/group1-shard2of4 -------------------------------------------------------------------------------- /tfjs/static/emotion_detection_old/group1-shard3of4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tfjs/static/emotion_detection_old/group1-shard3of4 -------------------------------------------------------------------------------- /tfjs/static/emotion_detection_old/group1-shard4of4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tfjs/static/emotion_detection_old/group1-shard4of4 -------------------------------------------------------------------------------- /tfjs/static/face_detection/tiny_face_detector_model-shard1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tfjs/static/face_detection/tiny_face_detector_model-shard1 -------------------------------------------------------------------------------- /tfjs/static/face_detection/tiny_face_detector_model-shard1.weight: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliotAndres/tensorflow-2-run-on-mobile-devices-ios-android-browser/684c1399059090dabb6f84a09419032ff046e23b/tfjs/static/face_detection/tiny_face_detector_model-shard1.weight -------------------------------------------------------------------------------- /tfjs/static/face_detection/tiny_face_detector_model-weights_manifest.json: -------------------------------------------------------------------------------- 1 | [{"weights":[{"name":"conv0/filters","shape":[3,3,3,16],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.009007044399485869,"min":-1.2069439495311063}},{"name":"conv0/bias","shape":[16],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.005263455241334205,"min":-0.9211046672334858}},{"name":"conv1/depthwise_filter","shape":[3,3,16,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.004001977630690033,"min":-0.5042491814669441}},{"name":"conv1/pointwise_filter","shape":[1,1,16,32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.013836609615999109,"min":-1.411334180831909}},{"name":"conv1/bias","shape":[32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0015159862590771096,"min":-0.30926119685173037}},{"name":"conv2/depthwise_filter","shape":[3,3,32,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002666276225856706,"min":-0.317286870876948}},{"name":"conv2/pointwise_filter","shape":[1,1,32,64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.015265831292844286,"min":-1.6792414422128714}},{"name":"conv2/bias","shape":[64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0020280554598453,"min":-0.37113414915168985}},{"name":"conv3/depthwise_filter","shape":[3,3,64,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006100742489683862,"min":-0.8907084034938438}},{"name":"conv3/pointwise_filter","shape":[1,1,64,128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.016276211832083907,"min":-2.0508026908425725}},{"name":"conv3/bias","shape":[128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.003394414279975143,"min":-0.7637432129944072}},{"name":"conv4/depthwise_filter","shape":[3,3,128,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006716050119961009,"min":-0.8059260143953211}},{"name":"conv4/pointwise_filter","shape":[1,1,128,256],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.021875603993733724,"min":-2.8875797271728514}},{"name":"conv4/bias","shape":[256],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0041141652009066415,"min":-0.8187188749804216}},{"name":"conv5/depthwise_filter","shape":[3,3,256,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.008423839597141042,"min":-0.9013508368940915}},{"name":"conv5/pointwise_filter","shape":[1,1,256,512],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.030007277283014035,"min":-3.8709387695088107}},{"name":"conv5/bias","shape":[512],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.008402082966823203,"min":-1.4871686851277068}},{"name":"conv8/filters","shape":[1,1,512,25],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.028336129469030042,"min":-4.675461362389957}},{"name":"conv8/bias","shape":[25],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002268134028303857,"min":-0.41053225912299807}}],"paths":["tiny_face_detector_model-shard1.weight"]}] 2 | --------------------------------------------------------------------------------