├── Sources ├── maskrcnn │ ├── Python │ │ ├── Conversion │ │ │ ├── .dockerignore │ │ │ ├── requirements.txt │ │ │ ├── Dockerfile │ │ │ └── task.py │ │ └── COCOEval │ │ │ ├── requirements.txt │ │ │ ├── Dockerfile │ │ │ └── task.py │ ├── main.swift │ ├── TrainCommand.swift │ ├── Docker.swift │ ├── DownloadCommand.swift │ ├── COCO.swift │ ├── ConvertCommand.swift │ ├── EvaluateCommand.swift │ └── results.pb.swift └── Mask-RCNN-CoreML │ ├── MaskRCNNConfig.swift │ ├── BoxUtils.swift │ ├── Detection.swift │ ├── TimeDistributedMaskLayer.swift │ ├── TimeDistributedClassifierLayer.swift │ ├── ProposalLayer.swift │ ├── Utils.swift │ ├── DetectionLayer.swift │ └── PyramidROIAlignLayer.swift ├── .gitignore ├── Example ├── Source │ ├── Assets.xcassets │ │ ├── Contents.json │ │ └── AppIcon.appiconset │ │ │ └── Contents.json │ ├── AppDelegate.swift │ ├── Base.lproj │ │ ├── LaunchScreen.storyboard │ │ └── Main.storyboard │ ├── Info.plist │ ├── DetectionRenderer.swift │ └── ViewController.swift ├── Screenshots │ ├── Screenshot1.png │ ├── Screenshot2.png │ ├── Screenshot3.png │ └── Screenshot4.png └── iOS Example.xcodeproj │ ├── project.xcworkspace │ ├── contents.xcworkspacedata │ └── xcshareddata │ │ └── IDEWorkspaceChecks.plist │ ├── xcshareddata │ └── xcschemes │ │ └── Example.xcscheme │ └── project.pbxproj ├── Package.swift ├── Package.resolved ├── LICENSE └── README.md /Sources/maskrcnn/Python/Conversion/.dockerignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | *.pyc 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.maskrcnn 2 | *.build 3 | *.idea 4 | *.xcuserdatad 5 | */venv/ 6 | */__pycache__/ 7 | *.pyc 8 | -------------------------------------------------------------------------------- /Example/Source/Assets.xcassets/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "info" : { 3 | "version" : 1, 4 | "author" : "xcode" 5 | } 6 | } -------------------------------------------------------------------------------- /Example/Screenshots/Screenshot1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/edouardlp/Mask-RCNN-CoreML/HEAD/Example/Screenshots/Screenshot1.png -------------------------------------------------------------------------------- /Example/Screenshots/Screenshot2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/edouardlp/Mask-RCNN-CoreML/HEAD/Example/Screenshots/Screenshot2.png -------------------------------------------------------------------------------- /Example/Screenshots/Screenshot3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/edouardlp/Mask-RCNN-CoreML/HEAD/Example/Screenshots/Screenshot3.png -------------------------------------------------------------------------------- /Example/Screenshots/Screenshot4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/edouardlp/Mask-RCNN-CoreML/HEAD/Example/Screenshots/Screenshot4.png -------------------------------------------------------------------------------- /Sources/maskrcnn/Python/Conversion/requirements.txt: -------------------------------------------------------------------------------- 1 | keras==2.1.6 2 | tensorflow==1.5.0 3 | coremltools 4 | git+git://github.com/edouardlp/Mask-RCNN-Keras#egg=maskrcnn 5 | -------------------------------------------------------------------------------- /Sources/maskrcnn/main.swift: -------------------------------------------------------------------------------- 1 | import SwiftCLI 2 | 3 | let mainCLI = CLI(name: "maskrcnn") 4 | mainCLI.commands = [ConvertCommand(),EvaluateCommand(),TrainCommand(),DownloadCommand()] 5 | mainCLI.goAndExit() 6 | -------------------------------------------------------------------------------- /Sources/maskrcnn/Python/Conversion/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6 2 | WORKDIR /usr/src/app 3 | COPY requirements.txt ./ 4 | RUN pip install --no-cache-dir -r requirements.txt 5 | COPY . . 6 | CMD [ "python", "./task.py" ] -------------------------------------------------------------------------------- /Sources/maskrcnn/Python/COCOEval/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | tensorflow==1.12 3 | scikit-image 4 | git+https://github.com/waleedka/coco.git#subdirectory=PythonAPI 5 | git+git://github.com/edouardlp/Mask-RCNN-Keras#egg=maskrcnn 6 | -------------------------------------------------------------------------------- /Example/iOS Example.xcodeproj/project.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /Sources/maskrcnn/Python/COCOEval/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6 2 | WORKDIR /usr/src/app 3 | RUN pip install Cython 4 | RUN pip install numpy 5 | COPY requirements.txt ./ 6 | RUN pip install --no-cache-dir -r requirements.txt 7 | COPY . . 8 | ENTRYPOINT [ "python", "./task.py" ] -------------------------------------------------------------------------------- /Example/iOS Example.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | IDEDidComputeMac32BitWarning 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /Sources/maskrcnn/TrainCommand.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import SwiftCLI 3 | 4 | class TrainCommand: Command { 5 | 6 | let name = "train" 7 | let shortDescription = "Trains model" 8 | let modelName = Parameter() 9 | 10 | func execute() throws { 11 | let name = modelName.value 12 | stdout <<< "Coming soon \(name)!." 13 | } 14 | 15 | } 16 | 17 | -------------------------------------------------------------------------------- /Sources/Mask-RCNN-CoreML/MaskRCNNConfig.swift: -------------------------------------------------------------------------------- 1 | // 2 | // MaskRCNNConfig.swift 3 | // Example 4 | // 5 | // Created by Edouard Lavery-Plante on 2018-12-07. 6 | // Copyright © 2018 Edouard Lavery Plante. All rights reserved. 7 | // 8 | import Foundation 9 | 10 | public class MaskRCNNConfig { 11 | 12 | public static let defaultConfig = MaskRCNNConfig() 13 | 14 | //TODO: generate the anchors on demand based on image shape, this will save 5mb 15 | public var anchorsURL:URL? 16 | public var compiledClassifierModelURL:URL? 17 | public var compiledMaskModelURL:URL? 18 | 19 | } 20 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:4.2 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "Mask-RCNN-CoreML", 8 | products: [ 9 | .library( 10 | name: "Mask-RCNN-CoreML", 11 | targets: ["Mask-RCNN-CoreML"]), 12 | ], 13 | dependencies: [ 14 | .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.2.0"), 15 | .package(url: "https://github.com/jakeheis/SwiftCLI", from: "5.2.1"), 16 | .package(url: "https://github.com/Alamofire/Alamofire.git", from: "4.8.0") 17 | ], 18 | targets: [ 19 | .target( 20 | name: "Mask-RCNN-CoreML", 21 | dependencies: []), 22 | .target( 23 | name: "maskrcnn", 24 | dependencies: ["SwiftProtobuf","SwiftCLI", "Mask-RCNN-CoreML", "Alamofire"]), 25 | ] 26 | ) 27 | -------------------------------------------------------------------------------- /Example/Source/AppDelegate.swift: -------------------------------------------------------------------------------- 1 | // 2 | // AppDelegate.swift 3 | // Example 4 | // 5 | // Created by Edouard Lavery-Plante on 2018-11-14. 6 | // Copyright © 2018 Edouard Lavery Plante. All rights reserved. 7 | // 8 | 9 | import UIKit 10 | 11 | @UIApplicationMain 12 | class AppDelegate: UIResponder, UIApplicationDelegate { 13 | 14 | var window: UIWindow? 15 | 16 | func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool { 17 | 18 | MaskRCNNConfig.defaultConfig.anchorsURL = Bundle.main.url(forResource: "anchors", withExtension: "bin")! 19 | MaskRCNNConfig.defaultConfig.compiledClassifierModelURL = Bundle.main.url(forResource: "Classifier", withExtension:"mlmodelc")! 20 | MaskRCNNConfig.defaultConfig.compiledMaskModelURL = Bundle.main.url(forResource: "Mask", withExtension:"mlmodelc")! 21 | 22 | return true 23 | } 24 | } 25 | 26 | -------------------------------------------------------------------------------- /Package.resolved: -------------------------------------------------------------------------------- 1 | { 2 | "object": { 3 | "pins": [ 4 | { 5 | "package": "Alamofire", 6 | "repositoryURL": "https://github.com/Alamofire/Alamofire.git", 7 | "state": { 8 | "branch": null, 9 | "revision": "32573d05b91d8b7349ca31b4726e80212483f02a", 10 | "version": "4.8.0" 11 | } 12 | }, 13 | { 14 | "package": "SwiftProtobuf", 15 | "repositoryURL": "https://github.com/apple/swift-protobuf.git", 16 | "state": { 17 | "branch": null, 18 | "revision": "86c579d280d629416c7bd9d32a5dfacab8e0b0b4", 19 | "version": "1.2.0" 20 | } 21 | }, 22 | { 23 | "package": "SwiftCLI", 24 | "repositoryURL": "https://github.com/jakeheis/SwiftCLI", 25 | "state": { 26 | "branch": null, 27 | "revision": "fb076cba39c679da4e27813518d8860d8815a25b", 28 | "version": "5.2.1" 29 | } 30 | } 31 | ] 32 | }, 33 | "version": 1 34 | } 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Edouard Lavery-Plante 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Sources/maskrcnn/Docker.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import SwiftCLI 3 | 4 | class Docker { 5 | 6 | class var installed:Bool { 7 | do { 8 | let _ = try SwiftCLI.capture("docker", 9 | arguments:["version"]) 10 | return true 11 | } catch { 12 | return false 13 | } 14 | } 15 | 16 | let name:String 17 | let buildURL:URL 18 | 19 | struct Mount { 20 | let source:URL 21 | let destination:String 22 | } 23 | 24 | init(name:String, 25 | buildURL:URL) { 26 | self.name = name 27 | self.buildURL = buildURL 28 | } 29 | 30 | func build(verbose:Bool = false) throws { 31 | let result = try SwiftCLI.capture("docker", 32 | arguments:["build", "-t", self.name, "."], 33 | directory:self.buildURL.relativePath) 34 | if(verbose) { 35 | print(result.stdout) 36 | } 37 | } 38 | 39 | func run(mounts:[Mount], arguments:[String] = [], verbose:Bool = false) throws { 40 | let uuid = UUID().uuidString 41 | var allArguments = ["run", "--rm", "--name", uuid] 42 | for mount in mounts { 43 | allArguments.append("--mount") 44 | allArguments.append("type=bind,source=\(mount.source.relativePath),target=\(mount.destination)") 45 | } 46 | allArguments.append(self.name) 47 | allArguments.append(contentsOf:arguments) 48 | let result = try SwiftCLI.capture("docker", arguments:allArguments) 49 | if(verbose) { 50 | print(result.stdout) 51 | } 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /Example/Source/Base.lproj/LaunchScreen.storyboard: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /Example/Source/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | $(DEVELOPMENT_LANGUAGE) 7 | CFBundleExecutable 8 | $(EXECUTABLE_NAME) 9 | CFBundleIdentifier 10 | $(PRODUCT_BUNDLE_IDENTIFIER) 11 | CFBundleInfoDictionaryVersion 12 | 6.0 13 | CFBundleName 14 | $(PRODUCT_NAME) 15 | CFBundlePackageType 16 | APPL 17 | CFBundleShortVersionString 18 | 1.0 19 | CFBundleVersion 20 | 1 21 | LSRequiresIPhoneOS 22 | 23 | NSCameraUsageDescription 24 | This app uses the camera to demo Mask-RCNN capabilities 25 | UILaunchStoryboardName 26 | LaunchScreen 27 | UIMainStoryboardFile 28 | Main 29 | UIRequiredDeviceCapabilities 30 | 31 | armv7 32 | 33 | UISupportedInterfaceOrientations 34 | 35 | UIInterfaceOrientationPortrait 36 | UIInterfaceOrientationLandscapeLeft 37 | UIInterfaceOrientationLandscapeRight 38 | 39 | UISupportedInterfaceOrientations~ipad 40 | 41 | UIInterfaceOrientationPortrait 42 | UIInterfaceOrientationPortraitUpsideDown 43 | UIInterfaceOrientationLandscapeLeft 44 | UIInterfaceOrientationLandscapeRight 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /Sources/maskrcnn/DownloadCommand.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import SwiftCLI 3 | import Alamofire 4 | 5 | class DownloadCommand: Command { 6 | 7 | let name = "download" 8 | let shortDescription = "Downloads resources" 9 | let downloadName = Parameter() 10 | let version = "0.2" 11 | 12 | func execute() throws { 13 | let name = downloadName.value 14 | stdout <<< "Downloading \(name) resources. This may take a while..." 15 | self.download(files:["anchors.bin","MaskRCNN.mlmodel","Mask.mlmodel","Classifier.mlmodel"]) 16 | stdout <<< "Done." 17 | } 18 | 19 | func download(files:[String]){ 20 | let queue = DispatchQueue.global(qos: .userInitiated) 21 | let group = DispatchGroup() 22 | for file in files { 23 | download(file:file, group:group, queue:queue) 24 | } 25 | group.wait() 26 | } 27 | 28 | func download(file:String, 29 | group:DispatchGroup, 30 | queue:DispatchQueue) { 31 | 32 | let urlString = "https://github.com/edouardlp/Mask-RCNN-CoreML/releases/download/\(self.version)/\(file)" 33 | 34 | let currentDirectoryPath = FileManager.default.currentDirectoryPath 35 | let currentDirectoryURL = URL(fileURLWithPath: currentDirectoryPath) 36 | 37 | let modelURL = currentDirectoryURL.appendingPathComponent(".maskrcnn/models").appendingPathComponent("coco") 38 | let defaultProductsDirectoryURL = modelURL.appendingPathComponent("products/") 39 | 40 | let destination: DownloadRequest.DownloadFileDestination = { _, _ in 41 | let fileURL = defaultProductsDirectoryURL.appendingPathComponent(file) 42 | return (fileURL, [.removePreviousFile, .createIntermediateDirectories]) 43 | } 44 | group.enter() 45 | Alamofire.download(urlString, to: destination).response(queue:queue) { 46 | _ in 47 | group.leave() 48 | } 49 | } 50 | 51 | } 52 | 53 | -------------------------------------------------------------------------------- /Example/Source/Assets.xcassets/AppIcon.appiconset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "idiom" : "iphone", 5 | "size" : "20x20", 6 | "scale" : "2x" 7 | }, 8 | { 9 | "idiom" : "iphone", 10 | "size" : "20x20", 11 | "scale" : "3x" 12 | }, 13 | { 14 | "idiom" : "iphone", 15 | "size" : "29x29", 16 | "scale" : "2x" 17 | }, 18 | { 19 | "idiom" : "iphone", 20 | "size" : "29x29", 21 | "scale" : "3x" 22 | }, 23 | { 24 | "idiom" : "iphone", 25 | "size" : "40x40", 26 | "scale" : "2x" 27 | }, 28 | { 29 | "idiom" : "iphone", 30 | "size" : "40x40", 31 | "scale" : "3x" 32 | }, 33 | { 34 | "idiom" : "iphone", 35 | "size" : "60x60", 36 | "scale" : "2x" 37 | }, 38 | { 39 | "idiom" : "iphone", 40 | "size" : "60x60", 41 | "scale" : "3x" 42 | }, 43 | { 44 | "idiom" : "ipad", 45 | "size" : "20x20", 46 | "scale" : "1x" 47 | }, 48 | { 49 | "idiom" : "ipad", 50 | "size" : "20x20", 51 | "scale" : "2x" 52 | }, 53 | { 54 | "idiom" : "ipad", 55 | "size" : "29x29", 56 | "scale" : "1x" 57 | }, 58 | { 59 | "idiom" : "ipad", 60 | "size" : "29x29", 61 | "scale" : "2x" 62 | }, 63 | { 64 | "idiom" : "ipad", 65 | "size" : "40x40", 66 | "scale" : "1x" 67 | }, 68 | { 69 | "idiom" : "ipad", 70 | "size" : "40x40", 71 | "scale" : "2x" 72 | }, 73 | { 74 | "idiom" : "ipad", 75 | "size" : "76x76", 76 | "scale" : "1x" 77 | }, 78 | { 79 | "idiom" : "ipad", 80 | "size" : "76x76", 81 | "scale" : "2x" 82 | }, 83 | { 84 | "idiom" : "ipad", 85 | "size" : "83.5x83.5", 86 | "scale" : "2x" 87 | }, 88 | { 89 | "idiom" : "ios-marketing", 90 | "size" : "1024x1024", 91 | "scale" : "1x" 92 | } 93 | ], 94 | "info" : { 95 | "version" : 1, 96 | "author" : "xcode" 97 | } 98 | } -------------------------------------------------------------------------------- /Sources/Mask-RCNN-CoreML/BoxUtils.swift: -------------------------------------------------------------------------------- 1 | // 2 | // BoxUtils.swift 3 | // Mask-RCNN-CoreML 4 | // 5 | // Created by Edouard Lavery-Plante on 2018-11-16. 6 | // Copyright © 2018 Edouard Lavery Plante. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import Accelerate 11 | 12 | typealias BoxArray = [Float] 13 | 14 | extension Array where Element == Float { 15 | mutating func boxReference() -> BoxReference { 16 | return BoxReference(pointer: UnsafeMutablePointer(&self), boxCount: self.count/4) 17 | } 18 | } 19 | 20 | struct BoxReference { 21 | 22 | let pointer:UnsafeMutablePointer 23 | let boxCount:Int 24 | 25 | var count:Int { 26 | return boxCount * 4 27 | } 28 | } 29 | 30 | extension BoxReference { 31 | 32 | func applyBoxDeltas(_ deltas:BoxArray){ 33 | 34 | precondition(self.count == deltas.count) 35 | 36 | for i in 0 ..< self.boxCount { 37 | 38 | let boxPointer = self.pointer.advanced(by: i*4) 39 | 40 | let y1 = boxPointer.pointee 41 | let x1 = boxPointer.advanced(by: 1).pointee 42 | let y2 = boxPointer.advanced(by: 2).pointee 43 | let x2 = boxPointer.advanced(by: 3).pointee 44 | 45 | let deltaY1 = deltas[i*4] 46 | let deltaX1 = deltas[i*4+1] 47 | let deltaY2 = deltas[i*4+2] 48 | let deltaX2 = deltas[i*4+3] 49 | 50 | var height = y2 - y1 51 | var width = x2 - x1 52 | var centerY = y1 + 0.5 * height 53 | var centerX = x1 + 0.5 * width 54 | 55 | centerY += deltaY1 * height 56 | centerX += deltaX1 * width 57 | 58 | height *= exp(deltaY2) 59 | width *= exp(deltaX2) 60 | 61 | let resultY1 = centerY - 0.5 * height 62 | let resultX1 = centerX - 0.5 * width 63 | let resultY2 = resultY1 + height 64 | let resultX2 = resultX1 + width 65 | 66 | boxPointer.assign(repeating: resultY1, count: 1) 67 | boxPointer.advanced(by: 1).assign(repeating: resultX1, count: 1) 68 | boxPointer.advanced(by: 2).assign(repeating: resultY2, count: 1) 69 | boxPointer.advanced(by: 3).assign(repeating: resultX2, count: 1) 70 | } 71 | } 72 | 73 | func clip() { 74 | var minimum:Float = 0.0 75 | let minimumPointer = UnsafeMutablePointer(&minimum) 76 | var maximum:Float = 1.0 77 | let maximumPointer = UnsafeMutablePointer(&maximum) 78 | //Clip all balues between 0 and 1 79 | vDSP_vclip(self.pointer, 1, minimumPointer, maximumPointer, self.pointer, 1, vDSP_Length(self.count)) 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /Sources/maskrcnn/COCO.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | class COCO { 4 | 5 | let instances:COCOInstances 6 | 7 | struct Index { 8 | 9 | let annotationsByImageIds:[Int:[COCOAnnotation]] 10 | 11 | static func build(instances:COCOInstances) -> Index { 12 | 13 | var annotationsByImageIds:[Int:[COCOAnnotation]] = [:] 14 | 15 | for annotation in instances.annotations { 16 | 17 | if annotationsByImageIds[annotation.imageId] == nil { 18 | annotationsByImageIds[annotation.imageId] = [COCOAnnotation]() 19 | } 20 | 21 | annotationsByImageIds[annotation.imageId]?.append(annotation) 22 | } 23 | 24 | return Index(annotationsByImageIds:annotationsByImageIds) 25 | } 26 | } 27 | 28 | struct ImageIterator:IteratorProtocol { 29 | 30 | let images:[COCOImage] 31 | var imageIterator:IndexingIterator<[COCOImage]> 32 | let index:Index 33 | 34 | init(images:[COCOImage], index:Index) { 35 | self.images = images 36 | self.imageIterator = self.images.makeIterator() 37 | self.index = index 38 | } 39 | 40 | mutating func next() -> (COCOImage,[COCOAnnotation])? { 41 | guard let nextImage = self.imageIterator.next() 42 | else { return nil } 43 | let annotations = self.index.annotationsByImageIds[nextImage.id] ?? [] 44 | return (nextImage,annotations) 45 | } 46 | 47 | } 48 | 49 | lazy var index:Index = { 50 | return Index.build(instances:self.instances) 51 | }() 52 | 53 | init(url:URL) throws { 54 | let decoder = JSONDecoder() 55 | decoder.keyDecodingStrategy = .convertFromSnakeCase 56 | let data = try Data(contentsOf:url) 57 | self.instances = try decoder.decode(COCOInstances.self, from: data) 58 | } 59 | 60 | func makeImageIterator(limit:Int? = nil, sortById:Bool = false) -> ImageIterator { 61 | let images:[COCOImage] = { 62 | () -> [COCOImage] in 63 | var sorted = self.instances.images 64 | if(sortById) { 65 | sorted = sorted.sorted(by: { 66 | l, r in 67 | return l.id < r.id 68 | }) 69 | } 70 | var limited = sorted 71 | if let limit = limit { 72 | limited = Array(limited[0.. UIImage { 14 | 15 | UIGraphicsBeginImageContext(size) 16 | 17 | UIGraphicsGetCurrentContext()?.clip(to: CGRect(origin: CGPoint.zero, size: size), mask: mask) 18 | UIGraphicsGetCurrentContext()?.setFillColor(color.cgColor) 19 | UIGraphicsGetCurrentContext()?.fill(CGRect(origin: CGPoint.zero, size: size)) 20 | 21 | let image = UIGraphicsGetImageFromCurrentImageContext() 22 | UIGraphicsEndImageContext() 23 | return UIImage(cgImage: image!.cgImage!, scale: 1.0, orientation: UIImage.Orientation.downMirrored) 24 | } 25 | 26 | static func renderDetection(detection:Detection, inSize size:CGSize, color:UIColor) -> UIImage { 27 | 28 | UIGraphicsBeginImageContext(size) 29 | 30 | let transform = CGAffineTransform(scaleX: size.width, y: size.height) 31 | let scaledBoundingBox = detection.boundingBox.applying(transform) 32 | 33 | let path = UIBezierPath(rect:scaledBoundingBox) 34 | UIGraphicsGetCurrentContext()?.setStrokeColor(color.cgColor) 35 | path.lineWidth = 3.0 36 | path.stroke() 37 | 38 | if let mask = detection.mask { 39 | let maskImage = renderMask(mask: mask, inSize: scaledBoundingBox.size, color: color) 40 | maskImage.draw(at: scaledBoundingBox.origin) 41 | } 42 | 43 | let image = UIGraphicsGetImageFromCurrentImageContext() 44 | UIGraphicsEndImageContext() 45 | return image! 46 | } 47 | 48 | static func renderDetections(detections:[Detection], 49 | onImage image:UIImage, 50 | size:CGSize) -> UIImage { 51 | 52 | UIGraphicsGetCurrentContext()?.saveGState() 53 | let colors = [UIColor.red, UIColor.blue, UIColor.green, UIColor.yellow] 54 | 55 | var i = -1 56 | let detectionImages = detections.map { (detection) -> UIImage in 57 | i += 1 58 | return renderDetection(detection: detection, inSize: size, color: colors[i%4]) 59 | } 60 | 61 | UIGraphicsBeginImageContext(size) 62 | 63 | let horizontalScaleFactor = size.width/image.size.width 64 | let verticalScaleFactor = size.height/image.size.height 65 | 66 | let fitsHorizontally = image.size.height*horizontalScaleFactor <= size.height 67 | 68 | let scaleFactor = fitsHorizontally ? horizontalScaleFactor : verticalScaleFactor 69 | 70 | let imageSize = CGSize(width: image.size.width*scaleFactor, height: image.size.height*scaleFactor) 71 | 72 | let horizontalPadding = size.width-imageSize.width 73 | let verticalPadding = size.height-imageSize.height 74 | 75 | image.draw(in: CGRect(origin: CGPoint(x: horizontalPadding/2, y: verticalPadding/2), size: imageSize)) 76 | 77 | for detectionImage in detectionImages { 78 | detectionImage.draw(at: CGPoint(x: 0, y: 0)) 79 | } 80 | 81 | let outputImage = UIGraphicsGetImageFromCurrentImageContext() 82 | UIGraphicsEndImageContext() 83 | UIGraphicsGetCurrentContext()?.restoreGState() 84 | 85 | return outputImage ?? image 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /Example/iOS Example.xcodeproj/xcshareddata/xcschemes/Example.xcscheme: -------------------------------------------------------------------------------- 1 | 2 | 5 | 8 | 9 | 15 | 21 | 22 | 23 | 24 | 25 | 30 | 31 | 32 | 33 | 39 | 40 | 41 | 42 | 43 | 44 | 55 | 57 | 63 | 64 | 65 | 66 | 67 | 68 | 74 | 76 | 82 | 83 | 84 | 85 | 87 | 88 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /Sources/maskrcnn/ConvertCommand.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import SwiftCLI 3 | 4 | class ConvertCommand: Command { 5 | 6 | let name = "convert" 7 | let shortDescription = "Converts trained model to CoreML" 8 | let modelName = Parameter() 9 | let configFilePath = Key("--config", description: "Path to config JSON file") 10 | let weightsFilePath = Key("--weights", description: "Path to HDF5 weights file") 11 | let outputDirectoryPath = Key("--output_dir", description: "Path to output directory") 12 | 13 | func execute() throws { 14 | 15 | guard #available(macOS 10.11, *) else { 16 | stdout <<< "eval requires macOS >= 10.11" 17 | return 18 | } 19 | 20 | guard Docker.installed else { 21 | stdout <<< "Docker is required to run this script." 22 | return 23 | } 24 | 25 | let name = self.modelName.value 26 | 27 | stdout <<< "Converting '\(name)' model to CoreML." 28 | 29 | let verbose = false 30 | 31 | let currentDirectoryPath = FileManager.default.currentDirectoryPath 32 | let currentDirectoryURL = URL(fileURLWithPath: currentDirectoryPath) 33 | 34 | let buildURL = currentDirectoryURL.appendingPathComponent("Sources/maskrcnn/Python/Conversion") 35 | let modelURL = currentDirectoryURL.appendingPathComponent(".maskrcnn/models").appendingPathComponent(name) 36 | let defaultModelDirectoryURL = modelURL.appendingPathComponent("model/") 37 | let defaultProductsDirectoryURL = modelURL.appendingPathComponent("products/") 38 | 39 | var mounts = [Docker.Mount]() 40 | 41 | let dockerConfigDestinationPath = "/usr/src/app/model/config.json" 42 | 43 | if let configFilePath = self.configFilePath.value { 44 | let configURL = URL(fileURLWithPath:configFilePath, isDirectory:false, relativeTo:currentDirectoryURL) 45 | mounts.append(Docker.Mount(source:configURL.standardizedFileURL, destination:dockerConfigDestinationPath)) 46 | } else { 47 | mounts.append(Docker.Mount(source:defaultModelDirectoryURL.appendingPathComponent("config.json"), 48 | destination:dockerConfigDestinationPath)) 49 | } 50 | 51 | let dockerWeightsDestinationPath = "/usr/src/app/model/weights.h5" 52 | 53 | if let weightsFilePath = self.weightsFilePath.value { 54 | let weightsURL = URL(fileURLWithPath:weightsFilePath, isDirectory:false, relativeTo:currentDirectoryURL) 55 | mounts.append(Docker.Mount(source:weightsURL.standardizedFileURL, destination:dockerWeightsDestinationPath)) 56 | } else { 57 | mounts.append(Docker.Mount(source:defaultModelDirectoryURL.appendingPathComponent("weights.h5"), 58 | destination:dockerWeightsDestinationPath)) 59 | } 60 | 61 | let dockerProductsDestinationPath = "/usr/src/app/products" 62 | 63 | if let outputDirectoryPath = self.outputDirectoryPath.value { 64 | let productsURL = URL(fileURLWithPath:outputDirectoryPath, isDirectory:true, relativeTo:currentDirectoryURL) 65 | mounts.append(Docker.Mount(source:productsURL.standardizedFileURL, destination:dockerProductsDestinationPath)) 66 | } else { 67 | mounts.append(Docker.Mount(source:defaultProductsDirectoryURL, 68 | destination:dockerProductsDestinationPath)) 69 | } 70 | 71 | let docker = Docker(name:"mask-rcnn-convert", buildURL:buildURL) 72 | 73 | try docker.build(verbose:verbose) 74 | try docker.run(mounts:mounts,verbose:verbose) 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /Sources/Mask-RCNN-CoreML/Detection.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Detection.swift 3 | // Mask-RCNN-CoreML 4 | // 5 | // Created by Edouard Lavery-Plante on 2018-11-14. 6 | // Copyright © 2018 Edouard Lavery Plante. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import CoreGraphics 11 | import CoreML 12 | import CoreImage 13 | 14 | @available(iOS 12.0, macOS 10.14, *) 15 | public struct Detection { 16 | 17 | public let index:Int 18 | public let boundingBox:CGRect 19 | public let classId:Int 20 | public let score:Double 21 | public let mask:CGImage? 22 | 23 | public static func detectionsFromFeatureValue(featureValue:MLFeatureValue, 24 | maskFeatureValue:MLFeatureValue?) -> [Detection] { 25 | 26 | guard let rawDetections = featureValue.multiArrayValue else { 27 | return [] 28 | } 29 | 30 | let detectionsCount = Int(truncating: rawDetections.shape[0]) 31 | let detectionStride = Int(truncating: rawDetections.strides[0]) 32 | 33 | var detections = [Detection]() 34 | 35 | for i in 0 ..< detectionsCount { 36 | 37 | let score = Double(truncating: rawDetections[i*detectionStride+5]) 38 | if(score > 0.7) { 39 | 40 | let classId = Int(truncating: rawDetections[i*detectionStride+4]) 41 | let y1 = CGFloat(truncating: rawDetections[i*detectionStride]) 42 | let x1 = CGFloat(truncating: rawDetections[i*detectionStride+1]) 43 | let y2 = CGFloat(truncating: rawDetections[i*detectionStride+2]) 44 | let x2 = CGFloat(truncating: rawDetections[i*detectionStride+3]) 45 | let width = x2-x1 46 | let height = y2-y1 47 | 48 | let mask:CGImage? = { 49 | if let maskFeatureValue = maskFeatureValue { 50 | return maskFromFeatureValue(maskFeatureValue: maskFeatureValue, atIndex: i) 51 | } 52 | return nil 53 | }() 54 | 55 | let detection = Detection(index:i, boundingBox: CGRect(x: x1, y: y1, width: width, height: height), classId: classId, score: score, mask:mask) 56 | detections.append(detection) 57 | } 58 | 59 | } 60 | 61 | return detections 62 | } 63 | 64 | static func maskFromFeatureValue(maskFeatureValue:MLFeatureValue, atIndex index:Int) -> CGImage? { 65 | 66 | guard let rawMasks = maskFeatureValue.multiArrayValue else { 67 | return nil 68 | } 69 | 70 | let maskCount = Int(truncating: rawMasks.shape[0]) 71 | 72 | guard maskCount > index else { 73 | return nil 74 | } 75 | 76 | let maskStride = Int(truncating: rawMasks.strides[0]) 77 | assert(rawMasks.dataType == .double) 78 | var maskData = Array(repeating: 0.0, count: maskStride) 79 | let maskDataPointer = UnsafeMutableRawPointer(&maskData) 80 | let elementSize = MemoryLayout.size 81 | maskDataPointer.copyMemory(from: rawMasks.dataPointer.advanced(by: maskStride*elementSize*index), byteCount: maskStride*elementSize) 82 | 83 | var intMaskData = maskData.map { (doubleValue) -> UInt8 in 84 | return UInt8(255-(doubleValue/2*255)) 85 | } 86 | 87 | let intMaskDataPointer = UnsafeMutablePointer(&intMaskData) 88 | let data = CFDataCreate(nil, intMaskDataPointer, maskData.count)! 89 | 90 | let image = CGImage(maskWidth: 28, 91 | height: 28, 92 | bitsPerComponent: 8, 93 | bitsPerPixel: 8, 94 | bytesPerRow: 28, 95 | provider: CGDataProvider(data: data)!, 96 | decode: nil, 97 | shouldInterpolate: false) 98 | return image 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /Sources/maskrcnn/Python/COCOEval/task.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | 4 | from maskrcnn.model import Config 5 | from maskrcnn.model import MaskRCNNModel 6 | from maskrcnn.datasets.coco import COCODataset 7 | from maskrcnn import results_pb2 8 | 9 | if __name__ == '__main__': 10 | parser = argparse.ArgumentParser() 11 | 12 | parser.add_argument('--config_path', 13 | help='Path to config file', 14 | default="model/config.json", 15 | required=False) 16 | 17 | parser.add_argument('--weights_path', 18 | help='Path to weights file', 19 | default="model/weights.h5", 20 | required=False) 21 | 22 | parser.add_argument('--model_dir', 23 | help='Path to model dir', 24 | default="products/model_dir", 25 | required=False) 26 | 27 | parser.add_argument('--results_path', 28 | help='Path to results', 29 | required=True) 30 | 31 | parser.add_argument('--dataset_path', 32 | help='Path to dataset', 33 | default="data/coco/data.tfrecords", 34 | required=False) 35 | 36 | parser.add_argument('--coco_annotations_dir', 37 | help='Path to COCO annotations directory', 38 | default="data/coco", 39 | required=False) 40 | 41 | parser.add_argument('--coco_images_dir', 42 | help='Path to COCO images directory', 43 | default="data/coco/val2017", 44 | required=False) 45 | 46 | parser.add_argument('--coco_year', 47 | help='COCO Year', 48 | default="2017", 49 | required=False) 50 | 51 | parser.add_argument('--coco_type', 52 | help='COCO Type', 53 | default="val", 54 | required=False) 55 | 56 | parser.add_argument('--compare', 57 | help='Compare to model', 58 | action='store_true') 59 | 60 | args = parser.parse_args() 61 | params = args.__dict__ 62 | 63 | config_path = params.pop('config_path') 64 | weights_path = params.pop('weights_path') 65 | model_dir = params.pop('model_dir') 66 | results_path = params.pop('results_path') 67 | dataset_path = params.pop('dataset_path') 68 | coco_annotations_dir = params.pop('coco_annotations_dir') 69 | coco_images_dir = params.pop('coco_images_dir') 70 | compare = params.pop('compare') 71 | 72 | coco_year = params.pop('coco_year') 73 | coco_type = params.pop('coco_type') 74 | print(coco_year) 75 | print(coco_type) 76 | print(results_path) 77 | config = Config() 78 | with open(config_path) as file: 79 | config_dict = json.load(file) 80 | config.__dict__.update(config_dict) 81 | 82 | model = MaskRCNNModel(config_path, 83 | model_dir=model_dir, 84 | initial_keras_weights=weights_path) 85 | 86 | coco_dataset = COCODataset(path=dataset_path, 87 | type=coco_type, 88 | year=coco_year, 89 | annotations_dir=coco_annotations_dir, 90 | images_dir=coco_images_dir, 91 | image_shape=(config.input_width, config.input_height, 3)) 92 | 93 | input_results = results_pb2.Results() 94 | f = open(results_path, "rb") 95 | input_results.ParseFromString(f.read()) 96 | f.close() 97 | print("Printing CoreML results") 98 | coco_dataset.evaluate_results(input_results) 99 | if compare: 100 | coco_dataset.preprocess(reprocess_if_exists=True, limit=5) 101 | input_fn = coco_dataset.make_input_fn(batch_size=1, limit=5) 102 | results = model.predict(dataset_id=coco_dataset.id, input_fn=input_fn, 103 | class_label_fn=coco_dataset.class_label_from_id) 104 | print("Printing Keras results") 105 | coco_dataset.evaluate_results(results) 106 | -------------------------------------------------------------------------------- /Sources/Mask-RCNN-CoreML/TimeDistributedMaskLayer.swift: -------------------------------------------------------------------------------- 1 | // 2 | // TimeDistributedMask.swift 3 | // Mask-RCNN-Demo 4 | // 5 | // Created by Edouard Lavery-Plante on 2018-10-31. 6 | // Copyright © 2018 Edouard Lavery Plante. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import CoreML 11 | import Accelerate 12 | 13 | @available(iOS 12.0, macOS 10.14, *) 14 | @objc(TimeDistributedMaskLayer) class TimeDistributedMaskLayer: NSObject, MLCustomLayer { 15 | 16 | let featureNames:[String] = ["feature_map"] 17 | 18 | required init(parameters: [String : Any]) throws { 19 | super.init() 20 | } 21 | 22 | func setWeightData(_ weights: [Data]) throws { 23 | //No-op 24 | } 25 | 26 | func outputShapes(forInputShapes inputShapes: [[NSNumber]]) throws -> [[NSNumber]] { 27 | let featureMapShape = inputShapes[0] 28 | let poolHeight = featureMapShape[3] 29 | let poolWidth = featureMapShape[4] 30 | let seq = 1 as NSNumber 31 | let batch = featureMapShape[1] 32 | let channel = featureMapShape[0] 33 | let height = Int(truncating: poolHeight)*2 as NSNumber 34 | let width = Int(truncating: poolWidth)*2 as NSNumber 35 | let outputShapes = [[seq,batch,channel,height,width]] 36 | return outputShapes 37 | } 38 | 39 | func evaluate(inputs: [MLMultiArray], outputs: [MLMultiArray]) throws { 40 | 41 | let log = OSLog(subsystem: "TimeDistributedMaskLayer", category: OSLog.Category.pointsOfInterest) 42 | os_signpost(OSSignpostType.begin, log: log, name: "TimeDistributedMask-Eval") 43 | 44 | assert(inputs[0].dataType == MLMultiArrayDataType.float32) 45 | let detections = inputs[1] 46 | let detectionCount = Int(truncating: detections.shape[0]) 47 | let detectionsStride = Int(truncating: detections.strides[0]) 48 | 49 | let model = try MLModel(contentsOf:MaskRCNNConfig.defaultConfig.compiledMaskModelURL!) 50 | let predictionOptions = MLPredictionOptions() 51 | 52 | let batchIn = MultiArrayBatchProvider(multiArrays: inputs, removeZeros:true, featureNames: self.featureNames) 53 | let batchOut = try model.predictions(from: batchIn, options: predictionOptions) 54 | let resultFeatureNames = ["masks"] 55 | let output = outputs[0] 56 | let outputStride = Int(truncating: output.strides[2]) 57 | 58 | for i in 0...size 69 | let resultStride = Int(truncating: resultArray.strides[0]) 70 | 71 | let classId = Int(truncating: detections[detectionsStride*i+4]) 72 | 73 | var doubleBuffer = Array(repeating:0.0, count:stride) 74 | let doubleBufferPointer = UnsafeMutableRawPointer(&doubleBuffer) 75 | doubleBufferPointer.copyMemory(from: resultArray.dataPointer.advanced(by: resultStride*classId*resultMemorySize), byteCount: stride*resultMemorySize) 76 | 77 | var floatBuffer = doubleBuffer.map { (doubleValue) -> Float in 78 | return Float(doubleValue) 79 | } 80 | let floatBufferPointer = UnsafeMutableRawPointer(&floatBuffer) 81 | 82 | let outputMemorySize = MemoryLayout.size 83 | outputMultiArray.dataPointer.advanced(by: stride*actualIndex*outputMemorySize).copyMemory(from: floatBufferPointer, byteCount: stride*outputMemorySize) 84 | } 85 | } 86 | 87 | let resultCount = batchOut.count 88 | let paddingCount = max(0,detectionCount-resultCount)*outputStride 89 | output.padTailWithZeros(startIndex: resultCount*outputStride, count: paddingCount) 90 | os_signpost(OSSignpostType.end, log: log, name: "TimeDistributedMask-Eval") 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /Example/Source/Base.lproj/Main.storyboard: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 27 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mask-RCNN for CoreML 2 | 3 | Mask-RCNN using Core ML, Metal 2 and Accelerate. 4 | 5 |
6 | Example 7 |
8 | 9 | ## Mask-RCNN 10 | 11 | Mask-RCNN is a general framework for object instance segmentation. It detects objects, the class they belong to, their bounding box and segmentation masks. 12 | 13 | ## Motivation 14 | 15 | Mask-RCNN is not fast, especially with the current ResNet101 + FPN backbone. 16 | 17 | There are much faster models for object detection such as SSDLite and YOLO. 18 | 19 | This model will only be useful if instance segmentation is valuable for your use-case. 20 | 21 | ## Examples 22 | 23 | ![Example 1](Example/Screenshots/Screenshot2.png) 24 | ![Example 2](Example/Screenshots/Screenshot3.png) 25 | ![Example 3](Example/Screenshots/Screenshot4.png) 26 | 27 | ## Requirements 28 | 29 | - Xcode 10.1 30 | - iOS 12 or macOS 10.14 device with Metal support 31 | - Swift 4.2 32 | - (More requirements details coming soon) 33 | 34 | ### Requirements for using the scripts 35 | - Docker 36 | - (More requirements details coming soon) 37 | 38 | ## iOS Example Project Usage 39 | 40 | 41 | 1. Checkout or download this repository 42 | 2. Open a shell and navigate (cd) to the root of the repository 43 | 3. Download the pre-trained model files using the command: 44 | 45 | ```bash 46 | $ swift run maskrcnn download example 47 | ``` 48 | 49 | 4. Open Example/iOS Example.xcodeproj 50 | 5. Build and run on an iOS 12 device with Metal support 51 | 52 | ## Installation 53 | 54 | ### Cocoapods 55 | 56 | Coming soon. See Roadmap. Install Manually instead. 57 | 58 | ### Carthage 59 | 60 | Coming soon. See Roadmap. Install Manually instead. 61 | 62 | ### Swift Package Manager 63 | 64 | Coming soon. See Roadmap. Install Manually instead. 65 | 66 | ### Manually 67 | 68 | 1. Import all of the Swift files in the Sources/Mask-RCNN-CoreML/ directory 69 | 2. If you have your own data to train or fine-tune a model, or if you have your own model weights, see Converting or training your own model. Otherwise, see Using COCO pre-trained model to use my model. 70 | 71 | ### Using COCO pre-trained model 72 | 73 | 1. Download the pre-trained model files from the [releases page](https://github.com/edouardlp/Mask-RCNN-CoreML/releases). (instructions for conversion coming soon) 74 | 2. Make the code you use is associated 75 | 3. Drag the four files into your Xcode project (anchors.bin, MaskRCNN.mlmodel, Mask.mlmodel, Classifier.mlmodel) 76 | 77 | ## Converting or training your own model 78 | 79 | If you have pre-trained model weights, or if you have data you want to train the model with, follow the instructions in this section. 80 | 81 | ### Converting pre-trained weights 82 | 83 | At the moment, only models trained using [Matterport's Mask-RCNN implementation](https://github.com/matterport/Mask_RCNN) are supported. If your model is trained differently, you may be able to get it to work by renaming your weights following Matterport's naming structure and exporting your model to the Keras HDF5 format. 84 | 85 | You should also specify configuration options in a JSON file. 86 | 87 | - *architecture* : The backbone architecture your model is trained with. "resnet101" or "resnet50". Defaults to "resnet101". 88 | - *input_image_shape* : The shape of the input image as a list of numbers. Defaults to [1024,1024,3]. 89 | - *num_classes* : The number of classes, including the background class. Defaults to 81. 90 | - *pre_nms_max_proposals* : The number of proposed regions to evaluate using NMS. Only the top *pre_nms_max_proposals* proposals by score will be evaluated. Defaults to 6000. 91 | - *max_proposals* : The number of proposals to classify. Only the top proposals by score, after NMS, will be evaluated. Defaults to 1000. 92 | - More options to come 93 | 94 | To use the default directory structure, place your files as such: 95 | 96 | ``` 97 | .maskrcnn/ 98 | models/ 99 | your_model_name/ 100 | model/ 101 | config.json 102 | weights.h5 103 | ``` 104 | 105 | The products of the conversion will be placed as such : 106 | 107 | ``` 108 | .maskrcnn/ 109 | models/ 110 | your_model_name/ 111 | products/ 112 | anchors.bin 113 | MaskRCNN.mlmodel 114 | Mask.mlmodel 115 | Classifier.model 116 | ``` 117 | 118 | Run : 119 | 120 | ```bash 121 | $ swift run maskrcnn convert 122 | ``` 123 | 124 | If you want to use input files located elsewhere, or to output the model to another directory, simply run : 125 | 126 | ```bash 127 | $ swift run maskrcnn convert --config= --weights= --output_dir= 128 | ``` 129 | 130 | ### Training or fine-tuning a model 131 | 132 | This is not supported at the moment, but the next item in my roadmap. 133 | 134 | ## Evaluating the model accuracy 135 | 136 | After conversion, or training you may want to evaluate the model accuracy. 137 | 138 | At the moment, only the COCO dataset can be used for evaluation. 139 | 140 | To use the default directory structure, place your files as such: 141 | 142 | ``` 143 | .maskrcnn/ 144 | data/ 145 | coco/ 146 | the_coco_annotation_files.json 147 | type_year(ex: val2017)/ 148 | the_images.jpg 149 | models/ 150 | your_model_name/ 151 | products/ 152 | anchors.bin 153 | MaskRCNN.mlmodel 154 | Mask.mlmodel 155 | Classifier.model 156 | ``` 157 | 158 | Run : 159 | 160 | ```bash 161 | $ swift run maskrcnn eval coco --year= --type= 162 | ``` 163 | 164 | If you want to compare against the model using Tensorflow, place the model files as such : 165 | 166 | ``` 167 | .maskrcnn/ 168 | models/ 169 | your_model_name/ 170 | model/ 171 | config.json 172 | weights.h5 173 | ``` 174 | 175 | Use -c to compare 176 | 177 | ```bash 178 | $ swift run maskrcnn eval coco -c --year= --type= 179 | ``` 180 | 181 | If you want to have your files in custom locations : 182 | 183 | ```bash 184 | $ swift run maskrcnn eval coco -c --year= --type= --config= --weights= --products_dir= 185 | ``` 186 | 187 | ## Roadmap 188 | 189 | - Training and fine-tuning support 190 | - Cocoapods, Carthage, Swift Package manager and improved documentation 191 | - Mobile-optimized backbone and other performance optimizations 192 | - Easy training support 193 | - Support for custom evaluation datasets 194 | - Support for pose estimation 195 | 196 | ## Author 197 | 198 | Édouard Lavery-Plante, ed@laveryplante.com 199 | 200 | ## Credits 201 | 202 | - [Original Paper](https://arxiv.org/abs/1703.06870) 203 | - [Matterport Implementation](https://github.com/matterport/Mask_RCNN/) 204 | - [Inspiration](http://machinethink.net/blog/) 205 | 206 | ## References 207 | 208 | - [Vision Framework](https://developer.apple.com/documentation/vision) 209 | - [CoreML Framework](https://developer.apple.com/documentation/coreml) 210 | - [coremltools](https://pypi.python.org/pypi/coremltools) 211 | -------------------------------------------------------------------------------- /Example/Source/ViewController.swift: -------------------------------------------------------------------------------- 1 | // 2 | // ViewController.swift 3 | // Example 4 | // 5 | // Created by Edouard Lavery-Plante on 2018-11-14. 6 | // Copyright © 2018 Edouard Lavery Plante. All rights reserved. 7 | // 8 | 9 | import UIKit 10 | import CoreML 11 | import Vision 12 | import os.signpost 13 | 14 | class ViewController: UIViewController, UIImagePickerControllerDelegate, UINavigationControllerDelegate { 15 | 16 | @IBOutlet weak var selectPhotoItem:UIBarButtonItem! 17 | @IBOutlet weak var imageView:UIImageView! 18 | @IBOutlet weak var label:UILabel! 19 | @IBOutlet weak var activityIndicator:UIActivityIndicatorView! 20 | 21 | private enum State { 22 | case initial 23 | case calculating 24 | case displayingResult 25 | } 26 | 27 | private var state:State = .initial { 28 | didSet { 29 | self.handleChangedState(state: self.state) 30 | } 31 | } 32 | 33 | private var selectedImage:UIImage? = nil 34 | 35 | private lazy var maskRCNNRequest: VNCoreMLRequest = { 36 | do { 37 | let model = try VNCoreMLModel(for: MaskRCNN().model) 38 | let request = VNCoreMLRequest(model: model, completionHandler: { 39 | [weak self] request, error in 40 | self?.processMaskRCNNRequest(for: request, error: error) 41 | }) 42 | request.imageCropAndScaleOption = .scaleFit 43 | return request 44 | } catch { 45 | fatalError("Failed to load Vision ML model: \(error)") 46 | } 47 | }() 48 | 49 | override func viewDidLoad() { 50 | super.viewDidLoad() 51 | self.title = "Mask-RCNN Example" 52 | } 53 | 54 | //MARK: Actions 55 | 56 | @IBAction func selectPhoto(sender:Any?) { 57 | 58 | guard UIImagePickerController.isSourceTypeAvailable(.camera) else { 59 | self.presentPicker(sourceType: .photoLibrary) 60 | return 61 | } 62 | 63 | let alertController = UIAlertController(title: nil, message: nil, preferredStyle: .actionSheet) 64 | 65 | let cameraAction = UIAlertAction(title: "Take a Photo", style: .default) { 66 | [weak self](_) in 67 | self?.presentPicker(sourceType: .camera) 68 | } 69 | alertController.addAction(cameraAction) 70 | 71 | let libraryAction = UIAlertAction(title: "Choose from Library", style: .default) { 72 | [weak self](_) in 73 | self?.presentPicker(sourceType: .photoLibrary) 74 | } 75 | alertController.addAction(libraryAction) 76 | 77 | let cancelAction = UIAlertAction(title: "Cancel", style: .cancel, handler: nil) 78 | alertController.addAction(cancelAction) 79 | 80 | self.present(alertController, animated: true, completion: nil) 81 | } 82 | 83 | //MARK: UIImagePickerControllerDelegate 84 | 85 | @objc func imagePickerController(_ picker: UIImagePickerController, 86 | didFinishPickingMediaWithInfo info: [UIImagePickerController.InfoKey : Any]) { 87 | 88 | guard state != .calculating else { 89 | return 90 | } 91 | 92 | picker.dismiss(animated: true, completion: nil) 93 | 94 | self.state = .calculating 95 | 96 | guard let image = info[UIImagePickerController.InfoKey.originalImage] as? UIImage else { 97 | return 98 | } 99 | 100 | self.selectedImage = image 101 | 102 | let orientation = CGImagePropertyOrientation(rawValue: UInt32(image.imageOrientation.rawValue))! 103 | guard let ciImage = CIImage(image: image) else { fatalError("Unable to create \(CIImage.self) from \(image).") } 104 | 105 | DispatchQueue.global(qos: .userInitiated).async { 106 | let handler = VNImageRequestHandler(ciImage: ciImage, orientation: orientation) 107 | do { 108 | 109 | let request = self.maskRCNNRequest 110 | let log = OSLog(subsystem: "Mask-RCNN", category: OSLog.Category.pointsOfInterest) 111 | os_signpost(OSSignpostType.begin, log: log, name: "Mask-RCNN-Eval") 112 | try handler.perform([request]) 113 | os_signpost(OSSignpostType.end, log: log, name: "Mask-RCNN-Eval") 114 | } catch { 115 | DispatchQueue.main.async { 116 | print("Failed to perform Mask-RCNN.\n\(error.localizedDescription)") 117 | self.handleFailure() 118 | } 119 | } 120 | } 121 | } 122 | 123 | //MARK: Private Methods 124 | 125 | private func presentPicker(sourceType:UIImagePickerController.SourceType) { 126 | 127 | let picker = UIImagePickerController() 128 | picker.delegate = self 129 | picker.sourceType = sourceType 130 | self.present(picker, animated: true, completion: nil) 131 | 132 | } 133 | 134 | private func handleChangedState(state:State) { 135 | 136 | switch(state) { 137 | case .initial: 138 | 139 | self.imageView.image = nil 140 | self.activityIndicator.stopAnimating() 141 | self.label.text = "Choose a Photo" 142 | self.label.isHidden = false 143 | self.selectPhotoItem.isEnabled = true 144 | 145 | case .calculating: 146 | 147 | self.imageView.image = nil 148 | self.activityIndicator.startAnimating() 149 | self.label.text = "Calculating..." 150 | self.label.isHidden = false 151 | self.selectPhotoItem.isEnabled = false 152 | 153 | case .displayingResult: 154 | 155 | self.activityIndicator.stopAnimating() 156 | self.label.isHidden = true 157 | self.selectPhotoItem.isEnabled = true 158 | 159 | } 160 | 161 | } 162 | 163 | private func processMaskRCNNRequest(for request: VNRequest, error: Error?) { 164 | 165 | guard let selectedImage = self.selectedImage, 166 | let results = request.results as? [VNCoreMLFeatureValueObservation], 167 | let detectionsFeatureValue = results.first?.featureValue, 168 | let maskFeatureValue = results.last?.featureValue else { 169 | DispatchQueue.main.async { 170 | print("Failed to perform Mask-RCNN.\n\(error?.localizedDescription ?? "")") 171 | self.handleFailure() 172 | } 173 | return 174 | } 175 | 176 | let detections = Detection.detectionsFromFeatureValue(featureValue: detectionsFeatureValue, maskFeatureValue:maskFeatureValue) 177 | 178 | print(detections) 179 | 180 | let resultImage = DetectionRenderer.renderDetections(detections: detections, 181 | onImage: selectedImage, 182 | size:CGSize(width:1024,height:1024)) 183 | 184 | DispatchQueue.main.async { 185 | self.handleSuccess(image: resultImage) 186 | } 187 | } 188 | 189 | private func handleSuccess(image:UIImage) { 190 | self.state = .displayingResult 191 | self.selectedImage = nil 192 | self.imageView.image = image 193 | } 194 | 195 | private func handleFailure() { 196 | 197 | self.state = .initial 198 | 199 | let alertController = UIAlertController(title: "Error", message: "An error occurred attempting to run Mask-RCNN", preferredStyle: .alert) 200 | 201 | let cancelAction = UIAlertAction(title: "Ok", style: .cancel, handler: nil) 202 | alertController.addAction(cancelAction) 203 | 204 | self.present(alertController, animated: true, completion: nil) 205 | } 206 | 207 | } 208 | 209 | -------------------------------------------------------------------------------- /Sources/Mask-RCNN-CoreML/TimeDistributedClassifierLayer.swift: -------------------------------------------------------------------------------- 1 | // 2 | // TimeDistributedClassifierLayer.swift 3 | // Example 4 | // 5 | // Created by Edouard Lavery-Plante on 2018-11-20. 6 | // Copyright © 2018 Edouard Lavery Plante. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import CoreML 11 | import Accelerate 12 | 13 | @available(iOS 12.0, macOS 10.14, *) 14 | @objc(TimeDistributedClassifierLayer) class TimeDistributedClassifierLayer: NSObject, MLCustomLayer { 15 | 16 | let featureNames:[String] = ["feature_map"] 17 | 18 | required init(parameters: [String : Any]) throws { 19 | super.init() 20 | } 21 | 22 | func setWeightData(_ weights: [Data]) throws { 23 | //No-op 24 | } 25 | 26 | func outputShapes(forInputShapes inputShapes: [[NSNumber]]) throws -> [[NSNumber]] { 27 | let featureMapShape = inputShapes[0] 28 | let batch = featureMapShape[1] 29 | let channel = featureMapShape[0] 30 | let outputShapes = [[channel,batch,1,1,6]] 31 | return outputShapes 32 | } 33 | 34 | func evaluate(inputs: [MLMultiArray], outputs: [MLMultiArray]) throws { 35 | 36 | let log = OSLog(subsystem: "TimeDistributedClassifierLayer", category: OSLog.Category.pointsOfInterest) 37 | os_signpost(OSSignpostType.begin, log: log, name: "TimeDistributedClassifierLayer-Eval") 38 | 39 | assert(inputs[0].dataType == MLMultiArrayDataType.float32) 40 | 41 | let model = try MLModel(contentsOf:MaskRCNNConfig.defaultConfig.compiledClassifierModelURL!) 42 | let predictionOptions = MLPredictionOptions() 43 | 44 | let batchIn = MultiArrayBatchProvider(multiArrays: inputs, removeZeros:false, featureNames: self.featureNames) 45 | let batchOut = try model.predictions(from: batchIn, options: predictionOptions) 46 | let resultFeatureNames = ["probabilities", "bounding_boxes"] 47 | 48 | os_signpost(OSSignpostType.begin, log: log, name: "TimeDistributedClassifierLayer-ProcessOutput") 49 | 50 | for i in 0...size 60 | let resultShape = Int(truncating: resultArray.shape[0]) 61 | 62 | let outputMultiArray = outputs[0] 63 | let outputStride = Int(truncating: outputMultiArray.strides[2]) 64 | 65 | var doubleBuffer = Array(repeating:0.0, count:resultShape) 66 | let doubleBufferPointer = UnsafeMutableRawPointer(&doubleBuffer) 67 | doubleBufferPointer.copyMemory(from: resultArray.dataPointer, byteCount: resultShape*resultMemorySize) 68 | 69 | var floatBuffer = doubleBuffer.map { (doubleValue) -> Float in 70 | return Float(doubleValue) 71 | } 72 | 73 | //TODO: attempt to get CoreML to do this instead 74 | 75 | if(j==0){ 76 | let (value, index) = maximumValueWithIndex(values: floatBuffer) 77 | lastIndex = Int(index) 78 | //Write the argmax and score to indices 4 and 5 79 | outputMultiArray[outputStride*actualIndex+4] = index as NSNumber 80 | outputMultiArray[outputStride*actualIndex+5] = value as NSNumber 81 | } else { 82 | for z in 0 ..< 4 { 83 | let deltaIndex = lastIndex*4+z 84 | outputMultiArray[outputStride*actualIndex+z] = floatBuffer[deltaIndex] as NSNumber 85 | } 86 | } 87 | } 88 | } 89 | os_signpost(OSSignpostType.end, log: log, name: "TimeDistributedClassifierLayer-ProcessOutput") 90 | os_signpost(OSSignpostType.end, log: log, name: "TimeDistributedClassifierLayer-Eval") 91 | } 92 | } 93 | 94 | @available(iOS 12.0, macOS 10.14, *) 95 | class MultiArrayBatchProvider : MLBatchProvider 96 | { 97 | let multiArrays:[MLMultiArray] 98 | 99 | var featureNames: [String] 100 | 101 | let indexMapping:[Int:Int] 102 | 103 | public var count: Int { 104 | return indexMapping.count 105 | } 106 | 107 | init(multiArrays:[MLMultiArray], 108 | removeZeros:Bool, 109 | featureNames:[String]) { 110 | 111 | self.multiArrays = multiArrays 112 | self.featureNames = featureNames 113 | var mapping = [Int:Int]() 114 | let stride = Int(truncating: multiArrays[0].strides[0]) 115 | var index = 0 116 | if(removeZeros){ 117 | var buffer = Array(repeating: 0.0, count: stride) 118 | let bufferPointer = UnsafeMutableRawPointer(&buffer) 119 | for i in 0 ..< Int(truncating:multiArrays[0].shape[0]) { 120 | bufferPointer.copyMemory(from: multiArrays[0].dataPointer.advanced(by: stride*i*4), byteCount: stride*4) 121 | if(buffer.allSatisfy({ (value) -> Bool in 122 | return value != 0 123 | })) { 124 | mapping[index] = i 125 | index += 1 126 | } 127 | } 128 | } else { 129 | for i in 0 ..< Int(truncating:multiArrays[0].shape[0]) { 130 | mapping[index] = i 131 | index += 1 132 | } 133 | } 134 | self.indexMapping = mapping 135 | } 136 | 137 | public func features(at index: Int) -> MLFeatureProvider { 138 | let mappedIndex = self.indexMapping[index]! 139 | return MultiArrayFeatureProvider(multiArrays: self.multiArrays, featureNames: self.featureNames, index: mappedIndex) 140 | } 141 | } 142 | 143 | @available(iOS 12.0, macOS 10.14, *) 144 | class MultiArrayFeatureProvider : MLFeatureProvider 145 | { 146 | let multiArrays:[MLMultiArray] 147 | var featureNames: Set { 148 | return Set(orderedFeatureNames) 149 | } 150 | var orderedFeatureNames: [String] 151 | let index:Int 152 | 153 | init(multiArrays:[MLMultiArray], 154 | featureNames:[String], 155 | index:Int) { 156 | self.multiArrays = multiArrays 157 | self.orderedFeatureNames = featureNames 158 | self.index = index 159 | } 160 | 161 | func featureValue(for featureName: String) -> MLFeatureValue? { 162 | guard let featureIndex = self.orderedFeatureNames.firstIndex(of: featureName) else 163 | { 164 | return nil 165 | } 166 | let multiArray = self.multiArrays[featureIndex] 167 | let outputComponentsSize = MemoryLayout.size 168 | guard let outputMultiArray = try? MLMultiArray(dataPointer: multiArray.dataPointer.advanced(by: Int(truncating: multiArray.strides[0])*index*outputComponentsSize), shape: Array(multiArray.shape[2...4]), dataType: multiArray.dataType, strides: Array(multiArray.strides[2...4]), deallocator: nil) else 169 | { 170 | return nil 171 | } 172 | return MLFeatureValue(multiArray: outputMultiArray) 173 | } 174 | } 175 | 176 | 177 | func maximumValueWithIndex(values:[Float]) -> (Float,UInt) { 178 | 179 | var values = values 180 | var resultValue:Float = 0.0 181 | let resultValuePointer = UnsafeMutablePointer(&resultValue) 182 | var resultIndex:UInt = 0 183 | let resultIndexPointer = UnsafeMutablePointer(&resultIndex) 184 | 185 | vDSP_maxvi(UnsafeMutablePointer(&values), 186 | 1, 187 | resultValuePointer, 188 | resultIndexPointer, 189 | vDSP_Length(values.count)) 190 | 191 | return (resultValue,resultIndex) 192 | } 193 | -------------------------------------------------------------------------------- /Sources/maskrcnn/Python/Conversion/task.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import os 4 | 5 | import tensorflow as tf 6 | 7 | from coremltools.converters.keras import convert as convert_keras_to_coreml 8 | from coremltools.models.utils import convert_neural_network_weights_to_fp16 9 | from coremltools.models.utils import save_spec 10 | from coremltools.proto import NeuralNetwork_pb2 11 | 12 | from maskrcnn.model import Config 13 | from maskrcnn.model import MaskRCNNModel 14 | 15 | def export_models(config, 16 | mask_rcnn_model, 17 | classifier_model, 18 | mask_model, 19 | export_main_path, 20 | export_mask_path, 21 | export_anchors_path): 22 | license = "MIT" 23 | author = "Édouard Lavery-Plante" 24 | 25 | def convert_proposal(layer): 26 | params = NeuralNetwork_pb2.CustomLayerParams() 27 | params.className = "ProposalLayer" 28 | params.description = "Proposes regions of interests and performs NMS." 29 | params.parameters["bboxStdDev_count"].intValue = len(layer.bounding_box_std_dev) 30 | for idx, value in enumerate(layer.bounding_box_std_dev): 31 | params.parameters["bboxStdDev_" + str(idx)].doubleValue = value 32 | params.parameters["preNMSMaxProposals"].intValue = layer.pre_nms_max_proposals 33 | params.parameters["maxProposals"].intValue = layer.max_proposals 34 | params.parameters["nmsIOUThreshold"].doubleValue = layer.nms_threshold 35 | return params 36 | 37 | def convert_pyramid(layer): 38 | params = NeuralNetwork_pb2.CustomLayerParams() 39 | params.className = "PyramidROIAlignLayer" 40 | params.parameters["poolSize"].intValue = layer.pool_shape[0] 41 | params.parameters["imageWidth"].intValue = layer.image_shape[0] 42 | params.parameters["imageHeight"].intValue = layer.image_shape[1] 43 | params.description = "Extracts feature maps based on the regions of interest." 44 | return params 45 | 46 | def convert_time_distributed_classifier(layer): 47 | params = NeuralNetwork_pb2.CustomLayerParams() 48 | params.className = "TimeDistributedClassifierLayer" 49 | return params 50 | 51 | def convert_time_distributed(layer): 52 | params = NeuralNetwork_pb2.CustomLayerParams() 53 | params.className = "TimeDistributedMaskLayer" 54 | params.description = "Applies the Mask graph to each detections along the time dimension." 55 | return params 56 | 57 | def convert_detection(layer): 58 | params = NeuralNetwork_pb2.CustomLayerParams() 59 | params.className = "DetectionLayer" 60 | params.parameters["bboxStdDev_count"].intValue = len(layer.bounding_box_std_dev) 61 | for idx, value in enumerate(layer.bounding_box_std_dev): 62 | params.parameters["bboxStdDev_" + str(idx)].doubleValue = value 63 | params.parameters["maxDetections"].intValue = layer.max_detections 64 | params.parameters["scoreThreshold"].doubleValue = layer.detection_min_confidence 65 | params.parameters["nmsIOUThreshold"].doubleValue = layer.detection_nms_threshold 66 | params.description = "Outputs detections based on confidence and performs NMS." 67 | return params 68 | 69 | mask_rcnn_model = convert_keras_to_coreml(mask_rcnn_model, 70 | input_names=["image"], 71 | image_input_names=['image'], 72 | output_names=["detections", "mask"], 73 | red_bias=-123.7, 74 | green_bias=-116.8, 75 | blue_bias=-103.9, 76 | add_custom_layers=True, 77 | custom_conversion_functions={ 78 | "ProposalLayer": convert_proposal, 79 | "PyramidROIAlign": convert_pyramid, 80 | "TimeDistributedClassifier": convert_time_distributed_classifier, 81 | "TimeDistributedMask": convert_time_distributed, 82 | "DetectionLayer": convert_detection}) 83 | 84 | mask_rcnn_model.author = author 85 | mask_rcnn_model.license = license 86 | mask_rcnn_model.short_description = "Mask-RCNN" 87 | mask_rcnn_model.input_description["image"] = "Input image" 88 | mask_rcnn_model.output_description["detections"] = "Detections (y1,x1,y2,x2,classId,score)" 89 | mask_rcnn_model.output_description["mask"] = "Masks for the detections" 90 | half_model = convert_neural_network_weights_to_fp16(mask_rcnn_model) 91 | half_spec = half_model.get_spec() 92 | save_spec(half_spec, export_main_path) 93 | 94 | mask_model_coreml = convert_keras_to_coreml(mask_model, 95 | input_names=["feature_map"], 96 | output_names=["masks"]) 97 | mask_model_coreml.author = author 98 | mask_model_coreml.license = license 99 | mask_model_coreml.short_description = "Generates a mask for each class for a given feature map" 100 | mask_model_coreml.input_description["feature_map"] = "Fully processed feature map, ready for mask generation." 101 | mask_model_coreml.output_description["masks"] = "Masks corresponding to each class" 102 | half_mask_model = convert_neural_network_weights_to_fp16(mask_model_coreml) 103 | half_mask_spec = half_mask_model.get_spec() 104 | save_spec(half_mask_spec, export_mask_path) 105 | 106 | classifier_model_coreml = convert_keras_to_coreml(classifier_model, 107 | input_names=["feature_map"], 108 | output_names=["probabilities", "bounding_boxes"]) 109 | 110 | classifier_model_coreml.author = author 111 | classifier_model_coreml.license = license 112 | classifier_model_coreml.short_description = "" 113 | classifier_model_coreml.input_description["feature_map"] = "Fully processed feature map, ready for classification." 114 | half_classifier_model = convert_neural_network_weights_to_fp16(classifier_model_coreml) 115 | half_classifier_spec = half_classifier_model.get_spec() 116 | save_spec(half_classifier_spec, "products/Classifier.mlmodel") 117 | 118 | if __name__ == '__main__': 119 | parser = argparse.ArgumentParser() 120 | 121 | parser.add_argument( 122 | '--config_path', 123 | help='Path to config file', 124 | default="model/config.json", 125 | required=False 126 | ) 127 | 128 | parser.add_argument( 129 | '--weights_path', 130 | help='Path to weights file', 131 | default="model/weights.h5", 132 | required=False 133 | ) 134 | 135 | parser.add_argument( 136 | '--export_main_path', 137 | help='Path to export main file', 138 | default="products/MaskRCNN.mlmodel", 139 | required=False 140 | ) 141 | 142 | parser.add_argument( 143 | '--export_mask_path', 144 | help='Path to export mask file', 145 | default="products/Mask.mlmodel", 146 | required=False 147 | ) 148 | 149 | parser.add_argument( 150 | '--export_anchors_path', 151 | help='Path to export anchors file', 152 | default="products/anchors.bin", 153 | required=False 154 | ) 155 | 156 | args = parser.parse_args() 157 | params = args.__dict__ 158 | 159 | config_path = params.pop('config_path') 160 | weights_path = params.pop('weights_path') 161 | export_main_path = params.pop('export_main_path') 162 | export_mask_path = params.pop('export_mask_path') 163 | #TODO: remove and generate instead 164 | export_anchors_path = params.pop('export_anchors_path') 165 | 166 | config = Config() 167 | with open(config_path) as file: 168 | config_dict = json.load(file) 169 | config.__dict__.update(config_dict) 170 | 171 | model = MaskRCNNModel(config_path, initial_keras_weights=weights_path) 172 | 173 | mask_rcnn_model, classifier_model, mask_model, anchors = model.get_trained_keras_models() 174 | export_models(config, mask_rcnn_model, classifier_model, mask_model, export_main_path, export_mask_path, 175 | export_anchors_path) 176 | anchors.tofile(export_anchors_path) 177 | 178 | 179 | -------------------------------------------------------------------------------- /Sources/Mask-RCNN-CoreML/ProposalLayer.swift: -------------------------------------------------------------------------------- 1 | // 2 | // ProposalLayer.swift 3 | // Mask-RCNN-CoreML 4 | // 5 | // Created by Edouard Lavery-Plante on 2018-10-26. 6 | // Copyright © 2018 Edouard Lavery Plante. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import CoreML 11 | import Accelerate 12 | 13 | /** 14 | 15 | ProposalLayer is a Custom ML Layer that proposes regions of interests. 16 | 17 | ProposalLayer proposes regions of interest based on the probability of objects 18 | being detected in each region. Regions that overlap more than a given 19 | threshold are removed through a process called Non-Max Supression (NMS). 20 | 21 | Regions correspond to predefined "anchors" that are not inputs to the layer. 22 | Anchors are generated based on the image shape using a heuristic that maximizes 23 | the likelihood of bounding objects in the image. The process of generating anchors 24 | can be though of as a hyperparameter. 25 | 26 | Anchors are adjusted using deltas provided as input to refine how they enclose the 27 | detected objects. 28 | 29 | The layer takes two inputs : 30 | 31 | - Probabilities of each region containing an object. Shape : (#regions, 2). 32 | - Anchor deltas to refine the anchors shape. Shape : (#regions,4) 33 | 34 | The probabilities input's last dimension corresponds to the mutually exclusive 35 | probabilities of the region being background (index 0) or an object (index 1). 36 | 37 | The anchor deltas are layed out as follows : (dy,dx,log(dh),log(dw)). 38 | 39 | The layer takes four parameters : 40 | 41 | - boundingBoxRefinementStandardDeviation : Anchor deltas refinement standard deviation 42 | - preNMSMaxProposals : Maximum # of regions to evaluate for non max supression 43 | - maxProposals : Maximum # of regions to output 44 | - nmsIOUThreshold : Threshold below which to supress regions that overlap 45 | 46 | The layer has one ouput : 47 | 48 | - Regions of interest (y1,x1,y2,x2). Shape : (#regionsOut,4), 49 | 50 | */ 51 | @available(iOS 12.0, macOS 10.14, *) 52 | @objc(ProposalLayer) class ProposalLayer: NSObject, MLCustomLayer { 53 | 54 | var anchorData:Data! 55 | 56 | //Anchor deltas refinement standard deviation 57 | var boundingBoxRefinementStandardDeviation:[Float] = [0.1, 0.1, 0.2, 0.2] 58 | //Maximum # of regions to evaluate for non max supression 59 | var preNMSMaxProposals = 6000 60 | //Maximum # of regions to output 61 | var maxProposals = 1000 62 | //Threshold below which to supress regions that overlap 63 | var nmsIOUThreshold:Float = 0.7 64 | 65 | required init(parameters: [String : Any]) throws { 66 | super.init() 67 | 68 | self.anchorData = try Data(contentsOf: MaskRCNNConfig.defaultConfig.anchorsURL!) 69 | 70 | if let bboxStdDevCount = parameters["bboxStdDev_count"] as? Int { 71 | var bboxStdDev = [Float]() 72 | for i in 0.. [[NSNumber]] { 98 | var outputshape = inputShapes[1] 99 | outputshape[0] = NSNumber(integerLiteral: self.maxProposals) 100 | return [outputshape] 101 | } 102 | 103 | func evaluate(inputs: [MLMultiArray], outputs: [MLMultiArray]) throws { 104 | 105 | let log = OSLog(subsystem: "ProposalLayer", category: OSLog.Category.pointsOfInterest) 106 | os_signpost(OSSignpostType.begin, log: log, name: "Proposal-Eval") 107 | 108 | assert(inputs[0].dataType == MLMultiArrayDataType.float32) 109 | assert(inputs[1].dataType == MLMultiArrayDataType.float32) 110 | 111 | //Probabilities of each region containing an object. Shape : (#regions, 2). 112 | let classProbabilities = inputs[0] 113 | //Anchor deltas to refine the anchors shape. Shape : (#regions,4) 114 | let anchorDeltas = inputs[1] 115 | 116 | let preNonMaxLimit = self.preNMSMaxProposals 117 | let maxProposals = self.maxProposals 118 | 119 | let totalNumberOfElements = Int(truncating:classProbabilities.shape[0]) 120 | let numberOfElementsToProcess = min(totalNumberOfElements, preNonMaxLimit) 121 | 122 | os_signpost(OSSignpostType.begin, log: log, name: "Proposal-StridedSlice") 123 | //We extract only the object probabilities, which are always at the odd indices of the array 124 | let objectProbabilities = classProbabilities.floatDataPointer().stridedSlice(begin: 1, count: totalNumberOfElements, stride: 2) 125 | os_signpost(OSSignpostType.end, log: log, name: "Proposal-StridedSlice") 126 | 127 | 128 | os_signpost(OSSignpostType.begin, log: log, name: "Proposal-Sorting") 129 | //We sort the probabilities in descending order and get the index so as to reorder the other arrays. 130 | //We also clip to the limit. 131 | //This is the slowest operation of the layer, taking avg of 45 ms on the ResNet101 backbone. 132 | //Would this be solved with a scatter/gather distributed merge sort? probably not 133 | let sortedProbabilityIndices = objectProbabilities.sortedIndices(ascending: false)[0 ..< numberOfElementsToProcess].toFloat() 134 | os_signpost(OSSignpostType.end, log: log, name: "Proposal-Sorting") 135 | 136 | os_signpost(OSSignpostType.begin, log: log, name: "Proposal-Gathering") 137 | 138 | //We broadcast the probability indices so that they index the boxes (anchor deltas and anchors) 139 | let boxElementLength = 4 140 | let boxIndices = broadcastedIndices(indices: sortedProbabilityIndices, toElementLength: boxElementLength) 141 | 142 | //We sort the deltas and the anchors 143 | 144 | var sortedDeltas:BoxArray = anchorDeltas.floatDataPointer().indexed(indices: boxIndices) 145 | 146 | var sortedAnchors:BoxArray = self.anchorData.withUnsafeBytes { 147 | (data:UnsafePointer) -> [Float] in 148 | return data.indexed(indices: boxIndices) 149 | } 150 | 151 | os_signpost(OSSignpostType.end, log: log, name: "Proposal-Gathering") 152 | 153 | os_signpost(OSSignpostType.begin, log: log, name: "Proposal-Compute") 154 | //For each element of deltas, multiply by stdev 155 | 156 | var stdDev = self.boundingBoxRefinementStandardDeviation 157 | let stdDevPointer = UnsafeMutablePointer(&stdDev) 158 | elementWiseMultiply(matrixPointer: UnsafeMutablePointer(&sortedDeltas), vectorPointer: stdDevPointer, height:numberOfElementsToProcess, width: stdDev.count) 159 | 160 | //We apply the box deltas and clip the results in place to the image boundaries 161 | let anchorsReference = sortedAnchors.boxReference() 162 | anchorsReference.applyBoxDeltas(sortedDeltas) 163 | anchorsReference.clip() 164 | os_signpost(OSSignpostType.end, log: log, name: "Proposal-Compute") 165 | 166 | //We apply Non Max Supression to the result boxes 167 | os_signpost(OSSignpostType.begin, log: log, name: "Proposal-NMS") 168 | 169 | let resultIndices = nonMaxSupression(boxes: sortedAnchors, 170 | indices: Array(0 ..< sortedAnchors.count), 171 | iouThreshold: self.nmsIOUThreshold, 172 | max: maxProposals) 173 | os_signpost(OSSignpostType.end, log: log, name: "Proposal-NMS") 174 | 175 | //We copy the result boxes corresponding to the resultIndices to the output 176 | os_signpost(OSSignpostType.begin, log: log, name: "Proposal-Copy") 177 | 178 | let output = outputs[0] 179 | let outputElementStride = Int(truncating: output.strides[0]) 180 | 181 | for (i,resultIndex) in resultIndices.enumerated() { 182 | for j in 0 ..< 4 { 183 | output[i*outputElementStride+j] = sortedAnchors[resultIndex*4+j] as NSNumber 184 | } 185 | } 186 | os_signpost(OSSignpostType.end, log: log, name: "Proposal-Copy") 187 | 188 | //Zero-pad the rest since CoreML does not erase the memory between evaluations 189 | 190 | let proposalCount = resultIndices.count 191 | let paddingCount = max(0,maxProposals-proposalCount)*outputElementStride 192 | output.padTailWithZeros(startIndex: proposalCount*outputElementStride, count: paddingCount) 193 | 194 | os_signpost(OSSignpostType.end, log: log, name: "Proposal-Eval") 195 | } 196 | 197 | } 198 | 199 | 200 | -------------------------------------------------------------------------------- /Sources/Mask-RCNN-CoreML/Utils.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Utils.swift 3 | // Mask-RCNN-CoreML 4 | // 5 | // Created by Edouard Lavery-Plante on 2018-11-06. 6 | // Copyright © 2018 Edouard Lavery Plante. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import Accelerate 11 | import CoreML 12 | 13 | extension String: Error {} 14 | 15 | extension UnsafePointer where Pointee == Float { 16 | 17 | func stridedSlice(begin:Int, count:Int, stride:Int, length:Int = 1) -> [Float] { 18 | let dataPointer = self.advanced(by: begin) 19 | let size = count * length 20 | var result = Array(repeating:0.0, count: size) 21 | let resultPointer = UnsafeMutablePointer(&result) 22 | for l in 0 ..< length { 23 | cblas_scopy(Int32(count), dataPointer.advanced(by: l), Int32(stride), resultPointer.advanced(by: l), Int32(length)) 24 | } 25 | return result 26 | } 27 | 28 | func indexed(indices:[Float]) -> [Float] { 29 | var results = Array(repeating: 0, count: indices.count) 30 | var indices = indices 31 | vDSP_vindex(self, 32 | UnsafeMutablePointer(&indices), 33 | 1, 34 | UnsafeMutablePointer(&results), 35 | 1, 36 | vDSP_Length(indices.count)) 37 | return results 38 | } 39 | } 40 | 41 | extension UnsafeMutablePointer where Pointee == Float { 42 | 43 | func toUnmutable() -> UnsafePointer { 44 | return UnsafePointer(self) 45 | } 46 | 47 | func indexed(indices:[Float]) -> [Float] { 48 | return self.toUnmutable().indexed(indices: indices) 49 | } 50 | 51 | } 52 | 53 | 54 | extension Array where Element == Float { 55 | 56 | func sortedIndices(ascending:Bool) -> [UInt] { 57 | var array = self 58 | let totalElements = vDSP_Length(array.count) 59 | var indices = Array(0 ..< totalElements) 60 | vDSP_vsorti(UnsafeMutablePointer(&array), 61 | UnsafeMutablePointer(&indices), 62 | nil, 63 | vDSP_Length(array.count), 64 | ascending ? 1 : -1) 65 | return indices 66 | } 67 | 68 | } 69 | 70 | extension Array where Element == UInt { 71 | 72 | func toFloat() -> [Float] { 73 | return Array(self.map({ (integerValue) -> Float in 74 | return Float(integerValue) 75 | })) 76 | } 77 | 78 | } 79 | 80 | extension ArraySlice where Element == UInt { 81 | 82 | func toFloat() -> [Float] { 83 | return Array(self.map({ (integerValue) -> Float in 84 | return Float(integerValue) 85 | })) 86 | } 87 | 88 | } 89 | 90 | @available(iOS 12.0, macOS 10.14, *) 91 | extension MLMultiArray { 92 | 93 | func floatDataPointer() -> UnsafePointer { 94 | assert(self.dataType == .float32) 95 | let dataPointer = UnsafePointer(OpaquePointer(self.dataPointer)) 96 | return dataPointer 97 | } 98 | 99 | func padTailWithZeros(startIndex:Int, count:Int) { 100 | 101 | guard count > 0 else { 102 | return 103 | } 104 | 105 | var paddingBuffer = Array(repeating: 0.0, count: count) 106 | let paddingBufferPointer = UnsafeMutableRawPointer(&paddingBuffer) 107 | self.dataPointer.advanced(by: startIndex*MemoryLayout.size).copyMemory(from: paddingBufferPointer, byteCount: count*MemoryLayout.size) 108 | } 109 | 110 | } 111 | 112 | func broadcastedIndices(indices:[Float], toElementLength elementLength:Int) -> [Float] { 113 | 114 | var indices:[Float] = indices 115 | let resultCount = elementLength*indices.count 116 | var resultIndices:[Float] = Array(repeating: 0, count: resultCount) 117 | 118 | for i in 0 ..< Int(elementLength) { 119 | cblas_scopy(Int32(indices.count), 120 | UnsafeMutablePointer(&indices), 121 | 1, 122 | UnsafeMutablePointer(&resultIndices).advanced(by: i), 123 | Int32(elementLength)) 124 | } 125 | 126 | var multiplicationScalar = Float(elementLength) 127 | vDSP_vsmul(UnsafeMutablePointer(&resultIndices), 128 | 1, 129 | UnsafeMutablePointer(&multiplicationScalar), 130 | UnsafeMutablePointer(&resultIndices), 131 | 1, 132 | vDSP_Length(resultCount)) 133 | 134 | for i in 1 ..< Int(elementLength) { 135 | 136 | var shift:Float = Float(i) 137 | let shiftPointer = UnsafeMutablePointer(&shift) 138 | 139 | vDSP_vsadd(UnsafeMutablePointer(&resultIndices).advanced(by: i), 140 | vDSP_Stride(elementLength), 141 | shiftPointer, 142 | UnsafeMutablePointer(&resultIndices).advanced(by: i), 143 | vDSP_Stride(elementLength), 144 | vDSP_Length(indices.count)) 145 | } 146 | 147 | return resultIndices 148 | } 149 | 150 | func computeIndices(fromIndicesPointer:UnsafeMutablePointer, 151 | toIndicesPointer:UnsafeMutablePointer, 152 | elementLength:UInt, 153 | elementCount:UInt) { 154 | cblas_scopy(Int32(elementCount), fromIndicesPointer, 1, toIndicesPointer, Int32(elementLength)) 155 | 156 | for i in 1 ..< Int(elementLength) { 157 | cblas_scopy(Int32(elementCount), fromIndicesPointer, 1, toIndicesPointer.advanced(by: i), Int32(elementLength)) 158 | } 159 | 160 | var multiplicationScalar:Float = Float(elementLength) 161 | let multiplicationScalarPointer = UnsafeMutablePointer(&multiplicationScalar) 162 | vDSP_vsmul(toIndicesPointer, 1, multiplicationScalarPointer, toIndicesPointer, 1, elementLength*elementCount) 163 | 164 | for i in 1 ..< Int(elementLength) { 165 | 166 | var shift:Float = Float(i) 167 | let shiftPointer = UnsafeMutablePointer(&shift) 168 | 169 | vDSP_vsadd(toIndicesPointer.advanced(by: i), vDSP_Stride(elementLength), shiftPointer, toIndicesPointer.advanced(by: i), vDSP_Stride(elementLength), elementCount) 170 | } 171 | } 172 | 173 | func elementWiseMultiply(matrixPointer:UnsafeMutablePointer, 174 | vectorPointer:UnsafeMutablePointer, 175 | height:Int, 176 | width:Int) { 177 | for i in 0 ..< width { 178 | vDSP_vsmul(matrixPointer.advanced(by: i), width, vectorPointer.advanced(by: i), matrixPointer.advanced(by: i), width, vDSP_Length(height)) 179 | } 180 | } 181 | 182 | //nonMaxSupression Adapted from https://github.com/hollance/CoreMLHelpers 183 | 184 | //This is also a bottle neck, could we find a way to vectorize so parts of it? Also, is it worth to compute these without having to transform the floats into CG structures? 185 | func nonMaxSupression(boxes:[Float], 186 | indices:[Int], 187 | iouThreshold:Float, 188 | max:Int) -> [Int] { 189 | var selected:[Int] = [] 190 | 191 | for index in indices { 192 | if selected.count >= max { return selected } 193 | 194 | let anchorA = CGRect(anchorDatum: boxes[index*4.. 0 && anchorA.height > 0 196 | 197 | if(shouldSelect) { 198 | // Does the current box overlap one of the selected anchors more than the 199 | // given threshold amount? Then it's too similar, so don't keep it. 200 | for j in selected { 201 | 202 | let anchorB = CGRect(anchorDatum: boxes[j*4.. iouThreshold { 204 | shouldSelect = false 205 | break 206 | } 207 | 208 | } 209 | } 210 | // This bounding box did not overlap too much with any previously selected 211 | // bounding box, so we'll keep it. 212 | if shouldSelect { 213 | selected.append(index) 214 | } 215 | } 216 | 217 | return selected 218 | } 219 | 220 | extension CGRect 221 | { 222 | init(anchorDatum:ArraySlice) { 223 | let index = anchorDatum.startIndex 224 | let y1 = CGFloat(anchorDatum[index]) 225 | let x1 = CGFloat(anchorDatum[index+1]) 226 | let y2 = CGFloat(anchorDatum[index+2]) 227 | let x2 = CGFloat(anchorDatum[index+3]) 228 | self = CGRect(x: x1, y:y1, width: x2-x1, height: y2-y1) 229 | } 230 | } 231 | 232 | public func IOU(_ a: CGRect, _ b: CGRect) -> Float { 233 | let areaA = a.width * a.height 234 | if areaA <= 0 { return 0 } 235 | 236 | let areaB = b.width * b.height 237 | if areaB <= 0 { return 0 } 238 | 239 | let intersectionMinX = max(a.minX, b.minX) 240 | let intersectionMinY = max(a.minY, b.minY) 241 | let intersectionMaxX = min(a.maxX, b.maxX) 242 | let intersectionMaxY = min(a.maxY, b.maxY) 243 | let intersectionArea = max(intersectionMaxY - intersectionMinY, 0) * 244 | max(intersectionMaxX - intersectionMinX, 0) 245 | return Float(intersectionArea / (areaA + areaB - intersectionArea)) 246 | } 247 | -------------------------------------------------------------------------------- /Sources/maskrcnn/EvaluateCommand.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Vision 3 | import CoreML 4 | import QuartzCore 5 | 6 | import SwiftCLI 7 | import Mask_RCNN_CoreML 8 | 9 | class EvaluateCommand: Command { 10 | 11 | let name = "evaluate" 12 | let shortDescription = "Evaluates CoreML model against validation data" 13 | 14 | let modelName = Parameter() 15 | let evalDataset = Parameter() 16 | let configFilePath = Key("--config", description: "Path to config JSON file") 17 | let weightsFilePath = Key("--weights", description: "Path to HDF5 weights file") 18 | let productsDirectoryPath = Key("--products_dir", description: "Path to products directory") 19 | let yearOption = Key("--year", description: "COCO dataset year") 20 | let typeOption = Key("--type", description: "COCO dataset type") 21 | let compareFlag = Flag("-c", "--compare") 22 | 23 | func execute() throws { 24 | 25 | guard #available(macOS 10.14, *) else { 26 | stdout <<< "eval requires macOS >= 10.14" 27 | return 28 | } 29 | 30 | guard Docker.installed else { 31 | stdout <<< "Docker is required to run this script." 32 | return 33 | } 34 | 35 | let name = self.modelName.value 36 | let evalDataset = self.evalDataset.value 37 | 38 | stdout <<< "Evaluating \(name) using \(evalDataset)" 39 | 40 | let currentDirectoryPath = FileManager.default.currentDirectoryPath 41 | let currentDirectoryURL = URL(fileURLWithPath: currentDirectoryPath) 42 | 43 | let buildURL = currentDirectoryURL.appendingPathComponent("Sources/maskrcnn/Python/COCOEval") 44 | 45 | let verbose = true 46 | let docker = Docker(name:"mask-rcnn-evaluate", buildURL:buildURL) 47 | try docker.build(verbose:verbose) 48 | 49 | let defaultModelsURL = currentDirectoryURL.appendingPathComponent(".maskrcnn/models").appendingPathComponent(name) 50 | let defaultDataURL = currentDirectoryURL.appendingPathComponent(".maskrcnn/data") 51 | let defaultModelDirectoryURL = defaultModelsURL.appendingPathComponent("model") 52 | 53 | let productsURL:URL = { 54 | () -> URL in 55 | guard let productsDirectoryPath = productsDirectoryPath.value else { 56 | return defaultModelsURL.appendingPathComponent("products") 57 | } 58 | return URL(fileURLWithPath:productsDirectoryPath, isDirectory:false, relativeTo:currentDirectoryURL).standardizedFileURL 59 | }() 60 | 61 | let mainModelURL = productsURL.appendingPathComponent("MaskRCNN.mlmodel") 62 | let classifierModelURL = productsURL.appendingPathComponent("Classifier.mlmodel") 63 | let maskModelURL = productsURL.appendingPathComponent("Mask.mlmodel") 64 | let anchorsURL = productsURL.appendingPathComponent("anchors.bin") 65 | 66 | let cocoURL = defaultDataURL.appendingPathComponent("coco_eval") 67 | let annotationsDirectoryURL = cocoURL 68 | 69 | let year = yearOption.value ?? "2017" 70 | let type = typeOption.value ?? "val" 71 | let imagesDirectoryURL = cocoURL.appendingPathComponent("\(type)\(year)") 72 | 73 | var mounts = [Docker.Mount]() 74 | 75 | let dockerConfigDestinationPath = "/usr/src/app/model/config.json" 76 | 77 | if let configFilePath = self.configFilePath.value { 78 | let configURL = URL(fileURLWithPath:configFilePath, isDirectory:false, relativeTo:currentDirectoryURL) 79 | mounts.append(Docker.Mount(source:configURL.standardizedFileURL, destination:dockerConfigDestinationPath)) 80 | } else { 81 | mounts.append(Docker.Mount(source:defaultModelDirectoryURL.appendingPathComponent("config.json"), 82 | destination:dockerConfigDestinationPath)) 83 | } 84 | 85 | let dockerWeightsDestinationPath = "/usr/src/app/model/weights.h5" 86 | 87 | if let weightsFilePath = self.weightsFilePath.value { 88 | let weightsURL = URL(fileURLWithPath:weightsFilePath, isDirectory:false, relativeTo:currentDirectoryURL) 89 | mounts.append(Docker.Mount(source:weightsURL.standardizedFileURL, destination:dockerWeightsDestinationPath)) 90 | } else { 91 | mounts.append(Docker.Mount(source:defaultModelDirectoryURL.appendingPathComponent("weights.h5"), 92 | destination:dockerWeightsDestinationPath)) 93 | } 94 | 95 | let dockerProductsDestinationPath = "/usr/src/app/products" 96 | mounts.append(Docker.Mount(source:productsURL, 97 | destination:dockerProductsDestinationPath)) 98 | 99 | let dockerCocoDataDestinationPath = "/usr/src/app/data/coco" 100 | mounts.append(Docker.Mount(source:cocoURL, 101 | destination:dockerCocoDataDestinationPath)) 102 | 103 | let temporaryDirectory = currentDirectoryURL.appendingPathComponent(".maskrcnn/tmp") 104 | try Foundation.FileManager.default.createDirectory(at: temporaryDirectory, withIntermediateDirectories: true, attributes: nil) 105 | let fileName = NSUUID().uuidString+".pb" 106 | let outputDataURL = temporaryDirectory.appendingPathComponent(fileName) 107 | 108 | let output = try evaluate(datasetId:evalDataset, 109 | modelURL:mainModelURL, 110 | classifierModelURL:classifierModelURL, 111 | maskModelURL:maskModelURL, 112 | anchorsURL:anchorsURL, 113 | annotationsDirectoryURL:annotationsDirectoryURL, 114 | imagesDirectoryURL:imagesDirectoryURL, 115 | year:year, 116 | type:type) 117 | let outputData = try output.serializedData() 118 | try outputData.write(to:outputDataURL) 119 | 120 | let dockerResultsDestinationPath = "/usr/src/app/results" 121 | mounts.append(Docker.Mount(source:outputDataURL.standardizedFileURL, 122 | destination:dockerResultsDestinationPath)) 123 | var arguments = ["--coco_year", year, "--coco_type", type, "--results_path", "results"] 124 | if(compareFlag.value){ 125 | arguments.append("--compare") 126 | arguments.append("--compare") 127 | } 128 | try docker.run(mounts:mounts, arguments:arguments, verbose:verbose) 129 | try Foundation.FileManager.default.removeItem(at:outputDataURL) 130 | } 131 | } 132 | 133 | @available(macOS 10.14, *) 134 | func evaluate(datasetId:String, 135 | modelURL:URL, 136 | classifierModelURL:URL, 137 | maskModelURL:URL, 138 | anchorsURL:URL, 139 | annotationsDirectoryURL:URL, 140 | imagesDirectoryURL:URL, 141 | year:String, 142 | type:String) throws -> Results { 143 | 144 | MaskRCNNConfig.defaultConfig.anchorsURL = anchorsURL 145 | 146 | let compiledClassifierUrl = try MLModel.compileModel(at: classifierModelURL) 147 | MaskRCNNConfig.defaultConfig.compiledClassifierModelURL = compiledClassifierUrl 148 | 149 | let compiledMaskUrl = try MLModel.compileModel(at: maskModelURL) 150 | MaskRCNNConfig.defaultConfig.compiledMaskModelURL = compiledMaskUrl 151 | 152 | let compiledUrl = try MLModel.compileModel(at: modelURL) 153 | let model = try MLModel(contentsOf: compiledUrl) 154 | 155 | let vnModel = try VNCoreMLModel(for:model) 156 | let request = VNCoreMLRequest(model: vnModel) 157 | request.imageCropAndScaleOption = .scaleFit 158 | 159 | let instancesURL = annotationsDirectoryURL.appendingPathComponent("instances_\(type)\(year).json") 160 | 161 | let coco = try COCO(url:instancesURL) 162 | 163 | var outputResults = [Result]() 164 | 165 | var iterator = coco.makeImageIterator(limit:5, sortById:true) 166 | while let item = iterator.next() { 167 | let start = Date().timeIntervalSinceReferenceDate 168 | let image = item.0 169 | let imageURL = imagesDirectoryURL.appendingPathComponent(image.fileName) 170 | let ciImage = CIImage(contentsOf:imageURL)! 171 | let handler = VNImageRequestHandler(ciImage: ciImage) 172 | try handler.perform([request]) 173 | 174 | guard let results = request.results as? [VNCoreMLFeatureValueObservation], 175 | let detectionsFeatureValue = results.first?.featureValue, 176 | let maskFeatureValue = results.last?.featureValue else { 177 | throw "Error during prediction" 178 | } 179 | let end = Date().timeIntervalSinceReferenceDate 180 | let detections = detectionsFromFeatureValue(featureValue: detectionsFeatureValue, maskFeatureValue:maskFeatureValue) 181 | 182 | let result = Result.with { 183 | let image = Result.ImageInfo.with { 184 | $0.id = String(image.id) 185 | $0.datasetID = datasetId 186 | $0.width = Int32(image.width) 187 | $0.height = Int32(image.height) 188 | } 189 | $0.imageInfo = image 190 | $0.detections = detections 191 | } 192 | outputResults.append(result) 193 | print(end-start) 194 | } 195 | 196 | let output = Results.with { 197 | $0.results = outputResults 198 | } 199 | return output 200 | } 201 | 202 | @available(macOS 10.14, *) 203 | func detectionsFromFeatureValue(featureValue: MLFeatureValue, 204 | maskFeatureValue:MLFeatureValue) -> [Result.Detection] { 205 | 206 | guard let rawDetections = featureValue.multiArrayValue else { 207 | return [] 208 | } 209 | 210 | let detectionsCount = Int(truncating: rawDetections.shape[0]) 211 | let detectionStride = Int(truncating: rawDetections.strides[0]) 212 | var detections = [Result.Detection]() 213 | 214 | for i in 0 ..< detectionsCount { 215 | 216 | let probability = Double(truncating: rawDetections[i*detectionStride+5]) 217 | if(probability > 0.7) { 218 | 219 | let classId = Int(truncating: rawDetections[i*detectionStride+4]) 220 | let classLabel = "test" 221 | let y1 = Double(truncating: rawDetections[i*detectionStride]) 222 | let x1 = Double(truncating: rawDetections[i*detectionStride+1]) 223 | let y2 = Double(truncating: rawDetections[i*detectionStride+2]) 224 | let x2 = Double(truncating: rawDetections[i*detectionStride+3]) 225 | let width = x2-x1 226 | let height = y2-y1 227 | 228 | let detection = Result.Detection.with { 229 | $0.probability = probability 230 | $0.classID = Int32(classId) 231 | $0.classLabel = classLabel 232 | $0.boundingBox = Result.Rect.with { 233 | $0.origin = Result.Origin.with { 234 | $0.x = x1 235 | $0.y = y1 236 | } 237 | $0.size = Result.Size.with { 238 | $0.width = width 239 | $0.height = height 240 | } 241 | } 242 | } 243 | detections.append(detection) 244 | } 245 | 246 | } 247 | return detections 248 | } 249 | -------------------------------------------------------------------------------- /Sources/Mask-RCNN-CoreML/DetectionLayer.swift: -------------------------------------------------------------------------------- 1 | // 2 | // DetectionLayer.swift 3 | // Mask-RCNN-CoreML 4 | // 5 | // Created by Edouard Lavery-Plante on 2018-11-06. 6 | // Copyright © 2018 Edouard Lavery Plante. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import CoreML 11 | import Accelerate 12 | 13 | /** 14 | 15 | DetectionLayer is a Custom ML Layer that outputs object detections. 16 | 17 | DetectionLayer outputs detections based on the confidence of an object 18 | (any non-background class) being detected in each region. 19 | Regions that overlap more than a given threshold are removed 20 | through a process called Non-Max Supression (NMS). 21 | 22 | The regions of interest are adjusted using deltas provided as input to refine 23 | how they enclose the detected objects. 24 | 25 | The layer takes three inputs : 26 | 27 | - Regions of interest. Shape : (#regions,4) 28 | - TODO 29 | 30 | The regions of interest are layed out as follows : (y1,x1,y2,x2). 31 | 32 | The class probability input is layed out such that the index 0 corresponds to the 33 | background class. 34 | 35 | The bounding box deltas are layed out as follows : (dy,dx,log(dh),log(dw)). 36 | 37 | The layer takes four parameters : 38 | 39 | - boundingBoxRefinementStandardDeviation : Bounding box deltas refinement standard deviation 40 | - detectionLimit : Maximum # of detections to output 41 | - lowConfidenceScoreThreshold : Threshold below which to discard regions 42 | - nmsIOUThreshold : Threshold below which to supress regions that overlap 43 | 44 | The layer has one ouput : 45 | 46 | - Detections (y1,x1,y2,x2,classId,score). Shape : (#regionsOut,6) 47 | 48 | The classIds are the argmax of each rows in the class probability input. 49 | 50 | */ 51 | @available(iOS 12.0, macOS 10.14, *) 52 | @objc(DetectionLayer) class DetectionLayer: NSObject, MLCustomLayer { 53 | 54 | //Bounding box deltas refinement standard deviation 55 | var boundingBoxRefinementStandardDeviation:[Float] = [0.1, 0.1, 0.2, 0.2] 56 | //Maximum # of detections to output 57 | var maxDetections = 100 58 | //Threshold below which to discard regions 59 | var lowConfidenceScoreThreshold:Float = 0.7 60 | //Threshold below which to supress regions that overlap 61 | var nmsIOUThreshold:Float = 0.3 62 | 63 | required init(parameters: [String : Any]) throws { 64 | 65 | super.init() 66 | 67 | if let bboxStdDevCount = parameters["bboxStdDev_count"] as? Int { 68 | var bboxStdDev = [Float]() 69 | for i in 0.. [[NSNumber]] { 95 | 96 | let roisShape = inputShapes[0] 97 | 98 | let seq = maxDetections as NSNumber 99 | let batch = roisShape[1] 100 | let channels:NSNumber = 6//(y1,x1,y2,x2,classId,score) 101 | let height:NSNumber = 1 102 | let width:NSNumber = 1 103 | let outputShapes = [[seq,batch,channels,height,width]] 104 | return outputShapes 105 | } 106 | 107 | func evaluate(inputs: [MLMultiArray], outputs: [MLMultiArray]) throws { 108 | 109 | let log = OSLog(subsystem: "DetectionLayer", category: OSLog.Category.pointsOfInterest) 110 | os_signpost(OSSignpostType.begin, log: log, name: "Detection-Eval") 111 | 112 | //Regions of interest. Shape : (#regions,4) 113 | //(y1,x1,y2,x2) 114 | let rois = inputs[0] 115 | assert(inputs[0].dataType == MLMultiArrayDataType.float32) 116 | 117 | //TODO: doc 118 | //dy,dx,log(dh),log(dw),classId,score) 119 | let classifications = inputs[1] 120 | assert(inputs[1].dataType == MLMultiArrayDataType.float32) 121 | 122 | let inputRegionsCount = Int(truncating:rois.shape[0]) 123 | let maxDetections = self.maxDetections 124 | 125 | var boundingBoxDeltas = classifications.floatDataPointer().stridedSlice(begin: 0, count: inputRegionsCount, stride: 6, length: 4) 126 | 127 | var classIds = classifications.floatDataPointer().stridedSlice(begin: 4, count: inputRegionsCount, stride: 6) 128 | var scores = classifications.floatDataPointer().stridedSlice(begin: 5, count: inputRegionsCount, stride: 6) 129 | 130 | //Start from all indices of rois and apply threshold 131 | var filteredIndices = indicesOfRoisWithHighScores(scores: UnsafeMutablePointer(&scores), 132 | threshold: self.lowConfidenceScoreThreshold, 133 | count: UInt(scores.count)) 134 | 135 | //Omit background class 136 | filteredIndices = filteredIndices.filter { (index) -> Bool in 137 | let intIndex = Int(index) 138 | let classId = classIds[intIndex] 139 | return classId > 0 140 | } 141 | 142 | //Gather rois based on filtered indices 143 | let boxElementLength = 4 144 | let boxIndices = broadcastedIndices(indices: filteredIndices, toElementLength: boxElementLength) 145 | var filteredRois = rois.floatDataPointer().indexed(indices: boxIndices) 146 | 147 | //Gather scores based on filtered indices 148 | let filteredScores = UnsafeMutablePointer(&scores).indexed(indices:filteredIndices) 149 | 150 | //Gather classes based on filtered indices 151 | let filteredClass = UnsafeMutablePointer(&classIds).indexed(indices:filteredIndices) 152 | 153 | //Gather deltas based on filtered indices 154 | var filteredDeltas = UnsafeMutablePointer(&boundingBoxDeltas).indexed(indices: boxIndices) 155 | 156 | //Multiply bounding box deltas by std dev 157 | var stdDev = self.boundingBoxRefinementStandardDeviation 158 | let stdDevPointer = UnsafeMutablePointer(&stdDev) 159 | elementWiseMultiply(matrixPointer: UnsafeMutablePointer(&filteredDeltas), vectorPointer: stdDevPointer, height:filteredClass.count, width: stdDev.count) 160 | 161 | //Apply deltas and clip rois 162 | let roisReference = filteredRois.boxReference() 163 | roisReference.applyBoxDeltas(filteredDeltas) 164 | roisReference.clip() 165 | 166 | var nmsBoxIds = [Int]() 167 | let classIdSet = Set(filteredClass) 168 | 169 | //Apply NMS for each class that's present 170 | for classId in classIdSet { 171 | 172 | let indicesOfClass = filteredClass.enumerated().filter { (_,thisClassId) -> Bool in 173 | return thisClassId == classId 174 | }.map { (offset, _) -> Int in 175 | return offset 176 | } 177 | 178 | let nmsResults = nonMaxSupression(boxes: filteredRois, 179 | indices: indicesOfClass, 180 | iouThreshold: self.nmsIOUThreshold, 181 | max: self.maxDetections) 182 | nmsBoxIds.append(contentsOf:nmsResults) 183 | } 184 | 185 | //Keep the top NOut 186 | let resultIndices:[Int] = { 187 | () -> [Int] in 188 | 189 | let maxElements = min(nmsBoxIds.count, self.maxDetections) 190 | 191 | var scores = Array(repeating: 0.0, count: nmsBoxIds.count) 192 | 193 | for (i,index) in nmsBoxIds.enumerated() { 194 | scores[i] = filteredScores[index] 195 | } 196 | 197 | let zippedIdsAndScores = zip(nmsBoxIds, scores) 198 | 199 | let sortedZippedIds = zippedIdsAndScores.sorted(by: { 200 | (a, b) -> Bool in 201 | return a.1 > b.1 202 | }) 203 | 204 | let clippedZippedIds = sortedZippedIds[0.. Int in 207 | return offset 208 | }) 209 | }() 210 | 211 | let boxLength = 4 212 | let output = outputs[0] 213 | let outputElementStride = Int(truncating: output.strides[0]) 214 | 215 | //Layout output is [NOut, (y1, x1, y2, x2, class_id, score)] 216 | 217 | for (i,resultIndex) in resultIndices.enumerated() { 218 | 219 | for j in 0 ..< boxLength { 220 | output[i*outputElementStride+j] = filteredRois[resultIndex*boxLength+j] as NSNumber 221 | } 222 | output[i*outputElementStride+4] = filteredClass[resultIndex] as NSNumber 223 | output[i*outputElementStride+5] = filteredScores[resultIndex] as NSNumber 224 | } 225 | 226 | //Zero-pad the rest as CoreML does not erase the memory between evaluations 227 | 228 | let detectionsCount = resultIndices.count 229 | let paddingCount = max(0,maxDetections-detectionsCount)*outputElementStride 230 | 231 | output.padTailWithZeros(startIndex: detectionsCount*outputElementStride, count: paddingCount) 232 | 233 | os_signpost(OSSignpostType.end, log: log, name: "Detection-Eval") 234 | } 235 | 236 | } 237 | 238 | func indicesOfRoisWithHighScores(scores:UnsafeMutablePointer, 239 | threshold:Float, 240 | count:UInt) -> [Float] { 241 | let temporaryBuffer = UnsafeMutablePointer.allocate(capacity: Int(count)) 242 | 243 | //Count how many scores below threshold 244 | var lowThreshold:Float = threshold 245 | var highThreshold:Float = 1.0 246 | var lowCount:UInt = 0 247 | var highCount:UInt = 0 248 | vDSP_vclipc(scores, 249 | 1, 250 | UnsafeMutablePointer(&lowThreshold), 251 | UnsafeMutablePointer(&highThreshold), 252 | temporaryBuffer, 253 | 1, 254 | count, 255 | UnsafeMutablePointer(&lowCount), 256 | UnsafeMutablePointer(&highCount)) 257 | 258 | //Set to 0 scores below threshold 259 | vDSP_vthres(scores, 1, UnsafeMutablePointer(&lowThreshold), scores, 1, count) 260 | 261 | //Vector ramp 262 | var initial:Float = 0 263 | var increment:Float = 1 264 | vDSP_vramp(UnsafeMutablePointer(&initial), UnsafeMutablePointer(&increment), temporaryBuffer, 1, count) 265 | var indices = Array(repeating: 0, count:Int(count-lowCount)) 266 | //Compress the ramp 267 | vDSP_vcmprs(temporaryBuffer, 268 | 1, 269 | scores, 270 | 1, 271 | UnsafeMutablePointer(&indices), 272 | 1, 273 | count) 274 | temporaryBuffer.deallocate() 275 | return indices 276 | } 277 | -------------------------------------------------------------------------------- /Sources/Mask-RCNN-CoreML/PyramidROIAlignLayer.swift: -------------------------------------------------------------------------------- 1 | // 2 | // PyramidROIAlignLayer.swift 3 | // Mask-RCNN-CoreML 4 | // 5 | // Created by Edouard Lavery-Plante on 2018-10-31. 6 | // Copyright © 2018 Edouard Lavery Plante. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import CoreML 11 | import Accelerate 12 | import MetalPerformanceShaders 13 | import os.signpost 14 | 15 | /** 16 | 17 | PyramidROIAlignLayer is a Custom ML Layer that extracts feature maps based on the regions of interest. 18 | 19 | PyramidROIAlignLayer outputs aligned feature maps by cropping and 20 | resizing portions of the input feature maps based on the regions of interest. 21 | 22 | The region's size determine which input feature map is used. 23 | 24 | The layer takes five inputs : 25 | - Regions of interest. Shape (#regions, 4) 26 | - 4 feature maps of different sizes. Shapes : (#channels,256,256), (#channels,128,128), 27 | (#channels,64,64),(#channels,32,32) 28 | 29 | The layer takes two parameters : 30 | - poolSize : The size of the output feature map 31 | - imageSize : The input image sizes 32 | 33 | The imageSize is used to determine which feature map to use 34 | 35 | The layer has one output 36 | - Feature maps. Shape (#regions, 1, #channels, poolSize, poolSize) 37 | 38 | */ 39 | @available(iOS 12.0, macOS 10.14, *) 40 | @objc(PyramidROIAlignLayer) class PyramidROIAlignLayer: NSObject, MLCustomLayer { 41 | 42 | let device = MTLCreateSystemDefaultDevice()! 43 | let maxBatchSize = 64 44 | let maxCommandBufferCount = 3 45 | var poolSize:Int = 7 46 | var imageSize = CGSize(width: 1024, height: 1024) 47 | 48 | required init(parameters: [String : Any]) throws { 49 | super.init() 50 | 51 | if let poolSize = parameters["poolSize"] as? Int { 52 | self.poolSize = poolSize 53 | } 54 | 55 | if let imageWidth = parameters["imageWidth"] as? CGFloat, 56 | let imageHeight = parameters["imageHeight"] as? CGFloat { 57 | self.imageSize = CGSize(width: imageWidth, height: imageHeight) 58 | } 59 | } 60 | 61 | func setWeightData(_ weights: [Data]) throws { 62 | //No-op 63 | } 64 | 65 | func outputShapes(forInputShapes inputShapes: [[NSNumber]]) throws -> [[NSNumber]] { 66 | 67 | let roisShape = inputShapes[0] 68 | let featureMapShape = inputShapes[1] 69 | 70 | let seq = roisShape[0] 71 | let batch = roisShape[1] 72 | let channel = featureMapShape[2] 73 | let height = self.poolSize as NSNumber 74 | let width = self.poolSize as NSNumber 75 | let outputShapes = [[seq,batch,channel,height,width]] 76 | return outputShapes 77 | } 78 | 79 | func evaluate(inputs: [MLMultiArray], outputs: [MLMultiArray]) throws { 80 | 81 | assert(inputs[0].dataType == MLMultiArrayDataType.float32) 82 | 83 | let log = OSLog(subsystem: "PyramidROIAlign", category: OSLog.Category.pointsOfInterest) 84 | os_signpost(OSSignpostType.begin, log: log, name: "PyramidROIAlign-Eval") 85 | 86 | let commandQueue:MTLCommandQueue = device.makeCommandQueue(maxCommandBufferCount: self.maxCommandBufferCount)! 87 | 88 | let rois = inputs[0] 89 | let featureMaps = Array(inputs[1.. MPSImage in 112 | let inputWidth = Int(truncating: featureMap.shape[4]) 113 | let inputHeight = Int(truncating: featureMap.shape[3]) 114 | let image = MPSImage(device: device, imageDescriptor: MPSImageDescriptor(channelFormat: MPSImageFeatureChannelFormat.float32, width: inputWidth, height: inputHeight, featureChannels: channels, numberOfImages:1, usage:MTLTextureUsage.shaderRead)) 115 | //This is the main bottleneck. Is there a more efficient approach? 116 | image.writeBytes(featureMap.dataPointer, dataLayout: MPSDataLayout.featureChannelsxHeightxWidth, imageIndex: 0) 117 | return image 118 | } 119 | 120 | os_signpost(OSSignpostType.end, log: log, name: "PyramidROIAlign-CreateInputRessources") 121 | 122 | let outputPointer = outputs[0].dataPointer 123 | 124 | let resultStride = Int(truncating: outputs[0].strides[0]) 125 | let metalRegion = MTLRegion(origin: MTLOrigin(x: 0, y: 0, z: 0), 126 | size: MTLSize(width: outputWidth, height: outputHeight, depth: 1)) 127 | 128 | 129 | os_signpost(OSSignpostType.begin, log: log, name: "PyramidROIAlign-CreateOutputRessources") 130 | 131 | //We write to the same images to improve performance 132 | var outputImagesByBuffers = Array>() 133 | for _ in 0 ..< maxCommandBufferCount { 134 | var outputImages = Array() 135 | for _ in 0 ..< self.maxBatchSize { 136 | outputImages.append(MPSImage(device: commandQueue.device, imageDescriptor: MPSImageDescriptor(channelFormat: MPSImageFeatureChannelFormat.float32, width: outputWidth, height: outputHeight, featureChannels: channels, numberOfImages:1, usage:[MTLTextureUsage.shaderWrite]))) 137 | } 138 | outputImagesByBuffers.append(outputImages) 139 | } 140 | 141 | os_signpost(OSSignpostType.end, log: log, name: "PyramidROIAlign-CreateOutputRessources") 142 | 143 | let commandBufferEncodingSemaphore = DispatchSemaphore(value: maxCommandBufferCount) 144 | let maxBatchSize = self.maxBatchSize 145 | 146 | let dispatchGroup = DispatchGroup() 147 | 148 | for (i,batch) in batches.enumerated() { 149 | 150 | dispatchGroup.enter() 151 | 152 | commandBufferEncodingSemaphore.wait() 153 | 154 | let outputImagesIndex = i % maxCommandBufferCount 155 | 156 | performBatch(commandQueue: commandQueue, 157 | featureMaps: featureMapImages, 158 | channelsSize:channels, 159 | outputWidth: outputWidth, 160 | outputHeight: outputHeight, 161 | batch: batch, 162 | outputImages:outputImagesByBuffers[outputImagesIndex]){ 163 | outputItems in 164 | 165 | copyOutput(items: outputItems, 166 | metalRegion: metalRegion, 167 | maxBatchSize: maxBatchSize, 168 | resultStride: resultStride, 169 | outputWidth: outputWidth, 170 | outputHeight: outputHeight, 171 | channels: channels, 172 | toPointer: outputPointer) 173 | 174 | commandBufferEncodingSemaphore.signal() 175 | dispatchGroup.leave() 176 | } 177 | 178 | } 179 | dispatchGroup.wait() 180 | os_signpost(OSSignpostType.end, log: log, name: "PyramidROIAlign-Eval") 181 | } 182 | 183 | } 184 | 185 | @available(iOS 12.0, macOS 10.14, *) 186 | func performBatch(commandQueue:MTLCommandQueue, 187 | featureMaps:[MPSImage], 188 | channelsSize:Int, 189 | outputWidth:Int, 190 | outputHeight:Int, 191 | batch:ROIAlignInputBatch, 192 | outputImages:[MPSImage], 193 | completionHandler:@escaping (_ items:[ROIAlignOutputItem])->Void){ 194 | 195 | 196 | let buffer = batch.computeSize > 0 ? commandQueue.makeCommandBufferWithUnretainedReferences() : nil 197 | let channels = channelsSize 198 | 199 | var outputItems = [ROIAlignOutputItem]() 200 | var computeOffset:Int = 0 201 | for group in batch.groups { 202 | switch(group.content){ 203 | case .regions(featureMapIndex: let featureMapIndex, regions: let regions): 204 | let inputImage = featureMaps[featureMapIndex] 205 | for (r,region) in regions.enumerated() { 206 | 207 | let outputImage = outputImages[computeOffset] 208 | let output = ROIAlignOutputItem(offset: group.offset+r, content: .image(image: outputImage)) 209 | outputItems.append(output) 210 | var regions = [region] 211 | let regionsPointer = UnsafeMutablePointer(®ions) 212 | let kernel = MPSNNCropAndResizeBilinear(device: commandQueue.device, 213 | resizeWidth: outputWidth, 214 | resizeHeight: outputHeight, 215 | numberOfRegions: 1, 216 | regions: regionsPointer) 217 | 218 | for slice in 0 ..< channels/4 { 219 | kernel.sourceFeatureChannelOffset = slice*4 220 | kernel.destinationFeatureChannelOffset = slice*4 221 | kernel.encode(commandBuffer: buffer!, 222 | sourceImage: inputImage, 223 | destinationImage: outputImage) 224 | } 225 | computeOffset += 1 226 | } 227 | case .padding(count: let paddingCount): 228 | let output = ROIAlignOutputItem(offset: group.offset, content: .padding(count: paddingCount)) 229 | outputItems.append(output) 230 | } 231 | } 232 | 233 | if(batch.computeSize == 0){ 234 | completionHandler(outputItems) 235 | return 236 | } 237 | 238 | buffer!.addCompletedHandler { _ in 239 | completionHandler(outputItems) 240 | } 241 | buffer!.commit() 242 | } 243 | 244 | @available(iOS 12.0, macOS 10.14, *) 245 | func copyOutput(items:[ROIAlignOutputItem], 246 | metalRegion:MTLRegion, 247 | maxBatchSize:Int, 248 | resultStride:Int, 249 | outputWidth:Int, 250 | outputHeight:Int, 251 | channels:Int, 252 | toPointer outputPointer:UnsafeMutableRawPointer) { 253 | 254 | var paddingBuffer = Array(repeating: 0.0, count: maxBatchSize) 255 | 256 | for output in items { 257 | let offset = output.offset * resultStride 258 | let pointer = outputPointer.advanced(by:offset*MemoryLayout.size) 259 | switch(output.content){ 260 | case .image(image: let image): 261 | image.readBytes(pointer, 262 | dataLayout: MPSDataLayout.featureChannelsxHeightxWidth, 263 | bytesPerRow: outputWidth*MemoryLayout.size, 264 | region: metalRegion, featureChannelInfo: MPSImageReadWriteParams(featureChannelOffset: 0, numberOfFeatureChannelsToReadWrite: channels), imageIndex: 0) 265 | case .padding(count: let paddingCount): 266 | if(paddingCount > paddingBuffer.count){ 267 | paddingBuffer = Array(repeating: 0.0, count: paddingCount) 268 | } 269 | 270 | let bufferPointer = UnsafeMutableRawPointer(&paddingBuffer) 271 | pointer.copyMemory(from: bufferPointer, byteCount: MemoryLayout.size*paddingCount) 272 | } 273 | } 274 | } 275 | 276 | @available(iOS 12.0, macOS 10.14, *) 277 | struct ROIAlignInputBatch { 278 | 279 | let groups:[ROIAlignInputGroup] 280 | 281 | var size:Int { 282 | return self.groups.reduce(0, { (result, group) -> Int in 283 | return result + group.size 284 | }) 285 | } 286 | 287 | var computeSize:Int { 288 | return self.groups.reduce(0, { (result, group) -> Int in 289 | return result + group.computeSize 290 | }) 291 | } 292 | 293 | } 294 | 295 | @available(iOS 12.0, macOS 10.14, *) 296 | struct ROIAlignInputGroup { 297 | 298 | enum Content { 299 | case regions(featureMapIndex:Int, regions:[MPSRegion]) 300 | case padding(count:Int) 301 | } 302 | 303 | let offset:Int 304 | let content:Content 305 | 306 | var size:Int { 307 | switch self.content { 308 | case .padding(count: let count): 309 | return count 310 | default: 311 | return self.computeSize 312 | } 313 | } 314 | 315 | var computeSize:Int { 316 | switch self.content { 317 | case .regions(featureMapIndex: _, regions: let regions): 318 | return regions.count 319 | default: 320 | return 0 321 | } 322 | } 323 | 324 | } 325 | 326 | @available(iOS 12.0, macOS 10.14, *) 327 | struct ROIAlignInputItem { 328 | 329 | enum Content { 330 | case region(featureMapIndex:Int, region:MPSRegion) 331 | case padding 332 | } 333 | 334 | let offset:Int 335 | let content:Content 336 | } 337 | 338 | @available(iOS 12.0, macOS 10.14, *) 339 | struct ROIAlignOutputItem { 340 | 341 | enum Content { 342 | case image(image:MPSImage) 343 | case padding(count:Int) 344 | } 345 | 346 | let offset:Int 347 | let content:Content 348 | } 349 | 350 | @available(iOS 12.0, macOS 10.14, *) 351 | func roisToInputItems(rois:MLMultiArray, 352 | featureMapSelectionFactor:CGFloat, 353 | imageSize:CGSize) -> [ROIAlignInputItem] { 354 | 355 | let totalCount = Int(truncating: rois.shape[0]) 356 | let stride = Int(truncating: rois.strides[0]) 357 | let ratio = Double(featureMapSelectionFactor/sqrt(imageSize.width*imageSize.height)) 358 | 359 | var results = [ROIAlignInputItem]() 360 | 361 | for i in 0 ..< totalCount { 362 | 363 | let roiIndex = stride*i 364 | 365 | let y1 = Double(truncating: rois[roiIndex]) 366 | let x1 = Double(truncating: rois[roiIndex+1]) 367 | let y2 = Double(truncating: rois[roiIndex+2]) 368 | let x2 = Double(truncating: rois[roiIndex+3]) 369 | 370 | let width = x2-x1 371 | let height = y2-y1 372 | 373 | let featureMapLevelFloat = log2(sqrt(width*height)/ratio)+4.0 374 | let regionIsValid = !featureMapLevelFloat.isNaN && !featureMapLevelFloat.isInfinite 375 | 376 | let featureMapLevel = (!regionIsValid) ? 2 : min(5,max(2,Int(round(featureMapLevelFloat)))) 377 | let featureMapIndex = featureMapLevel-2 378 | 379 | let region = MPSRegion(origin: MPSOrigin(x: x1, y: y1, z: 0), size: MPSSize(width: width, 380 | height: height, 381 | depth: 1.0)) 382 | 383 | let content:ROIAlignInputItem.Content = { 384 | () -> ROIAlignInputItem.Content in 385 | if(regionIsValid){ 386 | return .region(featureMapIndex: featureMapIndex, region: region) 387 | } 388 | return .padding 389 | }() 390 | 391 | let item = ROIAlignInputItem(offset: i, content: content) 392 | results.append(item) 393 | } 394 | 395 | return results 396 | } 397 | 398 | @available(iOS 12.0, macOS 10.14, *) 399 | func groupInputItemsByContent(items:[ROIAlignInputItem], maxComputeBatchSize:Int) -> [ROIAlignInputGroup] { 400 | 401 | var results = [ROIAlignInputGroup]() 402 | 403 | var offset:Int = 0 404 | var currentPaddingCount:Int? 405 | var currentFeatureMapIndex:Int? 406 | var currentRegions:[MPSRegion]? 407 | 408 | let closeRegionsGroupIfNecessary = { 409 | 410 | () -> Void in 411 | 412 | if let mapIndex = currentFeatureMapIndex, let regions = currentRegions { 413 | let group = ROIAlignInputGroup(offset: offset, content: ROIAlignInputGroup.Content.regions(featureMapIndex: mapIndex, regions: regions)) 414 | results.append(group) 415 | currentFeatureMapIndex = nil 416 | currentRegions = nil 417 | offset += group.size 418 | } 419 | } 420 | 421 | let closePaddingGroupIfNecessary = { 422 | 423 | () -> Void in 424 | if let paddingCount = currentPaddingCount { 425 | let group = ROIAlignInputGroup(offset: offset, content: ROIAlignInputGroup.Content.padding(count: paddingCount)) 426 | results.append(group) 427 | currentPaddingCount = nil 428 | offset += group.size 429 | } 430 | } 431 | 432 | for item in items { 433 | 434 | switch(item.content) 435 | { 436 | case .padding: 437 | 438 | closeRegionsGroupIfNecessary() 439 | 440 | if let paddingCount = currentPaddingCount { 441 | currentPaddingCount = paddingCount + 1 442 | } else { 443 | currentPaddingCount = 1 444 | } 445 | 446 | case .region(featureMapIndex: let mapIndex, region: let region): 447 | 448 | closePaddingGroupIfNecessary() 449 | 450 | if let currentFeatureMapIndex = currentFeatureMapIndex, currentFeatureMapIndex == mapIndex { 451 | currentRegions?.append(region) 452 | } else { 453 | closeRegionsGroupIfNecessary() 454 | currentFeatureMapIndex = mapIndex 455 | currentRegions = [region] 456 | } 457 | 458 | if(currentRegions?.count == maxComputeBatchSize) { 459 | closeRegionsGroupIfNecessary() 460 | } 461 | 462 | } 463 | 464 | } 465 | 466 | return results 467 | } 468 | 469 | @available(iOS 12.0, macOS 10.14, *) 470 | func batchInputGroups(groups:[ROIAlignInputGroup], maxComputeBatchSize:Int) -> [ROIAlignInputBatch] { 471 | 472 | var batches = [ROIAlignInputBatch]() 473 | 474 | var groupsInBatch = [ROIAlignInputGroup]() 475 | var currentComputeSize:Int = 0 476 | 477 | for group in groups { 478 | 479 | let nextComputeSize = currentComputeSize + group.computeSize 480 | 481 | if(nextComputeSize <= maxComputeBatchSize) { 482 | groupsInBatch.append(group) 483 | currentComputeSize = nextComputeSize 484 | } else { 485 | batches.append(ROIAlignInputBatch(groups: groupsInBatch)) 486 | groupsInBatch = [group] 487 | currentComputeSize = group.computeSize 488 | } 489 | 490 | } 491 | 492 | if(!groupsInBatch.isEmpty) { 493 | batches.append(ROIAlignInputBatch(groups: groupsInBatch)) 494 | } 495 | 496 | return batches 497 | 498 | } 499 | -------------------------------------------------------------------------------- /Sources/maskrcnn/results.pb.swift: -------------------------------------------------------------------------------- 1 | // DO NOT EDIT. 2 | // 3 | // Generated by the Swift generator plugin for the protocol buffer compiler. 4 | // Source: maskrcnn/results.proto 5 | // 6 | // For information on using the generated types, please see the documenation: 7 | // https://github.com/apple/swift-protobuf/ 8 | 9 | import Foundation 10 | import SwiftProtobuf 11 | 12 | // If the compiler emits an error on this type, it is because this file 13 | // was generated by a version of the `protoc` Swift plug-in that is 14 | // incompatible with the version of SwiftProtobuf to which you are linking. 15 | // Please ensure that your are building against the same version of the API 16 | // that was used to generate this file. 17 | fileprivate struct _GeneratedWithProtocGenSwiftVersion: SwiftProtobuf.ProtobufAPIVersionCheck { 18 | struct _2: SwiftProtobuf.ProtobufAPIVersion_2 {} 19 | typealias Version = _2 20 | } 21 | 22 | struct Result { 23 | // SwiftProtobuf.Message conformance is added in an extension below. See the 24 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 25 | // methods supported on all messages. 26 | 27 | var imageInfo: Result.ImageInfo { 28 | get {return _storage._imageInfo ?? Result.ImageInfo()} 29 | set {_uniqueStorage()._imageInfo = newValue} 30 | } 31 | /// Returns true if `imageInfo` has been explicitly set. 32 | var hasImageInfo: Bool {return _storage._imageInfo != nil} 33 | /// Clears the value of `imageInfo`. Subsequent reads from it will return its default value. 34 | mutating func clearImageInfo() {_uniqueStorage()._imageInfo = nil} 35 | 36 | var detections: [Result.Detection] { 37 | get {return _storage._detections} 38 | set {_uniqueStorage()._detections = newValue} 39 | } 40 | 41 | var unknownFields = SwiftProtobuf.UnknownStorage() 42 | 43 | struct Origin { 44 | // SwiftProtobuf.Message conformance is added in an extension below. See the 45 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 46 | // methods supported on all messages. 47 | 48 | var x: Double = 0 49 | 50 | var y: Double = 0 51 | 52 | var unknownFields = SwiftProtobuf.UnknownStorage() 53 | 54 | init() {} 55 | } 56 | 57 | struct Size { 58 | // SwiftProtobuf.Message conformance is added in an extension below. See the 59 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 60 | // methods supported on all messages. 61 | 62 | var width: Double = 0 63 | 64 | var height: Double = 0 65 | 66 | var unknownFields = SwiftProtobuf.UnknownStorage() 67 | 68 | init() {} 69 | } 70 | 71 | struct Rect { 72 | // SwiftProtobuf.Message conformance is added in an extension below. See the 73 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 74 | // methods supported on all messages. 75 | 76 | var origin: Result.Origin { 77 | get {return _storage._origin ?? Result.Origin()} 78 | set {_uniqueStorage()._origin = newValue} 79 | } 80 | /// Returns true if `origin` has been explicitly set. 81 | var hasOrigin: Bool {return _storage._origin != nil} 82 | /// Clears the value of `origin`. Subsequent reads from it will return its default value. 83 | mutating func clearOrigin() {_uniqueStorage()._origin = nil} 84 | 85 | var size: Result.Size { 86 | get {return _storage._size ?? Result.Size()} 87 | set {_uniqueStorage()._size = newValue} 88 | } 89 | /// Returns true if `size` has been explicitly set. 90 | var hasSize: Bool {return _storage._size != nil} 91 | /// Clears the value of `size`. Subsequent reads from it will return its default value. 92 | mutating func clearSize() {_uniqueStorage()._size = nil} 93 | 94 | var unknownFields = SwiftProtobuf.UnknownStorage() 95 | 96 | init() {} 97 | 98 | fileprivate var _storage = _StorageClass.defaultInstance 99 | } 100 | 101 | struct ImageInfo { 102 | // SwiftProtobuf.Message conformance is added in an extension below. See the 103 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 104 | // methods supported on all messages. 105 | 106 | var datasetID: String = String() 107 | 108 | var id: String = String() 109 | 110 | var width: Int32 = 0 111 | 112 | var height: Int32 = 0 113 | 114 | var unknownFields = SwiftProtobuf.UnknownStorage() 115 | 116 | init() {} 117 | } 118 | 119 | struct Detection { 120 | // SwiftProtobuf.Message conformance is added in an extension below. See the 121 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 122 | // methods supported on all messages. 123 | 124 | var probability: Double { 125 | get {return _storage._probability} 126 | set {_uniqueStorage()._probability = newValue} 127 | } 128 | 129 | var classID: Int32 { 130 | get {return _storage._classID} 131 | set {_uniqueStorage()._classID = newValue} 132 | } 133 | 134 | var classLabel: String { 135 | get {return _storage._classLabel} 136 | set {_uniqueStorage()._classLabel = newValue} 137 | } 138 | 139 | var boundingBox: Result.Rect { 140 | get {return _storage._boundingBox ?? Result.Rect()} 141 | set {_uniqueStorage()._boundingBox = newValue} 142 | } 143 | /// Returns true if `boundingBox` has been explicitly set. 144 | var hasBoundingBox: Bool {return _storage._boundingBox != nil} 145 | /// Clears the value of `boundingBox`. Subsequent reads from it will return its default value. 146 | mutating func clearBoundingBox() {_uniqueStorage()._boundingBox = nil} 147 | 148 | var unknownFields = SwiftProtobuf.UnknownStorage() 149 | 150 | init() {} 151 | 152 | fileprivate var _storage = _StorageClass.defaultInstance 153 | } 154 | 155 | init() {} 156 | 157 | fileprivate var _storage = _StorageClass.defaultInstance 158 | } 159 | 160 | struct Results { 161 | // SwiftProtobuf.Message conformance is added in an extension below. See the 162 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 163 | // methods supported on all messages. 164 | 165 | var results: [Result] = [] 166 | 167 | var unknownFields = SwiftProtobuf.UnknownStorage() 168 | 169 | init() {} 170 | } 171 | 172 | // MARK: - Code below here is support for the SwiftProtobuf runtime. 173 | 174 | extension Result: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 175 | static let protoMessageName: String = "Result" 176 | static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 177 | 1: .same(proto: "imageInfo"), 178 | 2: .same(proto: "detections"), 179 | ] 180 | 181 | fileprivate class _StorageClass { 182 | var _imageInfo: Result.ImageInfo? = nil 183 | var _detections: [Result.Detection] = [] 184 | 185 | static let defaultInstance = _StorageClass() 186 | 187 | private init() {} 188 | 189 | init(copying source: _StorageClass) { 190 | _imageInfo = source._imageInfo 191 | _detections = source._detections 192 | } 193 | } 194 | 195 | fileprivate mutating func _uniqueStorage() -> _StorageClass { 196 | if !isKnownUniquelyReferenced(&_storage) { 197 | _storage = _StorageClass(copying: _storage) 198 | } 199 | return _storage 200 | } 201 | 202 | mutating func decodeMessage(decoder: inout D) throws { 203 | _ = _uniqueStorage() 204 | try withExtendedLifetime(_storage) { (_storage: _StorageClass) in 205 | while let fieldNumber = try decoder.nextFieldNumber() { 206 | switch fieldNumber { 207 | case 1: try decoder.decodeSingularMessageField(value: &_storage._imageInfo) 208 | case 2: try decoder.decodeRepeatedMessageField(value: &_storage._detections) 209 | default: break 210 | } 211 | } 212 | } 213 | } 214 | 215 | func traverse(visitor: inout V) throws { 216 | try withExtendedLifetime(_storage) { (_storage: _StorageClass) in 217 | if let v = _storage._imageInfo { 218 | try visitor.visitSingularMessageField(value: v, fieldNumber: 1) 219 | } 220 | if !_storage._detections.isEmpty { 221 | try visitor.visitRepeatedMessageField(value: _storage._detections, fieldNumber: 2) 222 | } 223 | } 224 | try unknownFields.traverse(visitor: &visitor) 225 | } 226 | 227 | static func ==(lhs: Result, rhs: Result) -> Bool { 228 | if lhs._storage !== rhs._storage { 229 | let storagesAreEqual: Bool = withExtendedLifetime((lhs._storage, rhs._storage)) { (_args: (_StorageClass, _StorageClass)) in 230 | let _storage = _args.0 231 | let rhs_storage = _args.1 232 | if _storage._imageInfo != rhs_storage._imageInfo {return false} 233 | if _storage._detections != rhs_storage._detections {return false} 234 | return true 235 | } 236 | if !storagesAreEqual {return false} 237 | } 238 | if lhs.unknownFields != rhs.unknownFields {return false} 239 | return true 240 | } 241 | } 242 | 243 | extension Result.Origin: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 244 | static let protoMessageName: String = Result.protoMessageName + ".Origin" 245 | static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 246 | 1: .same(proto: "x"), 247 | 2: .same(proto: "y"), 248 | ] 249 | 250 | mutating func decodeMessage(decoder: inout D) throws { 251 | while let fieldNumber = try decoder.nextFieldNumber() { 252 | switch fieldNumber { 253 | case 1: try decoder.decodeSingularDoubleField(value: &self.x) 254 | case 2: try decoder.decodeSingularDoubleField(value: &self.y) 255 | default: break 256 | } 257 | } 258 | } 259 | 260 | func traverse(visitor: inout V) throws { 261 | if self.x != 0 { 262 | try visitor.visitSingularDoubleField(value: self.x, fieldNumber: 1) 263 | } 264 | if self.y != 0 { 265 | try visitor.visitSingularDoubleField(value: self.y, fieldNumber: 2) 266 | } 267 | try unknownFields.traverse(visitor: &visitor) 268 | } 269 | 270 | static func ==(lhs: Result.Origin, rhs: Result.Origin) -> Bool { 271 | if lhs.x != rhs.x {return false} 272 | if lhs.y != rhs.y {return false} 273 | if lhs.unknownFields != rhs.unknownFields {return false} 274 | return true 275 | } 276 | } 277 | 278 | extension Result.Size: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 279 | static let protoMessageName: String = Result.protoMessageName + ".Size" 280 | static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 281 | 1: .same(proto: "width"), 282 | 2: .same(proto: "height"), 283 | ] 284 | 285 | mutating func decodeMessage(decoder: inout D) throws { 286 | while let fieldNumber = try decoder.nextFieldNumber() { 287 | switch fieldNumber { 288 | case 1: try decoder.decodeSingularDoubleField(value: &self.width) 289 | case 2: try decoder.decodeSingularDoubleField(value: &self.height) 290 | default: break 291 | } 292 | } 293 | } 294 | 295 | func traverse(visitor: inout V) throws { 296 | if self.width != 0 { 297 | try visitor.visitSingularDoubleField(value: self.width, fieldNumber: 1) 298 | } 299 | if self.height != 0 { 300 | try visitor.visitSingularDoubleField(value: self.height, fieldNumber: 2) 301 | } 302 | try unknownFields.traverse(visitor: &visitor) 303 | } 304 | 305 | static func ==(lhs: Result.Size, rhs: Result.Size) -> Bool { 306 | if lhs.width != rhs.width {return false} 307 | if lhs.height != rhs.height {return false} 308 | if lhs.unknownFields != rhs.unknownFields {return false} 309 | return true 310 | } 311 | } 312 | 313 | extension Result.Rect: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 314 | static let protoMessageName: String = Result.protoMessageName + ".Rect" 315 | static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 316 | 1: .same(proto: "origin"), 317 | 2: .same(proto: "size"), 318 | ] 319 | 320 | fileprivate class _StorageClass { 321 | var _origin: Result.Origin? = nil 322 | var _size: Result.Size? = nil 323 | 324 | static let defaultInstance = _StorageClass() 325 | 326 | private init() {} 327 | 328 | init(copying source: _StorageClass) { 329 | _origin = source._origin 330 | _size = source._size 331 | } 332 | } 333 | 334 | fileprivate mutating func _uniqueStorage() -> _StorageClass { 335 | if !isKnownUniquelyReferenced(&_storage) { 336 | _storage = _StorageClass(copying: _storage) 337 | } 338 | return _storage 339 | } 340 | 341 | mutating func decodeMessage(decoder: inout D) throws { 342 | _ = _uniqueStorage() 343 | try withExtendedLifetime(_storage) { (_storage: _StorageClass) in 344 | while let fieldNumber = try decoder.nextFieldNumber() { 345 | switch fieldNumber { 346 | case 1: try decoder.decodeSingularMessageField(value: &_storage._origin) 347 | case 2: try decoder.decodeSingularMessageField(value: &_storage._size) 348 | default: break 349 | } 350 | } 351 | } 352 | } 353 | 354 | func traverse(visitor: inout V) throws { 355 | try withExtendedLifetime(_storage) { (_storage: _StorageClass) in 356 | if let v = _storage._origin { 357 | try visitor.visitSingularMessageField(value: v, fieldNumber: 1) 358 | } 359 | if let v = _storage._size { 360 | try visitor.visitSingularMessageField(value: v, fieldNumber: 2) 361 | } 362 | } 363 | try unknownFields.traverse(visitor: &visitor) 364 | } 365 | 366 | static func ==(lhs: Result.Rect, rhs: Result.Rect) -> Bool { 367 | if lhs._storage !== rhs._storage { 368 | let storagesAreEqual: Bool = withExtendedLifetime((lhs._storage, rhs._storage)) { (_args: (_StorageClass, _StorageClass)) in 369 | let _storage = _args.0 370 | let rhs_storage = _args.1 371 | if _storage._origin != rhs_storage._origin {return false} 372 | if _storage._size != rhs_storage._size {return false} 373 | return true 374 | } 375 | if !storagesAreEqual {return false} 376 | } 377 | if lhs.unknownFields != rhs.unknownFields {return false} 378 | return true 379 | } 380 | } 381 | 382 | extension Result.ImageInfo: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 383 | static let protoMessageName: String = Result.protoMessageName + ".ImageInfo" 384 | static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 385 | 1: .same(proto: "datasetId"), 386 | 2: .same(proto: "id"), 387 | 3: .same(proto: "width"), 388 | 4: .same(proto: "height"), 389 | ] 390 | 391 | mutating func decodeMessage(decoder: inout D) throws { 392 | while let fieldNumber = try decoder.nextFieldNumber() { 393 | switch fieldNumber { 394 | case 1: try decoder.decodeSingularStringField(value: &self.datasetID) 395 | case 2: try decoder.decodeSingularStringField(value: &self.id) 396 | case 3: try decoder.decodeSingularInt32Field(value: &self.width) 397 | case 4: try decoder.decodeSingularInt32Field(value: &self.height) 398 | default: break 399 | } 400 | } 401 | } 402 | 403 | func traverse(visitor: inout V) throws { 404 | if !self.datasetID.isEmpty { 405 | try visitor.visitSingularStringField(value: self.datasetID, fieldNumber: 1) 406 | } 407 | if !self.id.isEmpty { 408 | try visitor.visitSingularStringField(value: self.id, fieldNumber: 2) 409 | } 410 | if self.width != 0 { 411 | try visitor.visitSingularInt32Field(value: self.width, fieldNumber: 3) 412 | } 413 | if self.height != 0 { 414 | try visitor.visitSingularInt32Field(value: self.height, fieldNumber: 4) 415 | } 416 | try unknownFields.traverse(visitor: &visitor) 417 | } 418 | 419 | static func ==(lhs: Result.ImageInfo, rhs: Result.ImageInfo) -> Bool { 420 | if lhs.datasetID != rhs.datasetID {return false} 421 | if lhs.id != rhs.id {return false} 422 | if lhs.width != rhs.width {return false} 423 | if lhs.height != rhs.height {return false} 424 | if lhs.unknownFields != rhs.unknownFields {return false} 425 | return true 426 | } 427 | } 428 | 429 | extension Result.Detection: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 430 | static let protoMessageName: String = Result.protoMessageName + ".Detection" 431 | static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 432 | 1: .same(proto: "probability"), 433 | 2: .same(proto: "classId"), 434 | 3: .same(proto: "classLabel"), 435 | 4: .same(proto: "boundingBox"), 436 | ] 437 | 438 | fileprivate class _StorageClass { 439 | var _probability: Double = 0 440 | var _classID: Int32 = 0 441 | var _classLabel: String = String() 442 | var _boundingBox: Result.Rect? = nil 443 | 444 | static let defaultInstance = _StorageClass() 445 | 446 | private init() {} 447 | 448 | init(copying source: _StorageClass) { 449 | _probability = source._probability 450 | _classID = source._classID 451 | _classLabel = source._classLabel 452 | _boundingBox = source._boundingBox 453 | } 454 | } 455 | 456 | fileprivate mutating func _uniqueStorage() -> _StorageClass { 457 | if !isKnownUniquelyReferenced(&_storage) { 458 | _storage = _StorageClass(copying: _storage) 459 | } 460 | return _storage 461 | } 462 | 463 | mutating func decodeMessage(decoder: inout D) throws { 464 | _ = _uniqueStorage() 465 | try withExtendedLifetime(_storage) { (_storage: _StorageClass) in 466 | while let fieldNumber = try decoder.nextFieldNumber() { 467 | switch fieldNumber { 468 | case 1: try decoder.decodeSingularDoubleField(value: &_storage._probability) 469 | case 2: try decoder.decodeSingularInt32Field(value: &_storage._classID) 470 | case 3: try decoder.decodeSingularStringField(value: &_storage._classLabel) 471 | case 4: try decoder.decodeSingularMessageField(value: &_storage._boundingBox) 472 | default: break 473 | } 474 | } 475 | } 476 | } 477 | 478 | func traverse(visitor: inout V) throws { 479 | try withExtendedLifetime(_storage) { (_storage: _StorageClass) in 480 | if _storage._probability != 0 { 481 | try visitor.visitSingularDoubleField(value: _storage._probability, fieldNumber: 1) 482 | } 483 | if _storage._classID != 0 { 484 | try visitor.visitSingularInt32Field(value: _storage._classID, fieldNumber: 2) 485 | } 486 | if !_storage._classLabel.isEmpty { 487 | try visitor.visitSingularStringField(value: _storage._classLabel, fieldNumber: 3) 488 | } 489 | if let v = _storage._boundingBox { 490 | try visitor.visitSingularMessageField(value: v, fieldNumber: 4) 491 | } 492 | } 493 | try unknownFields.traverse(visitor: &visitor) 494 | } 495 | 496 | static func ==(lhs: Result.Detection, rhs: Result.Detection) -> Bool { 497 | if lhs._storage !== rhs._storage { 498 | let storagesAreEqual: Bool = withExtendedLifetime((lhs._storage, rhs._storage)) { (_args: (_StorageClass, _StorageClass)) in 499 | let _storage = _args.0 500 | let rhs_storage = _args.1 501 | if _storage._probability != rhs_storage._probability {return false} 502 | if _storage._classID != rhs_storage._classID {return false} 503 | if _storage._classLabel != rhs_storage._classLabel {return false} 504 | if _storage._boundingBox != rhs_storage._boundingBox {return false} 505 | return true 506 | } 507 | if !storagesAreEqual {return false} 508 | } 509 | if lhs.unknownFields != rhs.unknownFields {return false} 510 | return true 511 | } 512 | } 513 | 514 | extension Results: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 515 | static let protoMessageName: String = "Results" 516 | static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 517 | 1: .same(proto: "results"), 518 | ] 519 | 520 | mutating func decodeMessage(decoder: inout D) throws { 521 | while let fieldNumber = try decoder.nextFieldNumber() { 522 | switch fieldNumber { 523 | case 1: try decoder.decodeRepeatedMessageField(value: &self.results) 524 | default: break 525 | } 526 | } 527 | } 528 | 529 | func traverse(visitor: inout V) throws { 530 | if !self.results.isEmpty { 531 | try visitor.visitRepeatedMessageField(value: self.results, fieldNumber: 1) 532 | } 533 | try unknownFields.traverse(visitor: &visitor) 534 | } 535 | 536 | static func ==(lhs: Results, rhs: Results) -> Bool { 537 | if lhs.results != rhs.results {return false} 538 | if lhs.unknownFields != rhs.unknownFields {return false} 539 | return true 540 | } 541 | } 542 | -------------------------------------------------------------------------------- /Example/iOS Example.xcodeproj/project.pbxproj: -------------------------------------------------------------------------------- 1 | // !$*UTF8*$! 2 | { 3 | archiveVersion = 1; 4 | classes = { 5 | }; 6 | objectVersion = 50; 7 | objects = { 8 | 9 | /* Begin PBXBuildFile section */ 10 | 054539F721B9A63A005A6AED /* Utils.swift in Sources */ = {isa = PBXBuildFile; fileRef = 054539EF21B9A63A005A6AED /* Utils.swift */; }; 11 | 054539F821B9A63A005A6AED /* PyramidROIAlignLayer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 054539F021B9A63A005A6AED /* PyramidROIAlignLayer.swift */; }; 12 | 054539F921B9A63A005A6AED /* ProposalLayer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 054539F121B9A63A005A6AED /* ProposalLayer.swift */; }; 13 | 054539FA21B9A63A005A6AED /* BoxUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = 054539F221B9A63A005A6AED /* BoxUtils.swift */; }; 14 | 054539FB21B9A63A005A6AED /* DetectionLayer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 054539F321B9A63A005A6AED /* DetectionLayer.swift */; }; 15 | 054539FC21B9A63A005A6AED /* Detection.swift in Sources */ = {isa = PBXBuildFile; fileRef = 054539F421B9A63A005A6AED /* Detection.swift */; }; 16 | 054539FD21B9A63A005A6AED /* TimeDistributedMaskLayer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 054539F521B9A63A005A6AED /* TimeDistributedMaskLayer.swift */; }; 17 | 054539FE21B9A63A005A6AED /* TimeDistributedClassifierLayer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 054539F621B9A63A005A6AED /* TimeDistributedClassifierLayer.swift */; }; 18 | 05800F16219C9F6700832400 /* DetectionRenderer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 05800F15219C9F6700832400 /* DetectionRenderer.swift */; }; 19 | 05B55095219C877A00974728 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 05B55094219C877A00974728 /* AppDelegate.swift */; }; 20 | 05B55097219C877A00974728 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 05B55096219C877A00974728 /* ViewController.swift */; }; 21 | 05B5509A219C877A00974728 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 05B55098219C877A00974728 /* Main.storyboard */; }; 22 | 05B5509C219C877B00974728 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 05B5509B219C877B00974728 /* Assets.xcassets */; }; 23 | 05B5509F219C877B00974728 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 05B5509D219C877B00974728 /* LaunchScreen.storyboard */; }; 24 | 05F2CE2821BAC2E8004EBBC5 /* MaskRCNNConfig.swift in Sources */ = {isa = PBXBuildFile; fileRef = 05F2CE2721BAC2E8004EBBC5 /* MaskRCNNConfig.swift */; }; 25 | 05F2CE3621BAF3AD004EBBC5 /* MaskRCNN.mlmodel in Sources */ = {isa = PBXBuildFile; fileRef = 05F2CE3221BAF3AD004EBBC5 /* MaskRCNN.mlmodel */; }; 26 | 05F2CE3721BAF3AD004EBBC5 /* Mask.mlmodel in Sources */ = {isa = PBXBuildFile; fileRef = 05F2CE3321BAF3AD004EBBC5 /* Mask.mlmodel */; }; 27 | 05F2CE3821BAF3AD004EBBC5 /* anchors.bin in Resources */ = {isa = PBXBuildFile; fileRef = 05F2CE3421BAF3AD004EBBC5 /* anchors.bin */; }; 28 | 05F2CE3921BAF3AD004EBBC5 /* Classifier.mlmodel in Sources */ = {isa = PBXBuildFile; fileRef = 05F2CE3521BAF3AD004EBBC5 /* Classifier.mlmodel */; }; 29 | /* End PBXBuildFile section */ 30 | 31 | /* Begin PBXFileReference section */ 32 | 054539EF21B9A63A005A6AED /* Utils.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Utils.swift; path = "../../Sources/Mask-RCNN-CoreML/Utils.swift"; sourceTree = ""; }; 33 | 054539F021B9A63A005A6AED /* PyramidROIAlignLayer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = PyramidROIAlignLayer.swift; path = "../../Sources/Mask-RCNN-CoreML/PyramidROIAlignLayer.swift"; sourceTree = ""; }; 34 | 054539F121B9A63A005A6AED /* ProposalLayer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = ProposalLayer.swift; path = "../../Sources/Mask-RCNN-CoreML/ProposalLayer.swift"; sourceTree = ""; }; 35 | 054539F221B9A63A005A6AED /* BoxUtils.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = BoxUtils.swift; path = "../../Sources/Mask-RCNN-CoreML/BoxUtils.swift"; sourceTree = ""; }; 36 | 054539F321B9A63A005A6AED /* DetectionLayer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = DetectionLayer.swift; path = "../../Sources/Mask-RCNN-CoreML/DetectionLayer.swift"; sourceTree = ""; }; 37 | 054539F421B9A63A005A6AED /* Detection.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Detection.swift; path = "../../Sources/Mask-RCNN-CoreML/Detection.swift"; sourceTree = ""; }; 38 | 054539F521B9A63A005A6AED /* TimeDistributedMaskLayer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = TimeDistributedMaskLayer.swift; path = "../../Sources/Mask-RCNN-CoreML/TimeDistributedMaskLayer.swift"; sourceTree = ""; }; 39 | 054539F621B9A63A005A6AED /* TimeDistributedClassifierLayer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = TimeDistributedClassifierLayer.swift; path = "../../Sources/Mask-RCNN-CoreML/TimeDistributedClassifierLayer.swift"; sourceTree = ""; }; 40 | 05800F15219C9F6700832400 /* DetectionRenderer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DetectionRenderer.swift; sourceTree = ""; }; 41 | 05B55091219C877A00974728 /* Example.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Example.app; sourceTree = BUILT_PRODUCTS_DIR; }; 42 | 05B55094219C877A00974728 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 43 | 05B55096219C877A00974728 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = ""; }; 44 | 05B55099219C877A00974728 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; 45 | 05B5509B219C877B00974728 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 46 | 05B5509E219C877B00974728 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; 47 | 05B550A0219C877B00974728 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 48 | 05F2CE2721BAC2E8004EBBC5 /* MaskRCNNConfig.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = MaskRCNNConfig.swift; path = "../../Sources/Mask-RCNN-CoreML/MaskRCNNConfig.swift"; sourceTree = ""; }; 49 | 05F2CE3221BAF3AD004EBBC5 /* MaskRCNN.mlmodel */ = {isa = PBXFileReference; lastKnownFileType = file.mlmodel; name = MaskRCNN.mlmodel; path = ../../.maskrcnn/models/coco/products/MaskRCNN.mlmodel; sourceTree = ""; }; 50 | 05F2CE3321BAF3AD004EBBC5 /* Mask.mlmodel */ = {isa = PBXFileReference; lastKnownFileType = file.mlmodel; name = Mask.mlmodel; path = ../../.maskrcnn/models/coco/products/Mask.mlmodel; sourceTree = ""; }; 51 | 05F2CE3421BAF3AD004EBBC5 /* anchors.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; name = anchors.bin; path = ../../.maskrcnn/models/coco/products/anchors.bin; sourceTree = ""; }; 52 | 05F2CE3521BAF3AD004EBBC5 /* Classifier.mlmodel */ = {isa = PBXFileReference; lastKnownFileType = file.mlmodel; name = Classifier.mlmodel; path = ../../.maskrcnn/models/coco/products/Classifier.mlmodel; sourceTree = ""; }; 53 | /* End PBXFileReference section */ 54 | 55 | /* Begin PBXFrameworksBuildPhase section */ 56 | 05B5508E219C877A00974728 /* Frameworks */ = { 57 | isa = PBXFrameworksBuildPhase; 58 | buildActionMask = 2147483647; 59 | files = ( 60 | ); 61 | runOnlyForDeploymentPostprocessing = 0; 62 | }; 63 | /* End PBXFrameworksBuildPhase section */ 64 | 65 | /* Begin PBXGroup section */ 66 | 05800EFD219C8E5E00832400 /* Library */ = { 67 | isa = PBXGroup; 68 | children = ( 69 | 05F2CE2721BAC2E8004EBBC5 /* MaskRCNNConfig.swift */, 70 | 054539F421B9A63A005A6AED /* Detection.swift */, 71 | 054539F321B9A63A005A6AED /* DetectionLayer.swift */, 72 | 054539F121B9A63A005A6AED /* ProposalLayer.swift */, 73 | 054539F021B9A63A005A6AED /* PyramidROIAlignLayer.swift */, 74 | 054539F621B9A63A005A6AED /* TimeDistributedClassifierLayer.swift */, 75 | 054539F521B9A63A005A6AED /* TimeDistributedMaskLayer.swift */, 76 | 054539F221B9A63A005A6AED /* BoxUtils.swift */, 77 | 054539EF21B9A63A005A6AED /* Utils.swift */, 78 | ); 79 | name = Library; 80 | sourceTree = ""; 81 | }; 82 | 05800F0D219C8E9900832400 /* Models */ = { 83 | isa = PBXGroup; 84 | children = ( 85 | 05F2CE3421BAF3AD004EBBC5 /* anchors.bin */, 86 | 05F2CE3521BAF3AD004EBBC5 /* Classifier.mlmodel */, 87 | 05F2CE3321BAF3AD004EBBC5 /* Mask.mlmodel */, 88 | 05F2CE3221BAF3AD004EBBC5 /* MaskRCNN.mlmodel */, 89 | ); 90 | name = Models; 91 | sourceTree = ""; 92 | }; 93 | 05B55088219C877A00974728 = { 94 | isa = PBXGroup; 95 | children = ( 96 | 05B55093219C877A00974728 /* Source */, 97 | 05B55092219C877A00974728 /* Products */, 98 | ); 99 | sourceTree = ""; 100 | }; 101 | 05B55092219C877A00974728 /* Products */ = { 102 | isa = PBXGroup; 103 | children = ( 104 | 05B55091219C877A00974728 /* Example.app */, 105 | ); 106 | name = Products; 107 | sourceTree = ""; 108 | }; 109 | 05B55093219C877A00974728 /* Source */ = { 110 | isa = PBXGroup; 111 | children = ( 112 | 05B55094219C877A00974728 /* AppDelegate.swift */, 113 | 05B55096219C877A00974728 /* ViewController.swift */, 114 | 05800F15219C9F6700832400 /* DetectionRenderer.swift */, 115 | 05B55098219C877A00974728 /* Main.storyboard */, 116 | 05B5509B219C877B00974728 /* Assets.xcassets */, 117 | 05B5509D219C877B00974728 /* LaunchScreen.storyboard */, 118 | 05B550A0219C877B00974728 /* Info.plist */, 119 | 05800EFD219C8E5E00832400 /* Library */, 120 | 05800F0D219C8E9900832400 /* Models */, 121 | ); 122 | path = Source; 123 | sourceTree = ""; 124 | }; 125 | /* End PBXGroup section */ 126 | 127 | /* Begin PBXNativeTarget section */ 128 | 05B55090219C877A00974728 /* Example */ = { 129 | isa = PBXNativeTarget; 130 | buildConfigurationList = 05B550A3219C877B00974728 /* Build configuration list for PBXNativeTarget "Example" */; 131 | buildPhases = ( 132 | 05B5508D219C877A00974728 /* Sources */, 133 | 05B5508E219C877A00974728 /* Frameworks */, 134 | 05B5508F219C877A00974728 /* Resources */, 135 | ); 136 | buildRules = ( 137 | ); 138 | dependencies = ( 139 | ); 140 | name = Example; 141 | productName = Example; 142 | productReference = 05B55091219C877A00974728 /* Example.app */; 143 | productType = "com.apple.product-type.application"; 144 | }; 145 | /* End PBXNativeTarget section */ 146 | 147 | /* Begin PBXProject section */ 148 | 05B55089219C877A00974728 /* Project object */ = { 149 | isa = PBXProject; 150 | attributes = { 151 | LastSwiftUpdateCheck = 1010; 152 | LastUpgradeCheck = 1010; 153 | ORGANIZATIONNAME = "Edouard Lavery Plante"; 154 | TargetAttributes = { 155 | 05B55090219C877A00974728 = { 156 | CreatedOnToolsVersion = 10.1; 157 | }; 158 | }; 159 | }; 160 | buildConfigurationList = 05B5508C219C877A00974728 /* Build configuration list for PBXProject "iOS Example" */; 161 | compatibilityVersion = "Xcode 9.3"; 162 | developmentRegion = en; 163 | hasScannedForEncodings = 0; 164 | knownRegions = ( 165 | en, 166 | Base, 167 | ); 168 | mainGroup = 05B55088219C877A00974728; 169 | productRefGroup = 05B55092219C877A00974728 /* Products */; 170 | projectDirPath = ""; 171 | projectRoot = ""; 172 | targets = ( 173 | 05B55090219C877A00974728 /* Example */, 174 | ); 175 | }; 176 | /* End PBXProject section */ 177 | 178 | /* Begin PBXResourcesBuildPhase section */ 179 | 05B5508F219C877A00974728 /* Resources */ = { 180 | isa = PBXResourcesBuildPhase; 181 | buildActionMask = 2147483647; 182 | files = ( 183 | 05F2CE3821BAF3AD004EBBC5 /* anchors.bin in Resources */, 184 | 05B5509F219C877B00974728 /* LaunchScreen.storyboard in Resources */, 185 | 05B5509C219C877B00974728 /* Assets.xcassets in Resources */, 186 | 05B5509A219C877A00974728 /* Main.storyboard in Resources */, 187 | ); 188 | runOnlyForDeploymentPostprocessing = 0; 189 | }; 190 | /* End PBXResourcesBuildPhase section */ 191 | 192 | /* Begin PBXSourcesBuildPhase section */ 193 | 05B5508D219C877A00974728 /* Sources */ = { 194 | isa = PBXSourcesBuildPhase; 195 | buildActionMask = 2147483647; 196 | files = ( 197 | 054539FC21B9A63A005A6AED /* Detection.swift in Sources */, 198 | 05B55097219C877A00974728 /* ViewController.swift in Sources */, 199 | 05F2CE3621BAF3AD004EBBC5 /* MaskRCNN.mlmodel in Sources */, 200 | 054539FB21B9A63A005A6AED /* DetectionLayer.swift in Sources */, 201 | 054539F721B9A63A005A6AED /* Utils.swift in Sources */, 202 | 054539F921B9A63A005A6AED /* ProposalLayer.swift in Sources */, 203 | 05B55095219C877A00974728 /* AppDelegate.swift in Sources */, 204 | 054539FD21B9A63A005A6AED /* TimeDistributedMaskLayer.swift in Sources */, 205 | 054539FE21B9A63A005A6AED /* TimeDistributedClassifierLayer.swift in Sources */, 206 | 05F2CE2821BAC2E8004EBBC5 /* MaskRCNNConfig.swift in Sources */, 207 | 05F2CE3721BAF3AD004EBBC5 /* Mask.mlmodel in Sources */, 208 | 05F2CE3921BAF3AD004EBBC5 /* Classifier.mlmodel in Sources */, 209 | 054539F821B9A63A005A6AED /* PyramidROIAlignLayer.swift in Sources */, 210 | 05800F16219C9F6700832400 /* DetectionRenderer.swift in Sources */, 211 | 054539FA21B9A63A005A6AED /* BoxUtils.swift in Sources */, 212 | ); 213 | runOnlyForDeploymentPostprocessing = 0; 214 | }; 215 | /* End PBXSourcesBuildPhase section */ 216 | 217 | /* Begin PBXVariantGroup section */ 218 | 05B55098219C877A00974728 /* Main.storyboard */ = { 219 | isa = PBXVariantGroup; 220 | children = ( 221 | 05B55099219C877A00974728 /* Base */, 222 | ); 223 | name = Main.storyboard; 224 | sourceTree = ""; 225 | }; 226 | 05B5509D219C877B00974728 /* LaunchScreen.storyboard */ = { 227 | isa = PBXVariantGroup; 228 | children = ( 229 | 05B5509E219C877B00974728 /* Base */, 230 | ); 231 | name = LaunchScreen.storyboard; 232 | sourceTree = ""; 233 | }; 234 | /* End PBXVariantGroup section */ 235 | 236 | /* Begin XCBuildConfiguration section */ 237 | 05B550A1219C877B00974728 /* Debug */ = { 238 | isa = XCBuildConfiguration; 239 | buildSettings = { 240 | ALWAYS_SEARCH_USER_PATHS = NO; 241 | CLANG_ANALYZER_NONNULL = YES; 242 | CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; 243 | CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; 244 | CLANG_CXX_LIBRARY = "libc++"; 245 | CLANG_ENABLE_MODULES = YES; 246 | CLANG_ENABLE_OBJC_ARC = YES; 247 | CLANG_ENABLE_OBJC_WEAK = YES; 248 | CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; 249 | CLANG_WARN_BOOL_CONVERSION = YES; 250 | CLANG_WARN_COMMA = YES; 251 | CLANG_WARN_CONSTANT_CONVERSION = YES; 252 | CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; 253 | CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; 254 | CLANG_WARN_DOCUMENTATION_COMMENTS = YES; 255 | CLANG_WARN_EMPTY_BODY = YES; 256 | CLANG_WARN_ENUM_CONVERSION = YES; 257 | CLANG_WARN_INFINITE_RECURSION = YES; 258 | CLANG_WARN_INT_CONVERSION = YES; 259 | CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; 260 | CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; 261 | CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; 262 | CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; 263 | CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; 264 | CLANG_WARN_STRICT_PROTOTYPES = YES; 265 | CLANG_WARN_SUSPICIOUS_MOVE = YES; 266 | CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; 267 | CLANG_WARN_UNREACHABLE_CODE = YES; 268 | CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; 269 | CODE_SIGN_IDENTITY = "iPhone Developer"; 270 | COPY_PHASE_STRIP = NO; 271 | DEBUG_INFORMATION_FORMAT = dwarf; 272 | ENABLE_STRICT_OBJC_MSGSEND = YES; 273 | ENABLE_TESTABILITY = YES; 274 | GCC_C_LANGUAGE_STANDARD = gnu11; 275 | GCC_DYNAMIC_NO_PIC = NO; 276 | GCC_NO_COMMON_BLOCKS = YES; 277 | GCC_OPTIMIZATION_LEVEL = 0; 278 | GCC_PREPROCESSOR_DEFINITIONS = ( 279 | "DEBUG=1", 280 | "$(inherited)", 281 | ); 282 | GCC_WARN_64_TO_32_BIT_CONVERSION = YES; 283 | GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; 284 | GCC_WARN_UNDECLARED_SELECTOR = YES; 285 | GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; 286 | GCC_WARN_UNUSED_FUNCTION = YES; 287 | GCC_WARN_UNUSED_VARIABLE = YES; 288 | IPHONEOS_DEPLOYMENT_TARGET = 12.1; 289 | MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; 290 | MTL_FAST_MATH = YES; 291 | ONLY_ACTIVE_ARCH = YES; 292 | SDKROOT = iphoneos; 293 | SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; 294 | SWIFT_OPTIMIZATION_LEVEL = "-Onone"; 295 | }; 296 | name = Debug; 297 | }; 298 | 05B550A2219C877B00974728 /* Release */ = { 299 | isa = XCBuildConfiguration; 300 | buildSettings = { 301 | ALWAYS_SEARCH_USER_PATHS = NO; 302 | CLANG_ANALYZER_NONNULL = YES; 303 | CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; 304 | CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; 305 | CLANG_CXX_LIBRARY = "libc++"; 306 | CLANG_ENABLE_MODULES = YES; 307 | CLANG_ENABLE_OBJC_ARC = YES; 308 | CLANG_ENABLE_OBJC_WEAK = YES; 309 | CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; 310 | CLANG_WARN_BOOL_CONVERSION = YES; 311 | CLANG_WARN_COMMA = YES; 312 | CLANG_WARN_CONSTANT_CONVERSION = YES; 313 | CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; 314 | CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; 315 | CLANG_WARN_DOCUMENTATION_COMMENTS = YES; 316 | CLANG_WARN_EMPTY_BODY = YES; 317 | CLANG_WARN_ENUM_CONVERSION = YES; 318 | CLANG_WARN_INFINITE_RECURSION = YES; 319 | CLANG_WARN_INT_CONVERSION = YES; 320 | CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; 321 | CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; 322 | CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; 323 | CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; 324 | CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; 325 | CLANG_WARN_STRICT_PROTOTYPES = YES; 326 | CLANG_WARN_SUSPICIOUS_MOVE = YES; 327 | CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; 328 | CLANG_WARN_UNREACHABLE_CODE = YES; 329 | CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; 330 | CODE_SIGN_IDENTITY = "iPhone Developer"; 331 | COPY_PHASE_STRIP = NO; 332 | DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; 333 | ENABLE_NS_ASSERTIONS = NO; 334 | ENABLE_STRICT_OBJC_MSGSEND = YES; 335 | GCC_C_LANGUAGE_STANDARD = gnu11; 336 | GCC_NO_COMMON_BLOCKS = YES; 337 | GCC_WARN_64_TO_32_BIT_CONVERSION = YES; 338 | GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; 339 | GCC_WARN_UNDECLARED_SELECTOR = YES; 340 | GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; 341 | GCC_WARN_UNUSED_FUNCTION = YES; 342 | GCC_WARN_UNUSED_VARIABLE = YES; 343 | IPHONEOS_DEPLOYMENT_TARGET = 12.1; 344 | MTL_ENABLE_DEBUG_INFO = NO; 345 | MTL_FAST_MATH = YES; 346 | SDKROOT = iphoneos; 347 | SWIFT_COMPILATION_MODE = wholemodule; 348 | SWIFT_OPTIMIZATION_LEVEL = "-O"; 349 | VALIDATE_PRODUCT = YES; 350 | }; 351 | name = Release; 352 | }; 353 | 05B550A4219C877B00974728 /* Debug */ = { 354 | isa = XCBuildConfiguration; 355 | buildSettings = { 356 | ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; 357 | CODE_SIGN_STYLE = Automatic; 358 | DEVELOPMENT_TEAM = G96622GK2S; 359 | INFOPLIST_FILE = Source/Info.plist; 360 | LD_RUNPATH_SEARCH_PATHS = ( 361 | "$(inherited)", 362 | "@executable_path/Frameworks", 363 | ); 364 | PRODUCT_BUNDLE_IDENTIFIER = com.laveryplante.Example; 365 | PRODUCT_NAME = "$(TARGET_NAME)"; 366 | SWIFT_VERSION = 4.2; 367 | TARGETED_DEVICE_FAMILY = "1,2"; 368 | }; 369 | name = Debug; 370 | }; 371 | 05B550A5219C877B00974728 /* Release */ = { 372 | isa = XCBuildConfiguration; 373 | buildSettings = { 374 | ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; 375 | CODE_SIGN_STYLE = Automatic; 376 | DEVELOPMENT_TEAM = G96622GK2S; 377 | INFOPLIST_FILE = Source/Info.plist; 378 | LD_RUNPATH_SEARCH_PATHS = ( 379 | "$(inherited)", 380 | "@executable_path/Frameworks", 381 | ); 382 | PRODUCT_BUNDLE_IDENTIFIER = com.laveryplante.Example; 383 | PRODUCT_NAME = "$(TARGET_NAME)"; 384 | SWIFT_VERSION = 4.2; 385 | TARGETED_DEVICE_FAMILY = "1,2"; 386 | }; 387 | name = Release; 388 | }; 389 | /* End XCBuildConfiguration section */ 390 | 391 | /* Begin XCConfigurationList section */ 392 | 05B5508C219C877A00974728 /* Build configuration list for PBXProject "iOS Example" */ = { 393 | isa = XCConfigurationList; 394 | buildConfigurations = ( 395 | 05B550A1219C877B00974728 /* Debug */, 396 | 05B550A2219C877B00974728 /* Release */, 397 | ); 398 | defaultConfigurationIsVisible = 0; 399 | defaultConfigurationName = Release; 400 | }; 401 | 05B550A3219C877B00974728 /* Build configuration list for PBXNativeTarget "Example" */ = { 402 | isa = XCConfigurationList; 403 | buildConfigurations = ( 404 | 05B550A4219C877B00974728 /* Debug */, 405 | 05B550A5219C877B00974728 /* Release */, 406 | ); 407 | defaultConfigurationIsVisible = 0; 408 | defaultConfigurationName = Release; 409 | }; 410 | /* End XCConfigurationList section */ 411 | }; 412 | rootObject = 05B55089219C877A00974728 /* Project object */; 413 | } 414 | --------------------------------------------------------------------------------