├── .gitignore ├── .vscode └── settings.json ├── CMakeLists.txt ├── LICENSE ├── README.md ├── data ├── images │ ├── bb_face1.jpg │ ├── bb_face2.jpg │ ├── classify.jpg │ ├── classify_result.jpg │ ├── face.jpg │ ├── face0.jpg │ ├── face1.jpg │ ├── face2.jpg │ ├── face3.jpg │ ├── face_result.jpg │ ├── fbb.jpg │ ├── fbb1.jpg │ ├── helmet.jpg │ ├── helmet_result.jpg │ ├── object.jpg │ ├── object_result.jpg │ └── result.jpg └── models │ ├── RFB-320-quant-ADMM-32.mnn │ ├── RFB-320-quant-KL-5792.mnn │ ├── RFB-320.mnn │ ├── centerface.mnn │ ├── db │ ├── label.txt │ ├── mobilefacenet.mnn │ ├── mobilenet.mnn │ ├── mobilenetssd.mnn │ ├── pfld.mnn │ ├── slim-320-quant-ADMM-50.mnn │ ├── slim-320.mnn │ ├── zqlandmark.mnn │ └── zwnet.mnn ├── include └── MNN │ ├── AutoTime.hpp │ ├── ErrorCode.hpp │ ├── HalideRuntime.h │ ├── ImageProcess.hpp │ ├── Interpreter.hpp │ ├── MNNDefine.h │ ├── MNNForwardType.h │ ├── MNNSharedContext.h │ ├── Matrix.h │ ├── Rect.h │ ├── Tensor.hpp │ ├── VCS.h │ └── expr │ ├── Executor.hpp │ ├── Expr.hpp │ ├── ExprCreator.hpp │ ├── MathOp.hpp │ ├── NeuralNetWorkOp.hpp │ └── Optimizer.hpp ├── lib └── libMNN.so └── src ├── CMakeLists.txt ├── classifier ├── classifier.cpp ├── classifier.h └── test_classifier.cpp ├── common ├── common.cpp └── common.h ├── face ├── aligner │ ├── aligner.cpp │ └── aligner.h ├── database │ ├── face_database.cpp │ ├── face_database.h │ └── stream │ │ ├── file_stream.cpp │ │ └── file_stream.h ├── detecter │ ├── centerface │ │ ├── centerface.cpp │ │ └── centerface.h │ ├── detecter.cpp │ ├── detecter.h │ └── ultraface │ │ ├── ultraface.cpp │ │ └── ultraface.h ├── face_engine.cpp ├── face_engine.h ├── landmarker │ ├── landmarker.cpp │ ├── landmarker.h │ ├── pfldlandmarker │ │ ├── pfldlandmarker.cpp │ │ └── pfldlandmarker.h │ └── zqlandmarker │ │ ├── zqlandmarker.cpp │ │ └── zqlandmarker.h ├── recognizer │ ├── mobilefacenet │ │ ├── mobilefacenet.cpp │ │ └── mobilefacenet.h │ ├── recognizer.cpp │ └── recognizer.h └── test_face.cpp ├── object ├── mobilenetssd │ ├── mobilenetssd.cpp │ └── mobilenetssd.h ├── object_engine.cpp ├── object_engine.h └── test_object.cpp ├── vision_engine.cpp └── vision_engine.h /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "files.associations": { 3 | "array": "cpp", 4 | "*.tcc": "cpp", 5 | "cctype": "cpp", 6 | "chrono": "cpp", 7 | "clocale": "cpp", 8 | "cmath": "cpp", 9 | "cstdarg": "cpp", 10 | "cstdint": "cpp", 11 | "cstdio": "cpp", 12 | "cstdlib": "cpp", 13 | "ctime": "cpp", 14 | "cwchar": "cpp", 15 | "cwctype": "cpp", 16 | "deque": "cpp", 17 | "list": "cpp", 18 | "unordered_map": "cpp", 19 | "vector": "cpp", 20 | "exception": "cpp", 21 | "fstream": "cpp", 22 | "functional": "cpp", 23 | "initializer_list": "cpp", 24 | "iosfwd": "cpp", 25 | "iostream": "cpp", 26 | "istream": "cpp", 27 | "limits": "cpp", 28 | "mutex": "cpp", 29 | "new": "cpp", 30 | "ostream": "cpp", 31 | "numeric": "cpp", 32 | "ratio": "cpp", 33 | "sstream": "cpp", 34 | "stdexcept": "cpp", 35 | "streambuf": "cpp", 36 | "system_error": "cpp", 37 | "tuple": "cpp", 38 | "type_traits": "cpp", 39 | "utility": "cpp", 40 | "typeinfo": "cpp" 41 | } 42 | } -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.1) 2 | project(VisionEngine) 3 | 4 | Option(MIRROR_OPENMP "openmp support" ON) 5 | Option(MIRROR_BUILD_OBJECT "build object test" ON) 6 | Option(MIRROR_BUILD_CLASSIFIER "build classifier test" ON) 7 | Option(MIRROR_BUILD_FACE "build face test" ON) 8 | 9 | add_subdirectory(src) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 MirrorYuChen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mnn_example 2 | ## alibaba MNN, mobilenet classifier, centerface detector, pfld landmarker and zqlandmarker, mobilefacenet 3 | ## **1.更新日志** 4 | 时间|更新内容 5 | --|-- 6 | 20200621| 1.remove the interface of GetImg; 7 | 20200320| 1.fix bug in face aligner; 8 | 20200305| 1.add ultraface and blending nms; 9 | 20200221| 1.add mobilefacenet; 10 | 20200220| 1.use template to reduce the reaptly code in NMS; 11 | 20200218| 1.refacter the project; 12 | 20200217| 1.add zwnet and face database; 13 | 20200216| 1.refacter the project and add zqlandmarker; 14 | 20200215| 1.add pfld landmarker and face aligner; 15 | 20200214| 1.add centerface detector; 16 | 20200205| 1.add object detection; 17 | 20200128| 1.add image classification; 18 | ## **2.How to use?** 19 | - (1). convert model 20 | - - classifier model comes from: https://github.com/tensorflow/models/tree/master/research/slim 21 | - - object detection model comes from: https://github.com/C-Aniruddh/realtime_object_recognition 22 | ``` 23 | ./MNNConvert -f TF --modelFile mobilenet_v1_1.0_224_frozen.pb --MNNModel mobilenet.mnn --bizCode MNN 24 | ``` 25 | ``` 26 | ./MNNConvert -f CAFFE --modelFile MobileNetSSD_deploy.caffemodel --prototxt MobileNetSSD_deploy.prototxt --MNNModel mobilenetssd.mnn --bizCode MNN 27 | ``` 28 | - (2). build 29 | ``` 30 | mkdir build && cd build && make -j3 && cd src && ./classifier && ./object && ./face 31 | ``` 32 | - (3). result 33 | classifier result: 34 | ![图片](https://github.com/MirrorYuChen/MNN_mobilenet/blob/master/data/images/classify_result.jpg) 35 | object result: 36 | ![图片](https://github.com/MirrorYuChen/MNN_mobilenet/blob/master/data/images/object_result.jpg) 37 | face detection result: 38 | ![图片](https://github.com/MirrorYuChen/MNN_mobilenet/blob/master/data/images/face_result.jpg) 39 | ## **3. TODO:** 40 | - [x] add pose 41 | ## **4. reference:** 42 | ### MNN: https://github.com/alibaba/MNN 43 | ### ZQCNN: https://github.com/zuoqing1988/ZQCNN 44 | ### MNN_APPLICATION: https://github.com/xindongzhang/MNN-APPLICATIONS 45 | ### insightface: https://github.com/deepinsight/insightface 46 | ### centerface: https://github.com/Star-Clouds/CenterFace 47 | ### ultraface: https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB 48 | ### seetaface2: https://github.com/seetafaceengine/SeetaFace2 49 | ### csdn blog: https://blog.csdn.net/abcd740181246/article/details/90143848 50 | -------------------------------------------------------------------------------- /data/images/bb_face1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/bb_face1.jpg -------------------------------------------------------------------------------- /data/images/bb_face2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/bb_face2.jpg -------------------------------------------------------------------------------- /data/images/classify.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/classify.jpg -------------------------------------------------------------------------------- /data/images/classify_result.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/classify_result.jpg -------------------------------------------------------------------------------- /data/images/face.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/face.jpg -------------------------------------------------------------------------------- /data/images/face0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/face0.jpg -------------------------------------------------------------------------------- /data/images/face1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/face1.jpg -------------------------------------------------------------------------------- /data/images/face2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/face2.jpg -------------------------------------------------------------------------------- /data/images/face3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/face3.jpg -------------------------------------------------------------------------------- /data/images/face_result.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/face_result.jpg -------------------------------------------------------------------------------- /data/images/fbb.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/fbb.jpg -------------------------------------------------------------------------------- /data/images/fbb1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/fbb1.jpg -------------------------------------------------------------------------------- /data/images/helmet.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/helmet.jpg -------------------------------------------------------------------------------- /data/images/helmet_result.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/helmet_result.jpg -------------------------------------------------------------------------------- /data/images/object.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/object.jpg -------------------------------------------------------------------------------- /data/images/object_result.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/object_result.jpg -------------------------------------------------------------------------------- /data/images/result.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/images/result.jpg -------------------------------------------------------------------------------- /data/models/RFB-320-quant-ADMM-32.mnn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/models/RFB-320-quant-ADMM-32.mnn -------------------------------------------------------------------------------- /data/models/RFB-320-quant-KL-5792.mnn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/models/RFB-320-quant-KL-5792.mnn -------------------------------------------------------------------------------- /data/models/RFB-320.mnn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/models/RFB-320.mnn -------------------------------------------------------------------------------- /data/models/centerface.mnn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/models/centerface.mnn -------------------------------------------------------------------------------- /data/models/db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/models/db -------------------------------------------------------------------------------- /data/models/mobilefacenet.mnn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/models/mobilefacenet.mnn -------------------------------------------------------------------------------- /data/models/mobilenet.mnn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/models/mobilenet.mnn -------------------------------------------------------------------------------- /data/models/mobilenetssd.mnn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/models/mobilenetssd.mnn -------------------------------------------------------------------------------- /data/models/pfld.mnn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/models/pfld.mnn -------------------------------------------------------------------------------- /data/models/slim-320-quant-ADMM-50.mnn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/models/slim-320-quant-ADMM-50.mnn -------------------------------------------------------------------------------- /data/models/slim-320.mnn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/models/slim-320.mnn -------------------------------------------------------------------------------- /data/models/zqlandmark.mnn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/models/zqlandmark.mnn -------------------------------------------------------------------------------- /data/models/zwnet.mnn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/data/models/zwnet.mnn -------------------------------------------------------------------------------- /include/MNN/AutoTime.hpp: -------------------------------------------------------------------------------- 1 | // 2 | // AutoTime.hpp 3 | // MNN 4 | // 5 | // Created by MNN on 2018/07/27. 6 | // Copyright © 2018, Alibaba Group Holding Limited 7 | // 8 | 9 | #ifndef AutoTime_hpp 10 | #define AutoTime_hpp 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | namespace MNN { 17 | 18 | class MNN_PUBLIC Timer { 19 | public: 20 | Timer(); 21 | ~Timer(); 22 | Timer(const Timer&) = delete; 23 | Timer(const Timer&&) = delete; 24 | Timer& operator=(const Timer&) = delete; 25 | Timer& operator=(const Timer&&) = delete; 26 | 27 | // reset timer 28 | void reset(); 29 | // get duration (us) from init or latest reset. 30 | uint64_t durationInUs(); 31 | 32 | protected: 33 | uint64_t mLastResetTime; 34 | }; 35 | 36 | /** time tracing util. prints duration between init and deinit. */ 37 | class MNN_PUBLIC AutoTime : Timer { 38 | public: 39 | AutoTime(int line, const char* func); 40 | ~AutoTime(); 41 | AutoTime(const AutoTime&) = delete; 42 | AutoTime(const AutoTime&&) = delete; 43 | AutoTime& operator=(const AutoTime&) = delete; 44 | AutoTime& operator=(const AutoTime&&) = delete; 45 | 46 | private: 47 | int mLine; 48 | char* mName; 49 | }; 50 | } // namespace MNN 51 | 52 | #ifdef MNN_OPEN_TIME_TRACE 53 | #define AUTOTIME MNN::AutoTime ___t(__LINE__, __func__) 54 | #else 55 | #define AUTOTIME 56 | #endif 57 | 58 | #endif /* AutoTime_hpp */ 59 | -------------------------------------------------------------------------------- /include/MNN/ErrorCode.hpp: -------------------------------------------------------------------------------- 1 | // 2 | // ErrorCode.hpp 3 | // MNN 4 | // 5 | // Created by MNN on 2018/09/18. 6 | // Copyright © 2018, Alibaba Group Holding Limited 7 | // 8 | 9 | #ifndef ErrorCode_h 10 | #define ErrorCode_h 11 | 12 | namespace MNN { 13 | enum ErrorCode { 14 | #ifdef NO_ERROR 15 | #undef NO_ERROR 16 | #endif // NO_ERROR 17 | NO_ERROR = 0, 18 | OUT_OF_MEMORY = 1, 19 | NOT_SUPPORT = 2, 20 | COMPUTE_SIZE_ERROR = 3, 21 | NO_EXECUTION = 4, 22 | INVALID_VALUE = 5, 23 | 24 | // User error 25 | INPUT_DATA_ERROR = 10, 26 | CALL_BACK_STOP = 11, 27 | 28 | // Op Resize Error 29 | TENSOR_NOT_SUPPORT = 20, 30 | TENSOR_NEED_DIVIDE = 21, 31 | }; 32 | } // namespace MNN 33 | 34 | #endif /* ErrorCode_h */ 35 | -------------------------------------------------------------------------------- /include/MNN/HalideRuntime.h: -------------------------------------------------------------------------------- 1 | #ifndef HALIDE_HALIDERUNTIME_H 2 | #define HALIDE_HALIDERUNTIME_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #ifdef __cplusplus 9 | extern "C" { 10 | #endif 11 | 12 | // Note that you should not use "inline" along with HALIDE_ALWAYS_INLINE; 13 | // it is not necessary, and may produce warnings for some build configurations. 14 | #ifdef _MSC_VER 15 | #define HALIDE_ALWAYS_INLINE __forceinline 16 | #define HALIDE_NEVER_INLINE __declspec(noinline) 17 | #else 18 | #define HALIDE_ALWAYS_INLINE __attribute__((always_inline)) inline 19 | #define HALIDE_NEVER_INLINE __attribute__((noinline)) 20 | #endif 21 | 22 | /** \file 23 | * 24 | * This file declares the routines used by Halide internally in its 25 | * runtime. On platforms that support weak linking, these can be 26 | * replaced with user-defined versions by defining an extern "C" 27 | * function with the same name and signature. 28 | * 29 | * When doing Just In Time (JIT) compilation methods on the Func being 30 | * compiled must be called instead. The corresponding methods are 31 | * documented below. 32 | * 33 | * All of these functions take a "void *user_context" parameter as their 34 | * first argument; if the Halide kernel that calls back to any of these 35 | * functions has been compiled with the UserContext feature set on its Target, 36 | * then the value of that pointer passed from the code that calls the 37 | * Halide kernel is piped through to the function. 38 | * 39 | * Some of these are also useful to call when using the default 40 | * implementation. E.g. halide_shutdown_thread_pool. 41 | * 42 | * Note that even on platforms with weak linking, some linker setups 43 | * may not respect the override you provide. E.g. if the override is 44 | * in a shared library and the halide object files are linked directly 45 | * into the output, the builtin versions of the runtime functions will 46 | * be called. See your linker documentation for more details. On 47 | * Linux, LD_DYNAMIC_WEAK=1 may help. 48 | * 49 | */ 50 | 51 | // Forward-declare to suppress warnings if compiling as C. 52 | struct halide_buffer_t; 53 | 54 | /** Types in the halide type system. They can be ints, unsigned ints, 55 | * or floats (of various bit-widths), or a handle (which is always 64-bits). 56 | * Note that the int/uint/float values do not imply a specific bit width 57 | * (the bit width is expected to be encoded in a separate value). 58 | */ 59 | typedef enum halide_type_code_t 60 | { 61 | halide_type_int = 0, //!< signed integers 62 | halide_type_uint = 1, //!< unsigned integers 63 | halide_type_float = 2, //!< floating point numbers 64 | halide_type_handle = 3 //!< opaque pointer type (void *) 65 | } halide_type_code_t; 66 | 67 | // Note that while __attribute__ can go before or after the declaration, 68 | // __declspec apparently is only allowed before. 69 | #ifndef HALIDE_ATTRIBUTE_ALIGN 70 | #ifdef _MSC_VER 71 | #define HALIDE_ATTRIBUTE_ALIGN(x) __declspec(align(x)) 72 | #else 73 | #define HALIDE_ATTRIBUTE_ALIGN(x) __attribute__((aligned(x))) 74 | #endif 75 | #endif 76 | 77 | /** A runtime tag for a type in the halide type system. Can be ints, 78 | * unsigned ints, or floats of various bit-widths (the 'bits' 79 | * field). Can also be vectors of the same (by setting the 'lanes' 80 | * field to something larger than one). This struct should be 81 | * exactly 32-bits in size. */ 82 | struct halide_type_t { 83 | /** The basic type code: signed integer, unsigned integer, or floating point. */ 84 | #if __cplusplus >= 201103L 85 | HALIDE_ATTRIBUTE_ALIGN(1) halide_type_code_t code; // halide_type_code_t 86 | #else 87 | HALIDE_ATTRIBUTE_ALIGN(1) uint8_t code; // halide_type_code_t 88 | #endif 89 | 90 | /** The number of bits of precision of a single scalar value of this type. */ 91 | HALIDE_ATTRIBUTE_ALIGN(1) uint8_t bits; 92 | 93 | /** How many elements in a vector. This is 1 for scalar types. */ 94 | HALIDE_ATTRIBUTE_ALIGN(2) uint16_t lanes; 95 | 96 | #ifdef __cplusplus 97 | /** Construct a runtime representation of a Halide type from: 98 | * code: The fundamental type from an enum. 99 | * bits: The bit size of one element. 100 | * lanes: The number of vector elements in the type. */ 101 | HALIDE_ALWAYS_INLINE halide_type_t(halide_type_code_t code, uint8_t bits, uint16_t lanes = 1) 102 | : code(code), bits(bits), lanes(lanes) { 103 | } 104 | 105 | /** Default constructor is required e.g. to declare halide_trace_event 106 | * instances. */ 107 | HALIDE_ALWAYS_INLINE halide_type_t() : code((halide_type_code_t)0), bits(0), lanes(0) {} 108 | 109 | /** Compare two types for equality. */ 110 | HALIDE_ALWAYS_INLINE bool operator==(const halide_type_t &other) const { 111 | return (code == other.code && 112 | bits == other.bits && 113 | lanes == other.lanes); 114 | } 115 | 116 | HALIDE_ALWAYS_INLINE bool operator!=(const halide_type_t &other) const { 117 | return !(*this == other); 118 | } 119 | 120 | /** Size in bytes for a single element, even if width is not 1, of this type. */ 121 | HALIDE_ALWAYS_INLINE int bytes() const { return (bits + 7) / 8; } 122 | #endif 123 | }; 124 | 125 | /** An opaque struct containing per-GPU API implementations of the 126 | * device functions. */ 127 | struct halide_device_interface_impl_t; 128 | 129 | /** Each GPU API provides a halide_device_interface_t struct pointing 130 | * to the code that manages device allocations. You can access these 131 | * functions directly from the struct member function pointers, or by 132 | * calling the functions declared below. Note that the global 133 | * functions are not available when using Halide as a JIT compiler. 134 | * If you are using raw halide_buffer_t in that context you must use 135 | * the function pointers in the device_interface struct. 136 | * 137 | * The function pointers below are currently the same for every GPU 138 | * API; only the impl field varies. These top-level functions do the 139 | * bookkeeping that is common across all GPU APIs, and then dispatch 140 | * to more API-specific functions via another set of function pointers 141 | * hidden inside the impl field. 142 | */ 143 | struct halide_device_interface_t { 144 | int (*device_malloc)(void *user_context, struct halide_buffer_t *buf, 145 | const struct halide_device_interface_t *device_interface); 146 | int (*device_free)(void *user_context, struct halide_buffer_t *buf); 147 | int (*device_sync)(void *user_context, struct halide_buffer_t *buf); 148 | void (*device_release)(void *user_context, 149 | const struct halide_device_interface_t *device_interface); 150 | int (*copy_to_host)(void *user_context, struct halide_buffer_t *buf); 151 | int (*copy_to_device)(void *user_context, struct halide_buffer_t *buf, 152 | const struct halide_device_interface_t *device_interface); 153 | int (*device_and_host_malloc)(void *user_context, struct halide_buffer_t *buf, 154 | const struct halide_device_interface_t *device_interface); 155 | int (*device_and_host_free)(void *user_context, struct halide_buffer_t *buf); 156 | int (*buffer_copy)(void *user_context, struct halide_buffer_t *src, 157 | const struct halide_device_interface_t *dst_device_interface, struct halide_buffer_t *dst); 158 | int (*device_crop)(void *user_context, const struct halide_buffer_t *src, 159 | struct halide_buffer_t *dst); 160 | int (*device_release_crop)(void *user_context, struct halide_buffer_t *buf); 161 | int (*wrap_native)(void *user_context, struct halide_buffer_t *buf, uint64_t handle, 162 | const struct halide_device_interface_t *device_interface); 163 | int (*detach_native)(void *user_context, struct halide_buffer_t *buf); 164 | const struct halide_device_interface_impl_t *impl; 165 | }; 166 | 167 | typedef struct halide_dimension_t { 168 | int32_t min, extent, stride; 169 | 170 | // Per-dimension flags. None are defined yet (This is reserved for future use). 171 | uint32_t flags; 172 | 173 | #ifdef __cplusplus 174 | HALIDE_ALWAYS_INLINE halide_dimension_t() : min(0), extent(0), stride(0), flags(0) {} 175 | HALIDE_ALWAYS_INLINE halide_dimension_t(int32_t m, int32_t e, int32_t s, uint32_t f = 0) : 176 | min(m), extent(e), stride(s), flags(f) {} 177 | 178 | HALIDE_ALWAYS_INLINE bool operator==(const halide_dimension_t &other) const { 179 | return (min == other.min) && 180 | (extent == other.extent) && 181 | (stride == other.stride) && 182 | (flags == other.flags); 183 | } 184 | 185 | HALIDE_ALWAYS_INLINE bool operator!=(const halide_dimension_t &other) const { 186 | return !(*this == other); 187 | } 188 | #endif 189 | } halide_dimension_t; 190 | 191 | #ifdef __cplusplus 192 | } // extern "C" 193 | #endif 194 | 195 | typedef enum {halide_buffer_flag_host_dirty = 1, 196 | halide_buffer_flag_device_dirty = 2} halide_buffer_flags; 197 | 198 | /** 199 | * The raw representation of an image passed around by generated 200 | * Halide code. It includes some stuff to track whether the image is 201 | * not actually in main memory, but instead on a device (like a 202 | * GPU). For a more convenient C++ wrapper, use Halide::Buffer. */ 203 | typedef struct halide_buffer_t { 204 | /** A device-handle for e.g. GPU memory used to back this buffer. */ 205 | uint64_t device; 206 | 207 | /** The interface used to interpret the above handle. */ 208 | const struct halide_device_interface_t *device_interface; 209 | 210 | /** A pointer to the start of the data in main memory. In terms of 211 | * the Halide coordinate system, this is the address of the min 212 | * coordinates (defined below). */ 213 | uint8_t* host; 214 | 215 | /** flags with various meanings. */ 216 | uint64_t flags; 217 | 218 | /** The type of each buffer element. */ 219 | struct halide_type_t type; 220 | 221 | /** The dimensionality of the buffer. */ 222 | int32_t dimensions; 223 | 224 | /** The shape of the buffer. Halide does not own this array - you 225 | * must manage the memory for it yourself. */ 226 | halide_dimension_t *dim; 227 | 228 | /** Pads the buffer up to a multiple of 8 bytes */ 229 | void *padding; 230 | } halide_buffer_t; 231 | 232 | 233 | #ifdef __cplusplus 234 | 235 | namespace { 236 | template struct check_is_pointer; 237 | template struct check_is_pointer {}; 238 | } 239 | 240 | /** Construct the halide equivalent of a C type */ 241 | template 242 | HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { 243 | // Create a compile-time error if T is not a pointer (without 244 | // using any includes - this code goes into the runtime). 245 | check_is_pointer check; 246 | (void)check; 247 | return halide_type_t(halide_type_handle, 64); 248 | } 249 | 250 | template<> 251 | HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { 252 | return halide_type_t(halide_type_float, 32); 253 | } 254 | 255 | template<> 256 | HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { 257 | return halide_type_t(halide_type_float, 64); 258 | } 259 | 260 | template<> 261 | HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { 262 | return halide_type_t(halide_type_uint, 1); 263 | } 264 | 265 | template<> 266 | HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { 267 | return halide_type_t(halide_type_uint, 8); 268 | } 269 | 270 | template<> 271 | HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { 272 | return halide_type_t(halide_type_uint, 16); 273 | } 274 | 275 | template<> 276 | HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { 277 | return halide_type_t(halide_type_uint, 32); 278 | } 279 | 280 | template<> 281 | HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { 282 | return halide_type_t(halide_type_uint, 64); 283 | } 284 | 285 | template<> 286 | HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { 287 | return halide_type_t(halide_type_int, 8); 288 | } 289 | 290 | template<> 291 | HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { 292 | return halide_type_t(halide_type_int, 16); 293 | } 294 | 295 | template<> 296 | HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { 297 | return halide_type_t(halide_type_int, 32); 298 | } 299 | 300 | template<> 301 | HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { 302 | return halide_type_t(halide_type_int, 64); 303 | } 304 | 305 | #endif 306 | 307 | #endif // HALIDE_HALIDERUNTIME_H 308 | -------------------------------------------------------------------------------- /include/MNN/ImageProcess.hpp: -------------------------------------------------------------------------------- 1 | // 2 | // ImageProcess.hpp 3 | // MNN 4 | // 5 | // Created by MNN on 2018/09/19. 6 | // Copyright © 2018, Alibaba Group Holding Limited 7 | // 8 | 9 | #ifndef ImageProcess_hpp 10 | #define ImageProcess_hpp 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | namespace MNN { 17 | namespace CV { 18 | enum ImageFormat { 19 | RGBA = 0, 20 | RGB, 21 | BGR, 22 | GRAY, 23 | BGRA, 24 | YUV_NV21 = 11, 25 | YUV_NV12 = 12, 26 | }; 27 | 28 | enum Filter { NEAREST = 0, BILINEAR = 1, BICUBIC = 2 }; 29 | 30 | enum Wrap { CLAMP_TO_EDGE = 0, ZERO = 1, REPEAT = 2 }; 31 | 32 | /** 33 | * handle image process for tensor. 34 | * step: 35 | * 1: Do transform compute and get points 36 | * 2: Sample line and do format convert 37 | * 3: Turn RGBA to float tensor, and do sub and normalize 38 | */ 39 | class MNN_PUBLIC ImageProcess { 40 | public: 41 | struct Inside; 42 | struct Config { 43 | /** data filter */ 44 | Filter filterType = NEAREST; 45 | /** format of source data */ 46 | ImageFormat sourceFormat = RGBA; 47 | /** format of destination data */ 48 | ImageFormat destFormat = RGBA; 49 | 50 | // Only valid if the dest type is float 51 | float mean[4] = {0.0f, 0.0f, 0.0f, 0.0f}; 52 | float normal[4] = {1.0f, 1.0f, 1.0f, 1.0f}; 53 | 54 | /** edge wrapper */ 55 | Wrap wrap = CLAMP_TO_EDGE; 56 | }; 57 | 58 | public: 59 | /** 60 | * @brief create image process with given config for given tensor. 61 | * @param config given config. 62 | * @param dstTensor given tensor. 63 | * @return image processor. 64 | */ 65 | static ImageProcess* create(const Config& config, const Tensor* dstTensor = nullptr); 66 | 67 | /** 68 | * @brief create image process with given config for given tensor. 69 | * @param means given means 70 | * @param meanCount given means count 71 | * @param normals given normals 72 | * @param normalCount given normal count 73 | * @param sourceFormat format of source data 74 | * @param destFormat format of destination data 75 | * @param dstTensor given tensor. 76 | * @return image processor. 77 | */ 78 | static ImageProcess* create(const ImageFormat sourceFormat = RGBA, const ImageFormat destFormat = RGBA, 79 | const float* means = nullptr, const int meanCount = 0, const float* normals = nullptr, 80 | const int normalCount = 0, const Tensor* dstTensor = nullptr); 81 | 82 | ~ImageProcess(); 83 | 84 | /** 85 | * @brief get affine transform matrix. 86 | * @return affine transform matrix. 87 | */ 88 | inline const Matrix& matrix() const { 89 | return mTransform; 90 | } 91 | void setMatrix(const Matrix& matrix); 92 | 93 | /** 94 | * @brief convert source data to given tensor. 95 | * @param source source data. 96 | * @param iw source width. 97 | * @param ih source height. 98 | * @param stride number of elements per row. eg: 100 width RGB contains at least 300 elements. 99 | * @param dest given tensor. 100 | * @return result code. 101 | */ 102 | ErrorCode convert(const uint8_t* source, int iw, int ih, int stride, Tensor* dest); 103 | 104 | /** 105 | * @brief convert source data to given tensor. 106 | * @param source source data. 107 | * @param iw source width. 108 | * @param ih source height. 109 | * @param stride number of elements per row. eg: 100 width RGB contains at least 300 elements. 110 | * @param dest dest data. 111 | * @param ow output width. 112 | * @param oh output height. 113 | * @param outputBpp output bpp, if 0, set as the save and config.destFormat. 114 | * @param outputStride output stride, if 0, set as ow * outputBpp. 115 | * @param type Only support halide_type_of and halide_type_of. 116 | * @return result code. 117 | */ 118 | ErrorCode convert(const uint8_t* source, int iw, int ih, int stride, void* dest, int ow, int oh, int outputBpp = 0, 119 | int outputStride = 0, halide_type_t type = halide_type_of()); 120 | 121 | /** 122 | * @brief create tensor with given data. 123 | * @param w image width. 124 | * @param h image height. 125 | * @param bpp bytes per pixel. 126 | * @param p pixel data pointer. 127 | * @return created tensor. 128 | */ 129 | template 130 | static Tensor* createImageTensor(int w, int h, int bpp, void* p = nullptr) { 131 | return createImageTensor(halide_type_of(), w, h, bpp, p); 132 | } 133 | static Tensor* createImageTensor(halide_type_t type, int w, int h, int bpp, void* p = nullptr); 134 | 135 | private: 136 | ImageProcess(const Config& config); 137 | Matrix mTransform; 138 | Matrix mTransformInvert; 139 | Inside* mInside; 140 | }; 141 | } // namespace CV 142 | } // namespace MNN 143 | 144 | #endif /* ImageProcess_hpp */ 145 | -------------------------------------------------------------------------------- /include/MNN/Interpreter.hpp: -------------------------------------------------------------------------------- 1 | // 2 | // Interpreter.hpp 3 | // MNN 4 | // 5 | // Created by MNN on 2018/07/23. 6 | // Copyright © 2018, Alibaba Group Holding Limited 7 | // 8 | 9 | #ifndef Interpreter_hpp 10 | #define Interpreter_hpp 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | namespace MNN { 20 | 21 | /** session schedule config */ 22 | struct ScheduleConfig { 23 | /** which tensor should be kept */ 24 | std::vector saveTensors; 25 | /** forward type */ 26 | MNNForwardType type = MNN_FORWARD_CPU; 27 | /** number of threads in parallel */ 28 | int numThread = 4; 29 | 30 | /** subpath to run */ 31 | struct Path { 32 | std::vector inputs; 33 | std::vector outputs; 34 | 35 | enum Mode { 36 | /** 37 | * Op Mode 38 | * - inputs means the source op, can NOT be empty. 39 | * - outputs means the sink op, can be empty. 40 | * The path will start from source op, then flow when encounter the sink op. 41 | * The sink op will not be compute in this path. 42 | */ 43 | Op = 0, 44 | 45 | /** 46 | * Tensor Mode (NOT supported yet) 47 | * - inputs means the inputs tensors, can NOT be empty. 48 | * - outputs means the outputs tensors, can NOT be empty. 49 | * It will find the pipeline that compute outputs from inputs. 50 | */ 51 | Tensor = 1 52 | }; 53 | 54 | /** running mode */ 55 | Mode mode = Op; 56 | }; 57 | Path path; 58 | 59 | /** backup backend used to create execution when desinated backend do NOT support any op */ 60 | MNNForwardType backupType = MNN_FORWARD_CPU; 61 | 62 | /** extra backend config */ 63 | BackendConfig* backendConfig = nullptr; 64 | }; 65 | 66 | class Session; 67 | struct Content; 68 | class Tensor; 69 | class Backend; 70 | 71 | class MNN_PUBLIC OperatorInfo { 72 | struct Info; 73 | 74 | public: 75 | /** Operator's name*/ 76 | const std::string& name() const; 77 | 78 | /** Operator's type*/ 79 | const std::string& type() const; 80 | 81 | /** Operator's flops, in M*/ 82 | float flops() const; 83 | 84 | protected: 85 | OperatorInfo(); 86 | ~OperatorInfo(); 87 | Info* mContent; 88 | }; 89 | 90 | typedef std::function&, const std::string& /*opName*/)> TensorCallBack; 91 | typedef std::function&, const OperatorInfo*)> TensorCallBackWithInfo; 92 | 93 | /** net data holder. multiple sessions could share same net. */ 94 | class MNN_PUBLIC Interpreter { 95 | public: 96 | /** 97 | * @brief create net from file. 98 | * @param file given file. 99 | * @return created net if success, NULL otherwise. 100 | */ 101 | static Interpreter* createFromFile(const char* file); 102 | /** 103 | * @brief create net from buffer. 104 | * @param buffer given data buffer. 105 | * @param size size of data buffer. 106 | * @return created net if success, NULL otherwise. 107 | */ 108 | static Interpreter* createFromBuffer(const void* buffer, size_t size); 109 | ~Interpreter(); 110 | 111 | public: 112 | /** 113 | * @brief create session with schedule config. created session will be managed in net. 114 | * @param config session schedule config. 115 | * @return created session if success, NULL otherwise. 116 | */ 117 | Session* createSession(const ScheduleConfig& config); 118 | 119 | /** 120 | * @brief create multi-path session with schedule configs. created session will be managed in net. 121 | * @param configs session schedule configs. 122 | * @return created session if success, NULL otherwise. 123 | */ 124 | Session* createMultiPathSession(const std::vector& configs); 125 | 126 | /** 127 | * @brief release session. 128 | * @param session given session. 129 | * @return true if given session is held by net and is freed. 130 | */ 131 | bool releaseSession(Session* session); 132 | 133 | /** 134 | * @brief call this function to get tensors ready. output tensor buffer (host or deviceId) should be retrieved 135 | * after resize of any input tensor. 136 | * @param session given session. 137 | */ 138 | void resizeSession(Session* session); 139 | 140 | /** 141 | * @brief call this function if don't need resize or create session any more, it will save a few memory that equal 142 | * to the size of model buffer 143 | */ 144 | void releaseModel(); 145 | 146 | /** 147 | * @brief Get the model buffer for user to save 148 | * @return std::make_pair(modleBuffer, modelSize). 149 | * @example: 150 | * std::ofstream output("trainResult.alinn") 151 | * auto buffer = net->getModelBuffer(); 152 | * output.write((const char*)buffer.first, buffer.second); 153 | */ 154 | std::pair getModelBuffer() const; 155 | 156 | /** 157 | * @brief update Session's Tensor to model's Const Op 158 | * @param session given session. 159 | * @return result of running. 160 | */ 161 | ErrorCode updateSessionToModel(Session* session); 162 | 163 | /** 164 | * @brief run session. 165 | * @param session given session. 166 | * @return result of running. 167 | */ 168 | ErrorCode runSession(Session* session) const; 169 | 170 | /* 171 | * @brief run session. 172 | * @param session given session. 173 | * @param before callback before each op. return true to run the op; return false to skip the op. 174 | * @param after callback after each op. return true to continue running; return false to interrupt the session. 175 | * @param sync synchronously wait for finish of execution or not. 176 | * @return result of running. 177 | */ 178 | ErrorCode runSessionWithCallBack(const Session* session, const TensorCallBack& before, const TensorCallBack& end, 179 | bool sync = false) const; 180 | 181 | /* 182 | * @brief run session. 183 | * @param session given session. 184 | * @param before callback before each op. return true to run the op; return false to skip the op. 185 | * @param after callback after each op. return true to continue running; return false to interrupt the session. 186 | * @param sync synchronously wait for finish of execution or not. 187 | * @return result of running. 188 | */ 189 | ErrorCode runSessionWithCallBackInfo(const Session* session, const TensorCallBackWithInfo& before, 190 | const TensorCallBackWithInfo& end, bool sync = false) const; 191 | 192 | /** 193 | * @brief get input tensor for given name. 194 | * @param session given session. 195 | * @param name given name. if NULL, return first input. 196 | * @return tensor if found, NULL otherwise. 197 | */ 198 | Tensor* getSessionInput(const Session* session, const char* name); 199 | /** 200 | * @brief get output tensor for given name. 201 | * @param session given session. 202 | * @param name given name. if NULL, return first output. 203 | * @return tensor if found, NULL otherwise. 204 | */ 205 | Tensor* getSessionOutput(const Session* session, const char* name); 206 | 207 | /** 208 | * @brief get all input tensors. 209 | * @param session given session. 210 | * @return all input tensors mapped with name. 211 | */ 212 | const std::map& getSessionOutputAll(const Session* session) const; 213 | /** 214 | * @brief get all output tensors. 215 | * @param session given session. 216 | * @return all output tensors mapped with name. 217 | */ 218 | const std::map& getSessionInputAll(const Session* session) const; 219 | 220 | public: 221 | /** 222 | * @brief resize given tensor. 223 | * @param tensor given tensor. 224 | * @param dims new dims. at most 6 dims. 225 | */ 226 | void resizeTensor(Tensor* tensor, const std::vector& dims); 227 | 228 | /** 229 | * @brief resize given tensor by nchw. 230 | * @param batch / N. 231 | * @param channel / C. 232 | * @param height / H. 233 | * @param width / W 234 | */ 235 | void resizeTensor(Tensor* tensor, int batch, int channel, int height, int width); 236 | 237 | /** 238 | * @brief get backend used to create given tensor. 239 | * @param session given session. 240 | * @param tensor given tensor. 241 | * @return backend used to create given tensor, may be NULL. 242 | */ 243 | const Backend* getBackend(const Session* session, const Tensor* tensor) const; 244 | 245 | /** 246 | * @brief get business code (model identifier). 247 | * @return business code. 248 | */ 249 | const char* bizCode() const; 250 | 251 | private: 252 | static Interpreter* createFromBufferInternal(Content* net); 253 | 254 | Content* mNet = nullptr; 255 | Interpreter(Content* net); 256 | 257 | Interpreter(const Interpreter&) = delete; 258 | Interpreter(const Interpreter&&) = delete; 259 | Interpreter& operator=(const Interpreter&) = delete; 260 | Interpreter& operator=(const Interpreter&&) = delete; 261 | }; 262 | } // namespace MNN 263 | 264 | #endif /* Interpreter_hpp */ 265 | -------------------------------------------------------------------------------- /include/MNN/MNNDefine.h: -------------------------------------------------------------------------------- 1 | // 2 | // MNNDefine.h 3 | // MNN 4 | // 5 | // Created by MNN on 2018/08/09. 6 | // Copyright © 2018, Alibaba Group Holding Limited 7 | // 8 | 9 | #ifndef MNNDefine_h 10 | #define MNNDefine_h 11 | 12 | #include 13 | #include 14 | 15 | #if defined(__APPLE__) 16 | #include 17 | #if TARGET_OS_IPHONE 18 | #define MNN_BUILD_FOR_IOS 19 | #endif 20 | #endif 21 | 22 | #ifdef MNN_USE_LOGCAT 23 | #include 24 | #define MNN_ERROR(format, ...) __android_log_print(ANDROID_LOG_ERROR, "MNNJNI", format, ##__VA_ARGS__) 25 | #define MNN_PRINT(format, ...) __android_log_print(ANDROID_LOG_INFO, "MNNJNI", format, ##__VA_ARGS__) 26 | #else 27 | #define MNN_PRINT(format, ...) printf(format, ##__VA_ARGS__) 28 | #define MNN_ERROR(format, ...) printf(format, ##__VA_ARGS__) 29 | #endif 30 | 31 | #ifdef DEBUG 32 | #define MNN_ASSERT(x) \ 33 | { \ 34 | int res = (x); \ 35 | if (!res) { \ 36 | MNN_ERROR("Error for %s, %d\n", __FILE__, __LINE__); \ 37 | assert(res); \ 38 | } \ 39 | } 40 | #else 41 | #define MNN_ASSERT(x) \ 42 | { \ 43 | int res = (x); \ 44 | if (!res) { \ 45 | MNN_ERROR("Error for %s, %d\n", __FILE__, __LINE__); \ 46 | } \ 47 | } 48 | #endif 49 | 50 | #define FUNC_PRINT(x) MNN_PRINT(#x "=%d in %s, %d \n", x, __func__, __LINE__); 51 | #define FUNC_PRINT_ALL(x, type) MNN_PRINT(#x "=" #type " %" #type " in %s, %d \n", x, __func__, __LINE__); 52 | 53 | #define MNN_CHECK(success, log) \ 54 | if(!(success)){ \ 55 | MNN_ERROR("Check failed: %s ==> %s\n", #success, #log); \ 56 | } 57 | 58 | #if defined(_MSC_VER) 59 | #if defined(BUILDING_MNN_DLL) 60 | #define MNN_PUBLIC __declspec(dllexport) 61 | #elif defined(USING_MNN_DLL) 62 | #define MNN_PUBLIC __declspec(dllimport) 63 | #else 64 | #define MNN_PUBLIC 65 | #endif 66 | #else 67 | #define MNN_PUBLIC __attribute__((visibility("default"))) 68 | #endif 69 | 70 | #endif /* MNNDefine_h */ 71 | -------------------------------------------------------------------------------- /include/MNN/MNNForwardType.h: -------------------------------------------------------------------------------- 1 | // 2 | // MNNForwardType.h 3 | // MNN 4 | // 5 | // Created by MNN on 2019/01/19. 6 | // Copyright © 2018, Alibaba Group Holding Limited 7 | // 8 | 9 | #ifndef MNNForwardType_h 10 | #define MNNForwardType_h 11 | #include 12 | #include 13 | 14 | typedef enum { 15 | MNN_FORWARD_CPU = 0, 16 | 17 | /* 18 | Firtly find the first available backends not equal to CPU 19 | If no other backends, use cpu 20 | */ 21 | MNN_FORWARD_AUTO = 4, 22 | 23 | /*Hand write metal*/ 24 | MNN_FORWARD_METAL = 1, 25 | 26 | /*Use IOS's MPS instead of hand-write metal, Not Support yet*/ 27 | MNN_FORWARD_MPS = 2, 28 | 29 | /*Android / Common Device GPU API*/ 30 | MNN_FORWARD_OPENCL = 3, 31 | MNN_FORWARD_OPENGL = 6, 32 | MNN_FORWARD_VULKAN = 7, 33 | 34 | /*Android 8.1's NNAPI, Not Support yet*/ 35 | MNN_FORWARD_NN = 5, 36 | 37 | /*User can use API from Backend.hpp to add or search Backend*/ 38 | MNN_FORWARD_USER_0 = 8, 39 | MNN_FORWARD_USER_1 = 9, 40 | MNN_FORWARD_USER_2 = 10, 41 | MNN_FORWARD_USER_3 = 11, 42 | 43 | MNN_FORWARD_ALL 44 | } MNNForwardType; 45 | #ifdef __cplusplus 46 | namespace MNN { 47 | struct BackendConfig { 48 | enum MemoryMode { Memory_Normal = 0, Memory_High, Memory_Low }; 49 | 50 | MemoryMode memory = Memory_Normal; 51 | 52 | enum PowerMode { Power_Normal = 0, Power_High, Power_Low }; 53 | 54 | PowerMode power = Power_Normal; 55 | 56 | enum PrecisionMode { Precision_Normal = 0, Precision_High, Precision_Low }; 57 | 58 | PrecisionMode precision = Precision_Normal; 59 | 60 | /** user defined context */ 61 | union { 62 | void* sharedContext = nullptr; 63 | size_t flags; // Valid for CPU Backend 64 | }; 65 | }; 66 | }; // namespace MNN 67 | #endif 68 | #endif /* MNNForwardType_h */ 69 | -------------------------------------------------------------------------------- /include/MNN/MNNSharedContext.h: -------------------------------------------------------------------------------- 1 | // 2 | // MNNSharedContext.h 3 | // MNN 4 | // 5 | // Created by MNN on 2018/10/11. 6 | // Copyright © 2018, Alibaba Group Holding Limited 7 | // 8 | 9 | #ifndef MNNSharedContext_h 10 | #define MNNSharedContext_h 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | #include /*uint32_t*/ 16 | 17 | #ifndef VK_DEFINE_HANDLE 18 | #define VK_DEFINE_HANDLE(object) typedef struct object##_T* object; 19 | VK_DEFINE_HANDLE(VkInstance) 20 | VK_DEFINE_HANDLE(VkPhysicalDevice) 21 | VK_DEFINE_HANDLE(VkDevice) 22 | VK_DEFINE_HANDLE(VkQueue) 23 | #endif 24 | struct MNNVulkanContext { 25 | VkInstance pInstance; 26 | VkPhysicalDevice pPhysicalDevice; 27 | VkDevice pDevice; 28 | VkQueue pQueue; 29 | uint32_t iQueueFamilyIndex; 30 | }; 31 | #ifdef __cplusplus 32 | } 33 | #endif 34 | 35 | #endif /* MNNSharedContext_h */ 36 | -------------------------------------------------------------------------------- /include/MNN/Rect.h: -------------------------------------------------------------------------------- 1 | // 2 | // Rect.h 3 | // MNN 4 | // 5 | // Modified by jiangxiaotang on 2018/09/19. 6 | // Copyright © 2018, Alibaba Group Holding Limited 7 | // 8 | 9 | /* 10 | * Copyright 2006 The Android Open Source Project 11 | * 12 | * Use of this source code is governed by a BSD-style license that can be 13 | * found in the LICENSE file. 14 | */ 15 | 16 | /* Generated by tools/bookmaker from include/core/Rect.h and docs/SkRect_Reference.bmh 17 | on 2018-07-13 08:15:11. Additional documentation and examples can be found at: 18 | https://skia.org/user/api/SkRect_Reference 19 | 20 | You may edit either file directly. Structural changes to public interfaces require 21 | editing both files. After editing docs/SkRect_Reference.bmh, run: 22 | bookmaker -b docs -i include/core/Rect.h -p 23 | to create an updated version of this file. 24 | */ 25 | 26 | #ifndef SkRect_DEFINED 27 | #define SkRect_DEFINED 28 | 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | namespace MNN { 35 | namespace CV { 36 | 37 | struct Point { 38 | float fX; 39 | float fY; 40 | 41 | void set(float x, float y) { 42 | fX = x; 43 | fY = y; 44 | } 45 | }; 46 | 47 | /** \struct Rect 48 | Rect holds four float coordinates describing the upper and 49 | lower bounds of a rectangle. Rect may be created from outer bounds or 50 | from position, width, and height. Rect describes an area; if its right 51 | is less than or equal to its left, or if its bottom is less than or equal to 52 | its top, it is considered empty. 53 | */ 54 | struct MNN_PUBLIC Rect { 55 | float fLeft; //!< smaller x-axis bounds 56 | float fTop; //!< smaller y-axis bounds 57 | float fRight; //!< larger x-axis bounds 58 | float fBottom; //!< larger y-axis bounds 59 | 60 | /** Returns constructed Rect set to (0, 0, 0, 0). 61 | Many other rectangles are empty; if left is equal to or greater than right, 62 | or if top is equal to or greater than bottom. Setting all members to zero 63 | is a convenience, but does not designate a special empty rectangle. 64 | 65 | @return bounds (0, 0, 0, 0) 66 | */ 67 | static constexpr Rect MakeEmpty() { 68 | return Rect{0, 0, 0, 0}; 69 | } 70 | 71 | #ifdef SK_SUPPORT_LEGACY_RECTMAKELARGEST 72 | /** Deprecated. 73 | */ 74 | static Rect MakeLargest() { 75 | return {SK_ScalarMin, SK_ScalarMin, SK_ScalarMax, SK_ScalarMax}; 76 | } 77 | #endif 78 | 79 | /** Returns constructed Rect set to float values (0, 0, w, h). Does not 80 | validate input; w or h may be negative. 81 | 82 | Passing integer values may generate a compiler warning since Rect cannot 83 | represent 32-bit integers exactly. Use SkIRect for an exact integer rectangle. 84 | 85 | @param w float width of constructed Rect 86 | @param h float height of constructed Rect 87 | @return bounds (0, 0, w, h) 88 | */ 89 | static constexpr Rect MakeWH(float w, float h) { 90 | return Rect{0, 0, w, h}; 91 | } 92 | 93 | /** Returns constructed Rect set to integer values (0, 0, w, h). Does not validate 94 | input; w or h may be negative. 95 | 96 | Use to avoid a compiler warning that input may lose precision when stored. 97 | Use SkIRect for an exact integer rectangle. 98 | 99 | @param w integer width of constructed Rect 100 | @param h integer height of constructed Rect 101 | @return bounds (0, 0, w, h) 102 | */ 103 | static Rect MakeIWH(int w, int h) { 104 | Rect r; 105 | r.set(0, 0, (float)(w), (float)(h)); 106 | return r; 107 | } 108 | 109 | /** Returns constructed Rect set to (l, t, r, b). Does not sort input; Rect may 110 | result in fLeft greater than fRight, or fTop greater than fBottom. 111 | 112 | @param l float stored in fLeft 113 | @param t float stored in fTop 114 | @param r float stored in fRight 115 | @param b float stored in fBottom 116 | @return bounds (l, t, r, b) 117 | */ 118 | static constexpr Rect MakeLTRB(float l, float t, float r, float b) { 119 | return Rect{l, t, r, b}; 120 | } 121 | 122 | /** Returns constructed Rect set to (x, y, x + w, y + h). Does not validate input; 123 | w or h may be negative. 124 | 125 | @param x stored in fLeft 126 | @param y stored in fTop 127 | @param w added to x and stored in fRight 128 | @param h added to y and stored in fBottom 129 | @return bounds at (x, y) with width w and height h 130 | */ 131 | static constexpr Rect MakeXYWH(float x, float y, float w, float h) { 132 | return Rect{x, y, x + w, y + h}; 133 | } 134 | 135 | /** Returns true if fLeft is equal to or greater than fRight, or if fTop is equal 136 | to or greater than fBottom. Call sort() to reverse rectangles with negative 137 | width() or height(). 138 | 139 | @return true if width() or height() are zero or negative 140 | */ 141 | bool isEmpty() const { 142 | // We write it as the NOT of a non-empty rect, so we will return true if any values 143 | // are NaN. 144 | return !(fLeft < fRight && fTop < fBottom); 145 | } 146 | 147 | /** Returns true if fLeft is equal to or less than fRight, or if fTop is equal 148 | to or less than fBottom. Call sort() to reverse rectangles with negative 149 | width() or height(). 150 | 151 | @return true if width() or height() are zero or positive 152 | */ 153 | bool isSorted() const { 154 | return fLeft <= fRight && fTop <= fBottom; 155 | } 156 | 157 | /** Returns left edge of Rect, if sorted. Call isSorted() to see if Rect is valid. 158 | Call sort() to reverse fLeft and fRight if needed. 159 | 160 | @return fLeft 161 | */ 162 | float x() const { 163 | return fLeft; 164 | } 165 | 166 | /** Returns top edge of Rect, if sorted. Call isEmpty() to see if Rect may be invalid, 167 | and sort() to reverse fTop and fBottom if needed. 168 | 169 | @return fTop 170 | */ 171 | float y() const { 172 | return fTop; 173 | } 174 | 175 | /** Returns left edge of Rect, if sorted. Call isSorted() to see if Rect is valid. 176 | Call sort() to reverse fLeft and fRight if needed. 177 | 178 | @return fLeft 179 | */ 180 | float left() const { 181 | return fLeft; 182 | } 183 | 184 | /** Returns top edge of Rect, if sorted. Call isEmpty() to see if Rect may be invalid, 185 | and sort() to reverse fTop and fBottom if needed. 186 | 187 | @return fTop 188 | */ 189 | float top() const { 190 | return fTop; 191 | } 192 | 193 | /** Returns right edge of Rect, if sorted. Call isSorted() to see if Rect is valid. 194 | Call sort() to reverse fLeft and fRight if needed. 195 | 196 | @return fRight 197 | */ 198 | float right() const { 199 | return fRight; 200 | } 201 | 202 | /** Returns bottom edge of Rect, if sorted. Call isEmpty() to see if Rect may be invalid, 203 | and sort() to reverse fTop and fBottom if needed. 204 | 205 | @return fBottom 206 | */ 207 | float bottom() const { 208 | return fBottom; 209 | } 210 | 211 | /** Returns span on the x-axis. This does not check if Rect is sorted, or if 212 | result fits in 32-bit float; result may be negative or infinity. 213 | 214 | @return fRight minus fLeft 215 | */ 216 | float width() const { 217 | return fRight - fLeft; 218 | } 219 | 220 | /** Returns span on the y-axis. This does not check if Rect is sorted, or if 221 | result fits in 32-bit float; result may be negative or infinity. 222 | 223 | @return fBottom minus fTop 224 | */ 225 | float height() const { 226 | return fBottom - fTop; 227 | } 228 | 229 | /** Returns average of left edge and right edge. Result does not change if Rect 230 | is sorted. Result may overflow to infinity if Rect is far from the origin. 231 | 232 | @return midpoint in x 233 | */ 234 | float centerX() const { 235 | // don't use floatHalf(fLeft + fBottom) as that might overflow before the 0.5 236 | return 0.5f * (fLeft) + 0.5f * (fRight); 237 | } 238 | 239 | /** Returns average of top edge and bottom edge. Result does not change if Rect 240 | is sorted. 241 | 242 | @return midpoint in y 243 | */ 244 | float centerY() const { 245 | // don't use floatHalf(fTop + fBottom) as that might overflow before the 0.5 246 | return 0.5f * (fTop) + 0.5f * (fBottom); 247 | } 248 | 249 | /** Sets Rect to (0, 0, 0, 0). 250 | 251 | Many other rectangles are empty; if left is equal to or greater than right, 252 | or if top is equal to or greater than bottom. Setting all members to zero 253 | is a convenience, but does not designate a special empty rectangle. 254 | */ 255 | void setEmpty() { 256 | *this = MakeEmpty(); 257 | } 258 | 259 | /** Sets Rect to (left, top, right, bottom). 260 | left and right are not sorted; left is not necessarily less than right. 261 | top and bottom are not sorted; top is not necessarily less than bottom. 262 | 263 | @param left stored in fLeft 264 | @param top stored in fTop 265 | @param right stored in fRight 266 | @param bottom stored in fBottom 267 | */ 268 | void set(float left, float top, float right, float bottom) { 269 | fLeft = left; 270 | fTop = top; 271 | fRight = right; 272 | fBottom = bottom; 273 | } 274 | 275 | /** Sets Rect to (left, top, right, bottom). 276 | left and right are not sorted; left is not necessarily less than right. 277 | top and bottom are not sorted; top is not necessarily less than bottom. 278 | 279 | @param left stored in fLeft 280 | @param top stored in fTop 281 | @param right stored in fRight 282 | @param bottom stored in fBottom 283 | */ 284 | void setLTRB(float left, float top, float right, float bottom) { 285 | this->set(left, top, right, bottom); 286 | } 287 | 288 | /** Sets Rect to (left, top, right, bottom). 289 | All parameters are promoted from integer to scalar. 290 | left and right are not sorted; left is not necessarily less than right. 291 | top and bottom are not sorted; top is not necessarily less than bottom. 292 | 293 | @param left promoted to float and stored in fLeft 294 | @param top promoted to float and stored in fTop 295 | @param right promoted to float and stored in fRight 296 | @param bottom promoted to float and stored in fBottom 297 | */ 298 | void iset(int left, int top, int right, int bottom) { 299 | fLeft = (float)(left); 300 | fTop = (float)(top); 301 | fRight = (float)(right); 302 | fBottom = (float)(bottom); 303 | } 304 | 305 | /** Sets Rect to (0, 0, width, height). 306 | width and height may be zero or negative. width and height are promoted from 307 | integer to float, large values may lose precision. 308 | 309 | @param width promoted to float and stored in fRight 310 | @param height promoted to float and stored in fBottom 311 | */ 312 | void isetWH(int width, int height) { 313 | fLeft = fTop = 0; 314 | fRight = (float)(width); 315 | fBottom = (float)(height); 316 | } 317 | 318 | /** Sets Rect to (x, y, x + width, y + height). Does not validate input; 319 | width or height may be negative. 320 | 321 | @param x stored in fLeft 322 | @param y stored in fTop 323 | @param width added to x and stored in fRight 324 | @param height added to y and stored in fBottom 325 | */ 326 | void setXYWH(float x, float y, float width, float height) { 327 | fLeft = x; 328 | fTop = y; 329 | fRight = x + width; 330 | fBottom = y + height; 331 | } 332 | 333 | /** Sets Rect to (0, 0, width, height). Does not validate input; 334 | width or height may be negative. 335 | 336 | @param width stored in fRight 337 | @param height stored in fBottom 338 | */ 339 | void setWH(float width, float height) { 340 | fLeft = 0; 341 | fTop = 0; 342 | fRight = width; 343 | fBottom = height; 344 | } 345 | 346 | /** Returns Rect offset by (dx, dy). 347 | 348 | If dx is negative, Rect returned is moved to the left. 349 | If dx is positive, Rect returned is moved to the right. 350 | If dy is negative, Rect returned is moved upward. 351 | If dy is positive, Rect returned is moved downward. 352 | 353 | @param dx added to fLeft and fRight 354 | @param dy added to fTop and fBottom 355 | @return Rect offset on axes, with original width and height 356 | */ 357 | Rect makeOffset(float dx, float dy) const { 358 | return MakeLTRB(fLeft + dx, fTop + dy, fRight + dx, fBottom + dy); 359 | } 360 | 361 | /** Returns Rect, inset by (dx, dy). 362 | 363 | If dx is negative, Rect returned is wider. 364 | If dx is positive, Rect returned is narrower. 365 | If dy is negative, Rect returned is taller. 366 | If dy is positive, Rect returned is shorter. 367 | 368 | @param dx added to fLeft and subtracted from fRight 369 | @param dy added to fTop and subtracted from fBottom 370 | @return Rect inset symmetrically left and right, top and bottom 371 | */ 372 | Rect makeInset(float dx, float dy) const { 373 | return MakeLTRB(fLeft + dx, fTop + dy, fRight - dx, fBottom - dy); 374 | } 375 | 376 | /** Returns Rect, outset by (dx, dy). 377 | 378 | If dx is negative, Rect returned is narrower. 379 | If dx is positive, Rect returned is wider. 380 | If dy is negative, Rect returned is shorter. 381 | If dy is positive, Rect returned is taller. 382 | 383 | @param dx subtracted to fLeft and added from fRight 384 | @param dy subtracted to fTop and added from fBottom 385 | @return Rect outset symmetrically left and right, top and bottom 386 | */ 387 | Rect makeOutset(float dx, float dy) const { 388 | return MakeLTRB(fLeft - dx, fTop - dy, fRight + dx, fBottom + dy); 389 | } 390 | 391 | /** Offsets Rect by adding dx to fLeft, fRight; and by adding dy to fTop, fBottom. 392 | 393 | If dx is negative, moves Rect to the left. 394 | If dx is positive, moves Rect to the right. 395 | If dy is negative, moves Rect upward. 396 | If dy is positive, moves Rect downward. 397 | 398 | @param dx offset added to fLeft and fRight 399 | @param dy offset added to fTop and fBottom 400 | */ 401 | void offset(float dx, float dy) { 402 | fLeft += dx; 403 | fTop += dy; 404 | fRight += dx; 405 | fBottom += dy; 406 | } 407 | 408 | /** Offsets Rect so that fLeft equals newX, and fTop equals newY. width and height 409 | are unchanged. 410 | 411 | @param newX stored in fLeft, preserving width() 412 | @param newY stored in fTop, preserving height() 413 | */ 414 | void offsetTo(float newX, float newY) { 415 | fRight += newX - fLeft; 416 | fBottom += newY - fTop; 417 | fLeft = newX; 418 | fTop = newY; 419 | } 420 | 421 | /** Insets Rect by (dx, dy). 422 | 423 | If dx is positive, makes Rect narrower. 424 | If dx is negative, makes Rect wider. 425 | If dy is positive, makes Rect shorter. 426 | If dy is negative, makes Rect taller. 427 | 428 | @param dx added to fLeft and subtracted from fRight 429 | @param dy added to fTop and subtracted from fBottom 430 | */ 431 | void inset(float dx, float dy) { 432 | fLeft += dx; 433 | fTop += dy; 434 | fRight -= dx; 435 | fBottom -= dy; 436 | } 437 | 438 | /** Outsets Rect by (dx, dy). 439 | 440 | If dx is positive, makes Rect wider. 441 | If dx is negative, makes Rect narrower. 442 | If dy is positive, makes Rect taller. 443 | If dy is negative, makes Rect shorter. 444 | 445 | @param dx subtracted to fLeft and added from fRight 446 | @param dy subtracted to fTop and added from fBottom 447 | */ 448 | void outset(float dx, float dy) { 449 | this->inset(-dx, -dy); 450 | } 451 | 452 | private: 453 | static bool Intersects(float al, float at, float ar, float ab, float bl, float bt, float br, float bb) { 454 | float L = std::max(al, bl); 455 | float R = std::min(ar, br); 456 | float T = std::max(at, bt); 457 | float B = std::min(ab, bb); 458 | return L < R && T < B; 459 | } 460 | 461 | public: 462 | /** Constructs Rect to intersect from (left, top, right, bottom). Does not sort 463 | construction. 464 | 465 | Returns true if Rect intersects construction. 466 | Returns false if either construction or Rect is empty, or do not intersect. 467 | 468 | @param left x-axis minimum of constructed Rect 469 | @param top y-axis minimum of constructed Rect 470 | @param right x-axis maximum of constructed Rect 471 | @param bottom y-axis maximum of constructed Rect 472 | @return true if construction and Rect have area in common 473 | */ 474 | bool intersects(float left, float top, float right, float bottom) const { 475 | return Intersects(fLeft, fTop, fRight, fBottom, left, top, right, bottom); 476 | } 477 | 478 | /** Returns true if Rect intersects r. 479 | Returns false if either r or Rect is empty, or do not intersect. 480 | 481 | @param r Rect to intersect 482 | @return true if r and Rect have area in common 483 | */ 484 | bool intersects(const Rect& r) const { 485 | return Intersects(fLeft, fTop, fRight, fBottom, r.fLeft, r.fTop, r.fRight, r.fBottom); 486 | } 487 | 488 | /** Returns true if a intersects b. 489 | Returns false if either a or b is empty, or do not intersect. 490 | 491 | @param a Rect to intersect 492 | @param b Rect to intersect 493 | @return true if a and b have area in common 494 | */ 495 | static bool Intersects(const Rect& a, const Rect& b) { 496 | return Intersects(a.fLeft, a.fTop, a.fRight, a.fBottom, b.fLeft, b.fTop, b.fRight, b.fBottom); 497 | } 498 | 499 | /** Sets Rect to the union of itself and r. 500 | 501 | Asserts if r is empty and SK_DEBUG is defined. 502 | If Rect is empty, sets Rect to r. 503 | 504 | May produce incorrect results if r is empty. 505 | 506 | @param r expansion Rect 507 | */ 508 | void joinNonEmptyArg(const Rect& r) { 509 | MNN_ASSERT(!r.isEmpty()); 510 | // if we are empty, just assign 511 | if (fLeft >= fRight || fTop >= fBottom) { 512 | *this = r; 513 | } else { 514 | this->joinPossiblyEmptyRect(r); 515 | } 516 | } 517 | 518 | /** Sets Rect to the union of itself and the construction. 519 | 520 | May produce incorrect results if Rect or r is empty. 521 | 522 | @param r expansion Rect 523 | */ 524 | void joinPossiblyEmptyRect(const Rect& r) { 525 | fLeft = std::min(fLeft, r.left()); 526 | fTop = std::min(fTop, r.top()); 527 | fRight = std::max(fRight, r.right()); 528 | fBottom = std::max(fBottom, r.bottom()); 529 | } 530 | 531 | /** Returns true if: fLeft <= x < fRight && fTop <= y < fBottom. 532 | Returns false if Rect is empty. 533 | 534 | @param x test Point x-coordinate 535 | @param y test Point y-coordinate 536 | @return true if (x, y) is inside Rect 537 | */ 538 | bool contains(float x, float y) const { 539 | return x >= fLeft && x < fRight && y >= fTop && y < fBottom; 540 | } 541 | 542 | /** Swaps fLeft and fRight if fLeft is greater than fRight; and swaps 543 | fTop and fBottom if fTop is greater than fBottom. Result may be empty; 544 | and width() and height() will be zero or positive. 545 | */ 546 | void sort() { 547 | using std::swap; 548 | if (fLeft > fRight) { 549 | swap(fLeft, fRight); 550 | } 551 | 552 | if (fTop > fBottom) { 553 | swap(fTop, fBottom); 554 | } 555 | } 556 | 557 | /** Returns Rect with fLeft and fRight swapped if fLeft is greater than fRight; and 558 | with fTop and fBottom swapped if fTop is greater than fBottom. Result may be empty; 559 | and width() and height() will be zero or positive. 560 | 561 | @return sorted Rect 562 | */ 563 | Rect makeSorted() const { 564 | return MakeLTRB(std::min(fLeft, fRight), std::min(fTop, fBottom), std::max(fLeft, fRight), 565 | std::max(fTop, fBottom)); 566 | } 567 | 568 | /** Returns pointer to first scalar in Rect, to treat it as an array with four 569 | entries. 570 | 571 | @return pointer to fLeft 572 | */ 573 | const float* asScalars() const { 574 | return &fLeft; 575 | } 576 | }; 577 | 578 | } // namespace CV 579 | } // namespace MNN 580 | #endif 581 | -------------------------------------------------------------------------------- /include/MNN/Tensor.hpp: -------------------------------------------------------------------------------- 1 | // 2 | // Tensor.hpp 3 | // MNN 4 | // 5 | // Created by MNN on 2018/08/14. 6 | // Copyright © 2018, Alibaba Group Holding Limited 7 | // 8 | 9 | #ifndef Tensor_hpp 10 | #define Tensor_hpp 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | namespace MNN { 17 | 18 | /** 19 | * data container. 20 | * data for host tensor is saved in `host` field. its memory is allocated malloc directly. 21 | * data for device tensor is saved in `deviceId` field. its memory is allocated by session's backend. 22 | * usually, device tensors are created by engine (like net, session). 23 | * meanwhile, host tensors could be created by engine or user. 24 | */ 25 | class MNN_PUBLIC Tensor { 26 | public: 27 | struct InsideDescribe; 28 | 29 | /** dimension type used to create tensor */ 30 | enum DimensionType { 31 | /** for tensorflow net type. uses NHWC as data format. */ 32 | TENSORFLOW, 33 | /** for caffe net type. uses NCHW as data format. */ 34 | CAFFE, 35 | /** for caffe net type. uses NC4HW4 as data format. */ 36 | CAFFE_C4 37 | }; 38 | 39 | /** handle type */ 40 | enum HandleDataType { 41 | /** default handle type */ 42 | HANDLE_NONE = 0, 43 | /** string handle type */ 44 | HANDLE_STRING = 1 45 | }; 46 | 47 | public: 48 | /** 49 | * @brief create a tensor with dimension size and type without acquire memory for data. 50 | * @param dimSize dimension size. 51 | * @param type dimension type. 52 | */ 53 | Tensor(int dimSize = 4, DimensionType type = CAFFE); 54 | 55 | /** 56 | * @brief create a tensor with same shape as given tensor. 57 | * @param tensor shape provider. 58 | * @param type dimension type. 59 | * @param allocMemory acquire memory for data or not. 60 | * @warning tensor data won't be copied. 61 | */ 62 | Tensor(const Tensor* tensor, DimensionType type = CAFFE, bool allocMemory = true); 63 | 64 | /** deinitializer */ 65 | ~Tensor(); 66 | 67 | private: 68 | // remove all assignment operator 69 | Tensor(const Tensor& tensor) = delete; 70 | Tensor(const Tensor&& tensor) = delete; 71 | Tensor& operator=(const Tensor&) = delete; 72 | Tensor& operator=(const Tensor&&) = delete; 73 | 74 | public: 75 | /** 76 | * @brief create tensor with shape, data type and dimension type. 77 | * @param shape tensor shape. 78 | * @param type data type. 79 | * @param dimType dimension type. 80 | * @return created tensor. 81 | * @warning memory for data won't be acquired. call backend's onAcquireBuffer to get memory ready. 82 | */ 83 | static Tensor* createDevice(const std::vector& shape, halide_type_t type, DimensionType dimType = TENSORFLOW); 84 | 85 | /** 86 | * @brief create tensor with shape and dimension type. data type is represented by `T`. 87 | * @param shape tensor shape. 88 | * @param dimType dimension type. 89 | * @return created tensor. 90 | * @warning memory for data won't be acquired. call backend's onAcquireBuffer to get memory ready. 91 | */ 92 | template 93 | static Tensor* createDevice(const std::vector& shape, DimensionType dimType = TENSORFLOW) { 94 | return createDevice(shape, halide_type_of(), dimType); 95 | } 96 | 97 | /** 98 | * @brief create tensor with shape, data type, data and dimension type. 99 | * @param shape tensor shape. 100 | * @param type data type. 101 | * @param data data to save. 102 | * @param dimType dimension type. 103 | * @return created tensor. 104 | */ 105 | static Tensor* create(const std::vector& shape, halide_type_t type, void* data = NULL, 106 | DimensionType dimType = TENSORFLOW); 107 | 108 | /** 109 | * @brief create tensor with shape, data and dimension type. data type is represented by `T`. 110 | * @param shape tensor shape. 111 | * @param data data to save. 112 | * @param dimType dimension type. 113 | * @return created tensor. 114 | */ 115 | template 116 | static Tensor* create(const std::vector& shape, void* data = NULL, DimensionType dimType = TENSORFLOW) { 117 | return create(shape, halide_type_of(), data, dimType); 118 | } 119 | 120 | public: 121 | /** 122 | * @brief for DEVICE tensor, copy data from given host tensor. 123 | * @param hostTensor host tensor, the data provider. 124 | * @return true for DEVICE tensor, and false for HOST tensor. 125 | */ 126 | bool copyFromHostTensor(const Tensor* hostTensor); 127 | 128 | /** 129 | * @brief for DEVICE tensor, copy data to given host tensor. 130 | * @param hostTensor host tensor, the data consumer. 131 | * @return true for DEVICE tensor, and false for HOST tensor. 132 | */ 133 | bool copyToHostTensor(Tensor* hostTensor) const; 134 | 135 | /** 136 | * @brief create HOST tensor from DEVICE tensor, with or without data copying. 137 | * @param deviceTensor given device tensor. 138 | * @param copyData copy data or not. 139 | * @return created host tensor. 140 | */ 141 | static Tensor* createHostTensorFromDevice(const Tensor* deviceTensor, bool copyData = true); 142 | 143 | public: 144 | const halide_buffer_t& buffer() const { 145 | return mBuffer; 146 | } 147 | halide_buffer_t& buffer() { 148 | return mBuffer; 149 | } 150 | 151 | /** 152 | * @brief get dimension type. 153 | * @return dimension type. 154 | */ 155 | DimensionType getDimensionType() const; 156 | 157 | /** 158 | * @brief handle data type. used when data type code is halide_type_handle. 159 | * @return handle data type. 160 | */ 161 | HandleDataType getHandleDataType() const; 162 | 163 | /** 164 | * @brief set data type. 165 | * @param type data type defined in 'Type_generated.h'. 166 | */ 167 | void setType(int type); 168 | 169 | /** 170 | * @brief get data type. 171 | * @return data type. 172 | */ 173 | inline halide_type_t getType() const { 174 | return mBuffer.type; 175 | } 176 | 177 | /** 178 | * @brief visit host memory, data type is represented by `T`. 179 | * @return data point in `T` type. 180 | */ 181 | template 182 | T* host() const { 183 | return (T*)mBuffer.host; 184 | } 185 | 186 | /** 187 | * @brief visit device memory. 188 | * @return device data ID. what the ID means varies between backends. 189 | */ 190 | uint64_t deviceId() const { 191 | return mBuffer.device; 192 | } 193 | 194 | public: 195 | int dimensions() const { 196 | return mBuffer.dimensions; 197 | } 198 | 199 | /** 200 | * @brief get all dimensions' extent. 201 | * @return dimensions' extent. 202 | */ 203 | std::vector shape() const; 204 | 205 | /** 206 | * @brief calculate number of bytes needed to store data taking reordering flag into account. 207 | * @return bytes needed to store data 208 | */ 209 | int size() const; 210 | 211 | /** 212 | * @brief calculate number of elements needed to store data taking reordering flag into account. 213 | * @return elements needed to store data 214 | */ 215 | inline int elementSize() const { 216 | return size() / mBuffer.type.bytes(); 217 | } 218 | 219 | public: 220 | inline int width() const { 221 | if (getDimensionType() == TENSORFLOW) { 222 | return mBuffer.dim[2].extent; 223 | } 224 | 225 | return mBuffer.dim[3].extent; 226 | } 227 | inline int height() const { 228 | if (getDimensionType() == TENSORFLOW) { 229 | return mBuffer.dim[1].extent; 230 | } 231 | return mBuffer.dim[2].extent; 232 | } 233 | inline int channel() const { 234 | if (getDimensionType() == TENSORFLOW) { 235 | return mBuffer.dim[3].extent; 236 | } 237 | return mBuffer.dim[1].extent; 238 | } 239 | inline int batch() const { 240 | return mBuffer.dim[0].extent; 241 | } 242 | 243 | // visit dimension's extent & stride 244 | inline int stride(int index) const { 245 | return mBuffer.dim[index].stride; 246 | } 247 | inline int length(int index) const { 248 | return mBuffer.dim[index].extent; 249 | } 250 | inline void setStride(int index, int stride) { 251 | mBuffer.dim[index].stride = stride; 252 | } 253 | inline void setLength(int index, int length) { 254 | mBuffer.dim[index].extent = length; 255 | } 256 | 257 | public: 258 | /** 259 | * @brief print tensor data. for DEBUG use only. 260 | */ 261 | void print() const; 262 | 263 | private: 264 | halide_buffer_t mBuffer; 265 | struct InsideDescribe* mDescribe; 266 | 267 | private: 268 | friend class TensorUtils; 269 | }; 270 | } // namespace MNN 271 | 272 | #endif /* Tensor_hpp */ 273 | -------------------------------------------------------------------------------- /include/MNN/VCS.h: -------------------------------------------------------------------------------- 1 | #undef MNN_REVISION 2 | #undef MNN_REPOSITORY 3 | -------------------------------------------------------------------------------- /include/MNN/expr/Executor.hpp: -------------------------------------------------------------------------------- 1 | // 2 | // Executor.hpp 3 | // MNN 4 | // 5 | // Created by MNN on 2019/07/25. 6 | // Copyright © 2018, Alibaba Group Holding Limited 7 | // 8 | #ifndef Executor_hpp 9 | #define Executor_hpp 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | namespace MNN { 18 | class Backend; 19 | class Execution; 20 | namespace Express { 21 | class MNN_PUBLIC Executor { 22 | public: 23 | class ComputeCache { 24 | public: 25 | void setShapeDirty(); 26 | void setContentDirty(); 27 | 28 | ErrorCode compute(); 29 | ErrorCode resize(); 30 | Tensor* output(EXPRP outputExpr, int index, bool host = true) const; 31 | void dup(EXPRP src, EXPRP dst); 32 | void recycle(Expr* expr); 33 | struct TensorContent { 34 | std::shared_ptr tensor; 35 | int refCount = 0; 36 | void reset(); 37 | }; 38 | struct Unit { 39 | std::vector inputs; 40 | std::vector inputFromCache; 41 | std::vector outputs; 42 | const Expr* origin; 43 | std::shared_ptr exe; 44 | }; 45 | static void create(const std::vector& outputs, std::map& units, std::set>&& inputCaches, std::vector&& tensors, std::shared_ptr bn, std::shared_ptr backendBn); 46 | 47 | ~ ComputeCache(); 48 | void addLink(std::shared_ptr cache); 49 | bool valid() const { 50 | return !mOutputTensors.empty(); 51 | } 52 | private: 53 | ComputeCache(){}; 54 | std::set> mInputs; 55 | // First is Host Tensor, Second is Device Tensor 56 | std::map>> mOutputTensors; 57 | std::vector mTensors; 58 | std::vector mUnits; 59 | std::vector> mLinks; 60 | bool mContentDirty = true; 61 | bool mShapeDirty = true; 62 | std::shared_ptr mBackend; 63 | std::shared_ptr mBackupBackend; 64 | }; 65 | struct Requirement { 66 | std::vector contentNeedContent; 67 | std::vector shapeNeedContent; 68 | std::vector supportError; 69 | }; 70 | ~Executor(); 71 | Requirement getRequirement(Expr* expr) const; 72 | ErrorCode computeInfo(Expr* expr); 73 | void makeCache(std::vector expr); 74 | ErrorCode runCache(std::shared_ptr cache); 75 | void setGlobalExecutorConfig(MNNForwardType type, const BackendConfig& config, int numberThread); 76 | enum GCFlag { 77 | FULL, 78 | PART 79 | }; 80 | void gc(GCFlag flag = FULL); 81 | static std::shared_ptr getGlobalExecutor(); 82 | void resetProfile(); 83 | void dumpProfile(); 84 | void addOpCostTime(int op, float costTime); 85 | class Profiler; 86 | private: 87 | void _addToCache(const std::vector>& caches); 88 | void _resetCache(); 89 | void _visit(EXPRP expr, std::map& units, std::set>& inputCaches, std::vector& tensors); 90 | 91 | Executor(std::shared_ptr backend); 92 | std::shared_ptr mBackend; 93 | std::shared_ptr mBackupBackend; 94 | std::mutex mMutex; 95 | std::vector> mStack; 96 | std::vector mInputs; 97 | std::vector mOutputs; 98 | std::shared_ptr mProfiler; 99 | }; 100 | } // namespace Express 101 | } // namespace MNN 102 | #endif 103 | -------------------------------------------------------------------------------- /include/MNN/expr/Expr.hpp: -------------------------------------------------------------------------------- 1 | // 2 | // Expr.hpp 3 | // MNN 4 | // 5 | // Created by MNN on 2019/06/10. 6 | // Copyright © 2018, Alibaba Group Holding Limited 7 | // 8 | 9 | #ifndef Expr_hpp 10 | #define Expr_hpp 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | namespace MNN { 22 | struct OpT; 23 | struct Op; 24 | struct NetT; 25 | namespace Express { 26 | class Variable; 27 | class Expr; 28 | class Executor; 29 | typedef std::shared_ptr EXPRP; 30 | typedef std::weak_ptr WeakEXPRP; 31 | typedef std::vector INTS; 32 | enum Dimensionformat { NHWC, NC4HW4, NCHW }; 33 | class MNN_PUBLIC VARP { 34 | public: 35 | VARP() { 36 | // Do nothing 37 | } 38 | VARP(std::shared_ptr c) { 39 | mContent = std::move(c); 40 | } 41 | VARP(Variable* c) { 42 | mContent.reset(c); 43 | } 44 | Variable* get() const { 45 | return mContent.get(); 46 | } 47 | ~ VARP() { 48 | // Do nothing 49 | } 50 | VARP(const VARP& var) { 51 | mContent = var.mContent; 52 | } 53 | VARP(VARP&& var) { 54 | mContent = std::move(var.mContent); 55 | } 56 | VARP operator+(VARP var) const; 57 | VARP operator-(VARP var) const; 58 | VARP operator*(VARP var) const; 59 | VARP operator/(VARP var) const; 60 | VARP mean(INTS dims) const; 61 | VARP sum(INTS dims) const; 62 | 63 | bool operator==(const VARP& var) { 64 | return var.mContent == mContent; 65 | } 66 | bool operator<(const VARP& var) { 67 | return mContent < var.mContent; 68 | } 69 | bool operator<=(const VARP& var) { 70 | return mContent <= var.mContent; 71 | } 72 | VARP& operator=(const VARP& var) { 73 | mContent = var.mContent; 74 | return *this; 75 | } 76 | VARP& operator=(Variable* var) { 77 | mContent.reset(var); 78 | return *this; 79 | } 80 | Variable* operator->() const { 81 | return mContent.get(); 82 | } 83 | enum InputType { 84 | INPUT = 0, 85 | CONST = 1, 86 | TRAINABLE = 2, 87 | }; 88 | bool fix(InputType type) const; 89 | private: 90 | std::shared_ptr mContent; 91 | }; 92 | inline bool operator==(Variable* src, VARP dst) { 93 | return src == dst.get(); 94 | } 95 | inline bool operator!=(Variable* src, VARP dst) { 96 | return src != dst.get(); 97 | } 98 | inline bool operator<(VARP src, VARP dst) { 99 | return src.get() < dst.get(); 100 | } 101 | typedef std::vector VARPS; 102 | 103 | class MNN_PUBLIC Variable { 104 | public: 105 | struct Info { 106 | Dimensionformat order = NHWC; 107 | INTS dim; 108 | halide_type_t type; 109 | int size; 110 | void* ptr = nullptr; 111 | void syncSize(); 112 | }; 113 | const std::string& name() const; 114 | void setName(const std::string& name); 115 | std::pair expr() const { 116 | return std::make_pair(mFrom, mFromIndex); 117 | } 118 | // If compute info error, return nullptr 119 | const Info* getInfo(); 120 | bool resize(INTS dims); 121 | template 122 | const T* readMap() { 123 | return (const T*)readInternal(); 124 | } 125 | 126 | template 127 | T* writeMap() { 128 | return (T*)writeInternal(); 129 | } 130 | 131 | //Depecerate 132 | void unMap(); 133 | 134 | bool input(VARP src); 135 | static void replace(VARP dst, VARP src); 136 | 137 | static VARP create(EXPRP expr, int index = 0); 138 | 139 | static std::vector load(const char* fileName); 140 | static std::map loadMap(const char* fileName); 141 | static std::pair, std::map> getInputAndOutput(const std::map& allVariable); 142 | static std::vector mapToSequence(const std::map& source); 143 | static std::vector getExecuteOrder(const std::vector& output); 144 | static void save(const std::vector& vars, const char* fileName); 145 | static void save(const std::vector& vars, NetT* dest); 146 | 147 | // Pack a few Variable to compute in one pipeline 148 | static void prepareCompute(const std::vector& vars); 149 | 150 | size_t linkNumber() const; 151 | const std::vector& toExprs() const; 152 | void setExpr(EXPRP expr, int index) { 153 | mFrom = expr; 154 | mFromIndex = index; 155 | } 156 | private: 157 | Variable(EXPRP expr, int index) { 158 | mFrom = expr; 159 | mFromIndex = index; 160 | } 161 | 162 | void* readInternal(); 163 | void* writeInternal(bool inform=true); 164 | void informDirty(); 165 | 166 | friend class Expr; 167 | EXPRP mFrom; 168 | int mFromIndex; 169 | }; 170 | 171 | class MNN_PUBLIC Expr { 172 | public: 173 | struct Inside; 174 | static EXPRP create(Variable::Info&& info); 175 | static EXPRP create(const OpT* op, std::vector inputs, int outputSize = 1); 176 | static EXPRP create(std::pair, int> extra, std::vector&& inputs, int outputSize = 1); 177 | static EXPRP create(std::unique_ptr&& op, std::vector inputs, int outputSize = 1) { 178 | return create(op.get(), inputs, outputSize); 179 | } 180 | void setName(const std::string& name); 181 | 182 | const Op* get() const { 183 | return mOp; 184 | } 185 | const std::vector& inputs() const { 186 | return mInputs; 187 | } 188 | int outputSize() const { 189 | return mOutputNames.size(); 190 | } 191 | static void replace(EXPRP oldExpr, EXPRP newExpr); 192 | bool requireInfo(); 193 | void visitOutputs(const std::function& visit); 194 | static void visit(EXPRP expr, const std::function& before, const std::function& after); 195 | 196 | const std::vector& outputs() const { 197 | return mTo; 198 | } 199 | ~Expr(); 200 | 201 | bool visited() const { 202 | return mVisited; 203 | } 204 | void setVisited(bool visited) { 205 | mVisited = visited; 206 | } 207 | const std::string& name() const { 208 | return mName; 209 | } 210 | const std::string& outputName(int index) { 211 | return mOutputNames[index]; 212 | } 213 | 214 | VARP::InputType inputType() const {return mType;} 215 | Variable::Info* outputInfo(int index) const; 216 | std::pair, int> extra() const { 217 | return std::make_pair(mExtraBuffer, mOpBufferSize); 218 | } 219 | bool setInfoDirty(); 220 | std::shared_ptr inside() const { 221 | return mInside; 222 | } 223 | bool valid() const { 224 | return mValid; 225 | } 226 | bool infoDirty() const { 227 | return mInfoDirty; 228 | } 229 | private: 230 | static void _addLinkForInputs(EXPRP expr); 231 | 232 | Expr(int outputSize); 233 | 234 | friend class Variable; 235 | friend class VARP; 236 | VARP::InputType mType; 237 | const Op* mOp; 238 | std::vector mInputs; 239 | std::vector mOutputNames; 240 | 241 | bool mValid = true; 242 | bool mInfoDirty = true; 243 | std::shared_ptr mExtraBuffer; 244 | int mOpBufferSize = 0; 245 | std::string mName; 246 | std::shared_ptr mInside = nullptr; 247 | bool mVisited = false; 248 | std::vector mTo; 249 | }; 250 | } // namespace Express 251 | } // namespace MNN 252 | 253 | #endif /* Expr_hpp */ 254 | -------------------------------------------------------------------------------- /include/MNN/expr/ExprCreator.hpp: -------------------------------------------------------------------------------- 1 | // 2 | // ExprCreator.hpp 3 | // MNN 4 | // 5 | // Created by MNN on 2019/06/27. 6 | // Copyright © 2018, Alibaba Group Holding Limited 7 | // 8 | 9 | #ifndef ExprCreator_hpp 10 | #define ExprCreator_hpp 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /include/MNN/expr/MathOp.hpp: -------------------------------------------------------------------------------- 1 | // 2 | // MathOp.hpp 3 | // MNN 4 | // 5 | // Created by MNN on 2019/06/27. 6 | // Copyright © 2018, Alibaba Group Holding Limited 7 | // 8 | 9 | namespace MNN { 10 | namespace Express { 11 | //BinaryOPs 12 | MNN_PUBLIC VARP _Add(VARP x, VARP y); 13 | MNN_PUBLIC VARP _Subtract(VARP x, VARP y); 14 | MNN_PUBLIC VARP _Multiply(VARP x, VARP y); 15 | MNN_PUBLIC VARP _Divide(VARP x, VARP y); 16 | MNN_PUBLIC VARP _Pow(VARP x, VARP y); 17 | MNN_PUBLIC VARP _Minimum(VARP x, VARP y); 18 | MNN_PUBLIC VARP _Maximum(VARP x, VARP y); 19 | MNN_PUBLIC VARP _BiasAdd(VARP value, VARP bias); 20 | MNN_PUBLIC VARP _Greater(VARP x, VARP y); 21 | MNN_PUBLIC VARP _GreaterEqual(VARP x, VARP y); 22 | MNN_PUBLIC VARP _Less(VARP x, VARP y); 23 | MNN_PUBLIC VARP _FloorDiv(VARP x, VARP y); 24 | MNN_PUBLIC VARP _SquaredDifference(VARP x, VARP y); 25 | MNN_PUBLIC VARP _Equal(VARP x, VARP y); 26 | MNN_PUBLIC VARP _LessEqual(VARP x, VARP y); 27 | MNN_PUBLIC VARP _FloorMod(VARP x, VARP y); 28 | 29 | //UnaryOPs 30 | MNN_PUBLIC VARP _Sign(VARP a); 31 | MNN_PUBLIC VARP _Abs(VARP x); 32 | MNN_PUBLIC VARP _Negative(VARP x); 33 | MNN_PUBLIC VARP _Floor(VARP x); 34 | MNN_PUBLIC VARP _Round(VARP x); 35 | MNN_PUBLIC VARP _Ceil(VARP x); 36 | MNN_PUBLIC VARP _Square(VARP x); 37 | MNN_PUBLIC VARP _Sqrt(VARP x); 38 | MNN_PUBLIC VARP _Rsqrt(VARP x); 39 | MNN_PUBLIC VARP _Exp(VARP x); 40 | MNN_PUBLIC VARP _Log(VARP x); 41 | MNN_PUBLIC VARP _Sin(VARP x); 42 | MNN_PUBLIC VARP _Cos(VARP x); 43 | MNN_PUBLIC VARP _Tan(VARP x); 44 | MNN_PUBLIC VARP _Asin(VARP x); 45 | MNN_PUBLIC VARP _Acos(VARP x); 46 | MNN_PUBLIC VARP _Atan(VARP x); 47 | MNN_PUBLIC VARP _Reciprocal(VARP x); 48 | MNN_PUBLIC VARP _Log1p(VARP x); 49 | //Only one but not in UnaryOPs 50 | MNN_PUBLIC VARP _Tanh(VARP x); 51 | MNN_PUBLIC VARP _Sigmoid(VARP x); 52 | 53 | 54 | //ReduceOPs 55 | MNN_PUBLIC VARP _ReduceSum(VARP input_variable, INTS axis = {}, bool keepDims = false); 56 | MNN_PUBLIC VARP _ReduceMean(VARP input_variable, INTS axis = {}, bool keepDims = false); 57 | MNN_PUBLIC VARP _ReduceMax(VARP input_variable, INTS axis = {}, bool keepDims = false); 58 | MNN_PUBLIC VARP _ReduceMin(VARP input_variable, INTS axis = {}, bool keepDims = false); 59 | MNN_PUBLIC VARP _ReduceProd(VARP input_variable, INTS axis = {}, bool keepDims = false); 60 | MNN_PUBLIC VARP _ReduceAny(VARP input_variable, INTS axis = {}, bool keepDims = false); 61 | MNN_PUBLIC VARP _ReduceAll(VARP input_variable, INTS axis = {}, bool keepDims = false); 62 | 63 | //EltwiseOPs 64 | MNN_PUBLIC VARP _Prod(VARP a, VARP b, std::vector coeff); 65 | MNN_PUBLIC VARP _Sum(VARP a, VARP b, std::vector coeff); 66 | MNN_PUBLIC VARP _Max(VARP a, VARP b, std::vector coeff); 67 | MNN_PUBLIC VARP _Sub(VARP a, VARP b, std::vector coeff); 68 | 69 | //OtherOPs 70 | template 71 | MNN_PUBLIC VARP _Cast(VARP x) { 72 | return _Cast(x, halide_type_of()); 73 | } 74 | MNN_PUBLIC VARP _Cast(VARP x, halide_type_t dtype); 75 | MNN_PUBLIC VARP _MatMul(VARP a, VARP b, bool tranposeA = false, bool tranposeB = false); 76 | MNN_PUBLIC VARP _Normalize(VARP x, int32_t acrossSpatial, int32_t channelShared, float eps, std::vector scale); 77 | MNN_PUBLIC VARP _ArgMax(VARP input, int axis = 0); 78 | MNN_PUBLIC VARP _BatchMatMul(VARP x, VARP y, bool adj_x = false, bool adj_y = false); 79 | MNN_PUBLIC VARP _UnravelIndex(VARP indices, VARP dims); 80 | MNN_PUBLIC VARP _ScatterNd(VARP indices, VARP updates, VARP shape); 81 | MNN_PUBLIC VARP _OneHot(VARP indices, VARP depth, VARP onValue, VARP offValue, int axis = -1); 82 | MNN_PUBLIC VARP _BroadcastTo(VARP a, VARP shape); 83 | }; // namespace Express 84 | }; // namespace MNN 85 | -------------------------------------------------------------------------------- /include/MNN/expr/NeuralNetWorkOp.hpp: -------------------------------------------------------------------------------- 1 | // 2 | // NeuralNetWorkOp.hpp 3 | // MNN 4 | // 5 | // Created by MNN on 2019/06/27. 6 | // Copyright © 2018, Alibaba Group Holding Limited 7 | // 8 | 9 | namespace MNN { 10 | namespace Express { 11 | enum PaddingMode {CAFFE, VALID, SAME}; 12 | enum PoolingMode {MAXPOOL, AVEPOOL}; 13 | enum PadValueMode {CONSTANT, REFLECT, SYMMETRIC}; 14 | MNN_PUBLIC VARP _Input(INTS shape = {}, Dimensionformat data_format = NC4HW4, halide_type_t dtype = halide_type_of()) ; 15 | MNN_PUBLIC VARP _Clone(VARP source, bool deepCopy = false); 16 | 17 | MNN_PUBLIC VARP _Scalar(const void* ptr, halide_type_t type); 18 | 19 | template 20 | VARP _Scalar(T value) { 21 | return _Scalar(&value, halide_type_of()); 22 | } 23 | 24 | 25 | MNN_PUBLIC VARP _Const(float value, INTS shape = {}, Dimensionformat format = NHWC); 26 | MNN_PUBLIC VARP _Const(const void* ptr, INTS shape = {}, Dimensionformat format = NHWC, 27 | halide_type_t type = halide_type_of()); 28 | MNN_PUBLIC VARP _TrainableParam(float value, INTS dims, Dimensionformat format); 29 | MNN_PUBLIC VARP _TrainableParam(const void* ptr, INTS dims, Dimensionformat format, 30 | halide_type_t type = halide_type_of()); 31 | MNN_PUBLIC VARP _Conv(VARP weight, VARP bias, VARP x, PaddingMode pad = VALID, INTS stride = {1, 1}, 32 | INTS dilate = {1, 1}, int group = 1, INTS pads = {0, 0}); 33 | 34 | MNN_PUBLIC VARP _Conv(float weight, float bias, VARP x, INTS channel, INTS kernelSize, PaddingMode pad = VALID, 35 | INTS stride = {1, 1}, INTS dilate = {1, 1}, int group = 1); 36 | MNN_PUBLIC VARP _Conv(std::vector&& weight, std::vector&& bias, VARP x, INTS channel, INTS kernelSize, 37 | PaddingMode pad = VALID, INTS stride = {1, 1}, INTS dilate = {1, 1}, int group = 1, INTS pads = {0, 0}); 38 | MNN_PUBLIC VARP _Deconv(VARP weight, VARP bias, VARP x, PaddingMode pad = VALID, INTS stride = {1, 1}, 39 | INTS dilate = {1, 1}, int group = 1, INTS pads = {0, 0}); 40 | MNN_PUBLIC VARP _MaxPool(VARP x, INTS kernel, INTS stride = {1, 1}, PaddingMode pad = VALID, INTS pads= {0, 0}); 41 | MNN_PUBLIC VARP _AvePool(VARP x, INTS kernel, INTS stride = {1, 1}, PaddingMode pad = VALID, INTS pads= {0, 0}); 42 | MNN_PUBLIC VARP _Reshape(VARP x, INTS shape, Dimensionformat original_format = NHWC); 43 | MNN_PUBLIC VARP _Reshape(VARP x, VARP shape); 44 | MNN_PUBLIC VARP _Scale(VARP x, int channels, std::vector&& scales, std::vector&& bias); 45 | 46 | MNN_PUBLIC VARP _Relu(VARP x, float slope = 0.0f); 47 | MNN_PUBLIC VARP _Relu6(VARP x); 48 | MNN_PUBLIC VARP _PRelu(VARP x, std::vector &&slopes); 49 | MNN_PUBLIC VARP _Softmax(VARP logits, int axis = -1); 50 | MNN_PUBLIC VARP _Softplus(VARP features); 51 | MNN_PUBLIC VARP _Softsign(VARP features); 52 | MNN_PUBLIC std::vector _Split(VARP value, INTS size_splits, int axis = 0); 53 | MNN_PUBLIC VARP _Slice(VARP x, VARP starts, VARP sizes); 54 | MNN_PUBLIC VARP _StridedSlice(VARP x, VARP begin, VARP end, VARP strided, halide_type_t type, 55 | int32_t beginMask, int32_t endMask, int32_t ellipsisMask, 56 | int32_t newAxisMask, int32_t shrinkAxisMask); 57 | MNN_PUBLIC VARP _Concat(VARPS values, int axis); 58 | MNN_PUBLIC VARP _Convert(VARP input, Dimensionformat format); 59 | MNN_PUBLIC VARP _Transpose(VARP x, INTS perm); 60 | MNN_PUBLIC VARP _Transpose(VARP x, VARP perm); 61 | MNN_PUBLIC VARP _ChannelShuffle(VARP x, int group); 62 | MNN_PUBLIC VARP _ChangeInputFormat(VARP input, Dimensionformat format); 63 | MNN_PUBLIC VARP _Conv2DBackPropFilter(VARP weight, VARP input, VARP inputGrad, PaddingMode pad = VALID, INTS stride = {1, 1}, INTS dilate = {1, 1}, int group = 1, INTS pads = {0, 0}); 64 | MNN_PUBLIC VARP _PoolGrad(VARP originInput, VARP originOutput, VARP inputGrad, INTS kernel, INTS stride, PoolingMode type, PaddingMode pad = VALID, INTS pads= {0, 0}); 65 | // FIXME: move the api to Array Ops 66 | MNN_PUBLIC VARP _ReverseSequence(VARP x, VARP y, int batchDim, int seqDim); 67 | // FIXME: move the api to Image Ops 68 | MNN_PUBLIC VARP _Crop(VARP images, VARP size, int axis, INTS offset); 69 | MNN_PUBLIC VARP _Resize(VARP images, float xScale, float yScale); 70 | MNN_PUBLIC VARP _Pad(VARP x, VARP paddings, PadValueMode mode = CONSTANT); 71 | MNN_PUBLIC VARP _ExpandDims(VARP input, int axis); 72 | MNN_PUBLIC VARP _ExpandDims(VARP input, VARP axis); 73 | 74 | MNN_PUBLIC VARP _Shape(VARP input); 75 | MNN_PUBLIC VARP _Stack(VARPS values, int axis=0); 76 | enum InterpolationMethod {BILINEAR, NEAREST}; 77 | MNN_PUBLIC VARP _CropAndResize(VARP image, VARP boxes, VARP box_ind, VARP crop_size, 78 | InterpolationMethod method, float extrapolation_value = 0.0); 79 | MNN_PUBLIC VARP _Fill(VARP dims, VARP value); 80 | MNN_PUBLIC VARP _Tile(VARP input, VARP multiples); 81 | MNN_PUBLIC VARP _Gather(VARP params, VARP indices); 82 | MNN_PUBLIC VARP _GatherV2(VARP params, VARP indices, VARP axis = nullptr); 83 | MNN_PUBLIC VARP _Squeeze(VARP input, INTS axis = {}); 84 | MNN_PUBLIC VARP _Unsqueeze(VARP input, INTS axis = {}); 85 | MNN_PUBLIC VARP _BatchToSpaceND(VARP input, VARP block_shape, VARP crops); 86 | MNN_PUBLIC VARP _GatherND(VARP params, VARP indices); 87 | MNN_PUBLIC VARP _Selu(VARP features, float scale, float alpha); 88 | MNN_PUBLIC VARP _Size(VARP input); 89 | MNN_PUBLIC VARP _Elu(VARP features, float alpha=1.0); 90 | MNN_PUBLIC VARP _MatrixBandPart(VARP input, VARP num_lower, VARP num_upper); 91 | MNN_PUBLIC std::vector _Moments(VARP x, INTS axis, VARP shift, bool keepDims); 92 | MNN_PUBLIC VARP _SetDiff1D(VARP x, VARP y); 93 | MNN_PUBLIC VARP _SpaceToDepth(VARP input, int block_size); 94 | MNN_PUBLIC VARP _SpaceToBatchND(VARP input, VARP block_shape, VARP paddings); 95 | MNN_PUBLIC VARP _ZerosLike(VARP input); 96 | MNN_PUBLIC std::vector _Unstack(VARP value, int axis=0); 97 | MNN_PUBLIC VARP _Rank(VARP input); 98 | MNN_PUBLIC VARP _Range(VARP start, VARP limit, VARP delta); 99 | MNN_PUBLIC VARP _DepthToSpace(VARP input, int block_size); 100 | MNN_PUBLIC VARP _PriorBox(VARP feature, VARP image, 101 | std::vector min_size, std::vector max_size, std::vectoraspect_ratio, 102 | bool flip, bool clip, std::vectorvariance, 103 | unsigned int img_h, unsigned int img_w, float step_h, float step_w, float offset = 0.5); 104 | MNN_PUBLIC VARP _Permute(VARP input, INTS dims); 105 | MNN_PUBLIC VARP _DetectionOutput(VARP location, VARP confidence, VARP priorbox, 106 | unsigned int num_classes, bool share_location, int background_label_id, 107 | float nms_threshhold, int nms_topk, int code_type, 108 | bool variance_encoded_in_target, 109 | int keep_top_k, float confidence_threshold, float visualize_threshold); 110 | MNN_PUBLIC VARP _Interp(VARPS xs, float widthScale, float heightScale, int outputWidth, int outputHeight, int resizeType, bool alignCorners); 111 | 112 | MNN_PUBLIC VARP _ZeroGrad(VARP x); 113 | MNN_PUBLIC VARP _Conv(std::vector&& weight, std::vector&& bias, std::vector&& scale, VARP x, INTS channel, INTS kernelSize, 114 | PaddingMode pad, INTS stride, INTS dilate, int group, INTS pads); 115 | 116 | } // namespace Express 117 | } // namespace MNN 118 | -------------------------------------------------------------------------------- /include/MNN/expr/Optimizer.hpp: -------------------------------------------------------------------------------- 1 | // 2 | // Optimizer.hpp 3 | // MNN 4 | // 5 | // Created by MNN on 2019/08/20. 6 | // Copyright © 2018, Alibaba Group Holding Limited 7 | // 8 | #ifndef Optimizer_hpp 9 | #define Optimizer_hpp 10 | #include 11 | #include 12 | 13 | namespace MNN { 14 | namespace Express { 15 | class MNN_PUBLIC Optimizer { 16 | public: 17 | enum Device { 18 | CPU = 0, 19 | GPU = 1, 20 | OTHER = 2, 21 | AUTO = 3 22 | }; 23 | struct Config { 24 | Device device = CPU; 25 | MNNForwardType forwardType = MNN_FORWARD_ALL; 26 | int numThread = 4; 27 | }; 28 | static std::shared_ptr create(Config config); 29 | struct Cost { 30 | float compute; // MFlops 31 | float memory; // MB 32 | }; 33 | class Parameters { 34 | public: 35 | Parameters(int n); 36 | virtual ~Parameters(); 37 | 38 | float* get() const { 39 | return mValue; 40 | } 41 | int size() const { 42 | return mSize; 43 | } 44 | 45 | private: 46 | float* mValue; 47 | int mSize; 48 | }; 49 | virtual std::shared_ptr onGetParameters(const std::vector& outputs) { 50 | return nullptr; 51 | } 52 | 53 | //Given paramters and measure cost, the parameters must be the same as onGetParameters 54 | virtual Cost onMeasure(const std::vector& outputs, std::shared_ptr parameters = nullptr) = 0; 55 | 56 | //Modify the output directly, the parameters must be the same as onGetParameters 57 | virtual bool onExecute(const std::vector& outputs, std::shared_ptr parameters = nullptr) = 0; 58 | 59 | Optimizer() = default; 60 | virtual ~Optimizer() = default; 61 | }; 62 | } // namespace Express 63 | } // namespace MNN 64 | #endif 65 | -------------------------------------------------------------------------------- /lib/libMNN.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MirrorYuChen/mnn_example/c07285b685af504e9fff40b7a0cc99b0a72f47d8/lib/libMNN.so -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.1) 2 | project(MNNTest) 3 | 4 | find_package(OpenCV) 5 | message("Opencv Version: ${OPENCV_VERSION}" ) 6 | 7 | file(GLOB LIBS ${CMAKE_CURRENT_SOURCE_DIR}/../lib/*.so) 8 | 9 | set(SRC_FILES 10 | ${PROJECT_SOURCE_DIR}/common/common.cpp 11 | ${PROJECT_SOURCE_DIR}/classifier/classifier.cpp 12 | 13 | ${PROJECT_SOURCE_DIR}/object/mobilenetssd/mobilenetssd.cpp 14 | ${PROJECT_SOURCE_DIR}/object/object_engine.cpp 15 | 16 | ${PROJECT_SOURCE_DIR}/face/detecter/detecter.cpp 17 | ${PROJECT_SOURCE_DIR}/face/detecter/centerface/centerface.cpp 18 | ${PROJECT_SOURCE_DIR}/face/detecter/ultraface/ultraface.cpp 19 | ${PROJECT_SOURCE_DIR}/face/landmarker/landmarker.cpp 20 | ${PROJECT_SOURCE_DIR}/face/landmarker/pfldlandmarker/pfldlandmarker.cpp 21 | ${PROJECT_SOURCE_DIR}/face/landmarker/zqlandmarker/zqlandmarker.cpp 22 | ${PROJECT_SOURCE_DIR}/face/aligner/aligner.cpp 23 | ${PROJECT_SOURCE_DIR}/face/recognizer/recognizer.cpp 24 | ${PROJECT_SOURCE_DIR}/face/recognizer/mobilefacenet/mobilefacenet.cpp 25 | ${PROJECT_SOURCE_DIR}/face/database/face_database.cpp 26 | ${PROJECT_SOURCE_DIR}/face/database/stream/file_stream.cpp 27 | ${PROJECT_SOURCE_DIR}/face/face_engine.cpp 28 | 29 | ${PROJECT_SOURCE_DIR}/vision_engine.cpp 30 | ) 31 | add_library(VisionEngine STATIC ${SRC_FILES}) 32 | 33 | FIND_PACKAGE( OpenMP REQUIRED) 34 | if(OPENMP_FOUND) 35 | message("OPENMP FOUND") 36 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") 37 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") 38 | set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") 39 | endif() 40 | 41 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fPIC -O3 -std=c++11 -fopenmp") 42 | target_link_libraries(VisionEngine PUBLIC ${LIBS} ${OpenCV_LIBS}) 43 | 44 | target_include_directories(VisionEngine 45 | PUBLIC 46 | $ 47 | $ 48 | $ 49 | $ 50 | 51 | $ 52 | $ 53 | 54 | $ 55 | $ 56 | $ 57 | $ 58 | $ 59 | $ 60 | $ 61 | $ 62 | $ 63 | $ 64 | $ 65 | $ 66 | ) 67 | 68 | if(MIRROR_BUILD_CLASSIFIER) 69 | add_executable(classifier ${CMAKE_CURRENT_SOURCE_DIR}/classifier/test_classifier.cpp) 70 | target_link_libraries(classifier PUBLIC VisionEngine) 71 | endif() 72 | 73 | if(MIRROR_BUILD_OBJECT) 74 | add_executable(object ${CMAKE_CURRENT_SOURCE_DIR}/object/test_object.cpp) 75 | target_link_libraries(object PUBLIC VisionEngine) 76 | endif() 77 | 78 | if(MIRROR_BUILD_FACE) 79 | add_executable(face ${CMAKE_CURRENT_SOURCE_DIR}/face/test_face.cpp) 80 | target_link_libraries(face PUBLIC VisionEngine) 81 | endif() 82 | 83 | -------------------------------------------------------------------------------- /src/classifier/classifier.cpp: -------------------------------------------------------------------------------- 1 | #include "classifier.h" 2 | #include 3 | #include 4 | 5 | #include "opencv2/imgproc.hpp" 6 | 7 | namespace mirror { 8 | 9 | Classifier::Classifier() { 10 | labels_.clear(); 11 | initialized_ = false; 12 | topk_ = 5; 13 | } 14 | 15 | Classifier::~Classifier() { 16 | classifier_interpreter_->releaseModel(); 17 | classifier_interpreter_->releaseSession(classifier_sess_); 18 | } 19 | 20 | int Classifier::Init(const char* root_path) { 21 | std::cout << "start Init." << std::endl; 22 | std::string model_file = std::string(root_path) + "/mobilenet.mnn"; 23 | classifier_interpreter_ = std::shared_ptr(MNN::Interpreter::createFromFile(model_file.c_str())); 24 | 25 | if (!classifier_interpreter_ || LoadLabels(root_path) != 0) { 26 | std::cout << "load model failed." << std::endl; 27 | return 10000; 28 | } 29 | 30 | MNN::ScheduleConfig schedule_config; 31 | schedule_config.type = MNN_FORWARD_CPU; 32 | schedule_config.numThread = 1; 33 | MNN::BackendConfig backend_config; 34 | backend_config.precision = MNN::BackendConfig::Precision_Normal; 35 | schedule_config.backendConfig = &backend_config; 36 | 37 | classifier_sess_ = classifier_interpreter_->createSession(schedule_config); 38 | input_tensor_ = classifier_interpreter_->getSessionInput(classifier_sess_, nullptr); 39 | 40 | classifier_interpreter_->resizeTensor(input_tensor_, {1, 3, inputSize_.height, inputSize_.width}); 41 | classifier_interpreter_->resizeSession(classifier_sess_); 42 | 43 | std::cout << "End Init." << std::endl; 44 | 45 | initialized_ = true; 46 | 47 | return 0; 48 | } 49 | 50 | int Classifier::Classify(const cv::Mat& img_src, std::vector* images) { 51 | std::cout << "start classify." << std::endl; 52 | images->clear(); 53 | if (!initialized_) { 54 | std::cout << "model uninitialized." << std::endl; 55 | return 10000; 56 | } 57 | 58 | if (img_src.empty()) { 59 | std::cout << "input empty." << std::endl; 60 | return 10001; 61 | } 62 | 63 | cv::Mat img_resized; 64 | cv::resize(img_src.clone(), img_resized, inputSize_); 65 | std::shared_ptr pretreat( 66 | MNN::CV::ImageProcess::create(MNN::CV::BGR, MNN::CV::RGB, meanVals, 3, normVals, 3) 67 | ); 68 | pretreat->convert((uint8_t*)img_resized.data, inputSize_.width, inputSize_.height, img_resized.step[0], input_tensor_); 69 | 70 | // forward 71 | classifier_interpreter_->runSession(classifier_sess_); 72 | 73 | // get output 74 | // mobilenet: "classifierV1/Predictions/Reshape_1" 75 | MNN::Tensor* output_score = classifier_interpreter_->getSessionOutput(classifier_sess_, nullptr); 76 | 77 | // copy to host 78 | MNN::Tensor score_host(output_score, output_score->getDimensionType()); 79 | output_score->copyToHostTensor(&score_host); 80 | 81 | auto score_ptr = score_host.host(); 82 | std::vector> scores; 83 | for (int i = 0; i < 1000; ++i) { 84 | float score = score_ptr[i]; 85 | scores.push_back(std::make_pair(score, i)); 86 | } 87 | 88 | std::partial_sort(scores.begin(), scores.begin() + topk_, scores.end(), std::greater< std::pair >()); 89 | for (int i = 0; i < topk_; ++i) { 90 | ImageInfo image_info; 91 | image_info.label_ = labels_[scores[i].second]; 92 | image_info.score_ = scores[i].first; 93 | images->push_back(image_info); 94 | } 95 | 96 | std::cout << "end classify." << std::endl; 97 | 98 | return 0; 99 | } 100 | 101 | 102 | int Classifier::LoadLabels(const char* root_path) { 103 | std::string label_file = std::string(root_path) + "/label.txt"; 104 | FILE* fp = fopen(label_file.c_str(), "r"); 105 | while (!feof(fp)) { 106 | char str[1024]; 107 | if (fgets(str, 1024, fp) == nullptr) continue; 108 | std::string str_s(str); 109 | 110 | if (str_s.length() > 0) { 111 | for (int i = 0; i < str_s.length(); i++) { 112 | if (str_s[i] == ' ') { 113 | std::string strr = str_s.substr(i, str_s.length() - i - 1); 114 | labels_.push_back(strr); 115 | i = str_s.length(); 116 | } 117 | } 118 | } 119 | } 120 | return 0; 121 | } 122 | 123 | 124 | 125 | } -------------------------------------------------------------------------------- /src/classifier/classifier.h: -------------------------------------------------------------------------------- 1 | #ifndef _CLASSIFIER_H_ 2 | #define _CLASSIFIER_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "MNN/Interpreter.hpp" 9 | #include "MNN/ImageProcess.hpp" 10 | #include "MNN/MNNDefine.h" 11 | #include "MNN/Tensor.hpp" 12 | 13 | #include "opencv2/core.hpp" 14 | #include "../common/common.h" 15 | 16 | namespace mirror { 17 | class Classifier { 18 | public: 19 | Classifier(); 20 | ~Classifier(); 21 | int Init(const char* root_path); 22 | int Classify(const cv::Mat& img_src, std::vector* images); 23 | 24 | private: 25 | bool initialized_; 26 | std::shared_ptr classifier_interpreter_; 27 | MNN::Session* classifier_sess_ = nullptr; 28 | MNN::Tensor* input_tensor_ = nullptr; 29 | 30 | const cv::Size inputSize_ = cv::Size(227, 227); 31 | const float meanVals[3] = { 103.94f, 116.78f, 123.68f }; 32 | const float normVals[3] = { 0.017f, 0.017f, 0.017f }; 33 | 34 | std::vector labels_; 35 | int topk_; 36 | 37 | int LoadLabels(const char* root_path); 38 | 39 | }; 40 | 41 | } 42 | 43 | 44 | 45 | #endif // !_CLASSIFIER_H_ 46 | -------------------------------------------------------------------------------- /src/classifier/test_classifier.cpp: -------------------------------------------------------------------------------- 1 | #include "opencv2/opencv.hpp" 2 | #include "opencv2/highgui.hpp" 3 | #include "../vision_engine.h" 4 | 5 | int main(int argc, char* argv[]) { 6 | const char* img_path = "../../data/images/classify.jpg"; 7 | cv::Mat img_src = cv::imread(img_path); 8 | 9 | const char* root_path = "../../data/models"; 10 | mirror::VisionEngine* vision_engine = new mirror::VisionEngine(); 11 | 12 | vision_engine->Init(root_path); 13 | std::vector images; 14 | vision_engine->Classify(img_src, &images); 15 | 16 | int topk = images.size(); 17 | for (int i = 0; i < topk; ++i) { 18 | cv::putText(img_src, images[i].label_, cv::Point(10, 10 + 30 * i), 19 | cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(255, 100, 0), 2, 2); 20 | } 21 | 22 | cv::imwrite("../../data/images/classify_result.jpg", img_src); 23 | cv::imshow("result", img_src); 24 | cv::waitKey(0); 25 | 26 | delete vision_engine; 27 | return 0; 28 | } -------------------------------------------------------------------------------- /src/common/common.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | #include 3 | #include 4 | #include "opencv2/imgproc.hpp" 5 | 6 | 7 | namespace mirror { 8 | float InterRectArea(const cv::Rect& a, const cv::Rect& b) { 9 | cv::Point left_top = cv::Point(MAX(a.x, b.x), MAX(a.y, b.y)); 10 | cv::Point right_bottom = cv::Point(MIN(a.br().x, b.br().x), MIN(a.br().y, b.br().y)); 11 | cv::Point diff = right_bottom - left_top; 12 | return (MAX(diff.x + 1, 0) * MAX(diff.y + 1, 0)); 13 | } 14 | 15 | int ComputeIOU(const cv::Rect& rect1, 16 | const cv::Rect& rect2, float* iou, 17 | const std::string& type) { 18 | 19 | float inter_area = InterRectArea(rect1, rect2); 20 | if (type == "MIN") { 21 | *iou = inter_area / MIN(rect1.area(), rect2.area()); 22 | } 23 | else { 24 | *iou = inter_area / (rect1.area() + rect2.area() - inter_area); 25 | } 26 | 27 | return 0; 28 | } 29 | 30 | int GenerateAnchors(const int & width, const int & height, 31 | const std::vector>& min_boxes, const std::vector& strides, 32 | std::vector>* anchors) { 33 | std::cout << "start generate priors." << std::endl; 34 | anchors->clear(); 35 | int num_strides = static_cast(strides.size()); 36 | for (int i = 0; i < num_strides; ++i) { 37 | auto stride = strides[i]; 38 | float scale_x = width / stride; 39 | float scale_y = height / stride; 40 | 41 | int num_x = ceil(width / stride); 42 | int num_y = ceil(height / stride); 43 | for (int y = 0; y < num_y; ++y) { 44 | for (int x = 0; x < num_x; ++x) { 45 | float center_x = (x + 0.5f) / scale_x; 46 | float center_y = (y + 0.5f) / scale_y; 47 | for (auto min_box : min_boxes[i]) { 48 | float center_w = min_box / width; 49 | float center_h = min_box / height; 50 | anchors->push_back({ Clip(center_x, 1.0f), Clip(center_y, 1.0f), 51 | Clip(center_w, 1.0f), Clip(center_h, 1.0f) }); 52 | } 53 | } 54 | } 55 | } 56 | 57 | std::cout << "end generate priors." << std::endl; 58 | 59 | return 0; 60 | } 61 | 62 | float CalculateSimilarity(const std::vector&feat1, const std::vector& feat2) { 63 | if (feat1.size() != feat2.size()) { 64 | std::cout << "feature size not match." << std::endl; 65 | return 10003; 66 | } 67 | float inner_product = 0.0f; 68 | float feat_norm1 = 0.0f; 69 | float feat_norm2 = 0.0f; 70 | #if defined(_OPENMP) 71 | #pragma omp parallel for num_threads(threads_num) 72 | #endif 73 | for(int i = 0; i < kFaceFeatureDim; ++i) { 74 | inner_product += feat1[i] * feat2[i]; 75 | feat_norm1 += feat1[i] * feat1[i]; 76 | feat_norm2 += feat2[i] * feat2[i]; 77 | } 78 | return inner_product / sqrt(feat_norm1) / sqrt(feat_norm2); 79 | } 80 | 81 | float Logists(const float& value) { 82 | return 1.0f / (1.0f + exp(-value));; 83 | } 84 | 85 | } -------------------------------------------------------------------------------- /src/common/common.h: -------------------------------------------------------------------------------- 1 | #ifndef _VISION_COMMON_H_ 2 | #define _VISION_COMMON_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "opencv2/core.hpp" 8 | 9 | namespace mirror { 10 | #define kFaceFeatureDim 128 11 | #define kFaceNameDim 256 12 | #define Clip(x, y) (x < 0 ? 0 : (x > y ? y : x)) 13 | const int threads_num = 2; 14 | 15 | struct ImageInfo { 16 | std::string label_; 17 | float score_; 18 | }; 19 | 20 | struct ObjectInfo { 21 | std::string name_; 22 | cv::Rect location_; 23 | float score_; 24 | }; 25 | 26 | struct FaceInfo { 27 | cv::Rect location_; 28 | float score_; 29 | float keypoints_[10]; 30 | }; 31 | 32 | struct QueryResult { 33 | std::string name_; 34 | float sim_; 35 | }; 36 | 37 | float InterRectArea(const cv::Rect& a, const cv::Rect& b); 38 | int ComputeIOU(const cv::Rect& rect1, const cv::Rect& rect2, float* iou, const std::string& type = "UNION"); 39 | int GenerateAnchors(const int& width, const int& height, const std::vector>& min_boxes, 40 | const std::vector& strides, std::vector>* anchors); 41 | 42 | template 43 | int const NMS(const std::vector& inputs, std::vector* result, 44 | const float& threshold, const std::string& type = "UNION") { 45 | result->clear(); 46 | if (inputs.size() == 0) 47 | return -1; 48 | 49 | std::vector inputs_tmp; 50 | inputs_tmp.assign(inputs.begin(), inputs.end()); 51 | std::sort(inputs_tmp.begin(), inputs_tmp.end(), 52 | [](const T& a, const T& b) { 53 | return a.score_ > b.score_; 54 | }); 55 | 56 | std::vector indexes(inputs_tmp.size()); 57 | 58 | for (int i = 0; i < indexes.size(); i++) { 59 | indexes[i] = i; 60 | } 61 | 62 | while (indexes.size() > 0) { 63 | int index_good = indexes[0]; 64 | std::vector indexes_tmp = indexes; 65 | indexes.clear(); 66 | std::vector indexes_nms; 67 | indexes_nms.push_back(index_good); 68 | float total = exp(inputs_tmp[index_good].score_); 69 | for (int i = 1; i < static_cast(indexes_tmp.size()); ++i) { 70 | int index_tmp = indexes_tmp[i]; 71 | float iou = 0.0f; 72 | ComputeIOU(inputs_tmp[index_good].location_, inputs_tmp[index_tmp].location_, &iou, type); 73 | if (iou <= threshold) { 74 | indexes.push_back(index_tmp); 75 | } else { 76 | indexes_nms.push_back(index_tmp); 77 | total += exp(inputs_tmp[index_tmp].score_); 78 | } 79 | } 80 | if ("BLENDING" == type) { 81 | T t; 82 | memset(&t, 0, sizeof(t)); 83 | for (auto index : indexes_nms) { 84 | float rate = exp(inputs_tmp[index].score_) / total; 85 | t.score_ += rate * inputs_tmp[index].score_; 86 | t.location_.x += rate * inputs_tmp[index].location_.x; 87 | t.location_.y += rate * inputs_tmp[index].location_.y; 88 | t.location_.width += rate * inputs_tmp[index].location_.width; 89 | t.location_.height += rate * inputs_tmp[index].location_.height; 90 | } 91 | result->push_back(t); 92 | } else { 93 | result->push_back(inputs_tmp[index_good]); 94 | } 95 | 96 | } 97 | return 0; 98 | } 99 | 100 | float CalculateSimilarity(const std::vector&feat1, const std::vector& feat2); 101 | float Logists(const float& value); 102 | 103 | } 104 | 105 | 106 | 107 | #endif // !_VISION_COMMON_H_ 108 | -------------------------------------------------------------------------------- /src/face/aligner/aligner.cpp: -------------------------------------------------------------------------------- 1 | #include "aligner.h" 2 | #include 3 | #include "opencv2/core.hpp" 4 | #include "opencv2/imgproc.hpp" 5 | 6 | namespace mirror { 7 | class Aligner::Impl { 8 | public: 9 | int AlignFace(const cv::Mat& img_src, const std::vector& keypoints, cv::Mat* face_aligned); 10 | 11 | private: 12 | cv::Mat MeanAxis0(const cv::Mat &src); 13 | cv::Mat ElementwiseMinus(const cv::Mat &A, const cv::Mat &B); 14 | cv::Mat VarAxis0(const cv::Mat &src); 15 | int MatrixRank(const cv::Mat& M); 16 | cv::Mat SimilarTransform(const cv::Mat& src, const cv::Mat& dst); 17 | 18 | float points_dst[5][2] = { 19 | { 30.2946f + 8.0f, 51.6963f }, 20 | { 65.5318f + 8.0f, 51.5014f }, 21 | { 48.0252f + 8.0f, 71.7366f }, 22 | { 33.5493f + 8.0f, 92.3655f }, 23 | { 62.7299f + 8.0f, 92.2041f } 24 | }; 25 | 26 | float* GetSrcPoints(const std::vector& keypoints) { 27 | 28 | } 29 | }; 30 | 31 | 32 | Aligner::Aligner() { 33 | impl_ = new Impl(); 34 | } 35 | 36 | Aligner::~Aligner() { 37 | if (impl_) { 38 | delete impl_; 39 | } 40 | } 41 | 42 | int Aligner::AlignFace(const cv::Mat & img_src, 43 | const std::vector& keypoints, cv::Mat * face_aligned) { 44 | return impl_->AlignFace(img_src, keypoints, face_aligned); 45 | } 46 | 47 | int Aligner::Impl::AlignFace(const cv::Mat & img_src, 48 | const std::vector& keypoints, cv::Mat * face_aligned) { 49 | std::cout << "start align face." << std::endl; 50 | if (img_src.empty()) { 51 | std::cout << "input empty." << std::endl; 52 | return 10001; 53 | } 54 | if (keypoints.size() == 0) { 55 | std::cout << "keypoints empty." << std::endl; 56 | return 10001; 57 | } 58 | int num_keypoints = static_cast(keypoints.size()); 59 | std::vector points_src(5); 60 | switch (num_keypoints) { 61 | case 98: 62 | points_src[0] = keypoints[96]; 63 | points_src[1] = keypoints[97]; 64 | points_src[2] = keypoints[54]; 65 | points_src[3] = keypoints[76]; 66 | points_src[4] = keypoints[82]; 67 | break; 68 | case 106: 69 | points_src[0] = keypoints[104]; 70 | points_src[1] = keypoints[105]; 71 | points_src[2] = keypoints[46]; 72 | points_src[3] = keypoints[84]; 73 | points_src[4] = keypoints[90]; 74 | break; 75 | default: 76 | std::cout << "error." << std::endl; 77 | break; 78 | } 79 | 80 | cv::Mat src_mat(5, 2, CV_32FC1, &points_src[0]); 81 | cv::Mat dst_mat(5, 2, CV_32FC1, points_dst); 82 | cv::Mat transform = SimilarTransform(src_mat, dst_mat); 83 | face_aligned->create(112, 112, CV_32FC3); 84 | cv::Mat transfer_mat = transform(cv::Rect(0, 0, 3, 2)); 85 | cv::warpAffine(img_src.clone(), *face_aligned, transfer_mat, cv::Size(112, 112), 1, 0, 0); 86 | 87 | std::cout << "end align face." << std::endl; 88 | return 0; 89 | } 90 | 91 | cv::Mat Aligner::Impl::MeanAxis0(const cv::Mat & src) { 92 | int num = src.rows; 93 | int dim = src.cols; 94 | 95 | // x1 y1 96 | // x2 y2 97 | cv::Mat output(1, dim, CV_32FC1); 98 | for (int i = 0; i < dim; i++) { 99 | float sum = 0; 100 | for (int j = 0; j < num; j++) { 101 | sum += src.at(j, i); 102 | } 103 | output.at(0, i) = sum / num; 104 | } 105 | 106 | return output; 107 | } 108 | 109 | cv::Mat Aligner::Impl::ElementwiseMinus(const cv::Mat & A, const cv::Mat & B) { 110 | cv::Mat output(A.rows, A.cols, A.type()); 111 | assert(B.cols == A.cols); 112 | if (B.cols == A.cols) { 113 | for (int i = 0; i < A.rows; i++) { 114 | for (int j = 0; j < B.cols; j++) { 115 | output.at(i, j) = A.at(i, j) - B.at(0, j); 116 | } 117 | } 118 | } 119 | 120 | return output; 121 | } 122 | 123 | cv::Mat Aligner::Impl::VarAxis0(const cv::Mat & src) { 124 | cv::Mat temp_ = ElementwiseMinus(src, MeanAxis0(src)); 125 | cv::multiply(temp_, temp_, temp_); 126 | return MeanAxis0(temp_); 127 | } 128 | 129 | int Aligner::Impl::MatrixRank(const cv::Mat& M) { 130 | cv::Mat w, u, vt; 131 | cv::SVD::compute(M.clone(), w, u, vt); 132 | cv::Mat1b nonZeroSingularValues = w > 0.0001; 133 | int rank = countNonZero(nonZeroSingularValues); 134 | return rank; 135 | } 136 | 137 | /* 138 | References: "Least-squares estimation of transformation parameters between two point patterns", Shinji Umeyama, PAMI 1991, DOI: 10.1109/34.88573 139 | Anthor: Jack Yu 140 | */ 141 | cv::Mat Aligner::Impl::SimilarTransform(const cv::Mat & src, const cv::Mat & dst) { 142 | int num = src.rows; 143 | int dim = src.cols; 144 | cv::Mat src_mean = MeanAxis0(src); 145 | cv::Mat dst_mean = MeanAxis0(dst); 146 | cv::Mat src_demean = ElementwiseMinus(src, src_mean); 147 | cv::Mat dst_demean = ElementwiseMinus(dst, dst_mean); 148 | cv::Mat A = (dst_demean.t() * src_demean) / static_cast(num); 149 | cv::Mat d(dim, 1, CV_32F); 150 | d.setTo(1.0f); 151 | if (cv::determinant(A) < 0) { 152 | d.at(dim - 1, 0) = -1; 153 | 154 | } 155 | cv::Mat T = cv::Mat::eye(dim + 1, dim + 1, CV_32F); 156 | cv::Mat U, S, V; 157 | cv::SVD::compute(A, S, U, V); 158 | 159 | // the SVD function in opencv differ from scipy . 160 | 161 | int rank = MatrixRank(A); 162 | if (rank == 0) { 163 | assert(rank == 0); 164 | 165 | } else if (rank == dim - 1) { 166 | if (cv::determinant(U) * cv::determinant(V) > 0) { 167 | T.rowRange(0, dim).colRange(0, dim) = U * V; 168 | } else { 169 | int s = d.at(dim - 1, 0) = -1; 170 | d.at(dim - 1, 0) = -1; 171 | 172 | T.rowRange(0, dim).colRange(0, dim) = U * V; 173 | cv::Mat diag_ = cv::Mat::diag(d); 174 | cv::Mat twp = diag_ * V; //np.dot(np.diag(d), V.T) 175 | cv::Mat B = cv::Mat::zeros(3, 3, CV_8UC1); 176 | cv::Mat C = B.diag(0); 177 | T.rowRange(0, dim).colRange(0, dim) = U * twp; 178 | d.at(dim - 1, 0) = s; 179 | } 180 | } else { 181 | cv::Mat diag_ = cv::Mat::diag(d); 182 | cv::Mat twp = diag_ * V.t(); //np.dot(np.diag(d), V.T) 183 | cv::Mat res = U * twp; // U 184 | T.rowRange(0, dim).colRange(0, dim) = -U.t()* twp; 185 | } 186 | cv::Mat var_ = VarAxis0(src_demean); 187 | float val = cv::sum(var_).val[0]; 188 | cv::Mat res; 189 | cv::multiply(d, S, res); 190 | float scale = 1.0 / val * cv::sum(res).val[0]; 191 | T.rowRange(0, dim).colRange(0, dim) = -T.rowRange(0, dim).colRange(0, dim).t(); 192 | cv::Mat temp1 = T.rowRange(0, dim).colRange(0, dim); // T[:dim, :dim] 193 | cv::Mat temp2 = src_mean.t(); 194 | cv::Mat temp3 = temp1 * temp2; 195 | cv::Mat temp4 = scale * temp3; 196 | T.rowRange(0, dim).colRange(dim, dim + 1) = -(temp4 - dst_mean.t()); 197 | T.rowRange(0, dim).colRange(0, dim) *= scale; 198 | return T; 199 | } 200 | 201 | } 202 | -------------------------------------------------------------------------------- /src/face/aligner/aligner.h: -------------------------------------------------------------------------------- 1 | #ifndef _FACE_ALIGNER_H_ 2 | #define _FACE_ALIGNER_H_ 3 | 4 | #include "opencv2/core.hpp" 5 | 6 | namespace mirror { 7 | class Aligner { 8 | public: 9 | Aligner(); 10 | ~Aligner(); 11 | 12 | int AlignFace(const cv::Mat& img_src, 13 | const std::vector& keypoints, cv::Mat* face_aligned); 14 | 15 | private: 16 | class Impl; 17 | Impl* impl_; 18 | }; 19 | 20 | } 21 | 22 | #endif // !_FACE_ALIGNER_H_ 23 | 24 | -------------------------------------------------------------------------------- /src/face/database/face_database.cpp: -------------------------------------------------------------------------------- 1 | #include "face_database.h" 2 | #include 3 | 4 | namespace mirror { 5 | 6 | class FaceDatabase::Impl { 7 | public: 8 | Impl() { 9 | max_index_ = 0; 10 | features_db_.clear(); 11 | } 12 | 13 | ~Impl() { 14 | 15 | } 16 | 17 | bool Save(StreamWriter& writer) const { 18 | const uint64_t num_faces = db_.size(); 19 | const uint64_t dim_feat = kFaceFeatureDim; 20 | const uint64_t dim_name = kFaceNameDim; 21 | 22 | Write(writer, num_faces); 23 | for (auto& line : db_) { 24 | auto& name = line.first; 25 | auto& feat = line.second; 26 | 27 | char name_arr[kFaceNameDim]; 28 | sprintf(name_arr, "%s", name.c_str()); 29 | 30 | Write(writer, name_arr, size_t(dim_name)); 31 | Write(writer, &feat[0], size_t(dim_feat)); 32 | 33 | } 34 | 35 | std::cout << "FaceDatabase Loaded " << num_faces << " faces" << std::endl; 36 | return true; 37 | } 38 | 39 | bool Load(StreamReader& reader) { 40 | uint64_t num_faces = 0; 41 | const uint64_t dim_feat = kFaceFeatureDim; 42 | const uint64_t dim_name = kFaceNameDim; 43 | 44 | Read(reader, num_faces); 45 | std::cout << "number faces is: " << num_faces << std::endl; 46 | 47 | db_.clear(); 48 | max_index_ = -1; 49 | 50 | for (size_t i = 0; i < num_faces; ++i) { 51 | char name_arr[kFaceNameDim]; 52 | Read(reader, name_arr, size_t(dim_name)); 53 | std::cout << "name is: " << name_arr << std::endl; 54 | 55 | std::vector feat(kFaceFeatureDim); 56 | Read(reader, &feat[0], size_t(dim_feat)); 57 | 58 | db_.insert(std::make_pair(std::string(name_arr), feat)); 59 | max_index_ = (max_index_ > i ? max_index_ : i); 60 | } 61 | ++max_index_; 62 | 63 | std::cout << "FaceDatabase Loaded " << num_faces << " faces" << std::endl; 64 | 65 | return true; 66 | } 67 | 68 | 69 | int64_t Insert(const std::string& name, const std::vector& feat) { 70 | int64_t new_index = max_index_++; 71 | std::cout << "new index is: " << new_index << std::endl; 72 | db_.insert(std::make_pair(name, feat)); 73 | return new_index; 74 | } 75 | 76 | int Delete(const std::string& name) { 77 | std::map>::iterator it = db_.find(name); 78 | if (it != db_.end()) { 79 | db_.erase(it); 80 | std::cout << "Delete: " << name << " successed." << std::endl; 81 | } 82 | return 0; 83 | } 84 | 85 | void Clear() { 86 | db_.clear(); 87 | max_index_ = 0; 88 | } 89 | 90 | 91 | 92 | float CalculateSimilarity(const std::vector& feat1, 93 | const std::vector& feat2) { 94 | double dot = 0; 95 | double norm1 = 0; 96 | double norm2 = 0; 97 | for (size_t i = 0; i < kFaceFeatureDim; ++i) { 98 | dot += feat1[i] * feat2[i]; 99 | norm1 += feat1[i] * feat1[i]; 100 | norm2 += feat2[i] * feat2[i]; 101 | } 102 | 103 | return dot / (sqrt(norm1 * norm2) + 1e-5); 104 | } 105 | 106 | bool Compare(const std::vector& feat1, 107 | const std::vector& feat2, float *similarity) { 108 | if (feat1.size() == 0 || feat2.size() == 0 || !similarity) return false; 109 | *similarity = CalculateSimilarity(feat1, feat2); 110 | return true; 111 | } 112 | 113 | size_t QueryTop(const std::vector& feat, 114 | QueryResult *query_result) { 115 | std::vector> result(db_.size()); { 116 | size_t i = 0; 117 | for (auto &line : db_) { 118 | result[i].first = line.first; 119 | Compare(feat, line.second, &result[i].second); 120 | i++; 121 | } 122 | } 123 | 124 | std::partial_sort(result.begin(), result.begin() + 1, result.end(), []( 125 | const std::pair &a, const std::pair &b) -> bool { 126 | return a.second > b.second; 127 | }); 128 | 129 | query_result->name_ = result[0].first; 130 | query_result->sim_ = result[0].second; 131 | 132 | return 0; 133 | } 134 | 135 | 136 | private: 137 | std::map> features_db_; 138 | std::map names_db_; 139 | std::map> db_; 140 | int64_t max_index_ = 0; 141 | }; 142 | 143 | FaceDatabase::FaceDatabase() { 144 | impl_ = new FaceDatabase::Impl(); 145 | } 146 | 147 | FaceDatabase::~FaceDatabase() { 148 | if (impl_) { 149 | delete impl_; 150 | impl_ = nullptr; 151 | } 152 | } 153 | 154 | bool FaceDatabase::Save(const char* path) const { 155 | std::cout << "start save data." << std::endl; 156 | std::string db_name = std::string(path) + "/db"; 157 | FileWriter ofile(db_name.c_str(), FileWriter::Binary); 158 | if (!ofile.is_opened()){ 159 | std::cout << "open database failed." << std::endl; 160 | return false; 161 | } 162 | return impl_->Save(ofile); 163 | } 164 | 165 | bool FaceDatabase::Load(const char* path) { 166 | std::string db_name = std::string(path) + "/db"; 167 | FileReader ifile(db_name.c_str(), FileWriter::Binary); 168 | if (!ifile.is_opened()) return false; 169 | return impl_->Load(ifile); 170 | } 171 | 172 | int64_t FaceDatabase::Insert(const std::vector& feat, const std::string& name) { 173 | return impl_->Insert(name, feat); 174 | } 175 | 176 | int FaceDatabase::Delete(const std::string& name) { 177 | return impl_->Delete(name); 178 | } 179 | 180 | int64_t FaceDatabase::QueryTop(const std::vector& feat, 181 | QueryResult* query_result) { 182 | return impl_->QueryTop(feat, query_result); 183 | } 184 | 185 | void FaceDatabase::Clear() { 186 | impl_->Clear(); 187 | } 188 | 189 | 190 | 191 | } 192 | 193 | -------------------------------------------------------------------------------- /src/face/database/face_database.h: -------------------------------------------------------------------------------- 1 | #ifndef _FACE_DATABASE_H_ 2 | #define _FACE_DATABASE_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "opencv2/core.hpp" 9 | #include "./stream/file_stream.h" 10 | #include "../../common/common.h" 11 | 12 | namespace mirror { 13 | 14 | class FaceDatabase { 15 | public: 16 | FaceDatabase(); 17 | ~FaceDatabase(); 18 | 19 | bool Save(const char* path) const; 20 | bool Load(const char* path); 21 | int64_t Insert(const std::vector& feat, const std::string& name); 22 | int Delete(const std::string& name); 23 | int64_t QueryTop(const std::vector& feat, QueryResult* query_result = nullptr); 24 | 25 | void Clear(); 26 | 27 | private: 28 | FaceDatabase(const FaceDatabase &other) = delete; 29 | const FaceDatabase &operator=(const FaceDatabase &other) = delete; 30 | 31 | private: 32 | class Impl; 33 | Impl* impl_; 34 | 35 | }; 36 | } 37 | 38 | 39 | #endif // !_FACE_DATABASE_H_ 40 | 41 | -------------------------------------------------------------------------------- /src/face/database/stream/file_stream.cpp: -------------------------------------------------------------------------------- 1 | #include "file_stream.h" 2 | 3 | namespace mirror { 4 | bool FileStream::open(const std::string & path, int mode) { 5 | close(); 6 | std::string mode_str; 7 | if ((mode & Input) && (mode & Output)) { 8 | mode_str += "a+"; 9 | } else { 10 | if (mode & Input) { 11 | mode_str += "r"; 12 | } else { 13 | mode_str += "w"; 14 | } 15 | } 16 | if (mode & Binary) mode_str += "b"; 17 | #if _MSC_VER >= 1600 18 | fopen_s(&iofile_, path.c_str(), mode_str.c_str()); 19 | #else 20 | iofile_ = std::fopen(path.c_str(), mode_str.c_str()); 21 | #endif 22 | return iofile_ != nullptr; 23 | } 24 | 25 | void FileStream::close() { 26 | if (iofile_ != nullptr) std::fclose(iofile_); 27 | } 28 | 29 | bool FileStream::is_opened() const { 30 | return iofile_ != nullptr; 31 | } 32 | 33 | size_t FileStream::write(const char* data, size_t length) { 34 | if (iofile_ == nullptr) return 0; 35 | auto result = std::fwrite(data, 1, length, iofile_); 36 | return size_t(result); 37 | } 38 | 39 | size_t FileStream::read(char* data, size_t length) { 40 | if (iofile_ == nullptr) return 0; 41 | auto result = std::fread(data, 1, length, iofile_); 42 | return size_t(result); 43 | } 44 | 45 | 46 | } 47 | 48 | 49 | -------------------------------------------------------------------------------- /src/face/database/stream/file_stream.h: -------------------------------------------------------------------------------- 1 | #ifndef _FILE_STREAM_H_ 2 | #define _FILE_STREAM_H_ 3 | 4 | #include 5 | 6 | namespace mirror { 7 | 8 | class StreamWriter { 9 | public: 10 | using self = StreamWriter; 11 | StreamWriter() {} 12 | virtual ~StreamWriter() = default; 13 | virtual size_t write(const char* data, size_t length) = 0; 14 | 15 | }; 16 | 17 | class StreamReader { 18 | public: 19 | using self = StreamReader; 20 | StreamReader() {} 21 | virtual ~StreamReader() = default; 22 | virtual size_t read(char* data, size_t length) = 0; 23 | }; 24 | 25 | class FileStream : public StreamWriter, public StreamReader { 26 | public: 27 | enum Mode { 28 | Input = 0x01, 29 | Output = 0x01 << 1, 30 | Binary = 0x01 << 2, 31 | }; 32 | 33 | FileStream() {} 34 | explicit FileStream(const std::string &path, int mode = Output) { 35 | open(path, mode); 36 | } 37 | 38 | FileStream(FileStream &&other) { 39 | std::swap(iofile_, other.iofile_); 40 | } 41 | 42 | const FileStream &operator=(FileStream &&other) { 43 | std::swap(iofile_, other.iofile_); 44 | return *this; 45 | } 46 | virtual ~FileStream() { 47 | close(); 48 | } 49 | 50 | bool open(const std::string &path, int mode = Output); 51 | void close(); 52 | bool is_opened() const; 53 | size_t write(const char *data, size_t length) override; 54 | size_t read(char *data, size_t length) override; 55 | 56 | private: 57 | FileStream(const FileStream& other) = delete; 58 | FILE* iofile_ = nullptr; 59 | 60 | }; 61 | 62 | class FileWriter : public FileStream { 63 | public: 64 | using self = FileWriter; 65 | using supper = FileStream; 66 | FileWriter() {}; 67 | explicit FileWriter(const std::string& path, int mode = Output) 68 | : FileStream(path, (mode & (~Input)) | Output) { 69 | } 70 | ~FileWriter() {}; 71 | 72 | bool open(const std::string& path, int mode = Output) { 73 | return supper::open(path, (mode & (!Input)) | Output); 74 | } 75 | 76 | }; 77 | 78 | class FileReader : public FileStream { 79 | public: 80 | using self = FileReader; 81 | using supper = FileStream; 82 | 83 | FileReader() {} 84 | explicit FileReader(const std::string& path, int mode = Input) 85 | : FileStream(path, (mode & (~Output)) | Input) { 86 | 87 | } 88 | 89 | bool open(const std::string& path, int mode = Input) { 90 | return supper::open(path, (mode & (~Output)) | Input); 91 | } 92 | 93 | 94 | }; 95 | 96 | 97 | // read and write value 98 | template 99 | static size_t Write(StreamWriter &writer, const T &value) { 100 | return writer.write(reinterpret_cast(&value), sizeof(T)); 101 | } 102 | 103 | template 104 | static size_t Read(StreamReader &reader, T &value) { 105 | return reader.read(reinterpret_cast(&value), sizeof(T)); 106 | } 107 | 108 | // read and write array 109 | template 110 | static size_t Write(StreamWriter &writer, const T *arr, size_t size) { 111 | return writer.write(reinterpret_cast(arr), sizeof(T) * size); 112 | } 113 | 114 | template 115 | static size_t Read(StreamReader &reader, T *arr, size_t size) { 116 | return reader.read(reinterpret_cast(arr), sizeof(T) * size); 117 | } 118 | 119 | 120 | 121 | } 122 | 123 | 124 | 125 | 126 | 127 | 128 | #endif // !_FILE_STREAM_H_ 129 | 130 | -------------------------------------------------------------------------------- /src/face/detecter/centerface/centerface.cpp: -------------------------------------------------------------------------------- 1 | #include "centerface.h" 2 | 3 | #include 4 | #include 5 | 6 | #include "opencv2/imgproc.hpp" 7 | 8 | namespace mirror { 9 | 10 | Centerface::Centerface() { 11 | initialized_ = false; 12 | } 13 | 14 | Centerface::~Centerface() { 15 | centerface_interpreter_->releaseModel(); 16 | centerface_interpreter_->releaseSession(centerface_sess_); 17 | } 18 | 19 | int Centerface::Init(const char * model_path) { 20 | std::cout << "start init." << std::endl; 21 | std::string model_file = std::string(model_path) + "/centerface.mnn"; 22 | centerface_interpreter_ = std::unique_ptr(MNN::Interpreter::createFromFile(model_file.c_str())); 23 | if (nullptr == centerface_interpreter_) { 24 | std::cout << "load centerface failed." << std::endl; 25 | return 10000; 26 | } 27 | 28 | MNN::ScheduleConfig schedule_config; 29 | schedule_config.type = MNN_FORWARD_CPU; 30 | schedule_config.numThread = 1; 31 | MNN::BackendConfig backend_config; 32 | backend_config.power = MNN::BackendConfig::Power_High; 33 | backend_config.precision = MNN::BackendConfig::Precision_High; 34 | schedule_config.backendConfig = &backend_config; 35 | 36 | // create session 37 | centerface_sess_ = centerface_interpreter_->createSession(schedule_config); 38 | input_tensor_ = centerface_interpreter_->getSessionInput(centerface_sess_, nullptr); 39 | 40 | MNN::CV::Matrix trans; 41 | trans.setScale(1.0f, 1.0f); 42 | MNN::CV::ImageProcess::Config img_config; 43 | img_config.filterType = MNN::CV::BICUBIC; 44 | ::memcpy(img_config.mean, meanVals_, sizeof(meanVals_)); 45 | ::memcpy(img_config.normal, normVals_, sizeof(normVals_)); 46 | img_config.sourceFormat = MNN::CV::BGR; 47 | img_config.destFormat = MNN::CV::RGB; 48 | pretreat_ = std::shared_ptr(MNN::CV::ImageProcess::create(img_config)); 49 | pretreat_->setMatrix(trans); 50 | 51 | initialized_ = true; 52 | 53 | std::cout << "end init." << std::endl; 54 | return 0; 55 | } 56 | 57 | int Centerface::DetectFace(const cv::Mat & img_src, std::vector* faces) { 58 | std::cout << "start detect." << std::endl; 59 | faces->clear(); 60 | if (!initialized_) { 61 | std::cout << "model uninitialized." << std::endl; 62 | return 10000; 63 | } 64 | if (img_src.empty()) { 65 | std::cout << "input empty." << std::endl; 66 | return 10001; 67 | } 68 | 69 | int width = img_src.cols; 70 | int height = img_src.rows; 71 | int width_resized = width / 32 * 32; 72 | int height_resized = height / 32 * 32; 73 | float scale_x = static_cast(width) / width_resized; 74 | float scale_y = static_cast(height) / height_resized; 75 | 76 | centerface_interpreter_->resizeTensor(input_tensor_, {1, 3, height_resized, width_resized}); 77 | centerface_interpreter_->resizeSession(centerface_sess_); 78 | 79 | cv::Mat img_resized; 80 | cv::resize(img_src, img_resized, cv::Size(width_resized, height_resized)); 81 | pretreat_->convert(img_resized.data, width_resized, height_resized, 0, input_tensor_); 82 | 83 | // run session 84 | centerface_interpreter_->runSession(centerface_sess_); 85 | 86 | // get output 87 | auto tensor_heatmap = centerface_interpreter_->getSessionOutput(centerface_sess_, "537"); 88 | auto tensor_scale = centerface_interpreter_->getSessionOutput(centerface_sess_, "538"); 89 | auto tensor_offset = centerface_interpreter_->getSessionOutput(centerface_sess_, "539"); 90 | auto tensor_landmark = centerface_interpreter_->getSessionOutput(centerface_sess_, "540"); 91 | 92 | // copy to host 93 | MNN::Tensor heatmap_host(tensor_heatmap, tensor_heatmap->getDimensionType()); 94 | MNN::Tensor scale_host(tensor_scale, tensor_scale->getDimensionType()); 95 | MNN::Tensor offset_host(tensor_offset, tensor_offset->getDimensionType()); 96 | MNN::Tensor landmark_host(tensor_landmark, tensor_landmark->getDimensionType()); 97 | tensor_heatmap->copyToHostTensor(&heatmap_host); 98 | tensor_scale->copyToHostTensor(&scale_host); 99 | tensor_offset->copyToHostTensor(&offset_host); 100 | tensor_landmark->copyToHostTensor(&landmark_host); 101 | 102 | int output_width = heatmap_host.width(); 103 | int output_height = heatmap_host.height(); 104 | int channel_step = output_width * output_height; 105 | std::vector faces_tmp; 106 | for (int h = 0; h < output_height; ++h) { 107 | for (int w = 0; w < output_width; ++w) { 108 | int index = h * output_width + w; 109 | float score = heatmap_host.host()[index]; 110 | if (score < scoreThreshold_) { 111 | continue; 112 | } 113 | float s0 = 4 * exp(scale_host.host()[index]); 114 | float s1 = 4 * exp(scale_host.host()[index + channel_step]); 115 | float o0 = offset_host.host()[index]; 116 | float o1 = offset_host.host()[index + channel_step]; 117 | 118 | float ymin = MAX(0, 4 * (h + o0 + 0.5) - 0.5 * s0); 119 | float xmin = MAX(0, 4 * (w + o1 + 0.5) - 0.5 * s1); 120 | float ymax = MIN(ymin + s0, height_resized); 121 | float xmax = MIN(xmin + s1, width_resized); 122 | 123 | cv::Rect face = cv::Rect(scale_x * xmin, scale_y * ymin, scale_x * (xmax - xmin), scale_y * (ymax - ymin)); 124 | float center_x = face.x + face.width / 2.0f; 125 | float center_y = face.y + face.height / 2.0f; 126 | int max_side = MAX(face.width, face.height); 127 | 128 | FaceInfo face_info; 129 | face_info.score_ = score; 130 | face_info.location_.x = center_x - 0.5f * max_side; 131 | face_info.location_.y = center_y - 0.5f * max_side; 132 | face_info.location_.width = max_side; 133 | face_info.location_.height = max_side; 134 | face_info.location_ = face_info.location_ & cv::Rect(0, 0, width, height); 135 | 136 | for (int num = 0; num < 5; ++num) { 137 | face_info.keypoints_[2 * num] = scale_x * (s1 * landmark_host.host()[(2 * num + 1) * channel_step + index] + xmin); 138 | face_info.keypoints_[2 * num + 1] = scale_y * (s0 * landmark_host.host()[(2 * num + 0) * channel_step + index] + ymin); 139 | } 140 | faces_tmp.push_back(face_info); 141 | } 142 | } 143 | NMS(faces_tmp, faces, nmsThreshold_); 144 | std::cout << "end detect." << std::endl; 145 | return 0; 146 | } 147 | 148 | 149 | } 150 | 151 | -------------------------------------------------------------------------------- /src/face/detecter/centerface/centerface.h: -------------------------------------------------------------------------------- 1 | #ifndef _FACE_CENTERFACE_H_ 2 | #define _FACE_CENTERFACE_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "MNN/Interpreter.hpp" 8 | #include "MNN/ImageProcess.hpp" 9 | #include "MNN/MNNDefine.h" 10 | #include "MNN/Tensor.hpp" 11 | 12 | #include "opencv2/core.hpp" 13 | #include "opencv2/imgproc.hpp" 14 | #include "../detecter.h" 15 | #include "../../common/common.h" 16 | 17 | namespace mirror { 18 | class Centerface : public Detecter { 19 | public: 20 | Centerface(); 21 | ~Centerface(); 22 | int Init(const char* model_path); 23 | int DetectFace(const cv::Mat& img_src, std::vector* faces); 24 | 25 | private: 26 | bool initialized_; 27 | std::shared_ptr pretreat_ = nullptr; 28 | std::shared_ptr centerface_interpreter_ = nullptr; 29 | MNN::Session* centerface_sess_ = nullptr; 30 | MNN::Tensor* input_tensor_ = nullptr; 31 | 32 | const float meanVals_[3] = { 0.0f, 0.0f, 0.0f }; 33 | const float normVals_[3] = { 1.0f, 1.0f, 1.0f }; 34 | const float scoreThreshold_ = 0.5f; 35 | const float nmsThreshold_ = 0.5f; 36 | 37 | }; 38 | 39 | } 40 | 41 | 42 | #endif // !_FACE_CENTERFACE_H_ 43 | -------------------------------------------------------------------------------- /src/face/detecter/detecter.cpp: -------------------------------------------------------------------------------- 1 | #include "detecter.h" 2 | #include "./centerface/centerface.h" 3 | #include "./ultraface/ultraface.h" 4 | 5 | namespace mirror { 6 | Detecter* CenterfaceFactory::CreateDetecter() { 7 | return new Centerface(); 8 | } 9 | 10 | Detecter* UltrafaceFactory::CreateDetecter() { 11 | return new UltraFace(); 12 | } 13 | 14 | 15 | } -------------------------------------------------------------------------------- /src/face/detecter/detecter.h: -------------------------------------------------------------------------------- 1 | #ifndef _FACE_DETECTER_H_ 2 | #define _FACE_DETECTER_H_ 3 | 4 | #include "../../common/common.h" 5 | 6 | namespace mirror { 7 | class Detecter { 8 | public: 9 | virtual int Init(const char* model_path) = 0; 10 | virtual int DetectFace(const cv::Mat& img_src, 11 | std::vector* faces) = 0; 12 | virtual ~Detecter() {} 13 | }; 14 | 15 | class DetecterFactory { 16 | public: 17 | virtual Detecter* CreateDetecter() = 0; 18 | virtual ~DetecterFactory() {} 19 | }; 20 | 21 | class CenterfaceFactory : public DetecterFactory { 22 | public: 23 | CenterfaceFactory() {} 24 | Detecter* CreateDetecter(); 25 | ~CenterfaceFactory() {} 26 | }; 27 | 28 | class UltrafaceFactory : public DetecterFactory { 29 | public: 30 | UltrafaceFactory() {} 31 | Detecter* CreateDetecter(); 32 | ~UltrafaceFactory() {} 33 | }; 34 | 35 | } 36 | 37 | 38 | #endif // !_FACE_DETECTER_H_ -------------------------------------------------------------------------------- /src/face/detecter/ultraface/ultraface.cpp: -------------------------------------------------------------------------------- 1 | #include "ultraface.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "opencv2/imgproc.hpp" 8 | 9 | #define Clip(x, y) (x < 0 ? 0 : (x > y ? y : x)) 10 | 11 | namespace mirror { 12 | UltraFace::UltraFace() { 13 | anchors_.clear(); 14 | initialized_ = false; 15 | } 16 | 17 | UltraFace::~UltraFace() { 18 | ultraface_interpreter_->releaseModel(); 19 | ultraface_interpreter_->releaseSession(ultraface_sess_); 20 | } 21 | 22 | int UltraFace::Init(const char * model_path) { 23 | std::cout << "start init ultraface." << std::endl; 24 | std::string model_file = std::string(model_path) + "/RFB-320.mnn"; 25 | std::cout << "model path: " << model_file << std::endl; 26 | ultraface_interpreter_ = std::unique_ptr(MNN::Interpreter::createFromFile(model_file.c_str())); 27 | 28 | if (nullptr == ultraface_interpreter_) { 29 | std::cout << "load model failed." << std::endl; 30 | return 10000; 31 | } 32 | 33 | MNN::ScheduleConfig schedule_config; 34 | schedule_config.type = MNN_FORWARD_CPU; 35 | schedule_config.numThread = 1; 36 | MNN::BackendConfig bacckend_config; 37 | bacckend_config.memory = MNN::BackendConfig::Memory_Normal; 38 | bacckend_config.power = MNN::BackendConfig::Power_Normal; 39 | bacckend_config.precision = MNN::BackendConfig::Precision_Normal; 40 | schedule_config.backendConfig = &bacckend_config; 41 | 42 | ultraface_sess_ = ultraface_interpreter_->createSession(schedule_config); 43 | input_tensor_ = ultraface_interpreter_->getSessionInput(ultraface_sess_, nullptr); 44 | ultraface_interpreter_->resizeTensor(input_tensor_, {1, 3, inputSize_.height, inputSize_.width}); 45 | ultraface_interpreter_->resizeSession(ultraface_sess_); 46 | 47 | MNN::CV::Matrix trans; 48 | trans.setScale(1.0f, 1.0f); 49 | MNN::CV::ImageProcess::Config img_config; 50 | img_config.filterType = MNN::CV::BICUBIC; 51 | ::memcpy(img_config.mean, meanVals_, sizeof(meanVals_)); 52 | ::memcpy(img_config.normal, normVals_, sizeof(normVals_)); 53 | img_config.sourceFormat = MNN::CV::BGR; 54 | img_config.destFormat = MNN::CV::RGB; 55 | pretreat_ = std::shared_ptr(MNN::CV::ImageProcess::create(img_config)); 56 | pretreat_->setMatrix(trans); 57 | 58 | GenerateAnchors(inputSize_.width, inputSize_.height, minBoxes_, strides_, &anchors_); 59 | 60 | initialized_ = true; 61 | 62 | 63 | std::cout << "end init ultraface." << std::endl; 64 | return 0; 65 | } 66 | 67 | int UltraFace::DetectFace(const cv::Mat & img_src, std::vector* faces) { 68 | std::cout << "start detect face." << std::endl; 69 | faces->clear(); 70 | if (!initialized_) { 71 | std::cout << "model uninitialized." << std::endl; 72 | return 10000; 73 | } 74 | if (img_src.empty()) { 75 | std::cout << "model empty." << std::endl; 76 | return 10001; 77 | } 78 | 79 | int width = img_src.cols; 80 | int height = img_src.rows; 81 | 82 | cv::Mat img_resized; 83 | cv::resize(img_src, img_resized, inputSize_); 84 | pretreat_->convert(img_resized.data, inputSize_.width, inputSize_.height, img_resized.step[0], input_tensor_); 85 | 86 | ultraface_interpreter_->runSession(ultraface_sess_); 87 | 88 | auto tensor_score = ultraface_interpreter_->getSessionOutput(ultraface_sess_, "scores"); 89 | auto tensor_bbox = ultraface_interpreter_->getSessionOutput(ultraface_sess_, "boxes"); 90 | MNN::Tensor host_score(tensor_score, tensor_score->getDimensionType()); 91 | MNN::Tensor host_bbox(tensor_bbox, tensor_bbox->getDimensionType()); 92 | tensor_score->copyToHostTensor(&host_score); 93 | tensor_bbox->copyToHostTensor(&host_bbox); 94 | 95 | int num_anchors = static_cast(anchors_.size()); 96 | std::vector faces_tmp; 97 | for (int i = 0; i < num_anchors; ++i) { 98 | float score = host_score.host()[2 * i + 1]; 99 | if (score <= scoreThreshold_) continue; 100 | FaceInfo face_info; 101 | float center_x = host_bbox.host()[4 * i] * centerVariance_ * anchors_[i][2] + anchors_[i][0]; 102 | float center_y = host_bbox.host()[4 * i + 1] * centerVariance_ * anchors_[i][3] + anchors_[i][1]; 103 | float center_w = exp(host_bbox.host()[4 * i + 2] * sizeVariance_) * anchors_[i][2]; 104 | float center_h = exp(host_bbox.host()[4 * i + 3] * sizeVariance_) * anchors_[i][3]; 105 | cv::Rect face; 106 | face.x = static_cast(Clip(center_x - center_w / 2.0f, 1.0f) * width); 107 | face.y = static_cast(Clip(center_y - center_h / 2.0f, 1.0f) * height); 108 | face.width = static_cast(Clip(center_w, 1.0f) * width); 109 | face.height = static_cast(Clip(center_h, 1.0f) * height); 110 | 111 | int max_side = MAX(face.width, face.height); 112 | face_info.location_.x = face.x + 0.5f * face.width - 0.5f * max_side; 113 | face_info.location_.y = face.y + 0.5f * face.height - 0.5f * max_side; 114 | face_info.location_.width = max_side; 115 | face_info.location_.height = max_side; 116 | face_info.location_ = face_info.location_ & cv::Rect(0, 0, width, height); 117 | 118 | face_info.score_ = Clip(score, 1.0f); 119 | faces_tmp.push_back(face_info); 120 | } 121 | NMS(faces_tmp, faces, iouThreshold_, "BLENDING"); 122 | 123 | std::cout << "end detect face." << std::endl; 124 | 125 | return 0; 126 | } 127 | 128 | 129 | 130 | 131 | } -------------------------------------------------------------------------------- /src/face/detecter/ultraface/ultraface.h: -------------------------------------------------------------------------------- 1 | #ifndef _FACE_ULTRAFACE_H_ 2 | #define _FACE_ULTRAFACE_H_ 3 | 4 | #include 5 | #include 6 | #include "opencv2/core.hpp" 7 | 8 | #include "MNN/Interpreter.hpp" 9 | #include "MNN/ImageProcess.hpp" 10 | #include "MNN/MNNDefine.h" 11 | #include "MNN/Tensor.hpp" 12 | 13 | #include "../detecter.h" 14 | #include "../../common/common.h" 15 | 16 | namespace mirror { 17 | class UltraFace : public Detecter { 18 | public: 19 | UltraFace(); 20 | ~UltraFace(); 21 | int Init(const char* model_path); 22 | int DetectFace(const cv::Mat& img_src, std::vector* faces); 23 | 24 | private: 25 | bool initialized_; 26 | std::shared_ptr pretreat_ = nullptr; 27 | std::shared_ptr ultraface_interpreter_ = nullptr; 28 | MNN::Session* ultraface_sess_ = nullptr; 29 | MNN::Tensor* input_tensor_ = nullptr; 30 | 31 | const cv::Size inputSize_ = { 320, 240 }; 32 | const float meanVals_[3] = { 127.0f, 127.0f, 127.0f }; 33 | const float normVals_[3] = { 0.0078125f, 0.0078125f, 0.0078125f }; 34 | const float centerVariance_ = 0.1f; 35 | const float sizeVariance_ = 0.2f; 36 | const std::vector> minBoxes_ = { 37 | { 10.0f, 16.0f, 24.0f }, 38 | { 32.0f, 48.0f }, 39 | { 64.0f, 96.0f }, 40 | { 128.0f, 192.0f, 256.0f } 41 | }; 42 | const std::vector strides_ = { 8.0f, 16.0f, 32.0f, 64.0f }; 43 | std::vector> anchors_ = {}; 44 | 45 | const float scoreThreshold_ = 0.65f; 46 | const float iouThreshold_ = 0.3f; 47 | }; 48 | 49 | } 50 | 51 | 52 | #endif // !_FACE_ULTRAFACE_H_ 53 | -------------------------------------------------------------------------------- /src/face/face_engine.cpp: -------------------------------------------------------------------------------- 1 | #include "face_engine.h" 2 | #include 3 | 4 | #include "detecter/detecter.h" 5 | #include "landmarker/landmarker.h" 6 | #include "aligner/aligner.h" 7 | #include "recognizer/recognizer.h" 8 | #include "database/face_database.h" 9 | 10 | 11 | namespace mirror { 12 | class FaceEngine::Impl { 13 | public: 14 | Impl() { 15 | detecter_factory_ = new UltrafaceFactory(); 16 | landmarker_factory_ = new ZQLandmarkerFactory(); 17 | recognizer_factory_ = new MobilefacenetFactory(); 18 | 19 | detecter_ = detecter_factory_->CreateDetecter(); 20 | landmarker_ = landmarker_factory_->CreateLandmarker(); 21 | recognizer_ = recognizer_factory_->CreateRecognizer(); 22 | 23 | aligner_ = new Aligner(); 24 | database_ = new FaceDatabase(); 25 | initialized_ = false; 26 | } 27 | 28 | ~Impl() { 29 | if (detecter_) { 30 | delete detecter_; 31 | detecter_ = nullptr; 32 | } 33 | 34 | if (landmarker_) { 35 | delete landmarker_; 36 | landmarker_ = nullptr; 37 | } 38 | 39 | if (recognizer_) { 40 | delete recognizer_; 41 | recognizer_ = nullptr; 42 | } 43 | 44 | if (database_) { 45 | delete database_; 46 | database_ = nullptr; 47 | } 48 | 49 | if (detecter_factory_) { 50 | delete detecter_factory_; 51 | detecter_factory_ = nullptr; 52 | } 53 | 54 | if (landmarker_factory_) { 55 | delete landmarker_factory_; 56 | landmarker_factory_ = nullptr; 57 | } 58 | 59 | if (recognizer_factory_) { 60 | delete recognizer_factory_; 61 | recognizer_factory_ = nullptr; 62 | } 63 | } 64 | 65 | int Init(const char* root_path) { 66 | if (detecter_->Init(root_path) != 0) { 67 | std::cout << "Init face detecter failed." << std::endl; 68 | return 10000; 69 | } 70 | 71 | if (landmarker_->Init(root_path) != 0) { 72 | std::cout << "Init face landmarker failed." << std::endl; 73 | return 10000; 74 | } 75 | 76 | if (recognizer_->Init(root_path) != 0) { 77 | std::cout << "Init face recognizer failed." << std::endl; 78 | return 10000; 79 | } 80 | 81 | db_name_ = std::string(root_path); 82 | initialized_ = true; 83 | 84 | return 0; 85 | } 86 | inline int DetectFace(const cv::Mat& img_src, std::vector* faces) { 87 | return detecter_->DetectFace(img_src, faces); 88 | } 89 | inline int ExtractKeypoints(const cv::Mat& img_src, const cv::Rect& face, std::vector* keypoints) { 90 | return landmarker_->ExtractKeypoints(img_src, face, keypoints); 91 | } 92 | inline int AlignFace(const cv::Mat& img_src, const std::vector& keypoints, cv::Mat* face_aligned) { 93 | return aligner_->AlignFace(img_src, keypoints, face_aligned); 94 | } 95 | inline int ExtractFeature(const cv::Mat& img_face, std::vector* feat) { 96 | return recognizer_->ExtractFeature(img_face, feat); 97 | } 98 | 99 | inline int Insert(const std::vector& feat, const std::string& name) { 100 | return database_->Insert(feat, name); 101 | } 102 | inline int Delete(const std::string& name) { 103 | return database_->Delete(name); 104 | } 105 | inline int64_t QueryTop(const std::vector& feat, QueryResult *query_result = nullptr) { 106 | return database_->QueryTop(feat, query_result); 107 | } 108 | inline int Save() { 109 | return database_->Save(db_name_.c_str()); 110 | } 111 | inline int Load() { 112 | return database_->Load(db_name_.c_str()); 113 | } 114 | 115 | private: 116 | DetecterFactory* detecter_factory_; 117 | LandmarkerFactory* landmarker_factory_; 118 | RecognizerFactory* recognizer_factory_; 119 | 120 | private: 121 | bool initialized_; 122 | std::string db_name_; 123 | Aligner* aligner_; 124 | Detecter* detecter_; 125 | Landmarker* landmarker_; 126 | Recognizer* recognizer_; 127 | FaceDatabase* database_; 128 | }; 129 | 130 | FaceEngine::FaceEngine() { 131 | impl_ = new FaceEngine::Impl(); 132 | } 133 | 134 | FaceEngine::~FaceEngine() { 135 | if (impl_) { 136 | delete impl_; 137 | impl_ = nullptr; 138 | } 139 | } 140 | 141 | int FaceEngine::Init(const char* root_path) { 142 | return impl_->Init(root_path); 143 | } 144 | 145 | int FaceEngine::DetectFace(const cv::Mat& img_src, std::vector* faces) { 146 | return impl_->DetectFace(img_src, faces); 147 | } 148 | 149 | int FaceEngine::ExtractKeypoints(const cv::Mat& img_src, const cv::Rect& face, std::vector* keypoints) { 150 | return impl_->ExtractKeypoints(img_src, face, keypoints); 151 | } 152 | 153 | int FaceEngine::AlignFace(const cv::Mat& img_src, const std::vector& keypoints, cv::Mat* face_aligned) { 154 | return impl_->AlignFace(img_src, keypoints, face_aligned); 155 | } 156 | 157 | int FaceEngine::ExtractFeature(const cv::Mat& img_face, std::vector* feat) { 158 | return impl_->ExtractFeature(img_face, feat); 159 | } 160 | 161 | int FaceEngine::Insert(const std::vector& feat, const std::string& name) { 162 | return impl_->Insert(feat, name); 163 | } 164 | 165 | int FaceEngine::Delete(const std::string& name) { 166 | return impl_->Delete(name); 167 | } 168 | 169 | int64_t FaceEngine::QueryTop(const std::vector& feat, 170 | QueryResult* query_result) { 171 | return impl_->QueryTop(feat, query_result); 172 | } 173 | 174 | int FaceEngine::Save() { 175 | return impl_->Save(); 176 | } 177 | 178 | int FaceEngine::Load() { 179 | return impl_->Load(); 180 | } 181 | 182 | } -------------------------------------------------------------------------------- /src/face/face_engine.h: -------------------------------------------------------------------------------- 1 | #ifndef _FACE_ENGINE_H_ 2 | #define _FACE_ENGINE_H_ 3 | 4 | #include 5 | #include "../common/common.h" 6 | 7 | namespace mirror { 8 | class FaceEngine { 9 | public: 10 | FaceEngine(); 11 | ~FaceEngine(); 12 | int Init(const char* root_path); 13 | int DetectFace(const cv::Mat& img_src, std::vector* faces); 14 | int ExtractKeypoints(const cv::Mat& img_src, const cv::Rect& face, std::vector* keypoints); 15 | int AlignFace(const cv::Mat& img_src, const std::vector& keypoints, cv::Mat* face_aligned); 16 | int ExtractFeature(const cv::Mat& img_face, std::vector* feat); 17 | 18 | int Insert(const std::vector& feat, const std::string& name); 19 | int Delete(const std::string& name); 20 | int64_t QueryTop(const std::vector& feat, QueryResult *query_result = nullptr); 21 | int Save(); 22 | int Load(); 23 | 24 | private: 25 | class Impl; 26 | Impl* impl_; 27 | 28 | }; 29 | 30 | 31 | } 32 | 33 | 34 | 35 | 36 | #endif // !_FACE_ENGINE_H_ -------------------------------------------------------------------------------- /src/face/landmarker/landmarker.cpp: -------------------------------------------------------------------------------- 1 | #include "landmarker.h" 2 | #include "pfldlandmarker/pfldlandmarker.h" 3 | #include "zqlandmarker/zqlandmarker.h" 4 | 5 | namespace mirror { 6 | Landmarker* PFLDLandmarkerFactory::CreateLandmarker() { 7 | return new PFLDLandmarker(); 8 | } 9 | 10 | Landmarker* ZQLandmarkerFactory::CreateLandmarker() { 11 | return new ZQLandmarker(); 12 | } 13 | 14 | } -------------------------------------------------------------------------------- /src/face/landmarker/landmarker.h: -------------------------------------------------------------------------------- 1 | #ifndef _FACE_LANDMARKER_H_ 2 | #define _FACE_LANDMARKER_H_ 3 | 4 | #include "../../common/common.h" 5 | 6 | namespace mirror { 7 | class Landmarker { 8 | public: 9 | virtual int Init(const char* model_path) = 0; 10 | virtual int ExtractKeypoints(const cv::Mat& img_src, const cv::Rect& face, std::vector* keypoints) = 0; 11 | virtual ~Landmarker() {} 12 | }; 13 | 14 | class LandmarkerFactory { 15 | public: 16 | virtual Landmarker* CreateLandmarker() = 0; 17 | virtual ~LandmarkerFactory() {} 18 | }; 19 | 20 | class PFLDLandmarkerFactory : public LandmarkerFactory { 21 | public: 22 | PFLDLandmarkerFactory() {} 23 | Landmarker* CreateLandmarker(); 24 | ~PFLDLandmarkerFactory() {} 25 | }; 26 | 27 | class ZQLandmarkerFactory : public LandmarkerFactory { 28 | public: 29 | ZQLandmarkerFactory() {} 30 | Landmarker* CreateLandmarker(); 31 | ~ZQLandmarkerFactory() {} 32 | }; 33 | 34 | 35 | 36 | 37 | } 38 | 39 | 40 | 41 | 42 | 43 | #endif // !_FACE_LANDMARKER_H_ -------------------------------------------------------------------------------- /src/face/landmarker/pfldlandmarker/pfldlandmarker.cpp: -------------------------------------------------------------------------------- 1 | #include "pfldlandmarker.h" 2 | 3 | #include 4 | #include 5 | 6 | #include "opencv2/imgproc.hpp" 7 | #include "../../common/common.h" 8 | 9 | namespace mirror { 10 | PFLDLandmarker::PFLDLandmarker() { 11 | initialized_ = false; 12 | } 13 | 14 | PFLDLandmarker::~PFLDLandmarker() { 15 | pfld_interpreter_->releaseModel(); 16 | pfld_interpreter_->releaseSession(pfld_sess_); 17 | } 18 | 19 | int PFLDLandmarker::Init(const char* model_path) { 20 | std::cout << "start init." << std::endl; 21 | std::string model_file = std::string(model_path) + "/pfld.mnn"; 22 | pfld_interpreter_ = std::unique_ptr(MNN::Interpreter::createFromFile(model_file.c_str())); 23 | if (nullptr == pfld_interpreter_) { 24 | std::cout << "load model failed." << std::endl; 25 | return 10000; 26 | } 27 | 28 | // create session 29 | MNN::ScheduleConfig schedule_config; 30 | schedule_config.type = MNN_FORWARD_CPU; 31 | schedule_config.numThread = 1; 32 | MNN::BackendConfig backend_config; 33 | backend_config.memory = MNN::BackendConfig::Memory_Normal; 34 | backend_config.power = MNN::BackendConfig::Power_Normal; 35 | backend_config.precision = MNN::BackendConfig::Precision_Normal; 36 | schedule_config.backendConfig = &backend_config; 37 | pfld_sess_ = pfld_interpreter_->createSession(schedule_config); 38 | input_tensor_ = pfld_interpreter_->getSessionInput(pfld_sess_, nullptr); 39 | pfld_interpreter_->resizeTensor(input_tensor_, {1, 3, inputSize_.height, inputSize_.width}); 40 | pfld_interpreter_->resizeSession(pfld_sess_); 41 | 42 | MNN::CV::Matrix trans; 43 | trans.setScale(1.0f, 1.0f); 44 | MNN::CV::ImageProcess::Config img_config; 45 | img_config.filterType = MNN::CV::BICUBIC; 46 | ::memcpy(img_config.mean, meanVals_, sizeof(meanVals_)); 47 | ::memcpy(img_config.normal, normVals_, sizeof(normVals_)); 48 | img_config.sourceFormat = MNN::CV::BGR; 49 | img_config.destFormat = MNN::CV::RGB; 50 | pretreat_ = std::shared_ptr(MNN::CV::ImageProcess::create(img_config)); 51 | pretreat_->setMatrix(trans); 52 | 53 | initialized_ = true; 54 | 55 | std::cout << "end init." << std::endl; 56 | 57 | return 0; 58 | } 59 | 60 | int PFLDLandmarker::ExtractKeypoints(const cv::Mat& img_src, const cv::Rect& face, std::vector* keypoints) { 61 | std::cout << "start extract keypoints." << std::endl; 62 | keypoints->clear(); 63 | if (!initialized_) { 64 | std::cout << "model uninitialized." << std::endl; 65 | return 10000; 66 | } 67 | if (img_src.empty()) { 68 | std::cout << "input empty." << std::endl; 69 | return 10001; 70 | } 71 | cv::Mat img_face = img_src(face).clone(); 72 | int width = img_face.cols; 73 | int height = img_face.rows; 74 | cv::Mat img_resized; 75 | cv::resize(img_face, img_resized, inputSize_); 76 | float scale_x = static_cast(width) / inputSize_.width; 77 | float scale_y = static_cast(height) / inputSize_.height; 78 | pretreat_->convert(img_resized.data, inputSize_.width, inputSize_.height, 0, input_tensor_); 79 | 80 | // run session 81 | pfld_interpreter_->runSession(pfld_sess_); 82 | 83 | // get output 84 | std::string output_name = "conv5_fwd"; 85 | auto output_landmark = pfld_interpreter_->getSessionOutput(pfld_sess_, output_name.c_str()); 86 | MNN::Tensor landmark_tensor(output_landmark, output_landmark->getDimensionType()); 87 | output_landmark->copyToHostTensor(&landmark_tensor); 88 | 89 | for (int i = 0; i < 98; ++i) { 90 | cv::Point2f curr_pt(landmark_tensor.host()[2 * i + 0] * scale_x + face.x, 91 | landmark_tensor.host()[2 * i + 1] * scale_y + face.y); 92 | keypoints->push_back(curr_pt); 93 | } 94 | 95 | std::cout << "end extract keypoints." << std::endl; 96 | 97 | return 0; 98 | } 99 | 100 | 101 | } -------------------------------------------------------------------------------- /src/face/landmarker/pfldlandmarker/pfldlandmarker.h: -------------------------------------------------------------------------------- 1 | #ifndef _FACE_PFLDLANDMARKER_H_ 2 | #define _FACE_PFLDLANDMARKER_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "opencv2/core.hpp" 8 | #include "../landmarker.h" 9 | 10 | #include "MNN/Interpreter.hpp" 11 | #include "MNN/ImageProcess.hpp" 12 | #include "MNN/MNNDefine.h" 13 | #include "MNN/Tensor.hpp" 14 | 15 | namespace mirror { 16 | class PFLDLandmarker : public Landmarker{ 17 | public: 18 | PFLDLandmarker(); 19 | ~PFLDLandmarker(); 20 | int Init(const char* model_path); 21 | int ExtractKeypoints(const cv::Mat& img_src, const cv::Rect& face, std::vector* keypoints); 22 | 23 | private: 24 | bool initialized_; 25 | std::shared_ptr pretreat_ = nullptr; 26 | std::shared_ptr pfld_interpreter_ = nullptr; 27 | MNN::Session* pfld_sess_ = nullptr; 28 | MNN::Tensor* input_tensor_ = nullptr; 29 | 30 | const cv::Size inputSize_ = cv::Size(96, 96); 31 | const float meanVals_[3] = { 123.0f, 123.0f, 123.0f }; 32 | const float normVals_[3] = { 0.01724f, 0.01724f, 0.01724f }; 33 | 34 | }; 35 | 36 | 37 | } 38 | 39 | 40 | 41 | 42 | #endif // !_FACE_PFLDLANDMARKER_H_ 43 | -------------------------------------------------------------------------------- /src/face/landmarker/zqlandmarker/zqlandmarker.cpp: -------------------------------------------------------------------------------- 1 | #include "zqlandmarker.h" 2 | 3 | #include 4 | #include 5 | 6 | #include "opencv2/imgproc.hpp" 7 | 8 | namespace mirror { 9 | ZQLandmarker::ZQLandmarker() { 10 | initialized_ = false; 11 | } 12 | 13 | ZQLandmarker::~ZQLandmarker() { 14 | zq_interpreter_->releaseModel(); 15 | zq_interpreter_->releaseSession(zq_sess_); 16 | } 17 | 18 | int ZQLandmarker::Init(const char* model_path) { 19 | std::cout << "start init." << std::endl; 20 | std::string model_file = std::string(model_path) + "/zqlandmark.mnn"; 21 | zq_interpreter_ = std::unique_ptr(MNN::Interpreter::createFromFile(model_file.c_str())); 22 | if (nullptr == zq_interpreter_) { 23 | std::cout << "load model failed." << std::endl; 24 | return 10000; 25 | } 26 | 27 | // create session 28 | MNN::ScheduleConfig schedule_config; 29 | schedule_config.type = MNN_FORWARD_CPU; 30 | schedule_config.numThread = 1; 31 | MNN::BackendConfig backend_config; 32 | backend_config.memory = MNN::BackendConfig::Memory_Normal; 33 | backend_config.power = MNN::BackendConfig::Power_Normal; 34 | backend_config.precision = MNN::BackendConfig::Precision_Normal; 35 | schedule_config.backendConfig = &backend_config; 36 | zq_sess_ = zq_interpreter_->createSession(schedule_config); 37 | input_tensor_ = zq_interpreter_->getSessionInput(zq_sess_, nullptr); 38 | zq_interpreter_->resizeTensor(input_tensor_, {1, 3, inputSize_.height, inputSize_.width}); 39 | zq_interpreter_->resizeSession(zq_sess_); 40 | 41 | MNN::CV::Matrix trans; 42 | trans.setScale(1.0f, 1.0f); 43 | MNN::CV::ImageProcess::Config img_config; 44 | img_config.filterType = MNN::CV::BICUBIC; 45 | ::memcpy(img_config.mean, meanVals_, sizeof(meanVals_)); 46 | ::memcpy(img_config.normal, normVals_, sizeof(normVals_)); 47 | img_config.sourceFormat = MNN::CV::BGR; 48 | img_config.destFormat = MNN::CV::RGB; 49 | pretreat_ = std::shared_ptr(MNN::CV::ImageProcess::create(img_config)); 50 | pretreat_->setMatrix(trans); 51 | 52 | initialized_ = true; 53 | 54 | std::cout << "end init." << std::endl; 55 | 56 | return 0; 57 | } 58 | 59 | int ZQLandmarker::ExtractKeypoints(const cv::Mat& img_src, const cv::Rect& face, std::vector* keypoints) { 60 | std::cout << "start extract keypoints." << std::endl; 61 | keypoints->clear(); 62 | if (!initialized_) { 63 | std::cout << "model uninitialized." << std::endl; 64 | return 10000; 65 | } 66 | if (img_src.empty()) { 67 | std::cout << "input empty." << std::endl; 68 | return 10001; 69 | } 70 | cv::Mat img_face = img_src(face).clone(); 71 | int width = img_face.cols; 72 | int height = img_face.rows; 73 | cv::Mat img_resized; 74 | cv::resize(img_face, img_resized, inputSize_); 75 | pretreat_->convert(img_resized.data, inputSize_.width, inputSize_.height, 0, input_tensor_); 76 | 77 | // run session 78 | zq_interpreter_->runSession(zq_sess_); 79 | 80 | // get output 81 | std::string output_name = "conv6-3"; 82 | auto output_landmark = zq_interpreter_->getSessionOutput(zq_sess_, output_name.c_str()); 83 | MNN::Tensor landmark_tensor(output_landmark, output_landmark->getDimensionType()); 84 | output_landmark->copyToHostTensor(&landmark_tensor); 85 | 86 | for (int i = 0; i < 106; ++i) { 87 | cv::Point2f curr_pt(landmark_tensor.host()[2 * i + 0] * width + face.x, 88 | landmark_tensor.host()[2 * i + 1] * height + face.y); 89 | keypoints->push_back(curr_pt); 90 | } 91 | 92 | std::cout << "end extract keypoints." << std::endl; 93 | 94 | return 0; 95 | } 96 | 97 | 98 | } -------------------------------------------------------------------------------- /src/face/landmarker/zqlandmarker/zqlandmarker.h: -------------------------------------------------------------------------------- 1 | #ifndef _FACE_ZQLANDMARKER_H_ 2 | #define _FACE_ZQLANDMARKER_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "opencv2/core.hpp" 8 | #include "../landmarker.h" 9 | 10 | #include "MNN/Interpreter.hpp" 11 | #include "MNN/ImageProcess.hpp" 12 | #include "MNN/MNNDefine.h" 13 | #include "MNN/Tensor.hpp" 14 | 15 | namespace mirror { 16 | class ZQLandmarker : public Landmarker { 17 | public: 18 | ZQLandmarker(); 19 | ~ZQLandmarker(); 20 | int Init(const char* model_path); 21 | int ExtractKeypoints(const cv::Mat& img_src, const cv::Rect& face, std::vector* keypoints); 22 | 23 | private: 24 | bool initialized_; 25 | std::shared_ptr pretreat_ = nullptr; 26 | std::shared_ptr zq_interpreter_ = nullptr; 27 | MNN::Session* zq_sess_ = nullptr; 28 | MNN::Tensor* input_tensor_ = nullptr; 29 | 30 | const cv::Size inputSize_ = cv::Size(112, 112); 31 | const float meanVals_[3] = { 127.5f, 127.5f, 127.5f }; 32 | const float normVals_[3] = { 0.0078125f, 0.0078125f, 0.0078125f }; 33 | 34 | }; 35 | 36 | 37 | } 38 | 39 | 40 | 41 | 42 | #endif // !_FACE_ZQLANDMARKER_H_ 43 | -------------------------------------------------------------------------------- /src/face/recognizer/mobilefacenet/mobilefacenet.cpp: -------------------------------------------------------------------------------- 1 | #include "mobilefacenet.h" 2 | 3 | #include 4 | #include 5 | 6 | #include "opencv2/imgproc.hpp" 7 | 8 | namespace mirror { 9 | Mobilefacenet::Mobilefacenet() { 10 | initialized_ = false; 11 | } 12 | 13 | Mobilefacenet::~Mobilefacenet() { 14 | mobilefacenet_interpreter_->releaseModel(); 15 | mobilefacenet_interpreter_->releaseSession(mobilefacenet_sess_); 16 | } 17 | 18 | int Mobilefacenet::Init(const char* model_path) { 19 | std::cout << "start init." << std::endl; 20 | std::string model_file = std::string(model_path) + "/mobilefacenet.mnn"; 21 | mobilefacenet_interpreter_ = std::unique_ptr(MNN::Interpreter::createFromFile(model_file.c_str())); 22 | if (nullptr == mobilefacenet_interpreter_) { 23 | std::cout << "load model failed." << std::endl; 24 | return 10000; 25 | } 26 | 27 | // create session 28 | MNN::ScheduleConfig schedule_config; 29 | schedule_config.type = MNN_FORWARD_CPU; 30 | schedule_config.numThread = 1; 31 | MNN::BackendConfig backend_config; 32 | backend_config.memory = MNN::BackendConfig::Memory_Normal; 33 | backend_config.power = MNN::BackendConfig::Power_Normal; 34 | backend_config.precision = MNN::BackendConfig::Precision_Normal; 35 | schedule_config.backendConfig = &backend_config; 36 | mobilefacenet_sess_ = mobilefacenet_interpreter_->createSession(schedule_config); 37 | input_tensor_ = mobilefacenet_interpreter_->getSessionInput(mobilefacenet_sess_, nullptr); 38 | 39 | initialized_ = true; 40 | 41 | std::cout << "end init." << std::endl; 42 | 43 | return 0; 44 | } 45 | 46 | int Mobilefacenet::ExtractFeature(const cv::Mat& img_face, std::vector* feat) { 47 | std::cout << "start extract feature." << std::endl; 48 | feat->clear(); 49 | if (!initialized_) { 50 | std::cout << "model uninitialized." << std::endl; 51 | return 10000; 52 | } 53 | if (img_face.empty()) { 54 | std::cout << "input empty." << std::endl; 55 | return 10001; 56 | } 57 | 58 | cv::Mat img_resized; 59 | cv::resize(img_face, img_resized, inputSize_); 60 | img_resized.convertTo(img_resized, CV_32FC3); 61 | 62 | // MNN ImageProcess不支持dstformat为MNN_DATA_FORMAT_NCHW 63 | // 需要自己定义TENSORFLOW或CAFFE_C4格式tensor进行赋值 64 | auto nhwc_tensor = MNN::Tensor::create({ 1,inputSize_.height, inputSize_.width, 3 }, NULL, MNN::Tensor::TENSORFLOW); 65 | auto nhwc_data = nhwc_tensor->host(); 66 | auto nhwc_size = nhwc_tensor->size(); 67 | ::memcpy(nhwc_data, img_resized.data, nhwc_size); 68 | input_tensor_->copyFromHostTensor(nhwc_tensor); 69 | delete nhwc_tensor; 70 | 71 | mobilefacenet_interpreter_->runSession(mobilefacenet_sess_); 72 | 73 | // get output 74 | std::string output_name = "fc1"; 75 | auto output_feat = mobilefacenet_interpreter_->getSessionOutput(mobilefacenet_sess_, output_name.c_str()); 76 | MNN::Tensor feat_tensor(output_feat, output_feat->getDimensionType()); 77 | output_feat->copyToHostTensor(&feat_tensor); 78 | 79 | std::cout << "channel: " << feat_tensor.channel() << std::endl 80 | << "width: " << feat_tensor.width() << std::endl 81 | << "height: " << feat_tensor.height() << std::endl; 82 | feat->resize(kFaceFeatureDim); 83 | for (int i = 0; i < kFaceFeatureDim; ++i) { 84 | feat->at(i) = feat_tensor.host()[i]; 85 | } 86 | 87 | std::cout << "end extract feature." << std::endl; 88 | return 0; 89 | } 90 | 91 | 92 | 93 | 94 | } -------------------------------------------------------------------------------- /src/face/recognizer/mobilefacenet/mobilefacenet.h: -------------------------------------------------------------------------------- 1 | #ifndef _FACE_MOBILEFACENET_H_ 2 | #define _FACE_MOBILEFACENET_H_ 3 | 4 | #include 5 | 6 | #include "../recognizer.h" 7 | #include "MNN/Interpreter.hpp" 8 | #include "MNN/ImageProcess.hpp" 9 | #include "MNN/MNNDefine.h" 10 | #include "MNN/Tensor.hpp" 11 | 12 | 13 | namespace mirror { 14 | class Mobilefacenet : public Recognizer { 15 | public: 16 | Mobilefacenet(); 17 | ~Mobilefacenet(); 18 | int Init(const char* model_path); 19 | int ExtractFeature(const cv::Mat& img_face, std::vector* feat); 20 | 21 | private: 22 | bool initialized_; 23 | std::shared_ptr mobilefacenet_interpreter_ = nullptr; 24 | MNN::Session* mobilefacenet_sess_ = nullptr; 25 | MNN::Tensor* input_tensor_ = nullptr; 26 | 27 | const cv::Size inputSize_ = cv::Size(112, 112); 28 | const float meanVals_[3] = { 127.5f, 127.5f, 127.5f }; 29 | const float normVals_[3] = { 0.0078125f, 0.0078125f, 0.0078125f }; 30 | }; 31 | 32 | 33 | } 34 | 35 | 36 | #endif // !_FACE_MOBILEFACENET_H_ 37 | -------------------------------------------------------------------------------- /src/face/recognizer/recognizer.cpp: -------------------------------------------------------------------------------- 1 | #include "recognizer.h" 2 | #include "./mobilefacenet/mobilefacenet.h" 3 | 4 | namespace mirror { 5 | Recognizer* MobilefacenetFactory::CreateRecognizer() { 6 | return new Mobilefacenet(); 7 | } 8 | 9 | } -------------------------------------------------------------------------------- /src/face/recognizer/recognizer.h: -------------------------------------------------------------------------------- 1 | #ifndef _FACE_RECOGNIZER_H_ 2 | #define _FACE_RECOGNIZER_H_ 3 | 4 | #include 5 | #include "opencv2/core.hpp" 6 | #include "../../common/common.h" 7 | 8 | namespace mirror { 9 | class Recognizer { 10 | public: 11 | virtual int Init(const char* model_path) = 0; 12 | virtual int ExtractFeature(const cv::Mat& img_face, std::vector* feat) = 0; 13 | virtual ~Recognizer() {} 14 | }; 15 | 16 | class RecognizerFactory { 17 | public: 18 | virtual Recognizer* CreateRecognizer() = 0; 19 | virtual ~RecognizerFactory() {} 20 | }; 21 | 22 | class MobilefacenetFactory : public RecognizerFactory { 23 | public: 24 | MobilefacenetFactory() {} 25 | Recognizer* CreateRecognizer(); 26 | ~MobilefacenetFactory() {} 27 | }; 28 | 29 | } 30 | 31 | 32 | 33 | #endif // !_FACE_RECOGNIZER_H_ -------------------------------------------------------------------------------- /src/face/test_face.cpp: -------------------------------------------------------------------------------- 1 | #include "../vision_engine.h" 2 | #include "opencv2/opencv.hpp" 3 | 4 | int TestLandmarker(int argc, char* argv[]) { 5 | const char* img_file = "../../data/images/face.jpg"; 6 | cv::Mat img_src = cv::imread(img_file); 7 | 8 | mirror::VisionEngine* vision_engine = new mirror::VisionEngine(); 9 | const char* root_path = "../../data/models"; 10 | vision_engine->Init(root_path); 11 | std::vector faces; 12 | vision_engine->DetectFace(img_src, &faces); 13 | int num_faces = static_cast(faces.size()); 14 | for (int i = 0; i < num_faces; ++i) { 15 | #if 0 16 | for (int j = 0; j < 5; ++j) { 17 | cv::Point curr_pt = cv::Point(faces[i].keypoints_[2 * j], faces[i].keypoints_[2 * j + 1]); 18 | cv::circle(img_src, curr_pt, 2, cv::Scalar(255, 0, 255), 2); 19 | } 20 | #endif 21 | cv::Rect face = faces[i].location_; 22 | std::vector keypoints; 23 | vision_engine->ExtractKeypoints(img_src, face, &keypoints); 24 | cv::Mat face_aligned; 25 | vision_engine->AlignFace(img_src, keypoints, &face_aligned); 26 | std::string face_name = "../../data/images/face" + std::to_string(i) + ".jpg"; 27 | cv::imwrite(face_name.c_str(), face_aligned); 28 | 29 | cv::rectangle(img_src, face, cv::Scalar(0, 255, 0), 2); 30 | int num_keypoints = static_cast(keypoints.size()); 31 | for (int j = 0; j < num_keypoints; ++j) { 32 | cv::circle(img_src, keypoints[j], 1, cv::Scalar(255, 255, 0), 1); 33 | } 34 | } 35 | 36 | cv::imwrite("../../data/images/face_result.jpg", img_src); 37 | cv::imshow("result", img_src); 38 | cv::waitKey(0); 39 | 40 | delete vision_engine; 41 | 42 | return 0; 43 | } 44 | 45 | int TestRecognizer(int argc, char* argv[]) { 46 | const char* face1_path = "../../data/images/bb_face1.jpg"; 47 | const char* face2_path = "../../data/images/bb_face2.jpg"; 48 | cv::Mat face1 = cv::imread(face1_path); 49 | cv::Mat face2 = cv::imread(face2_path); 50 | 51 | const char* model_path = "../../data/models"; 52 | mirror::VisionEngine* vision_engine = new mirror::VisionEngine(); 53 | vision_engine->Init(model_path); 54 | std::vector feat1, feat2; 55 | vision_engine->ExtractFeature(face1, &feat1); 56 | vision_engine->ExtractFeature(face2, &feat2); 57 | float sim = mirror::CalculateSimilarity(feat1, feat2); 58 | std::cout << "similarity is: " << sim << std::endl; 59 | 60 | return 0; 61 | } 62 | 63 | int TestAligner(int argc, char* argv[]) { 64 | std::string img_path = "../../data/images/" + std::string(argv[1]); 65 | cv::Mat img_src = cv::imread(img_path); 66 | 67 | mirror::VisionEngine* vision_engine = new mirror::VisionEngine(); 68 | const char* root_path = "../../data/models"; 69 | vision_engine->Init(root_path); 70 | std::vector faces; 71 | vision_engine->DetectFace(img_src, &faces); 72 | int num_faces = static_cast(faces.size()); 73 | for (int i = 0; i < num_faces; ++i) { 74 | cv::Rect face = faces[i].location_; 75 | std::vector keypoints; 76 | vision_engine->ExtractKeypoints(img_src, face, &keypoints); 77 | cv::Mat face_aligned; 78 | vision_engine->AlignFace(img_src, keypoints, &face_aligned); 79 | std::string face_name = "../../data/images/face" + std::to_string(i) + ".jpg"; 80 | cv::imwrite(face_name.c_str(), face_aligned); 81 | 82 | cv::rectangle(img_src, face, cv::Scalar(0, 255, 0), 2); 83 | int num_keypoints = static_cast(keypoints.size()); 84 | for (int j = 0; j < num_keypoints; ++j) { 85 | cv::circle(img_src, keypoints[j], 1, cv::Scalar(255, 255, 0), 1); 86 | } 87 | } 88 | cv::imshow("result", img_src); 89 | cv::waitKey(0); 90 | 91 | delete vision_engine; 92 | } 93 | 94 | int TestDatabase(int argc, char* argv[]) { 95 | const char* img_path = "../../data/images/face.jpg"; 96 | cv::Mat img_src = cv::imread(img_path); 97 | if (img_src.empty()) { 98 | std::cout << "load image failed." << std::endl; 99 | return 10001; 100 | } 101 | 102 | const char* root_path = "../../data/models"; 103 | mirror::VisionEngine* vision_engine = new mirror::VisionEngine(); 104 | vision_engine->Init(root_path); 105 | vision_engine->Load(); 106 | std::vector faces; 107 | vision_engine->DetectFace(img_src, &faces); 108 | int faces_num = static_cast(faces.size()); 109 | std::cout << "faces number: " << faces_num << std::endl; 110 | for (int i = 0; i < faces_num; ++i) { 111 | cv::Rect face = faces.at(i).location_; 112 | std::vector keypoints; 113 | vision_engine->ExtractKeypoints(img_src, face, &keypoints); 114 | cv::Mat face_aligned; 115 | vision_engine->AlignFace(img_src, keypoints, &face_aligned); 116 | std::vector feat; 117 | vision_engine->ExtractFeature(face_aligned, &feat); 118 | 119 | #if 0 120 | vision_engine->Insert(feat, "face" + std::to_string(i)); 121 | #endif 122 | 123 | #if 1 124 | mirror::QueryResult query_result; 125 | vision_engine->QueryTop(feat, &query_result); 126 | std::cout << i << "-th face is: " << query_result.name_ << 127 | " similarity is: " << query_result.sim_ << std::endl; 128 | #endif 129 | cv::rectangle(img_src, faces.at(i).location_, cv::Scalar(0, 255, 0), 2); 130 | cv::Point2f offset = cv::Point2f(faces[i].location_.tl()); 131 | for (int j = 0; j < static_cast(keypoints.size()); ++j) { 132 | cv::circle(img_src, keypoints[j] + offset, 1, cv::Scalar(0, 0, 255), 1); 133 | } 134 | } 135 | vision_engine->Delete("face0"); 136 | vision_engine->Save(); 137 | cv::imwrite("../../data/images/result.jpg", img_src); 138 | 139 | return 0; 140 | } 141 | 142 | int main(int argc, char* argv[]) { 143 | // return TestRecognizer(argc, argv); 144 | // return ExtractFace(argc, argv); 145 | // return TestLandmarker(argc, argv); 146 | // return TestDatabase(argc, argv); 147 | return TestAligner(argc, argv); 148 | } -------------------------------------------------------------------------------- /src/object/mobilenetssd/mobilenetssd.cpp: -------------------------------------------------------------------------------- 1 | #include "mobilenetssd.h" 2 | #include 3 | #include 4 | 5 | #include "opencv2/imgproc.hpp" 6 | 7 | namespace mirror { 8 | MobilenetSSD::MobilenetSSD() { 9 | initialized_ = false; 10 | } 11 | 12 | MobilenetSSD::~MobilenetSSD() { 13 | mobilenetssd_interpreter_->releaseModel(); 14 | mobilenetssd_interpreter_->releaseSession(mobilenetssd_sess_); 15 | } 16 | 17 | int MobilenetSSD::Init(const char * root_path) { 18 | std::cout << "start Init." << std::endl; 19 | std::string model_file = std::string(root_path) + "/mobilenetssd.mnn"; 20 | mobilenetssd_interpreter_ = std::unique_ptr(MNN::Interpreter::createFromFile(model_file.c_str())); 21 | if (nullptr == mobilenetssd_interpreter_) { 22 | std::cout << "load model failed." << std::endl; 23 | return 10000; 24 | } 25 | 26 | MNN::ScheduleConfig schedule_config; 27 | schedule_config.type = MNN_FORWARD_CPU; 28 | schedule_config.numThread = 4; 29 | 30 | MNN::BackendConfig backend_config; 31 | backend_config.precision = MNN::BackendConfig::Precision_High; 32 | backend_config.power = MNN::BackendConfig::Power_High; 33 | schedule_config.backendConfig = &backend_config; 34 | 35 | mobilenetssd_sess_ = mobilenetssd_interpreter_->createSession(schedule_config); 36 | 37 | // image processer 38 | MNN::CV::Matrix trans; 39 | trans.setScale(1.0f, 1.0f); 40 | MNN::CV::ImageProcess::Config img_config; 41 | img_config.filterType = MNN::CV::BICUBIC; 42 | ::memcpy(img_config.mean, meanVals_, sizeof(meanVals_)); 43 | ::memcpy(img_config.normal, normVals_, sizeof(normVals_)); 44 | img_config.sourceFormat = MNN::CV::BGR; 45 | img_config.destFormat = MNN::CV::RGB; 46 | pretreat_data_ = std::shared_ptr(MNN::CV::ImageProcess::create(img_config)); 47 | pretreat_data_->setMatrix(trans); 48 | 49 | std::string input_name = "data"; 50 | input_tensor_ = mobilenetssd_interpreter_->getSessionInput(mobilenetssd_sess_, input_name.c_str()); 51 | mobilenetssd_interpreter_->resizeTensor(input_tensor_, dims_); 52 | mobilenetssd_interpreter_->resizeSession(mobilenetssd_sess_); 53 | 54 | initialized_ = true; 55 | 56 | std::cout << "end Init." << std::endl; 57 | return 0; 58 | } 59 | 60 | int MobilenetSSD::DetectObject(const cv::Mat & img_src, std::vector* objects) { 61 | std::cout << "start detect." << std::endl; 62 | if (!initialized_) { 63 | std::cout << "model uninitialized." << std::endl; 64 | return 10000; 65 | } 66 | if (img_src.empty()) { 67 | std::cout << "input empty." << std::endl; 68 | return 10001; 69 | } 70 | 71 | int width = img_src.cols; 72 | int height = img_src.rows; 73 | 74 | // preprocess 75 | cv::Mat img_resized; 76 | cv::resize(img_src, img_resized, inputSize_); 77 | pretreat_data_->convert(img_resized.data, inputSize_.width, inputSize_.height, 0, input_tensor_); 78 | 79 | mobilenetssd_interpreter_->runSession(mobilenetssd_sess_); 80 | std::string output_name = "detection_out"; 81 | MNN::Tensor* output_tensor = mobilenetssd_interpreter_->getSessionOutput(mobilenetssd_sess_, output_name.c_str()); 82 | 83 | // copy to host 84 | MNN::Tensor output_host(output_tensor, output_tensor->getDimensionType()); 85 | output_tensor->copyToHostTensor(&output_host); 86 | 87 | auto output_ptr = output_host.host(); 88 | std::vector objects_tmp; 89 | for (int i = 0; i < output_host.height(); ++i) { 90 | int index = i * output_host.width(); 91 | ObjectInfo object; 92 | object.name_ = class_names[int(output_ptr[index + 0])]; 93 | object.score_ = output_ptr[index + 1]; 94 | object.location_.x = output_ptr[index + 2] * width; 95 | object.location_.y = output_ptr[index + 3] * height; 96 | object.location_.width = output_ptr[index + 4] * width - object.location_.x; 97 | object.location_.height = output_ptr[index + 5] * height - object.location_.y; 98 | 99 | objects_tmp.push_back(object); 100 | } 101 | NMS(objects_tmp, objects, nmsThreshold_); 102 | 103 | std::cout << "end detect." << std::endl; 104 | 105 | return 0; 106 | } 107 | 108 | } 109 | -------------------------------------------------------------------------------- /src/object/mobilenetssd/mobilenetssd.h: -------------------------------------------------------------------------------- 1 | #ifndef _MOBILENET_SSD_H_ 2 | #define _MOBILENET_SSD_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "MNN/Interpreter.hpp" 8 | #include "MNN/MNNDefine.h" 9 | #include "MNN/Tensor.hpp" 10 | #include "MNN/ImageProcess.hpp" 11 | 12 | #include "opencv2/core.hpp" 13 | #include "opencv2/imgproc.hpp" 14 | #include "../common/common.h" 15 | 16 | namespace mirror { 17 | class MobilenetSSD { 18 | public: 19 | std::vector class_names = { 20 | "background", "aeroplane", "bicycle", "bird", "boat", 21 | "bottle", "bus", "car", "cat", "chair", 22 | "cow", "diningtable", "dog", "horse", 23 | "motorbike", "person", "pottedplant", 24 | "sheep", "sofa", "train", "tvmonitor" 25 | }; 26 | MobilenetSSD(); 27 | ~MobilenetSSD(); 28 | int Init(const char* root_path); 29 | int DetectObject(const cv::Mat& img_src, std::vector* objects); 30 | 31 | private: 32 | bool initialized_; 33 | const cv::Size inputSize_ = { 300, 300 }; 34 | std::vector dims_ = { 1, 3, 300, 300 }; 35 | const float meanVals_[3] = { 0.5f, 0.5f, 0.5f }; 36 | const float normVals_[3] = { 0.007843f, 0.007843f, 0.007843f }; 37 | const float nmsThreshold_ = 0.5f; 38 | 39 | std::shared_ptr mobilenetssd_interpreter_; 40 | MNN::Session* mobilenetssd_sess_ = nullptr; 41 | MNN::Tensor* input_tensor_ = nullptr; 42 | std::shared_ptr pretreat_data_ = nullptr; 43 | 44 | }; 45 | 46 | } 47 | 48 | 49 | #endif // !_MOBILENET_SSD_H_ 50 | -------------------------------------------------------------------------------- /src/object/object_engine.cpp: -------------------------------------------------------------------------------- 1 | #include "object_engine.h" 2 | 3 | #include 4 | #include 5 | 6 | namespace mirror { 7 | ObjectEngine::ObjectEngine() { 8 | mobilenetssd_ = new MobilenetSSD(); 9 | initialized_ = false; 10 | } 11 | 12 | ObjectEngine::~ObjectEngine() { 13 | if (mobilenetssd_) { 14 | delete mobilenetssd_; 15 | mobilenetssd_ = nullptr; 16 | } 17 | } 18 | 19 | int ObjectEngine::Init(const char* model_path) { 20 | std::cout << "start init." << std::endl; 21 | if (mobilenetssd_->Init(model_path) != 0) { 22 | return 10000; 23 | } 24 | 25 | initialized_ = true; 26 | std::cout << "end init." << std::endl; 27 | return 0; 28 | } 29 | 30 | int ObjectEngine::DetectObject(const cv::Mat& img_src, std::vector* objects) { 31 | return mobilenetssd_->DetectObject(img_src, objects); 32 | } 33 | 34 | 35 | } -------------------------------------------------------------------------------- /src/object/object_engine.h: -------------------------------------------------------------------------------- 1 | #ifndef _OBJECTER_H_ 2 | #define _OBJECTER_H_ 3 | 4 | #include "../../common/common.h" 5 | #include "./mobilenetssd/mobilenetssd.h" 6 | 7 | namespace mirror { 8 | class ObjectEngine { 9 | public: 10 | ObjectEngine(); 11 | ~ObjectEngine(); 12 | int Init(const char* root_path); 13 | int DetectObject(const cv::Mat& img_src, std::vector* objects); 14 | 15 | private: 16 | bool initialized_; 17 | MobilenetSSD* mobilenetssd_; 18 | 19 | }; 20 | 21 | } 22 | 23 | 24 | 25 | #endif // !_OBJECTER_H_ 26 | -------------------------------------------------------------------------------- /src/object/test_object.cpp: -------------------------------------------------------------------------------- 1 | #include "../vision_engine.h" 2 | #include "opencv2/opencv.hpp" 3 | 4 | int TestObject(int argc, char* argv[]){ 5 | const char* img_path = "../../data/images/object.jpg"; 6 | cv::Mat img_src = cv::imread(img_path); 7 | mirror::VisionEngine* vision_engine = new mirror::VisionEngine(); 8 | 9 | const char* root_path = "../../data/models"; 10 | vision_engine->Init(root_path); 11 | std::vector objects; 12 | vision_engine->DetectObject(img_src, &objects); 13 | 14 | int num_objects = static_cast(objects.size()); 15 | for (int i = 0; i < num_objects; ++i) { 16 | std::cout << "location: " << objects[i].location_ << std::endl; 17 | cv::rectangle(img_src, objects[i].location_, cv::Scalar(255, 0, 255), 2); 18 | char text[256]; 19 | sprintf(text, "%s %.1f%%", objects[i].name_.c_str(), objects[i].score_ * 100); 20 | int baseLine = 0; 21 | cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); 22 | cv::putText(img_src, text, cv::Point(objects[i].location_.x, 23 | objects[i].location_.y + label_size.height), 24 | cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0)); 25 | } 26 | cv::imwrite("../../data/images/object_result.jpg", img_src); 27 | cv::imshow("result", img_src); 28 | cv::waitKey(0); 29 | 30 | delete vision_engine; 31 | return 0; 32 | } 33 | 34 | int main(int argc, char* argv[]) { 35 | return TestObject(argc, argv); 36 | } 37 | -------------------------------------------------------------------------------- /src/vision_engine.cpp: -------------------------------------------------------------------------------- 1 | #include "vision_engine.h" 2 | #include 3 | 4 | #include "classifier/classifier.h" 5 | #include "object/object_engine.h" 6 | #include "face/face_engine.h" 7 | #include "object/object_engine.h" 8 | 9 | 10 | namespace mirror { 11 | class VisionEngine::Impl { 12 | public: 13 | Impl() { 14 | face_engine_ = new FaceEngine(); 15 | object_engine_ = new ObjectEngine(); 16 | classifier_ = new Classifier(); 17 | initialized_ = false; 18 | } 19 | 20 | ~Impl() { 21 | if (classifier_) { 22 | delete classifier_; 23 | classifier_ = nullptr; 24 | } 25 | 26 | if (face_engine_) { 27 | delete face_engine_; 28 | face_engine_ = nullptr; 29 | } 30 | if (object_engine_) { 31 | delete object_engine_; 32 | object_engine_ = nullptr; 33 | } 34 | } 35 | 36 | int Init(const char* root_path) { 37 | if (classifier_->Init(root_path) != 0) { 38 | std::cout << "init classifier failed." << std::endl; 39 | return 10000; 40 | } 41 | 42 | if (object_engine_->Init(root_path) != 0) { 43 | std::cout << "init object detector failed." << std::endl; 44 | return 10000; 45 | } 46 | 47 | if (face_engine_->Init(root_path) != 0) { 48 | std::cout << "Init face engine failed." << std::endl; 49 | return 10000; 50 | } 51 | 52 | initialized_ = true; 53 | 54 | return 0; 55 | } 56 | inline int Classify(const cv::Mat& img_src, std::vector* images) { 57 | return classifier_->Classify(img_src, images); 58 | } 59 | inline int DetectObject(const cv::Mat& img_src, std::vector* objects) { 60 | return object_engine_->DetectObject(img_src, objects); 61 | } 62 | 63 | inline int DetectFace(const cv::Mat& img_src, std::vector* faces) { 64 | return face_engine_->DetectFace(img_src, faces); 65 | } 66 | inline int ExtractKeypoints(const cv::Mat& img_src, const cv::Rect& face, std::vector* keypoints) { 67 | return face_engine_->ExtractKeypoints(img_src, face, keypoints); 68 | } 69 | inline int AlignFace(const cv::Mat& img_src, const std::vector& keypoints, cv::Mat* face_aligned) { 70 | return face_engine_->AlignFace(img_src, keypoints, face_aligned); 71 | } 72 | inline int ExtractFeature(const cv::Mat& img_face, std::vector* feat) { 73 | return face_engine_->ExtractFeature(img_face, feat); 74 | } 75 | 76 | inline int Insert(const std::vector& feat, const std::string& name) { 77 | return face_engine_->Insert(feat, name); 78 | } 79 | inline int Delete(const std::string& name) { 80 | return face_engine_->Delete(name); 81 | } 82 | inline int64_t QueryTop(const std::vector& feat, QueryResult *query_result = nullptr) { 83 | return face_engine_->QueryTop(feat, query_result); 84 | } 85 | inline int Save() { 86 | return face_engine_->Save(); 87 | } 88 | inline int Load() { 89 | return face_engine_->Load(); 90 | } 91 | 92 | private: 93 | bool initialized_; 94 | FaceEngine* face_engine_; 95 | ObjectEngine* object_engine_; 96 | Classifier* classifier_; 97 | 98 | }; 99 | 100 | VisionEngine::VisionEngine() { 101 | impl_ = new VisionEngine::Impl(); 102 | } 103 | 104 | VisionEngine::~VisionEngine() { 105 | if (impl_) { 106 | delete impl_; 107 | impl_ = nullptr; 108 | } 109 | } 110 | 111 | int VisionEngine::Init(const char* root_path) { 112 | return impl_->Init(root_path); 113 | } 114 | 115 | int VisionEngine::Classify(const cv::Mat& img_src, std::vector* images) { 116 | return impl_->Classify(img_src, images); 117 | } 118 | 119 | int VisionEngine::DetectObject(const cv::Mat& img_src, std::vector* objects) { 120 | return impl_->DetectObject(img_src, objects); 121 | } 122 | 123 | int VisionEngine::DetectFace(const cv::Mat& img_src, std::vector* faces) { 124 | return impl_->DetectFace(img_src, faces); 125 | } 126 | 127 | int VisionEngine::ExtractKeypoints(const cv::Mat& img_src, const cv::Rect& face, std::vector* keypoints) { 128 | return impl_->ExtractKeypoints(img_src, face, keypoints); 129 | } 130 | 131 | int VisionEngine::AlignFace(const cv::Mat& img_src, const std::vector& keypoints, cv::Mat* face_aligned) { 132 | return impl_->AlignFace(img_src, keypoints, face_aligned); 133 | } 134 | 135 | int VisionEngine::ExtractFeature(const cv::Mat& img_face, std::vector* feat) { 136 | return impl_->ExtractFeature(img_face, feat); 137 | } 138 | 139 | int VisionEngine::Insert(const std::vector& feat, const std::string& name) { 140 | return impl_->Insert(feat, name); 141 | } 142 | 143 | int VisionEngine::Delete(const std::string& name) { 144 | return impl_->Delete(name); 145 | } 146 | 147 | int64_t VisionEngine::QueryTop(const std::vector& feat, 148 | QueryResult* query_result) { 149 | return impl_->QueryTop(feat, query_result); 150 | } 151 | 152 | int VisionEngine::Save() { 153 | return impl_->Save(); 154 | } 155 | 156 | int VisionEngine::Load() { 157 | return impl_->Load(); 158 | } 159 | 160 | } -------------------------------------------------------------------------------- /src/vision_engine.h: -------------------------------------------------------------------------------- 1 | #ifndef _VISION_ENGINE_H_ 2 | #define _VISION_ENGINE_H_ 3 | 4 | #include 5 | #include "common/common.h" 6 | 7 | namespace mirror { 8 | class VisionEngine { 9 | public: 10 | VisionEngine(); 11 | ~VisionEngine(); 12 | int Init(const char* root_path); 13 | int Classify(const cv::Mat& img_src, std::vector* images); 14 | 15 | int DetectObject(const cv::Mat& img_src, std::vector* objects); 16 | 17 | int DetectFace(const cv::Mat& img_src, std::vector* faces); 18 | int ExtractKeypoints(const cv::Mat& img_src, const cv::Rect& face, std::vector* keypoints); 19 | int AlignFace(const cv::Mat& img_src, const std::vector& keypoints, cv::Mat* face_aligned); 20 | int ExtractFeature(const cv::Mat& img_face, std::vector* feat); 21 | 22 | int Insert(const std::vector& feat, const std::string& name); 23 | int Delete(const std::string& name); 24 | int64_t QueryTop(const std::vector& feat, QueryResult *query_result = nullptr); 25 | int Save(); 26 | int Load(); 27 | 28 | private: 29 | class Impl; 30 | Impl* impl_; 31 | 32 | }; 33 | 34 | 35 | } 36 | 37 | 38 | 39 | 40 | #endif // !_VISION_ENGINE_ --------------------------------------------------------------------------------