├── .github └── workflows │ └── docker-image.yml ├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README.md ├── cmake ├── DeepvacConfig.cmake.in └── DeepvacConfigVersion.cmake.in ├── core ├── CMakeLists.txt ├── include │ ├── deepvac.h │ └── deepvac_nv.h └── src │ ├── deepvac.cpp │ └── deepvac_nv.cpp ├── docker └── ubuntu20.04 │ ├── Dockerfile.cuda │ ├── Dockerfile.deepvac │ └── Dockerfile.deepvac.vision ├── examples ├── CMakeLists.txt ├── include │ ├── syszux_imagenet_classes.h │ └── test_extract_feature.h └── src │ ├── test_cls_mobile.cpp │ ├── test_detect_yolo.cpp │ ├── test_extract_feature.cpp │ ├── test_face_retina.cpp │ ├── test_face_retina_nv.cpp │ ├── test_ocr_db.cpp │ ├── test_ocr_pse.cpp │ ├── test_resnet_benchmark.cpp │ ├── test_seg_esp.cpp │ ├── test_syszux_alignface.cpp │ ├── test_syszux_nms.cpp │ ├── test_syszux_priorbox.cpp │ └── test_syszux_verifylandmark.cpp ├── loader ├── CMakeLists.txt ├── include │ └── deepvac_loader.h └── src │ └── deepvac_loader.cpp ├── modules ├── CMakeLists.txt ├── include │ ├── syszux_cls_mobile.h │ ├── syszux_cls_resnet.h │ ├── syszux_detect_yolo.h │ ├── syszux_face_id_nv.h │ ├── syszux_face_retina.h │ ├── syszux_face_retina_config.h │ ├── syszux_face_retina_nv.h │ ├── syszux_ocr_db.h │ ├── syszux_ocr_pse.h │ └── syszux_seg_esp.h └── src │ ├── syszux_cls_mobile.cpp │ ├── syszux_cls_resnet.cpp │ ├── syszux_detect_yolo.cpp │ ├── syszux_face_id_nv.cpp │ ├── syszux_face_retina.cpp │ ├── syszux_face_retina_config.cpp │ ├── syszux_face_retina_nv.cpp │ ├── syszux_ocr_db.cpp │ ├── syszux_ocr_pse.cpp │ └── syszux_seg_esp.cpp ├── python ├── CMakeLists.txt ├── include │ └── syszux_type_caster.h └── src │ └── README.md └── utils ├── CMakeLists.txt ├── include ├── gemfield.h ├── syszux_align_face.h ├── syszux_base64.h ├── syszux_clipper.h ├── syszux_decrypt.h ├── syszux_glab.h ├── syszux_img2tensor.h ├── syszux_nms.h ├── syszux_priorbox.h ├── syszux_stream_buffer.h ├── syszux_tensorrt_buffers.h └── syszux_verify_landmark.h └── src ├── syszux_align_face.cpp ├── syszux_base64.cpp ├── syszux_clipper.cpp ├── syszux_glab.cpp ├── syszux_img2tensor.cpp ├── syszux_nms.cpp ├── syszux_priorbox.cpp ├── syszux_stream_buffer.cpp └── syszux_verify_landmark.cpp /.github/workflows/docker-image.yml: -------------------------------------------------------------------------------- 1 | name: Docker Image CI 2 | on: 3 | push: 4 | branches: 5 | - 'gemfield/**' 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Get short SHA 11 | id: gemfield 12 | run: echo "::set-output name=sha8::$(echo ${GITHUB_SHA} | cut -c1-8)" 13 | - uses: actions/checkout@v2 14 | - name: Login to DockerHub 15 | uses: docker/login-action@v1 16 | with: 17 | username: ${{ secrets.HUB_USERNAME }} 18 | password: ${{ secrets.HUB_TOKEN }} 19 | - name: Build the base image 20 | run: docker build . --file docker/ubuntu20.04/Dockerfile.cuda --tag gemfield/cuda:11.0.3-cudnn8-devel-ubuntu20.04 21 | - name: Push Docker image 22 | run: docker push gemfield/cuda 23 | - name: Build the deepvac devel image 24 | run: docker build . --file docker/ubuntu20.04/Dockerfile.deepvac --tag gemfield/deepvac:11.0.3-cudnn8-devel-ubuntu20.04 25 | - name: Push Docker image 26 | run: docker push gemfield/deepvac 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | 34 | # build 35 | build 36 | 37 | # install 38 | install 39 | 40 | .vscode 41 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 Gemfield 2 | # This file is part of libdeepvac, licensed under the GPLv3 (the "License") 3 | # You may not use this file except in compliance with the License. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | set(DEEPVAC_VERSION 0.1.0) 7 | project(deepvac 8 | VERSION ${DEEPVAC_VERSION} 9 | DESCRIPTION "A library for using PyTorch model in C++." 10 | HOMEPAGE_URL "https://github.com/deepvac/libdeepvac" 11 | LANGUAGES CXX) 12 | 13 | #has opencv static library link issue when turn off BUILD_STATIC 14 | option(BUILD_STATIC "build static libdeepvac library" ON) 15 | 16 | option(USE_MKL "use MKL as pytorch LAPACK backend" OFF) 17 | option(USE_LOADER "build loader which needs c++17" OFF) 18 | option(USE_CUDA "Use CUDA" OFF) 19 | option(USE_NUMA "Use NUMA" OFF) 20 | option(USE_TENSORRT "Use TensorRT" OFF) 21 | option(BUILD_ALL_EXAMPLES "build all examples" OFF) 22 | option(USE_STATIC_LIBTORCH "use LibTorch static library" OFF) 23 | option(GARRULOUS_GEMFIELD "Build deepvac libraries garrulously" OFF) 24 | option(USE_MAGMA "use magma" ON) 25 | #You should customize the blas dependencies. 26 | message(STATUS "SYSTEM_LAPACK_LIBRARIES: ${SYSTEM_LAPACK_LIBRARIES}") 27 | 28 | if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") 29 | set(LINUX TRUE) 30 | else() 31 | set(LINUX FALSE) 32 | message(FATAL_ERROR "deepvac does not support platform: " ${CMAKE_SYSTEM_NAME}) 33 | endif() 34 | 35 | if(USE_MKL) 36 | if(NOT DEFINED MKL_HOME) 37 | if(USE_STATIC_LIBTORCH) 38 | set(MKL_HOME "/opt/intel/mkl") 39 | else() 40 | set(MKL_HOME "/opt/conda") 41 | endif() 42 | endif() 43 | if(NOT EXISTS ${MKL_HOME}) 44 | message(FATAL "unable to locate MKL library, either set MKL_HOME or install MKL") 45 | endif() 46 | 47 | if(CMAKE_SIZEOF_VOID_P EQUAL 8) 48 | set(MKL_ARCHITECTURES intel64) 49 | else() 50 | set(MKL_ARCHITECTURES ia32) 51 | endif() 52 | 53 | if(USE_STATIC_LIBTORCH) 54 | #must consistent with pytorch static library. 55 | set(INT_LIB "libmkl_intel_lp64.a") 56 | set(SEQ_LIB "libmkl_sequential.a") 57 | #set(THR_LIB "libmkl_intel_thread.a") 58 | set(THR_LIB "libmkl_gnu_thread.a") 59 | set(COR_LIB "libmkl_core.a") 60 | set(MKL_LIB_PATH ${MKL_HOME}/lib/${MKL_ARCHITECTURES}) 61 | find_path(MKL_INCLUDE_DIR NAMES mkl.h HINTS ${MKL_HOME}/include) 62 | else() 63 | #must consistent with pytorch static library. 64 | set(INT_LIB "libmkl_intel_lp64.so") 65 | set(SEQ_LIB "libmkl_sequential.so") 66 | #set(THR_LIB "libmkl_intel_thread.a") 67 | set(THR_LIB "libmkl_gnu_thread.so") 68 | set(COR_LIB "libmkl_core.so") 69 | set(MKL_LIB_PATH ${MKL_HOME}/lib) 70 | endif() 71 | 72 | find_library(MKL_INTERFACE_LIBRARY NAMES ${INT_LIB} PATHS ${MKL_LIB_PATH} NO_DEFAULT_PATH) 73 | find_library(MKL_SEQUENTIAL_LAYER_LIBRARY NAMES ${SEQ_LIB} PATHS ${MKL_LIB_PATH} NO_DEFAULT_PATH) 74 | find_library(MKL_THREAD_LAYER_LIBRARY NAMES ${THR_LIB} PATHS ${MKL_LIB_PATH} NO_DEFAULT_PATH) 75 | find_library(MKL_CORE_LIBRARY NAMES ${COR_LIB} PATHS ${MKL_LIB_PATH} NO_DEFAULT_PATH) 76 | 77 | set(MKL_INCLUDE_DIRS ${MKL_INCLUDE_DIR}) 78 | set(MKL_LIBRARIES ${MKL_CORE_LIBRARY} ${MKL_THREAD_LAYER_LIBRARY} ${MKL_INTERFACE_LIBRARY} ) 79 | if(USE_STATIC_LIBTORCH) 80 | set(MKL_LIBRARIES -Wl,--start-group ${MKL_LIBRARIES} -Wl,--end-group) 81 | endif() 82 | list(APPEND MKL_LIBRARIES "-fopenmp") 83 | list(APPEND MKL_LIBRARIES "-lpthread") 84 | list(APPEND MKL_LIBRARIES "-lm") 85 | list(APPEND MKL_LIBRARIES "-ldl") 86 | if(NOT USE_STATIC_LIBTORCH) 87 | list(APPEND TORCH_LIBRARIES ${MKL_LIBRARIES}) 88 | endif() 89 | else() 90 | if(NOT USE_STATIC_LIBTORCH) 91 | list(APPEND TORCH_LIBRARIES "${SYSTEM_LAPACK_LIBRARIES}") 92 | endif() 93 | endif() 94 | 95 | set(CMAKE_CXX_STANDARD 17) 96 | set(CMAKE_CXX_FLAGS "-Wno-error=deprecated-declarations -Wno-deprecated-declarations") 97 | #setup library output dir 98 | set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib) 99 | #setup binary output dir 100 | set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin) 101 | list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules) 102 | #replace with your libtorch dir 103 | #set(CMAKE_PREFIX_PATH /home/gemfield/libtorch/) 104 | if(GARRULOUS_GEMFIELD) 105 | message(STATUS "will build deepvac library garrulously...") 106 | add_definitions(-DGARRULOUS_GEMFIELD) 107 | endif() 108 | 109 | find_package(Torch REQUIRED) 110 | find_package(OpenCV REQUIRED) 111 | find_package(Eigen3 QUIET) 112 | 113 | if(USE_TENSORRT) 114 | find_path(TENSORRT_INCLUDE_DIR NvInfer.h HINTS ${TENSORRT_ROOT} ${CUDA_TOOLKIT_ROOT_DIR} PATH_SUFFIXES include) 115 | message(STATUS "Found TensorRT headers at ${TENSORRT_INCLUDE_DIR}") 116 | find_library(TENSORRT_LIBRARY_INFER nvinfer HINTS ${TENSORRT_ROOT} ${TENSORRT_BUILD} ${CUDA_TOOLKIT_ROOT_DIR} PATH_SUFFIXES lib lib64 lib/x64) 117 | find_library(TENSORRT_LIBRARY_INFER_PLUGIN nvinfer_plugin HINTS ${TENSORRT_ROOT} ${TENSORRT_BUILD} ${CUDA_TOOLKIT_ROOT_DIR} PATH_SUFFIXES lib lib64 lib/x64) 118 | find_library(TENSORRT_LIBRARY_INFER_PARSER nvparsers HINTS ${TENSORRT_ROOT} ${TENSORRT_BUILD} ${CUDA_TOOLKIT_ROOT_DIR} PATH_SUFFIXES lib lib64 lib/x64) 119 | 120 | set(TENSORRT_LIBRARY ${TENSORRT_LIBRARY_INFER} ${TENSORRT_LIBRARY_INFER_PLUGIN} ${TENSORRT_LIBRARY_INFER_PARSER}) 121 | MESSAGE(STATUS "Found TensorRT libs at ${TENSORRT_LIBRARY}") 122 | find_package_handle_standard_args(TENSORRT DEFAULT_MSG TENSORRT_INCLUDE_DIR TENSORRT_LIBRARY) 123 | if(NOT TENSORRT_FOUND) 124 | message(ERROR "Cannot find TensorRT library.") 125 | endif() 126 | endif() 127 | 128 | #libtorch static stuff 129 | if(USE_STATIC_LIBTORCH) 130 | message(STATUS "will find libtorch static library since USE_STATIC_LIBTORCH is ON") 131 | list(APPEND LIBTORCH_CUDA_LIBRARIES ${TORCH_LIBRARIES}) 132 | 133 | if(USE_CUDA AND USE_MAGMA) 134 | FIND_LIBRARY(MAGMA_LIBRARIES magma HINTS /opt/conda /usr/local/magma PATH_SUFFIXES lib) 135 | if(MAGMA_LIBRARIES) 136 | list(APPEND LIBTORCH_CUDA_LIBRARIES ${MAGMA_LIBRARIES}) 137 | else(MAGMA_LIBRARIES) 138 | message(FATAL "Cannot find magma, which is required for libdeepvac when USE_CUDA & USE_MAGMA are enabled.") 139 | endif(MAGMA_LIBRARIES) 140 | endif() 141 | 142 | if(USE_MKL) 143 | list(APPEND LIBTORCH_CUDA_LIBRARIES ${MKL_LIBRARIES}) 144 | else() 145 | list(APPEND LIBTORCH_CUDA_LIBRARIES "${SYSTEM_LAPACK_LIBRARIES}") 146 | endif() 147 | 148 | foreach(GEM ${LIBTORCH_CUDA_LIBRARIES}) 149 | string(FIND ${GEM} "_cuda.a" GEM_FIELD) 150 | if(NOT ${GEM_FIELD} EQUAL "-1") 151 | message(STATUS "found cuda library need to be removed from LIBTORCH_CPU_LIBRARIES: ${GEM}") 152 | continue() 153 | endif() 154 | list(APPEND LIBTORCH_CPU_LIBRARIES ${GEM}) 155 | endforeach() 156 | 157 | if(USE_CUDA) 158 | find_package(CUDA REQUIRED) 159 | message(STATUS "will build with CUDA: ${CUDA_TOOLKIT_ROOT_DIR}") 160 | find_library(CUDNN_LIBRARY cudnn 161 | HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR} 162 | PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64) 163 | 164 | find_library(cublas_LIBRARY cublas 165 | HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR} 166 | PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64) 167 | 168 | find_library(cublasLT_LIBRARY cublasLt 169 | HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR} 170 | PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64) 171 | 172 | list(APPEND LIBCUDA_LIBRARIES 173 | ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcurand.so 174 | ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcusparse.so 175 | ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcufft.so 176 | ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcusolver.so 177 | ${cublas_LIBRARY} 178 | ${cublasLT_LIBRARY} 179 | ${CUDA_nvToolsExt_LIBRARY} 180 | ${CUDNN_LIBRARY} 181 | ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcudart.so 182 | ) 183 | if(USE_NUMA) 184 | list(APPEND LIBCUDA_LIBRARIES -lnuma) 185 | endif() 186 | message(STATUS "LIBCUDA_LIBRARIES: ${LIBCUDA_LIBRARIES}") 187 | set(LIBTORCH_DEFAULT_LIBRARIES ${LIBTORCH_CUDA_LIBRARIES}) 188 | else() 189 | set(LIBTORCH_DEFAULT_LIBRARIES ${LIBTORCH_CPU_LIBRARIES}) 190 | endif() 191 | message(STATUS "LIBTORCH_CUDA_LIBRARIES: ${LIBTORCH_CUDA_LIBRARIES}") 192 | message(STATUS "LIBTORCH_CPU_LIBRARIES: ${LIBTORCH_CPU_LIBRARIES}") 193 | endif() 194 | 195 | macro (add_syszux_sources) 196 | file (RELATIVE_PATH _relPath "${PROJECT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}") 197 | foreach (_src ${ARGN}) 198 | list (APPEND SRC_LIST "${_src}") 199 | endforeach() 200 | if (_relPath) 201 | set (SRC_LIST ${SRC_LIST} PARENT_SCOPE) 202 | endif() 203 | endmacro() 204 | 205 | macro (add_header_dir) 206 | foreach (_header_dir ${ARGN}) 207 | list (APPEND HEADER_DIR_LIST "${_header_dir}") 208 | endforeach() 209 | set (HEADER_DIR_LIST ${HEADER_DIR_LIST} PARENT_SCOPE) 210 | endmacro() 211 | 212 | macro (add_syszux_headers) 213 | file (RELATIVE_PATH _relPath "${PROJECT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}") 214 | foreach (_src ${ARGN}) 215 | list (APPEND HEADER_FILE_LIST "${_src}") 216 | endforeach() 217 | if (_relPath) 218 | set (HEADER_FILE_LIST ${HEADER_FILE_LIST} PARENT_SCOPE) 219 | endif() 220 | endmacro() 221 | 222 | message(STATUS "OpenCV library status:") 223 | message(STATUS " config: ${OpenCV_DIR}") 224 | message(STATUS " version: ${OpenCV_VERSION}") 225 | message(STATUS " libraries: ${OpenCV_LIBS}") 226 | message(STATUS " include path: ${OpenCV_INCLUDE_DIRS}") 227 | 228 | message(STATUS "TORCH_LIBRARIES: ${TORCH_LIBRARIES}") 229 | message(STATUS "TORCH_INCLUDE_DIRS: ${TORCH_INCLUDE_DIRS}") 230 | message(STATUS "CUDA_INCLUDE_DIRS: ${CUDA_INCLUDE_DIRS}") 231 | 232 | set(DEEPVAC_LIBTORCH_INCLUDE_DIRS ${TORCH_INCLUDE_DIRS}) 233 | set(DEEPVAC_TENSORRT_INCLUDE_DIRS ${TENSORRT_INCLUDE_DIR}) 234 | set(DEEPVAC_CV_INCLUDE_DIRS ${OpenCV_INCLUDE_DIRS}) 235 | 236 | set(DEEPVAC_LIBRARIES deepvac) 237 | set(DEEPVAC_LIBTORCH_CPU_LIBRARIES ${LIBTORCH_CPU_LIBRARIES}) 238 | set(DEEPVAC_LIBTORCH_CUDA_LIBRARIES ${LIBTORCH_CUDA_LIBRARIES}) 239 | set(DEEPVAC_LIBTORCH_DEFAULT_LIBRARIES ${LIBTORCH_DEFAULT_LIBRARIES}) 240 | set(DEEPVAC_LIBCUDA_LIBRARIES ${LIBCUDA_LIBRARIES}) 241 | set(DEEPVAC_TENSORRT_LIBRARIES ${TENSORRT_LIBRARY}) 242 | set(DEEPVAC_CV_LIBRARIES ${OpenCV_LIBS}) 243 | 244 | message(STATUS "DEEPVAC_LIBTORCH_INCLUDE_DIRS: ${DEEPVAC_LIBTORCH_INCLUDE_DIRS}") 245 | message(STATUS "DEEPVAC_TENSORRT_INCLUDE_DIRS: ${DEEPVAC_TENSORRT_INCLUDE_DIRS}") 246 | message(STATUS "DEEPVAC_CV_INCLUDE_DIRS: ${DEEPVAC_CV_INCLUDE_DIRS}") 247 | 248 | message(STATUS "DEEPVAC_LIBRARIES: ${DEEPVAC_LIBRARIES}") 249 | message(STATUS "DEEPVAC_LIBTORCH_CPU_LIBRARIES: ${DEEPVAC_LIBTORCH_CPU_LIBRARIES}") 250 | message(STATUS "DEEPVAC_LIBTORCH_CUDA_LIBRARIES: ${DEEPVAC_LIBTORCH_CUDA_LIBRARIES}") 251 | message(STATUS "DEEPVAC_LIBTORCH_DEFAULT_LIBRARIES: ${DEEPVAC_LIBTORCH_DEFAULT_LIBRARIES}") 252 | message(STATUS "DEEPVAC_LIBCUDA_LIBRARIES: ${DEEPVAC_LIBCUDA_LIBRARIES}") 253 | message(STATUS "DEEPVAC_TENSORRT_LIBRARIES: ${DEEPVAC_TENSORRT_LIBRARIES}") 254 | message(STATUS "DEEPVAC_CV_LIBRARIES: ${DEEPVAC_CV_LIBRARIES}") 255 | 256 | # add subdiretories 257 | add_subdirectory(utils) 258 | if(USE_LOADER) 259 | message(STATUS "Will build loader module since you enabled USE_LOADER.") 260 | add_subdirectory(loader) 261 | endif() 262 | add_subdirectory(core) 263 | add_subdirectory(modules) 264 | 265 | #add library 266 | if(BUILD_STATIC) 267 | message(STATUS "will build libdeepvac static library") 268 | add_library(deepvac STATIC ${SRC_LIST}) 269 | else() 270 | message(STATUS "will build libdeepvac shared library") 271 | add_library(deepvac SHARED ${SRC_LIST}) 272 | if(USE_STATIC_LIBTORCH) 273 | if(USE_TENSORRT) 274 | target_link_libraries(deepvac PRIVATE ${DEEPVAC_LIBTORCH_DEFAULT_LIBRARIES} ${DEEPVAC_TENSORRT_LIBRARIES} ${DEEPVAC_LIBCUDA_LIBRARIES} ${DEEPVAC_CV_LIBRARIES}) 275 | else() 276 | target_link_libraries(deepvac PRIVATE ${DEEPVAC_LIBTORCH_DEFAULT_LIBRARIES} ${DEEPVAC_LIBCUDA_LIBRARIES} ${DEEPVAC_CV_LIBRARIES}) 277 | endif() 278 | endif() 279 | endif() 280 | 281 | target_include_directories(deepvac PUBLIC 282 | "$" 283 | "$" 284 | ) 285 | 286 | target_compile_options(deepvac PUBLIC -fopenmp) 287 | set_target_properties(deepvac PROPERTIES POSITION_INDEPENDENT_CODE ON) 288 | set_target_properties(deepvac PROPERTIES LINK_FLAGS_RELEASE -s) 289 | target_compile_options(deepvac PRIVATE -Werror) 290 | 291 | find_program(STRIP_COMMAND NAMES "strip") 292 | if(STRIP_COMMAND AND (NOT BUILD_STATIC)) 293 | message(STATUS "found strip command...") 294 | add_custom_command(TARGET deepvac POST_BUILD COMMAND strip ${LIBRARY_OUTPUT_PATH}/lib*.*) 295 | endif() 296 | 297 | add_subdirectory(python) 298 | add_subdirectory(examples) 299 | 300 | #config helper 301 | configure_file( 302 | ${PROJECT_SOURCE_DIR}/cmake/DeepvacConfigVersion.cmake.in 303 | ${PROJECT_BINARY_DIR}/DeepvacConfigVersion.cmake 304 | @ONLY) 305 | configure_file( 306 | ${PROJECT_SOURCE_DIR}/cmake/DeepvacConfig.cmake.in 307 | ${PROJECT_BINARY_DIR}/DeepvacConfig.cmake 308 | @ONLY) 309 | install(FILES 310 | ${PROJECT_BINARY_DIR}/DeepvacConfigVersion.cmake 311 | ${PROJECT_BINARY_DIR}/DeepvacConfig.cmake 312 | DESTINATION share/deepvac) 313 | # 314 | message(STATUS "install headers: " ${HEADER_FILE_LIST}) 315 | 316 | install(TARGETS deepvac EXPORT deepvac DESTINATION "lib") 317 | install(FILES ${HEADER_FILE_LIST} DESTINATION "include/deepvac") 318 | 319 | install(EXPORT deepvac DESTINATION "share/deepvac") 320 | install(FILES ${DEEPVAC_FILE_LIST} DESTINATION "lib/deepvac") 321 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # libdeepvac 2 | Use PyTorch model in C++ project. 3 | 4 | 这个项目定义了如何在C++项目中使用PyTorch训练的模型。 5 | 6 | # 简介 7 | 8 | 在MLab（云上炼丹师）实验室，我们使用[DeepVAC](https://github.com/DeepVAC/deepvac) 来训练获得新模型，使用本项目来部署模型。 9 | 10 | libdeepvac作为一个Linux库，在以下四个方面发挥了价值： 11 | - 向下封装了推理引擎，目前封装了LibTorch，即将封装TensorRT、NCNN、TNN； 12 | - 向上提供Deepvac类，方便用户继承并实现其自定义的模型； 13 | - 在modules目录下，MLab提供了经典网络的C++实现； 14 | - 在utils目录下，MLab提供了网络中常见helper函数的C++实现。 15 | 16 | # libdeepvac实现的模块 17 | ### SOTA网络的C++实现 18 | |类名 | 网络 | 作用 | 19 | |-----|------|------| 20 | |SyszuxFaceRetina| RetinaNet | 人脸检测| 21 | |SyszuxOcrPse | PSENet | 文字检测 | 22 | |SyszuxOcrDB | DB Net | 文字检测 | 23 | |SyszuxSegEsp | ESPNetV2 | 语义分割 | 24 | |SyszuxClsMobile | MobileNetV3 | 分类 | 25 | |SyszuxDetectYolo | YOLOV5 | 目标检测 | 26 | |SyszuxClsResnet | ResNet50 | 分类 | 27 | 28 | ### helper函数实现 29 | |类名/函数名 | 作用 | 30 | |-----|------| 31 | |AlignFace|人脸对齐| 32 | |nms | 检测框的非极大值抑制| 33 | |PriorBox| 生成目标检测的候选框 | 34 | 35 | 36 | 未来我们会持续在modules、utils目录下提供SOTA网络的C++实现。如果用户（你）需要什么网络的C++实现，可在issues里提交申请。 37 | 38 | # 编译平台的支持 39 | libdeepvac支持在以下平台上进行编译： 40 | - x86_64 GNU/Linux（或者叫 AMD64 GNU/Linux） 41 | - aarch64 GNU/Linux（或者叫 ARM64 GNU/Linux） 42 | - macOS 43 | 44 | 未来不太可能扩展到其它平台。 45 | 46 | 47 | # 编译目标的支持 48 | libdeepvac支持以下目标平台的编译： 49 | - x86_64 GNU/Linux（或者叫 AMD64 GNU/Linux） 50 | - x86_64 GNU/Linux with CUDA（或者叫 AMD64 GNU/Linux with CUDA） 51 | - aarch64 GNU/Linux（或者叫 ARM64 GNU/Linux） 52 | - Android 53 | - iOS 54 | - Nvidia Jetson Xavier NX（Volta,384 CUDA cores, 48 Tensor cores, 6-core, 8GB） 55 | - Nvidia Jetson AGX Xavier (Volta, 512 CUDA cores, 6-core, 32GB ) 56 | - Nvidia Jetson TX2 (Pascal, 256 CUDA cores, 2-core/4-core, 8GB) 57 | - Nvidia Jetson TX2 NX (Pascal, 256 CUDA cores, 2-core/4-core, 4GB) 58 | 59 | # 项目依赖 60 | libdeepvac的编译依赖C++14编译器、CMake、opencv、LibTorch。 61 | 最简便、高效的方式就是使用我们提供的[MLab HomePod](https://github.com/DeepVAC/MLab#mlab-homepod)。使用MLab HomePod也是我们推荐的方式。 62 | 63 | # 如何编译libdeepvac 64 | libdeepvac基于CMake进行构建。 65 | 66 | ## 编译开关 67 | 如果要开始编译libdeepvac，需要先熟悉如下几个CMake选项的作用： 68 | |CMake选项|默认值|常用值| 作用|备注| 69 | |--------|-----|-----|-----|---| 70 | |BUILD_STATIC|ON|ON/OFF| ON：编译静态libdeepvac
OFF: 编译动态libdeepvac|OFF时，链接OpenCV静态库会带来hidden symbol问题，此时需链接OpenCV动态库| 71 | |USE_STATIC_LIBTORCH|OFF|ON/OFF|ON: 使用libtorch静态库
OFF: 使用libtorch动态库|MLab HomePod中内置有libtorch动态库| 72 | |USE_MKL|OFF|ON/OFF| 是否使用Intel MKL作为LAPACK/BLAS实现|OFF的时候，需要使用SYSTEM_LAPACK_LIBRARIES指定另外的LAPACK/BLAS实现，比如openblas、Eigen等| 73 | |SYSTEM_LAPACK_LIBRARIES|""|"-lblas -llapack"| USE_MKL关闭后需要指定的LAPACK/BLAS库|在系统路径下安装有相应的开发环境| 74 | |USE_CUDA|OFF| ON/OFF| 是否使用CUDA|需要CUDA硬件，且系统中已经安装有CUDA ToolKit的开发时| 75 | |USE_TENSORRT|OFF|ON/OFF|是否使用TensorRT|需要CUDA硬件，且系统中已经安装有TensorRT的开发时| 76 | |USE_NUMA|OFF|ON/OFF| 是否链接-lnuma库|NA| 77 | |USE_LOADER|OFF|ON/OFF| 是否使用图片装载器|需要C++17编译器| 78 | |GARRULOUS_GEMFIELD|OFF|ON/OFF| 是否打开调试log|NA| 79 | |BUILD_ALL_EXAMPLES|OFF|ON/OFF|是否编译所有的examples |NA| 80 | 81 | 82 | ## 下载依赖 83 | **如果你使用的是MLab HomePod 2.0 pro（或者以上版本），则忽略此小节**。 84 | 如果你使用的是自定义环境，那么你至少需要下载opencv库、libtorch库： 85 | - opencv动态库：自行apt下载； 86 | - opencv静态库：https://github.com/DeepVAC/libdeepvac/releases/download/1.9.0/opencv4deepvac.tar.gz 87 | - libtorch动态库：自行从PyTorch官网下载； 88 | - libtorch静态库：https://github.com/DeepVAC/libdeepvac/releases/download/1.9.0/libtorch.tar.gz 89 | 90 | 你亦可以在MLab HomePod 2.0 pro上自行从源码编译上述的依赖库。 91 | 92 | 93 | ## CMake命令 94 | 以下命令所使用路径均基于MLab HomePod 2.0 pro（你可以根据自身环境自行更改）。 95 | #### 预备工作 96 | ```bash 97 | # update to latest libdeepvac 98 | gemfield@homepod2:/opt/gemfield/libdeepvac$ git pull --rebase 99 | # create build directory 100 | gemfield@homepod2:/opt/gemfield/libdeepvac$ mkdir build 101 | gemfield@homepod2:/opt/gemfield/libdeepvac$ cd build 102 | ``` 103 | 104 | #### CMake 105 | libdeepvac内置了诸多cmake开关以支持不同的软硬件开发栈： 106 | - 在X86_64 GPU服务器上，使用CUDA，使用libtorch静态库，且用MKL作为BLAS/LAPACK库 (MLab HomePod 2.0 pro支持)： 107 | ```bash 108 | cmake -DUSE_MKL=ON -DUSE_CUDA=ON -DUSE_STATIC_LIBTORCH=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="/opt/gemfield/libtorch;/opt/gemfield/opencv4deepvac/" -DCMAKE_INSTALL_PREFIX=../install .. 109 | ``` 110 | - 在X86_64 GPU服务器上，使用CUDA，使用libtorch动态库，且用MKL作为BLAS/LAPACK库 (MLab HomePod 2.0 pro支持)： 111 | ```bash 112 | cmake -DUSE_MKL=ON -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="/opt/gemfield/opencv4deepvac;/opt/conda/lib/python3.8/site-packages/torch/" -DCMAKE_INSTALL_PREFIX=../install .. 113 | ``` 114 | - 在X86_64 GPU服务器上，使用TensorRT和libtorch静态库，且用MKL作为BLAS/LAPACK库： 115 | ```bash 116 | cmake -DUSE_MKL=ON -DUSE_CUDA=ON -DUSE_MAGMA=ON -DUSE_STATIC_LIBTORCH=ON -DUSE_TENSORRT=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="/opt/gemfield/opencv4deepvac/;/opt/gemfield/libtorch" -DCMAKE_INSTALL_PREFIX=../install .. 117 | ``` 118 | - 在Nvidia Jetson Xavier NX上，使用TensorRT，且用系统的blas和lapack库： 119 | ```bash 120 | cmake -DUSE_CUDA=ON -DUSE_NUMA=ON -DUSE_TENSORRT=ON -DSYSTEM_LAPACK_LIBRARIES="-lblas -llapack" -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="/opt/gemfield/opencv4deepvac/;/opt/gemfield/libtorch" -DCMAKE_INSTALL_PREFIX=../install .. 121 | ``` 122 | 123 | #### 编译 124 | ```bash 125 | cmake --build . --config Release 126 | make install 127 | ``` 128 | 129 | # 如何使用libdeepvac库 130 | 如何在自己的项目中使用libdeepvac预编译库呢？ 131 | ## 1. 添加find_package(Deepvac REQUIRED) 132 | 在自己项目的CMakeLists.txt中，添加 133 | ```cmake 134 | find_package(Deepvac REQUIRED) 135 | ``` 136 | 当然，基于libdeepvac的项目也必然基于opencv和libtorch，因此，下面两个find_package也是必须的： 137 | ```cmake 138 | find_package(Torch REQUIRED) 139 | find_package(OpenCV REQUIRED) 140 | ``` 141 | 142 | ## 2. 使用libdeepvac提供的头文件cmake变量 143 | 在自己项目的CMakeLists.txt中，你可以使用如下cmake变量： 144 | - DEEPVAC_INCLUDE_DIRS：libdeepvac库的头文件目录； 145 | - DEEPVAC_LIBTORCH_INCLUDE_DIRS：libtorch库的头文件目录； 146 | - DEEPVAC_TENSORRT_INCLUDE_DIRS：TensorRT库的头文件目录； 147 | - DEEPVAC_CV_INCLUDE_DIRS：OpenCV库的头文件目录； 148 | 149 | ## 3. 使用libdeepvac提供的库文件cmake变量 150 | - DEEPVAC_LIBRARIES：libdeepvac库； 151 | - DEEPVAC_LIBTORCH_CPU_LIBRARIES：libtorch cpu版库； 152 | - DEEPVAC_LIBTORCH_CUDA_LIBRARIES：libtorch cuda版库； 153 | - DEEPVAC_LIBTORCH_DEFAULT_LIBRARIES：libtorch默认版库（编译时用的cpu还是cuda）； 154 | - DEEPVAC_LIBCUDA_LIBRARIES：Nvidia cuda runtime库； 155 | - DEEPVAC_TENSORRT_LIBRARIES：Nvidia TensorRT runtime库； 156 | - DEEPVAC_CV_LIBRARIES：OpenCV库； 157 | 158 | 使用举例： 159 | ``` 160 | #头文件 161 | target_include_directories(${your_target} "${DEEPVAC_LIBTORCH_INCLUDE_DIRS};${DEEPVAC_TENSORRT_INCLUDE_DIRS};${CMAKE_CURRENT_SOURCE_DIR}/include>") 162 | 163 | #库文件 164 | target_link_libraries( ${your_target} ${DEEPVAC_LIBRARIES} ${DEEPVAC_LIBTORCH_CUDA_LIBRARIES} ${DEEPVAC_LIBCUDA_LIBRARIES} ${DEEPVAC_CV_LIBRARIES}) 165 | ``` 166 | 167 | # Benchmark 168 | libdeepvac会提供不同目标平台及不同推理引擎的benchmark，当前仅支持libtorch推理引擎。 169 | 170 | ## 1. X86-64 Linux + LibTorch的benchmark步骤 171 | - 部署[MLab HomePod](https://github.com/DeepVAC/MLab#1-%E9%83%A8%E7%BD%B2); 172 | - 克隆本项目： 173 | ```bash 174 | # 如果是MLab HomePod 2.0 标准版 175 | git clone https://github.com/DeepVAC/libdeepvac && cd libdeepvac 176 | 177 | # 如果是MLab HomePod 2.0 pro版 178 | cd /opt/gemfield/libdeepvac && git pull --rebase 179 | ``` 180 | - 编译 181 | ```bash 182 | #新建编译目录 183 | mkdir build 184 | cd build 185 | #cmake(如果基于LibTorch动态库) 186 | cmake -DGARRULOUS_GEMFIELD=ON -DUSE_MKL=ON -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="/opt/gemfield/opencv4deepvac/;/opt/conda/lib/python3.8/site-packages/torch/" -DCMAKE_INSTALL_PREFIX=../install .. 187 | #cmake(如果基于LibTorch静态库) 188 | cmake -DGARRULOUS_GEMFIELD=ON -DUSE_MKL=ON -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="/opt/gemfield/opencv4deepvac/;/opt/gemfield/libtorch/" -DCMAKE_INSTALL_PREFIX=../install .. 189 | #编译 190 | make -j4 191 | ``` 192 | - 运行benchmark 193 | ```bash 194 | ./bin/test_resnet_benchmark cuda:0 195 | ``` 196 | 197 | ## 2. NA 198 | 199 | # 演示 200 | [SYSZUX-FACE](https://github.com/CivilNet/SYSZUX-FACE)基于本项目实现了人脸检测功能。 -------------------------------------------------------------------------------- /cmake/DeepvacConfig.cmake.in: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 Gemfield 2 | # This file is part of libdeepvac, licensed under the GPLv3 (the "License") 3 | # You may not use this file except in compliance with the License. 4 | get_filename_component(DEEPVAC_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) 5 | get_filename_component(DEEPVAC_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_DIR}/../../" ABSOLUTE) 6 | include("${CMAKE_CURRENT_LIST_DIR}/deepvac.cmake") 7 | 8 | set(DEEPVAC_INCLUDE_DIRS ${DEEPVAC_INSTALL_PREFIX}/include/deepvac ${DEEPVAC_INSTALL_PREFIX}/include/deepvac_torch) 9 | set(DEEPVAC_LIBTORCH_INCLUDE_DIRS "@DEEPVAC_LIBTORCH_INCLUDE_DIRS@") 10 | set(DEEPVAC_TENSORRT_INCLUDE_DIRS "@DEEPVAC_TENSORRT_INCLUDE_DIRS@") 11 | set(DEEPVAC_CV_INCLUDE_DIRS "@DEEPVAC_CV_INCLUDE_DIRS@") 12 | 13 | set(DEEPVAC_LIBRARIES "@DEEPVAC_LIBRARIES@") 14 | set(DEEPVAC_LIBTORCH_CPU_LIBRARIES "@DEEPVAC_LIBTORCH_CPU_LIBRARIES@") 15 | set(DEEPVAC_LIBTORCH_CUDA_LIBRARIES "@DEEPVAC_LIBTORCH_CUDA_LIBRARIES@") 16 | set(DEEPVAC_LIBTORCH_DEFAULT_LIBRARIES "@DEEPVAC_LIBTORCH_DEFAULT_LIBRARIES@") 17 | set(DEEPVAC_LIBCUDA_LIBRARIES "@DEEPVAC_LIBCUDA_LIBRARIES@") 18 | set(DEEPVAC_TENSORRT_LIBRARIES "@DEEPVAC_TENSORRT_LIBRARIES@") 19 | set(DEEPVAC_CV_LIBRARIES "@DEEPVAC_CV_LIBRARIES@") -------------------------------------------------------------------------------- /cmake/DeepvacConfigVersion.cmake.in: -------------------------------------------------------------------------------- 1 | set(PACKAGE_VERSION "@DEEPVAC_VERSION@") 2 | 3 | # Check whether the requested PACKAGE_FIND_VERSION is compatible 4 | if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") 5 | set(PACKAGE_VERSION_COMPATIBLE FALSE) 6 | else() 7 | set(PACKAGE_VERSION_COMPATIBLE TRUE) 8 | if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}") 9 | set(PACKAGE_VERSION_EXACT TRUE) 10 | endif() 11 | endif() 12 | 13 | -------------------------------------------------------------------------------- /core/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 Gemfield 2 | # This file is part of libdeepvac, licensed under the GPLv3 (the "License") 3 | # You may not use this file except in compliance with the License. 4 | 5 | file(GLOB CORE_SRC_LIST src/*.cpp) 6 | 7 | if(NOT USE_TENSORRT) 8 | list(FILTER CORE_SRC_LIST EXCLUDE REGEX ".*_nv.cpp$") 9 | endif() 10 | 11 | add_syszux_sources(${CORE_SRC_LIST}) 12 | message(STATUS "found CORE_SRC_LIST: " ${CORE_SRC_LIST}) 13 | 14 | file(GLOB HEADER_LIST include/*.h) 15 | add_syszux_headers(${HEADER_LIST}) 16 | 17 | add_header_dir(${CMAKE_CURRENT_SOURCE_DIR}/include) -------------------------------------------------------------------------------- /core/include/deepvac.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | #include 9 | #include 10 | #include 11 | #include "gemfield.h" 12 | 13 | #define SYSZUX_EXPORT __attribute__((__visibility__("default"))) 14 | #define SYSZUX_HIDDEN __attribute__((__visibility__("hidden"))) 15 | 16 | namespace deepvac { 17 | //from pytorch 1.2, script::Module is now a reference type 18 | class SYSZUX_EXPORT Deepvac{ 19 | public: 20 | Deepvac() = default; 21 | explicit Deepvac(const char* model_path, std::string device); 22 | explicit Deepvac(std::string model_path, std::string device):Deepvac(model_path.c_str(), device){} 23 | explicit Deepvac(std::vector&& buffer, std::string device); 24 | Deepvac(const Deepvac& rhs); 25 | Deepvac& operator=(const Deepvac& rhs); 26 | Deepvac(Deepvac&&) = default; 27 | Deepvac& operator=(Deepvac&&) = default; 28 | virtual ~Deepvac() = default; 29 | 30 | public: 31 | void setDevice(std::string device){device_ = device;} 32 | void setModel(std::string model_path); 33 | std::string getDevice(){return device_;} 34 | 35 | template 36 | T forward(at::Tensor& t); 37 | 38 | protected: 39 | //all data members must be movable !! 40 | //all data members need dynamic memory must be managed by smart ptr !! 41 | std::string device_; 42 | std::unique_ptr module_; 43 | }; 44 | 45 | }// namespace deepvac -------------------------------------------------------------------------------- /core/include/deepvac_nv.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | #include 9 | #include 10 | #include 11 | #include "gemfield.h" 12 | #include "syszux_tensorrt_buffers.h" 13 | #include "NvInfer.h" 14 | 15 | class TrtLogger : public nvinfer1::ILogger { 16 | public: 17 | void log(Severity severity, const char* msg) noexcept override{ 18 | // suppress info-level messages 19 | if (severity != Severity::kVERBOSE){ 20 | std::cout << msg << std::endl; 21 | } 22 | } 23 | }; 24 | 25 | struct InferDeleter{ 26 | template 27 | void operator()(T* obj) const{ 28 | if (obj){ 29 | obj->destroy(); 30 | } 31 | } 32 | }; 33 | 34 | namespace deepvac { 35 | class DeepvacNV{ 36 | template 37 | using SampleUniquePtr = std::unique_ptr; 38 | 39 | public: 40 | DeepvacNV() = default; 41 | explicit DeepvacNV(const char* model_path, std::string device); 42 | explicit DeepvacNV(std::string model_path, std::string device):DeepvacNV(model_path.c_str(), device){} 43 | explicit DeepvacNV(std::vector&& buffer, std::string device); 44 | DeepvacNV(const DeepvacNV& rhs) = delete; 45 | DeepvacNV& operator=(const DeepvacNV& rhs) = delete; 46 | DeepvacNV(DeepvacNV&&) = default; 47 | DeepvacNV& operator=(DeepvacNV&&) = default; 48 | virtual ~DeepvacNV() = default; 49 | 50 | public: 51 | void setDevice(std::string device){device_ = device;} 52 | void setModel(const char* model_path); 53 | virtual void setBinding(int io_num); 54 | void** forward(void** data) { 55 | bool s = trt_context_->executeV2(data); 56 | return data; 57 | } 58 | 59 | protected: 60 | std::vector prepareInputOutput(const at::Tensor& input); 61 | 62 | protected: 63 | //all data members must be movable !! 64 | //all data members need dynamic memory must be managed by smart ptr !! 65 | SampleUniquePtr trt_module_; 66 | SampleUniquePtr trt_context_; 67 | template 68 | SampleUniquePtr makeUnique(T* t){ 69 | return SampleUniquePtr{t}; 70 | } 71 | TrtLogger logger_; 72 | std::vector datas_; 73 | std::string device_; 74 | }; 75 | 76 | }// namespace deepvac 77 | -------------------------------------------------------------------------------- /core/src/deepvac.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "ATen/Device.h" 11 | #include "deepvac.h" 12 | #include "syszux_stream_buffer.h" 13 | #ifdef GARRULOUS_GEMFIELD 14 | #include 15 | #include 16 | #endif 17 | namespace deepvac { 18 | 19 | Deepvac::Deepvac(const char* model_path, std::string device){ 20 | GEMFIELD_SI; 21 | auto start = std::chrono::system_clock::now(); 22 | try{ 23 | device_ = device; 24 | module_ = std::make_unique(torch::jit::load(model_path, device_)); 25 | }catch(const c10::Error& e){ 26 | std::string msg = gemfield_org::format("%s: %s", "ERROR MODEL: ",e.what_without_backtrace() ); 27 | GEMFIELD_E(msg.c_str()); 28 | throw std::runtime_error(msg); 29 | }catch(...){ 30 | std::string msg = "Internal ERROR!"; 31 | GEMFIELD_E(msg.c_str()); 32 | throw std::runtime_error(msg); 33 | } 34 | std::chrono::duration model_loading_duration = std::chrono::system_clock::now() - start; 35 | std::string msg = gemfield_org::format("Model loading time: %f", model_loading_duration.count()); 36 | GEMFIELD_DI(msg.c_str()); 37 | module_->eval(); 38 | } 39 | 40 | Deepvac::Deepvac(std::vector&& buffer, std::string device){ 41 | GEMFIELD_SI; 42 | auto start = std::chrono::system_clock::now(); 43 | try{ 44 | SyszuxStreamBuffer databuf(buffer.data(), buffer.size()); 45 | std::istream is(&databuf); 46 | device_ = device; 47 | module_ = std::make_unique(torch::jit::load(is, device_)); 48 | }catch(const c10::Error& e){ 49 | std::string msg = gemfield_org::format("%s: %s", "ERROR MODEL: ",e.what_without_backtrace() ); 50 | GEMFIELD_E(msg.c_str()); 51 | throw std::runtime_error(msg); 52 | }catch(...){ 53 | std::string msg = "Internal ERROR!"; 54 | GEMFIELD_E(msg.c_str()); 55 | throw std::runtime_error(msg); 56 | } 57 | std::chrono::duration model_loading_duration = std::chrono::system_clock::now() - start; 58 | std::string msg = gemfield_org::format("Model loading time: %f", model_loading_duration.count()); 59 | GEMFIELD_DI(msg.c_str()); 60 | module_->eval(); 61 | } 62 | 63 | Deepvac::Deepvac(const Deepvac& rhs):module_(new torch::jit::script::Module(*rhs.module_)) { 64 | device_ = rhs.device_; 65 | } 66 | 67 | Deepvac& Deepvac::operator=(const Deepvac& rhs){ 68 | if(this == &rhs){ 69 | return *this; 70 | } 71 | module_.reset( new torch::jit::script::Module( *rhs.module_ ) ); 72 | device_ = rhs.device_; 73 | return *this; 74 | } 75 | 76 | void Deepvac::setModel(std::string model_path){ 77 | try{ 78 | module_ = std::make_unique(torch::jit::load(model_path, device_)); 79 | }catch(const c10::Error& e){ 80 | std::string msg = gemfield_org::format("%s: %s", "ERROR MODEL: ",e.what_without_backtrace() ); 81 | GEMFIELD_E(msg.c_str()); 82 | throw std::runtime_error(msg); 83 | }catch(...){ 84 | std::string msg = "Internal ERROR!"; 85 | GEMFIELD_E(msg.c_str()); 86 | throw std::runtime_error(msg); 87 | } 88 | module_->eval(); 89 | } 90 | 91 | template struct dependent_false : std::false_type {}; 92 | template 93 | T Deepvac::forward(at::Tensor& t){ 94 | GEMFIELD_SI; 95 | if(device_.length() <3 or !module_){ 96 | throw std::runtime_error("Deepvac initialized incorrectly! Are you forget to call setDevice() or setModel() before usage?"); 97 | } 98 | torch::NoGradGuard no_grad; 99 | std::vector inputs; 100 | if(c10::Device(device_) == t.device()){ 101 | inputs.push_back(t); 102 | }else{ 103 | std::cout<<"[GEMFIELD] warning: model device: "<){ 113 | output = module_->forward(inputs).toTensor(); 114 | }else if constexpr (std::is_same_v >){ 115 | output = module_->forward(inputs).toTuple()->elements(); 116 | }else{ 117 | static_assert(dependent_false::value, "[libdeepvac] invalid usage of forward."); 118 | } 119 | #ifdef GARRULOUS_GEMFIELD 120 | stream = at::cuda::getCurrentCUDAStream(); 121 | AT_CUDA_CHECK(cudaStreamSynchronize(stream)); 122 | std::chrono::duration forward_duration = std::chrono::system_clock::now() - start; 123 | std::string msg = gemfield_org::format("model forward time: %f", forward_duration.count() ); 124 | GEMFIELD_DI(msg.c_str()); 125 | #endif 126 | return output; 127 | } 128 | 129 | template at::Tensor Deepvac::forward(at::Tensor& t); 130 | template std::vector Deepvac::forward>(at::Tensor& t); 131 | 132 | } //namespace deepvac 133 | 134 | -------------------------------------------------------------------------------- /core/src/deepvac_nv.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "deepvac_nv.h" 12 | 13 | namespace deepvac { 14 | DeepvacNV::DeepvacNV(const char* path, std::string device){ 15 | GEMFIELD_SI; 16 | auto start = std::chrono::system_clock::now(); 17 | try{ 18 | device_ = device; 19 | setModel(path); 20 | }catch(...){ 21 | std::string msg = "Internal ERROR!"; 22 | GEMFIELD_E(msg.c_str()); 23 | throw std::runtime_error(msg); 24 | } 25 | std::chrono::duration model_loading_duration = std::chrono::system_clock::now() - start; 26 | std::string msg = gemfield_org::format("NV Model loading time: %f", model_loading_duration.count()); 27 | GEMFIELD_DI(msg.c_str()); 28 | } 29 | 30 | DeepvacNV::DeepvacNV(std::vector&& buffer, std::string device){ 31 | GEMFIELD_SI; 32 | auto start = std::chrono::system_clock::now(); 33 | try{ 34 | device_ = device; 35 | nvinfer1::IRuntime* runtime = nvinfer1::createInferRuntime(logger_); 36 | trt_module_ = makeUnique(runtime->deserializeCudaEngine((void*)buffer.data(), buffer.size(), nullptr)); 37 | assert(trt_module_ != nullptr); 38 | trt_context_ = makeUnique(trt_module_->createExecutionContext()); 39 | assert(trt_context_ != nullptr); 40 | runtime->destroy(); 41 | runtime = nullptr; 42 | }catch(...){ 43 | std::string msg = "Internal ERROR!"; 44 | GEMFIELD_E(msg.c_str()); 45 | throw std::runtime_error(msg); 46 | } 47 | std::chrono::duration model_loading_duration = std::chrono::system_clock::now() - start; 48 | std::string msg = gemfield_org::format("NV Model loading time: %f", model_loading_duration.count()); 49 | GEMFIELD_DI(msg.c_str()); 50 | } 51 | 52 | void DeepvacNV::setBinding(int io_num) { 53 | for(int i = 0; i < io_num; ++i) { 54 | datas_.emplace_back(gemfield_org::ManagedBuffer()); 55 | } 56 | } 57 | 58 | void DeepvacNV::setModel(const char* model_path) { 59 | std::ifstream in(model_path, std::ifstream::binary); 60 | if(in.is_open()) { 61 | auto const start_pos = in.tellg(); 62 | in.ignore(std::numeric_limits::max()); 63 | size_t bufCount = in.gcount(); 64 | in.seekg(start_pos); 65 | std::unique_ptr engineBuf(new char[bufCount]); 66 | in.read(engineBuf.get(), bufCount); 67 | //initLibNvInferPlugins(&logger_, ""); 68 | nvinfer1::IRuntime* runtime = nvinfer1::createInferRuntime(logger_); 69 | trt_module_ = makeUnique(runtime->deserializeCudaEngine((void*)engineBuf.get(), bufCount, nullptr)); 70 | assert(trt_module_ != nullptr); 71 | trt_context_ = makeUnique(trt_module_->createExecutionContext()); 72 | assert(trt_context_ != nullptr); 73 | //mBatchSize = trt_module_->getMaxBatchSize(); 74 | //spdlog::info("max batch size of deserialized engine: {}",mEngine->getMaxBatchSize()); 75 | runtime->destroy(); 76 | runtime = nullptr; 77 | } 78 | } 79 | 80 | std::vector DeepvacNV::prepareInputOutput(const at::Tensor& input) { 81 | auto sizes = input.sizes(); 82 | nvinfer1::Dims dim; 83 | dim.nbDims = sizes.size(); 84 | std::copy(sizes.begin(), sizes.end(), dim.d); 85 | trt_context_->setBindingDimensions(0, dim); 86 | auto nb_bind = trt_module_->getNbBindings(); 87 | //input 88 | datas_[0].deviceBuffer.fromTensor(input); 89 | //output 90 | for(auto i = 1; i < nb_bind; ++i) { 91 | auto dims = trt_context_->getBindingDimensions(i); 92 | std::vector shape; 93 | std::copy(dims.d, dims.d+dims.nbDims, std::back_inserter(shape)); 94 | datas_[i].deviceBuffer.resize(shape); 95 | } 96 | std::vector predicitonBindings; 97 | for(auto i = 0; i < nb_bind; ++i) { 98 | predicitonBindings.emplace_back(datas_[i].deviceBuffer.data()); 99 | } 100 | return predicitonBindings; 101 | } 102 | 103 | } //namespace deepvac 104 | -------------------------------------------------------------------------------- /docker/ubuntu20.04/Dockerfile.cuda: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:11.0.3-cudnn8-devel-ubuntu20.04 2 | LABEL maintainer "Gemfield " 3 | ARG PYTHON_VERSION=3.8 4 | 5 | RUN apt-get update && DEBIAN_FRONTEND="noninteractive" \ 6 | apt-get install -y --no-install-recommends build-essential \ 7 | vim wget curl bzip2 git unzip g++ binutils cmake locales \ 8 | ca-certificates apt-transport-https gnupg software-properties-common \ 9 | libjpeg-dev libpng-dev iputils-ping net-tools libgl1 libglib2.0-0 && \ 10 | rm -rf /var/lib/apt/lists/* 11 | 12 | RUN cd /tmp && wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ 13 | apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ 14 | echo "deb https://apt.repos.intel.com/mkl all main" > /etc/apt/sources.list.d/intel-mkl.list && \ 15 | apt update && apt install -y intel-mkl-64bit-2020.4-912 && \ 16 | rm -rf /var/lib/apt/lists/* 17 | 18 | RUN ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && dpkg-reconfigure -f noninteractive tzdata && locale-gen zh_CN.utf8 19 | 20 | ENV LC_ALL zh_CN.UTF-8 21 | ENV LANG zh_CN.UTF-8 22 | ENV LANGUAGE zh_CN.UTF-8 23 | 24 | #conda 25 | RUN curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ 26 | chmod +x ~/miniconda.sh && \ 27 | ~/miniconda.sh -b -p /opt/conda && \ 28 | rm ~/miniconda.sh && \ 29 | /opt/conda/bin/conda install -y python=$PYTHON_VERSION conda-build anaconda-client numpy pyyaml scipy ipython mkl mkl-include \ 30 | cffi ninja setuptools typing_extensions future six requests dataclasses cython typing conda-package-handling=1.6.0 && \ 31 | /opt/conda/bin/conda install -c pytorch magma-cuda110 && \ 32 | /opt/conda/bin/pip3 install --no-cache-dir Pillow opencv-python confluent_kafka easydict sklearn matplotlib tensorboard fonttools onnx-coreml coremltools && \ 33 | /opt/conda/bin/conda clean -ya && \ 34 | /opt/conda/bin/conda clean -y --force-pkgs-dirs 35 | 36 | ENV PATH /opt/conda/bin:$PATH 37 | RUN conda config --add channels pytorch 38 | 39 | WORKDIR /gemfield 40 | RUN curl -L https://github.com/opencv/opencv/archive/4.4.0.zip -o opencv.zip && \ 41 | unzip opencv.zip && rm opencv.zip && cd opencv-4.4.0 && \ 42 | mkdir /gemfield/opencv4deepvac && mkdir build && cd build && \ 43 | cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_LIST=core,imgproc,imgcodecs -DCMAKE_INSTALL_PREFIX=/gemfield/opencv4deepvac -DBUILD_SHARED_LIBS=OFF .. && \ 44 | make VERBOSE=1 && make install && cd /gemfield && rm -rf opencv-4.4.0 45 | 46 | -------------------------------------------------------------------------------- /docker/ubuntu20.04/Dockerfile.deepvac: -------------------------------------------------------------------------------- 1 | FROM gemfield/cuda:11.0.3-cudnn8-devel-ubuntu20.04 2 | LABEL maintainer "Gemfield " 3 | 4 | WORKDIR /gemfield 5 | RUN curl -L https://github.com/CivilNet/libtorch/releases/download/1.8.0/libtorch.tar.gz -o libtorch.tar.gz && \ 6 | tar zxvf libtorch.tar.gz && \ 7 | rm libtorch.tar.gz 8 | 9 | RUN conda install -y pytorch -c gemfield && \ 10 | conda clean -ya && \ 11 | /opt/conda/bin/conda clean -y --force-pkgs-dirs 12 | 13 | RUN git clone https://github.com/DeepVAC/libdeepvac libdeepvac && \ 14 | cd libdeepvac && \ 15 | mkdir build && \ 16 | mkdir install && \ 17 | cd build && \ 18 | cmake -DBUILD_STATIC=ON -DUSE_STATIC_LIBTORCH=ON -DUSE_MKL=ON -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release \ 19 | -DCMAKE_PREFIX_PATH="/gemfield/libtorch;/gemfield/opencv4deepvac/" -DCMAKE_INSTALL_PREFIX=../install .. && \ 20 | cmake --build . --config Release && \ 21 | make install && \ 22 | cd .. && \ 23 | rm -rf build 24 | 25 | RUN pip3 install --no-cache-dir --upgrade deepvac 26 | -------------------------------------------------------------------------------- /docker/ubuntu20.04/Dockerfile.deepvac.vision: -------------------------------------------------------------------------------- 1 | FROM gemfield/deepvac:11.0.3-cudnn8-devel-ubuntu20.04 2 | LABEL maintainer "Gemfield " 3 | 4 | WORKDIR /gemfield 5 | ENV TORCH_CUDA_ARCH_LIST="6.1;7.0;7.5;8.0" 6 | COPY vision /gemfield/vision 7 | 8 | RUN cd vision && pip install -v . 9 | 10 | #docker build -t gemfield/deepvac:vision-11.0.3-cudnn8-devel-ubuntu20.04 -f libdeepvac/docker/ubuntu20.04/Dockerfile.deepvac.vision . 11 | -------------------------------------------------------------------------------- /examples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 Gemfield 2 | # This file is part of libdeepvac, licensed under the GPLv3 (the "License") 3 | # You may not use this file except in compliance with the License. 4 | 5 | file(GLOB EXAMPLES_LIST src/*.cpp) 6 | macro (add_syszux_flags) 7 | foreach (_src ${ARGN}) 8 | list (APPEND FLAGS_FILE_LIST "${_src}") 9 | endforeach() 10 | set (FLAGS_FILE_LIST ${FLAGS_FILE_LIST} PARENT_SCOPE) 11 | endmacro() 12 | 13 | find_library(CRYPTO crypto) 14 | if(CRYPTO) 15 | message(STATUS "found crypto library: " ${CRYPTO}) 16 | endif() 17 | 18 | #test examples 19 | set(examples_dest "bin") 20 | 21 | file( GLOB EXAMPLES_LIST ${CMAKE_SOURCE_DIR}/examples/*/test_*.cpp ) 22 | file( GLOB EXAMPLES_LIST_MIN ${CMAKE_SOURCE_DIR}/examples/*/test_*retina*.cpp) 23 | file( GLOB EXAMPLES_BENCHMARK ${CMAKE_SOURCE_DIR}/examples/*/test_*benchmark.cpp) 24 | list(APPEND EXAMPLES_LIST_MIN ${EXAMPLES_BENCHMARK}) 25 | if(NOT USE_TENSORRT) 26 | list(FILTER EXAMPLES_LIST EXCLUDE REGEX ".*_nv.cpp$") 27 | list(FILTER EXAMPLES_LIST_MIN EXCLUDE REGEX ".*_nv.cpp$") 28 | endif() 29 | if(NOT USE_LOADER) 30 | list(FILTER EXAMPLES_LIST EXCLUDE REGEX ".*_extract_feature.cpp$") 31 | list(FILTER EXAMPLES_LIST_MIN EXCLUDE REGEX ".*_extract_feature.cpp$") 32 | endif() 33 | #internal test 34 | add_syszux_flags(face_retina_deepvac@${CMAKE_CURRENT_SOURCE_DIR}/src/test_face_retina.cpp) 35 | add_syszux_flags(ocr_pse_deepvac@${CMAKE_CURRENT_SOURCE_DIR}/src/test_ocr_pse.cpp) 36 | add_syszux_flags(seg_esp_deepvac@${CMAKE_CURRENT_SOURCE_DIR}/src/test_seg_esp.cpp) 37 | add_syszux_flags(cls_mobile_deepvac@${CMAKE_CURRENT_SOURCE_DIR}/src/test_cls_mobile.cpp) 38 | add_syszux_flags(detect_yolo_deepvac@${CMAKE_CURRENT_SOURCE_DIR}/src/test_detect_yolo.cpp) 39 | add_syszux_flags(cls_resnet_deepvac@${CMAKE_CURRENT_SOURCE_DIR}/src/test_cls_resnet.cpp) 40 | 41 | foreach(_src ${FLAGS_FILE_LIST}) 42 | string(REGEX MATCH "^[^@]+" def_name ${_src}) 43 | string(REPLACE "_" "." def_value ${def_name}) 44 | string(REPLACE "${def_name}@" "" src_name ${_src}) 45 | set_property( 46 | SOURCE ${src_name} 47 | APPEND 48 | PROPERTY COMPILE_DEFINITIONS 49 | ${def_name}="${CMAKE_INSTALL_PREFIX}/lib/deepvac/${def_value}" 50 | ) 51 | endforeach() 52 | 53 | if(BUILD_ALL_EXAMPLES) 54 | set(EXAMPLES_LIST_MIN ${EXAMPLES_LIST}) 55 | endif() 56 | 57 | if(NOT USE_TENSORRT) 58 | list(FILTER EXAMPLES_LIST_MIN EXCLUDE REGEX ".*_nv.cpp$") 59 | endif() 60 | 61 | message(STATUS "found EXAMPLES_SRC_LIST: " ${EXAMPLES_LIST_MIN}) 62 | 63 | foreach( testsyszuxfile ${EXAMPLES_LIST_MIN} ) 64 | get_filename_component(testfile "${testsyszuxfile}" NAME) 65 | message(STATUS "Add test binary: ${testfile}") 66 | string(REPLACE ".cpp" "" testname ${testfile} ) 67 | add_executable( ${testname} ${testsyszuxfile} ) 68 | 69 | if(USE_STATIC_LIBTORCH) 70 | target_compile_options(${testname} PRIVATE -fopenmp) 71 | target_link_options(${testname} PRIVATE -fopenmp) 72 | if(BUILD_STATIC) 73 | if(USE_CUDA) 74 | if(USE_TENSORRT) 75 | target_link_libraries( ${testname} ${DEEPVAC_LIBRARIES} ${DEEPVAC_CV_LIBRARIES} ${DEEPVAC_LIBTORCH_CUDA_LIBRARIES} ${DEEPVAC_LIBCUDA_LIBRARIES} ${DEEPVAC_TENSORRT_LIBRARIES}) 76 | else() 77 | target_link_libraries( ${testname} ${DEEPVAC_LIBRARIES} ${DEEPVAC_CV_LIBRARIES} ${DEEPVAC_LIBTORCH_CUDA_LIBRARIES} ${DEEPVAC_LIBCUDA_LIBRARIES}) 78 | endif() 79 | else() 80 | target_link_libraries( ${testname} ${DEEPVAC_LIBRARIES} ${DEEPVAC_CV_LIBRARIES} ${DEEPVAC_LIBTORCH_CPU_LIBRARIES}) 81 | endif() 82 | else() 83 | target_link_libraries( ${testname} ${DEEPVAC_LIBRARIES} ${DEEPVAC_CV_LIBRARIES} -lrt) 84 | endif() 85 | else() 86 | target_link_libraries( ${testname} ${DEEPVAC_LIBRARIES} ${DEEPVAC_CV_LIBRARIES} ${TORCH_LIBRARIES}) 87 | endif() 88 | 89 | target_include_directories(${testname} PUBLIC 90 | "$" 91 | "$" 92 | ) 93 | #install(TARGETS ${testname} DESTINATION "${examples_dest}") 94 | endforeach( testsyszuxfile ${EXAMPLES_LIST_MIN} ) 95 | -------------------------------------------------------------------------------- /examples/include/test_extract_feature.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | #include "deepvac_loader.h" 9 | #include "deepvac.h" 10 | 11 | namespace deepvac{ 12 | 13 | template 14 | class FeatureEmbBase{ 15 | public: 16 | FeatureEmbBase() = delete; 17 | FeatureEmbBase(const FeatureEmbBase&) = delete; 18 | FeatureEmbBase& operator=(const FeatureEmbBase&) = delete; 19 | FeatureEmbBase(FeatureEmbBase&&) = default; 20 | FeatureEmbBase& operator=(FeatureEmbBase&&) = default; 21 | virtual ~FeatureEmbBase() = default; 22 | FeatureEmbBase(Deepvac&& deepvac, DeepvacLoaderType loader): 23 | loader_(std::move(loader)), deepvac_(std::move(deepvac)){} 24 | virtual void operator() () {}; 25 | 26 | protected: 27 | DeepvacLoaderType loader_; 28 | Deepvac deepvac_; 29 | }; 30 | 31 | using pairloader = DeepvacRecursiveFileLoader; 32 | class FeatureEmbFromDir : public FeatureEmbBase{ 33 | public: 34 | FeatureEmbFromDir(Deepvac&& deepvac, const char* url = ""): 35 | FeatureEmbBase(std::move(deepvac), pairloader(url)) {} 36 | 37 | virtual void dumpEmb(std::string output_feature_file = "feature.gemfield"); 38 | virtual void operator()(std::string input_feature_file = "feature.gemfield"); 39 | private: 40 | at::Tensor feature_; 41 | std::vector emb_vec_; 42 | std::vector name_vec_; 43 | }; 44 | 45 | } //namespace deepvac -------------------------------------------------------------------------------- /examples/src/test_cls_mobile.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | #include "gemfield.h" 7 | #include "syszux_cls_mobile.h" 8 | #include "syszux_img2tensor.h" 9 | 10 | using namespace deepvac; 11 | int main(int argc, char** argv) 12 | { 13 | if (argc != 3) { 14 | GEMFIELD_E("usage: deepvac "); 15 | return -1; 16 | } 17 | 18 | std::string device = argv[1]; 19 | std::string img_path = argv[2]; 20 | SyszuxClsMobile cls(cls_mobile_deepvac, device); 21 | 22 | auto mat_opt = gemfield_org::img2CvMat(img_path); 23 | if(!mat_opt){ 24 | throw std::runtime_error("illegal image detected"); 25 | return 1; 26 | } 27 | auto mat_out = mat_opt.value(); 28 | 29 | auto cls_out_opt = cls.process(mat_out); 30 | if(!cls_out_opt) { 31 | return 0; 32 | } 33 | auto cls_out = cls_out_opt.value(); 34 | std::cout << "Index: " << cls_out.first << std::endl; 35 | std::cout << "Probability: " << cls_out.second << std::endl; 36 | return 0; 37 | } 38 | -------------------------------------------------------------------------------- /examples/src/test_detect_yolo.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | #include "gemfield.h" 7 | #include "syszux_detect_yolo.h" 8 | 9 | using namespace deepvac; 10 | int main(int argc, char** argv) 11 | { 12 | if (argc != 3) { 13 | GEMFIELD_E("usage: deepvac "); 14 | return -1; 15 | } 16 | 17 | std::string device = argv[1]; 18 | std::string img_path = argv[2]; 19 | 20 | int input_size = 512; 21 | float iou_thresh = 0.45; 22 | float score_thresh = 0.25; 23 | std::vector idx_to_cls = {"1", "2", "3", "4"}; 24 | SyszuxDetectYolo detect(detect_yolo_deepvac, device); 25 | detect.set(input_size, iou_thresh, score_thresh); 26 | 27 | cv::Mat img_raw = cv::imread(img_path); 28 | if(img_raw.data == nullptr){ 29 | GEMFIELD_E(path + " is not a image file! Please input a image path..."); 30 | return -1; 31 | } 32 | auto detect_out_opt = detect.process(img_raw); 33 | if(!detect_out_opt) { 34 | return 0; 35 | } 36 | auto detect_out = detect_out_opt.value(); 37 | 38 | std::cout << "size: " << detect_out.size() << std::endl; 39 | if (detect_out.size()==0) { 40 | std::cout << "detect none." << std::endl; 41 | return 0; 42 | } 43 | 44 | for (int j=0; j bbox_and_score = detect_out[j].second; 47 | std::cout << "class: " << idx_to_cls[idx] << std::endl; 48 | std::cout << "bbox_and_score: " << bbox_and_score << std::endl; 49 | } 50 | return 0; 51 | } 52 | -------------------------------------------------------------------------------- /examples/src/test_extract_feature.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "deepvac_loader.h" 13 | #include "test_extract_feature.h" 14 | #include 15 | #include "syszux_img2tensor.h" 16 | 17 | namespace deepvac{ 18 | 19 | std::string getLabelFromPath(std::string& file_path, char seperator = '/') 20 | { 21 | std::size_t sep_pos = file_path.rfind(seperator); 22 | if(sep_pos != std::string::npos){ 23 | return file_path.substr(sep_pos + 1, file_path.size() -1); 24 | } 25 | return "unknown"; 26 | } 27 | 28 | void FeatureEmbFromDir::dumpEmb(std::string output_feature_file){ 29 | torch::NoGradGuard no_grad; 30 | for (auto& f_m : loader_){ 31 | auto [f, m] = f_m; 32 | if(!m){ 33 | continue; 34 | } 35 | {GEMFIELD_DI2("input file: ", f.c_str());} 36 | at::Tensor emb = deepvac_.forward(*m); 37 | emb_vec_.push_back(emb); 38 | 39 | std::string base_dir = std::filesystem::path(f).parent_path(); 40 | name_vec_.push_back(getLabelFromPath(base_dir)); 41 | } 42 | 43 | feature_ = torch::cat(emb_vec_, 0); 44 | 45 | std::stringstream feature_size; 46 | feature_size << feature_.sizes(); 47 | std::string msg = gemfield_org::format("%s: %s | %s : %d", "load feature size: ", feature_size.str().c_str(), "load name vector size: ", name_vec_.size() ); 48 | GEMFIELD_I(msg.c_str()); 49 | 50 | std::ofstream name_output(output_feature_file + ".name"); 51 | for (auto &name : name_vec_){ 52 | name_output << name << "\n"; 53 | } 54 | torch::save(feature_ , output_feature_file); 55 | } 56 | 57 | void FeatureEmbFromDir::operator()(std::string input_feature_file){ 58 | torch::NoGradGuard no_grad; 59 | std::ifstream inputstream(input_feature_file + ".name"); 60 | 61 | std::copy(std::istream_iterator(inputstream), 62 | std::istream_iterator(),back_inserter(name_vec_)); 63 | 64 | torch::load(feature_,input_feature_file); 65 | feature_ = feature_.to(deepvac_.getDevice()); 66 | 67 | std::stringstream feature_size; 68 | feature_size << feature_.sizes(); 69 | std::string msg = gemfield_org::format("%s: %s | %s : %d", "load feature size: ", feature_size.str().c_str(), "load name vector size: ", name_vec_.size() ); 70 | GEMFIELD_I(msg.c_str()); 71 | 72 | for (auto& f_m : loader_){ 73 | auto [f, m] = f_m; 74 | if(!m){ 75 | continue; 76 | } 77 | 78 | {GEMFIELD_DI2("input file: ", f.c_str());} 79 | at::Tensor emb = deepvac_.forward(*m); 80 | at::Tensor distance = torch::norm(feature_ - emb, 2, 1); 81 | int min_index = torch::argmin(distance).item(); 82 | GEMFIELD_DI2("predict: ", name_vec_.at(min_index).c_str()); 83 | } 84 | } 85 | 86 | } //namespace deepvac 87 | 88 | using namespace deepvac; 89 | int main(int argc, const char* argv[]) { 90 | if (argc != 4) { 91 | GEMFIELD_E("usage: deepvac "); 92 | return -1; 93 | } 94 | 95 | FeatureEmbFromDir emb_fromdir( Deepvac(argv[2], "cuda:1"), argv[3]); 96 | std::string feature_file = "gemfield_org.feature"; 97 | std::string op = argv[1]; 98 | 99 | if(op == "dumpEmb"){ 100 | emb_fromdir.dumpEmb(feature_file); 101 | return 0; 102 | } 103 | 104 | if(op == "predict"){ 105 | emb_fromdir(feature_file); 106 | return 0; 107 | } 108 | 109 | GEMFIELD_E("usage: deepvac "); 110 | return -1; 111 | } -------------------------------------------------------------------------------- /examples/src/test_face_retina.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include "syszux_face_retina.h" 8 | #include "syszux_img2tensor.h" 9 | 10 | using namespace deepvac; 11 | int main(int argc, const char* argv[]) { 12 | if (argc != 3) { 13 | GEMFIELD_E("usage: deepvac "); 14 | return -1; 15 | } 16 | 17 | std::string device = argv[1]; 18 | std::string img_path = argv[2]; 19 | SyszuxFaceRetina face_detect(face_retina_deepvac, device); 20 | auto mat_opt = gemfield_org::img2CvMat(img_path); 21 | if(!mat_opt){ 22 | throw std::runtime_error("illegal image detected"); 23 | return 1; 24 | } 25 | auto mat_out = mat_opt.value(); 26 | auto detect_out_opt = face_detect.process(mat_out); 27 | 28 | if(!detect_out_opt){ 29 | throw std::runtime_error("no face detected"); 30 | } 31 | auto detect_out = detect_out_opt.value(); 32 | for (int i=0; i 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include 8 | #include "syszux_face_retina_nv.h" 9 | #include "syszux_face_id_nv.h" 10 | #include "syszux_img2tensor.h" 11 | 12 | using namespace deepvac; 13 | int main(int argc, const char* argv[]) { 14 | if (argc != 3) { 15 | GEMFIELD_E("usage: deepvac "); 16 | return -1; 17 | } 18 | 19 | std::string device = argv[1]; 20 | std::string img_path = argv[2]; 21 | SyszuxFaceRetinaNV face_detect("detect_official.trt", device); 22 | SyszuxFaceIdNV face_id("branch13_best.trt", device); 23 | 24 | auto start1 = std::chrono::system_clock::now(); 25 | std::string img_name = img_path; 26 | auto mat_opt = gemfield_org::img2CvMat(img_name); 27 | 28 | if(!mat_opt){ 29 | throw std::runtime_error("illegal image detected"); 30 | return 1; 31 | } 32 | 33 | auto mat_out = mat_opt.value(); 34 | auto detect_out_opt = face_detect.process(mat_out); 35 | std::chrono::duration model_loading_duration_d = std::chrono::system_clock::now() - start1; 36 | std::string msg = gemfield_org::format("Model loading time: %f", model_loading_duration_d.count()); 37 | std::cout << msg << std::endl; 38 | 39 | if(detect_out_opt){ 40 | auto detect_out = detect_out_opt.value(); 41 | std::vector frames; 42 | for (int i=0; i model_loading_duration_d1 = std::chrono::system_clock::now() - start1; 50 | std::string msg1 = gemfield_org::format("Model loading time: %f", model_loading_duration_d1.count()); 51 | std::cout << msg1 << std::endl; 52 | 53 | return 0; 54 | } 55 | -------------------------------------------------------------------------------- /examples/src/test_ocr_db.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include "syszux_ocr_db.h" 8 | #include "gemfield.h" 9 | #include 10 | 11 | using namespace deepvac; 12 | 13 | inline std::string BoolToString(bool b) 14 | { 15 | return b ? "true" : "false"; 16 | } 17 | 18 | int main(int argc, char** argv) 19 | { 20 | if (argc != 3) { 21 | GEMFIELD_E("usage: deepvac "); 22 | return -1; 23 | } 24 | std::string device = argv[1]; 25 | std::string path = argv[2]; 26 | int long_size = 1280; 27 | int crop_gap = 5; 28 | int text_min_area = 300; 29 | int text_min_size = 3; 30 | float text_mean_score = 0.5; 31 | float text_thresh = 0.3; 32 | float unclip_ratio=2; 33 | 34 | SyszuxOcrDB ocr_detect; 35 | ocr_detect.setDevice(device); 36 | ocr_detect.setModel("ocr.db.deepvac"); 37 | 38 | std::vector> perm; 39 | 40 | for(int i=0;i<16;i++){ 41 | std::bitset<4> flag = i; 42 | perm.push_back(flag); 43 | } 44 | auto mat_opt = gemfield_org::img2CvMat(path); 45 | if(!mat_opt){ 46 | throw std::runtime_error("illegal image detected"); 47 | } 48 | 49 | for(int idx=0; idx, std::vector>> detect_out = detect_out_opt.value(); 63 | std::vector crop_imgs = detect_out.first; 64 | std::vector> rects = detect_out.second; 65 | 66 | if (crop_imgs.size()==0) { 67 | std::cout << "no text detected" << std::endl; 68 | return 0; 69 | } 70 | for (int i=0; i vPolygonPoint; 78 | int pts = rects[i].size(); 79 | assert(pts%2==0); 80 | for(int j=0; j 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include "syszux_ocr_pse.h" 8 | #include "gemfield.h" 9 | #include 10 | 11 | using namespace deepvac; 12 | int main(int argc, char** argv) 13 | { 14 | if (argc != 3) { 15 | GEMFIELD_E("usage: deepvac "); 16 | return -1; 17 | } 18 | std::string device = argv[1]; 19 | std::string path = argv[2]; 20 | int long_size = 1280; 21 | int crop_gap = 5; 22 | int text_min_area = 300; 23 | float text_mean_score = 0.90; 24 | 25 | SyszuxOcrPse ocr_detect; 26 | ocr_detect.setDevice(device); 27 | ocr_detect.setModel(ocr_pse_deepvac); 28 | 29 | ocr_detect.set(long_size, crop_gap, text_min_area, text_mean_score); 30 | auto mat_opt = gemfield_org::img2CvMat(path); 31 | if(!mat_opt){ 32 | throw std::runtime_error("illegal image detected"); 33 | return 1; 34 | } 35 | auto mat_out = mat_opt.value(); 36 | auto detect_out_opt = ocr_detect.process(mat_out); 37 | if(!detect_out_opt){ 38 | throw std::runtime_error("no text detected"); 39 | } 40 | 41 | std::pair, std::vector>> detect_out = detect_out_opt.value(); 42 | std::vector crop_imgs = detect_out.first; 43 | std::vector> rects = detect_out.second; 44 | 45 | if (crop_imgs.size()==0) { 46 | std::cout << "no text detected" << std::endl; 47 | return 0; 48 | } 49 | for (int i=0; i vPolygonPoint; 55 | int pts = rects[i].size(); 56 | assert(pts%2==0); 57 | for(int j=0; j 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | #include "syszux_cls_resnet.h" 7 | #include "syszux_img2tensor.h" 8 | #include "gemfield.h" 9 | #include "deepvac.h" 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "syszux_imagenet_classes.h" 16 | 17 | using namespace deepvac; 18 | 19 | void validate(SyszuxClsResnet& civilnet, at::Tensor& t, std::string img_path){ 20 | std::cout<<"---------------VALIDATE BEGIN---------------"<(t.size(4)), static_cast(t.size(3))}); 22 | auto mat_opt = gemfield_org::img2CvMat(img_path); 23 | if(!mat_opt){ 24 | throw std::runtime_error("illegal image detected in validate!"); 25 | } 26 | auto mat_out = mat_opt.value(); 27 | 28 | auto stream = at::cuda::getCurrentCUDAStream(); 29 | AT_CUDA_CHECK(cudaStreamSynchronize(stream)); 30 | auto start = std::chrono::system_clock::now(); 31 | 32 | auto resnet_out_opt = civilnet.process(mat_out); 33 | 34 | stream = at::cuda::getCurrentCUDAStream(); 35 | AT_CUDA_CHECK(cudaStreamSynchronize(stream)); 36 | std::chrono::duration model_loading_duration = std::chrono::system_clock::now() - start; 37 | std::string msg = gemfield_org::format("Overall process time in validate: %f", model_loading_duration.count()); 38 | 39 | if(!resnet_out_opt){ 40 | throw std::runtime_error("return empty error!"); 41 | } 42 | 43 | auto resnet_out = resnet_out_opt.value(); 44 | std::cout <(t.size(1)),static_cast(t.size(2)),static_cast(t.size(3)), static_cast(t.size(4))},options); 55 | auto stream = at::cuda::getCurrentCUDAStream(); 56 | AT_CUDA_CHECK(cudaStreamSynchronize(stream)); 57 | auto start = std::chrono::system_clock::now(); 58 | auto resnet_out_opt = civilnet.process(ti); 59 | stream = at::cuda::getCurrentCUDAStream(); 60 | AT_CUDA_CHECK(cudaStreamSynchronize(stream)); 61 | std::chrono::duration model_loading_duration = std::chrono::system_clock::now() - start; 62 | std::string msg = gemfield_org::format("Overall process time in warmup: %f", model_loading_duration.count()); 63 | } 64 | std::cout<<"---------------WARMUP END---------------"< model_loading_duration = std::chrono::system_clock::now() - start; 85 | std::string header = "|Model|Engine|Input size|forward time|\n"; 86 | std::string header2 = "|-----|-------|----------|-----------|\n"; 87 | std::string msg = gemfield_org::format("|Resnet50|libtorch|%dx%d|%f|\n", t.size(4),t.size(3),model_loading_duration.count()/item_num); 88 | std::cout << header< "); 94 | return -1; 95 | } 96 | 97 | std::string device = argv[1]; 98 | std::string model_path = argv[2]; 99 | std::string img_path = argv[3]; 100 | 101 | std::cout << "device : " << device << std::endl; 102 | std::cout << "model_path : " << model_path << std::endl; 103 | std::cout << "img_path : " << img_path << std::endl; 104 | 105 | //at::init_num_threads(); 106 | //at::set_num_threads(16); 107 | //at::set_num_interop_threads(16); 108 | std::cout<<"userEnabledCuDNN: "< 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include "syszux_seg_esp.h" 8 | #include "syszux_img2tensor.h" 9 | 10 | using namespace deepvac; 11 | int main(int argc, const char* argv[]) { 12 | if (argc != 3) { 13 | GEMFIELD_E("usage: deepvac "); 14 | return -1; 15 | } 16 | 17 | std::string device = argv[1]; 18 | std::string img_path = argv[2]; 19 | std::vector image_size = {512, 256}; 20 | SyszuxSegEsp seg(seg_esp_deepvac, device); 21 | seg.set(image_size); 22 | auto mat_opt = gemfield_org::img2CvMat(img_path); 23 | if(!mat_opt){ 24 | throw std::runtime_error("illegal image detected"); 25 | return 0; 26 | } 27 | auto mat_out = mat_opt.value(); 28 | auto seg_out_opt = seg.process(mat_out); 29 | if(!seg_out_opt){ 30 | throw std::runtime_error("seg error!"); 31 | return 0; 32 | } 33 | auto seg_out = seg_out_opt.value(); 34 | 35 | cv::Mat seg_img = mat_out.clone(); 36 | for (int i = 0; i < seg_out.rows; ++i) { 37 | for (int j = 0; j < seg_out.cols; ++j) { 38 | if ((int)(seg_out.at(i, j)) == 1) { 39 | *(seg_img.data + seg_img.step[0] * i + seg_img.step[1] * j + seg_img.elemSize1() * 0) = 0; 40 | *(seg_img.data + seg_img.step[0] * i + seg_img.step[1] * j + seg_img.elemSize1() * 1) = 0; 41 | *(seg_img.data + seg_img.step[0] * i + seg_img.step[1] * j + seg_img.elemSize1() * 2) = 255; 42 | } 43 | } 44 | } 45 | cv::imwrite("./res.jpg", seg_img); 46 | return 0; 47 | } 48 | -------------------------------------------------------------------------------- /examples/src/test_syszux_alignface.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include "gemfield.h" 8 | #include "syszux_align_face.h" 9 | #include "syszux_img2tensor.h" 10 | #include 11 | 12 | int main(int argc, char** argv) 13 | { 14 | if (argc != 2) { 15 | GEMFIELD_E("usage: test_syszux_alignface "); 16 | return -1; 17 | } 18 | 19 | std::string img_path = argv[1]; 20 | auto mat_opt = gemfield_org::img2CvMat(img_path); 21 | if(!mat_opt){ 22 | throw std::runtime_error("illegal image detected"); 23 | return 1; 24 | } 25 | auto mat_out = mat_opt.value(); 26 | 27 | std::vector facial_5pts = {632.30804, 177.63857, 687.6927, 185.6925, 649.9065, 28 | 213.16966, 632.7789, 239.9633, 673.7113, 246.61923}; 29 | 30 | cv::Mat facial_5pts_mat(facial_5pts); 31 | facial_5pts_mat = facial_5pts_mat.t(); 32 | 33 | gemfield_org::AlignFace align_face; 34 | auto [dst_img, dst_points] = align_face(mat_out, facial_5pts_mat); 35 | std::cout << dst_img.rows << std::endl; 36 | cv::imwrite("./test_res.jpg", dst_img); 37 | } 38 | -------------------------------------------------------------------------------- /examples/src/test_syszux_nms.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include "syszux_nms.h" 8 | #include "syszux_img2tensor.h" 9 | #include 10 | 11 | int main(int argc, const char* argv[]) { 12 | float threshold = 0.95; 13 | torch::Tensor dets_tensor; 14 | float dets[10][5] = { 15 | {608.869629,131.506607,726.548950,276.543457,0.999467671}, 16 | {608.558289,132.701614,724.578796,275.906860,0.999419928}, 17 | {608.443909,128.678253,726.312439,278.324097,0.916760027}, 18 | {609.709534,129.584427,724.541199,273.925568,0.848726749}, 19 | {609.766479,130.341476,726.441101,274.966766,0.747644246}, 20 | {608.292908,123.771599,725.066101,276.210419,0.707930803}, 21 | {609.070374,126.515434,723.639038,276.863739,0.649771333}, 22 | {607.130554,132.601349,724.788513,279.786499,0.600724220}, 23 | {608.286377,133.966751,727.646606,279.250275,0.523717880}, 24 | {608.372070,129.631256,724.156494,277.742035,0.497736454} 25 | }; 26 | dets_tensor = torch::from_blob(dets, {10, 5}); 27 | auto keep = gemfield_org::nms(dets_tensor, threshold); 28 | std::cout << keep << std::endl; 29 | return 0; 30 | } 31 | -------------------------------------------------------------------------------- /examples/src/test_syszux_priorbox.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include "syszux_priorbox.h" 8 | #include "syszux_img2tensor.h" 9 | 10 | int main(int argc, const char* argv[]) { 11 | gemfield_org::PriorBox pb({{16,32},{64,128},{256,512}}, {8,16,32}); 12 | std::vector img_size = {224,312}; 13 | auto x = pb.forward(img_size); 14 | std::cout< 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | * */ 6 | 7 | #include "syszux_verify_landmark.h" 8 | 9 | int main(int argc, char** argv){ 10 | std::vector bbox = {608.8696,131.5066,726.54895,276.54346}; 11 | std::vector landmark = {632.30804,177.63857,687.6927,185.6925,649.9065, 12 | 213.16966,632.7789,239.9633,673.7113,246.61923}; 13 | int min_face_size = 48; 14 | bool res = gemfield_org::isValidLandmark(bbox, landmark); 15 | std::cout << res << std::endl; 16 | } 17 | -------------------------------------------------------------------------------- /loader/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 Gemfield 2 | # This file is part of libdeepvac, licensed under the GPLv3 (the "License") 3 | # You may not use this file except in compliance with the License. 4 | 5 | file(GLOB LOADER_SRC_LIST src/*.cpp) 6 | add_syszux_sources(${LOADER_SRC_LIST}) 7 | message(STATUS "found LOADER_SRC_LIST: " ${LOADER_SRC_LIST}) 8 | 9 | file(GLOB HEADER_LIST include/*.h) 10 | add_syszux_headers(${HEADER_LIST}) 11 | 12 | add_header_dir(${CMAKE_CURRENT_SOURCE_DIR}/include) -------------------------------------------------------------------------------- /loader/include/deepvac_loader.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | namespace deepvac{ 15 | 16 | template 17 | class DeepvacIterBase { 18 | public: 19 | DeepvacIterBase(){} 20 | DeepvacIterBase(SyszuxWalkerType walker): walker_(walker){} 21 | DeepvacIterBase& operator=(const DeepvacIterBase&) = delete; 22 | virtual ~DeepvacIterBase() = default; 23 | DeepvacIterBase operator++() { ++walker_; return *this; } 24 | bool operator!=(const DeepvacIterBase & other) const {return walker_ != other.walker_; } 25 | 26 | protected: 27 | SyszuxWalkerType walker_; 28 | }; 29 | 30 | using deepvac_sfrdi = std::filesystem::recursive_directory_iterator; 31 | class RecursiveFileIterBase : public DeepvacIterBase{ 32 | public: 33 | RecursiveFileIterBase(const char* path) : DeepvacIterBase(deepvac_sfrdi(path)) {} 34 | RecursiveFileIterBase() : DeepvacIterBase(deepvac_sfrdi()) {} 35 | }; 36 | 37 | class RecursiveFileIter : public RecursiveFileIterBase{ 38 | public: 39 | RecursiveFileIter(const char* path) : RecursiveFileIterBase(path) {} 40 | RecursiveFileIter() : RecursiveFileIterBase() {} 41 | virtual const std::string operator*() const; 42 | }; 43 | 44 | class CvMatIter : public RecursiveFileIterBase{ 45 | public: 46 | CvMatIter(const char* path, bool is_rgb=false) : RecursiveFileIterBase(path),is_rgb_(is_rgb) {} 47 | CvMatIter(bool is_rgb=false) : RecursiveFileIterBase(),is_rgb_(is_rgb) {} 48 | virtual const std::optional operator*() const; 49 | private: 50 | bool is_rgb_; 51 | }; 52 | 53 | class ImgFileInputTensorPairIter : public RecursiveFileIterBase{ 54 | public: 55 | ImgFileInputTensorPairIter(const char* path, bool is_rgb=false) : RecursiveFileIterBase(path),is_rgb_(is_rgb) {} 56 | ImgFileInputTensorPairIter(bool is_rgb=false) : RecursiveFileIterBase(),is_rgb_(is_rgb) {} 57 | virtual const std::tuple> operator*() const; 58 | private: 59 | bool is_rgb_; 60 | std::unordered_set suffix_ {".jpg",".jpeg",".png"}; 61 | }; 62 | 63 | template 64 | class DeepvacRecursiveFileLoader { 65 | public: 66 | DeepvacRecursiveFileLoader(const char* url): url_(url) {} 67 | DeepvacIterType begin() { return DeepvacIterType(url_.c_str()); } 68 | DeepvacIterType end() { return DeepvacIterType(); } 69 | 70 | protected: 71 | std::string url_; 72 | }; 73 | 74 | 75 | //loader that based on std container. 76 | using deepvac_sv = std::vector; 77 | class DeepvacVectorIter : public DeepvacIterBase{ 78 | public: 79 | DeepvacVectorIter(deepvac_sv::iterator sv): DeepvacIterBase(sv) {} 80 | virtual std::optional operator*() const; 81 | }; 82 | 83 | template 84 | class DeepvacContainerLoader { 85 | public: 86 | DeepvacContainerLoader(deepvac_sv sv) : sv_(std::move(sv)) {} 87 | DeepvacIterType begin() { return DeepvacIterType(sv_.begin()); } 88 | DeepvacIterType end() { return DeepvacIterType(sv_.end()); } 89 | private: 90 | deepvac_sv sv_; 91 | }; 92 | 93 | }//namespace deepvac -------------------------------------------------------------------------------- /loader/src/deepvac_loader.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "deepvac_loader.h" 12 | #include "gemfield.h" 13 | #include "syszux_img2tensor.h" 14 | 15 | namespace deepvac{ 16 | 17 | const std::string RecursiveFileIter::operator*() const { 18 | std::string p = (*walker_).path(); 19 | return p; 20 | } 21 | 22 | const std::optional CvMatIter::operator*() const { 23 | std::string p = (*walker_).path(); 24 | return gemfield_org::img2CvMat(p); 25 | } 26 | 27 | const std::tuple> ImgFileInputTensorPairIter::operator*() const { 28 | std::string p = (*walker_).path(); 29 | std::string suffix = std::filesystem::path(p).extension(); 30 | 31 | if (suffix.empty()){ 32 | return {p, std::nullopt}; 33 | } 34 | 35 | std::transform(suffix.begin(), suffix.end(), suffix.begin(),[](unsigned char c){ return std::tolower(c); }); 36 | 37 | if(suffix_.find(suffix) == suffix_.end() ){ 38 | return {p, std::nullopt}; 39 | } 40 | 41 | return {p, gemfield_org::img2Tensor(p)}; 42 | } 43 | 44 | //loader that based on std container. 45 | std::optional DeepvacVectorIter::operator*() const { 46 | return gemfield_org::img2CvMat(*walker_); 47 | } 48 | 49 | } //namespace deepvac 50 | -------------------------------------------------------------------------------- /modules/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 Gemfield 2 | # This file is part of libdeepvac, licensed under the GPLv3 (the "License") 3 | # You may not use this file except in compliance with the License. 4 | 5 | file(GLOB MODULES_SRC_LIST src/*.cpp) 6 | 7 | if(NOT USE_TENSORRT) 8 | list(FILTER MODULES_SRC_LIST EXCLUDE REGEX ".*_nv.cpp$") 9 | endif() 10 | 11 | add_syszux_sources(${MODULES_SRC_LIST}) 12 | message(STATUS "found MODULES_SRC_LIST: " ${MODULES_SRC_LIST}) 13 | 14 | file(GLOB HEADER_LIST include/*.h) 15 | add_syszux_headers(${HEADER_LIST}) 16 | 17 | add_header_dir(${CMAKE_CURRENT_SOURCE_DIR}/include) 18 | -------------------------------------------------------------------------------- /modules/include/syszux_cls_mobile.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | #include "opencv2/opencv.hpp" 9 | #include "deepvac.h" 10 | #include "syszux_img2tensor.h" 11 | 12 | namespace deepvac { 13 | class SyszuxClsMobile : public Deepvac { 14 | public: 15 | using Deepvac::Deepvac; 16 | SyszuxClsMobile(const SyszuxClsMobile&) = default; 17 | SyszuxClsMobile& operator=(const SyszuxClsMobile&) = default; 18 | SyszuxClsMobile(SyszuxClsMobile&&) = default; 19 | SyszuxClsMobile& operator=(SyszuxClsMobile&&) = default; 20 | virtual ~SyszuxClsMobile() = default; 21 | public: 22 | std::optional> process (cv::Mat frame); 23 | std::optional>> process(std::vector frame); 24 | }; 25 | } //namespace deepvac 26 | -------------------------------------------------------------------------------- /modules/include/syszux_cls_resnet.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | 9 | #include "deepvac.h" 10 | #include "syszux_img2tensor.h" 11 | 12 | namespace deepvac{ 13 | class SyszuxClsResnet : public Deepvac{ 14 | public: 15 | using Deepvac::Deepvac; 16 | SyszuxClsResnet(const SyszuxClsResnet&) = default; 17 | SyszuxClsResnet& operator=(const SyszuxClsResnet&) = default; 18 | SyszuxClsResnet(SyszuxClsResnet&&) = default; 19 | SyszuxClsResnet& operator=(SyszuxClsResnet&&) = default; 20 | virtual ~SyszuxClsResnet() = default; 21 | std::optional> process(cv::Mat frame); 22 | at::Tensor process(at::Tensor& t); 23 | void set(std::vector input_size); 24 | private: 25 | std::vector input_size_; 26 | }; 27 | } //namespace deepvac 28 | -------------------------------------------------------------------------------- /modules/include/syszux_detect_yolo.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | #include "opencv2/opencv.hpp" 9 | #include "deepvac.h" 10 | #include "syszux_img2tensor.h" 11 | #include "syszux_nms.h" 12 | 13 | namespace deepvac { 14 | class SyszuxDetectYolo : public Deepvac { 15 | public: 16 | using Deepvac::Deepvac; 17 | SyszuxDetectYolo(const SyszuxDetectYolo&) = default; 18 | SyszuxDetectYolo& operator=(const SyszuxDetectYolo&) = default; 19 | SyszuxDetectYolo(SyszuxDetectYolo&&) = default; 20 | SyszuxDetectYolo& operator=(SyszuxDetectYolo&&) = default; 21 | virtual ~SyszuxDetectYolo() = default; 22 | public: 23 | void set(int input_size, float iou_thresh, float score_thresh); 24 | std::optional>>> process (cv::Mat frame); 25 | private: 26 | torch::Tensor postProcess(torch::Tensor& preds); 27 | private: 28 | int input_size_; 29 | float iou_thresh_; 30 | float score_thresh_; 31 | std::vector idx_to_cls_; 32 | }; 33 | } //namespace deepvac 34 | -------------------------------------------------------------------------------- /modules/include/syszux_face_id_nv.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include "deepvac_nv.h" 7 | #include "syszux_tensorrt_buffers.h" 8 | #include "syszux_img2tensor.h" 9 | 10 | namespace deepvac{ 11 | class SyszuxFaceIdNV : public DeepvacNV{ 12 | public: 13 | SyszuxFaceIdNV(std::string path, std::string device = "cpu"); 14 | SyszuxFaceIdNV(std::vector&& buffer, std::string device = "cpu"); 15 | SyszuxFaceIdNV(const SyszuxFaceIdNV&) = delete; 16 | SyszuxFaceIdNV& operator=(const SyszuxFaceIdNV&) = delete; 17 | SyszuxFaceIdNV(SyszuxFaceIdNV&&) = default; 18 | SyszuxFaceIdNV& operator=(SyszuxFaceIdNV&&) = default; 19 | virtual ~SyszuxFaceIdNV() = default; 20 | virtual std::optional>> process(std::vector& frames); 21 | void initBinding(); 22 | 23 | private: 24 | void loadDB(std::string path); 25 | int size(); 26 | int cachedSize(); 27 | void commit(std::string path_prefix); 28 | int add(cv::Mat& frame, std::string name); 29 | 30 | 31 | private: 32 | int batch_size_; 33 | std::vector db2commit_vec_; 34 | std::vector id2commit_vec_; 35 | at::Tensor db_; 36 | std::vector id_vec_; 37 | }; 38 | }//namespace 39 | 40 | 41 | -------------------------------------------------------------------------------- /modules/include/syszux_face_retina.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | 9 | #include 10 | 11 | #include "deepvac.h" 12 | #include "syszux_priorbox.h" 13 | #include "syszux_nms.h" 14 | #include "syszux_align_face.h" 15 | #include "syszux_verify_landmark.h" 16 | #include "syszux_img2tensor.h" 17 | #include "syszux_face_retina_config.h" 18 | 19 | namespace deepvac{ 20 | class SyszuxFaceRetina : public Deepvac, public SyszuxFaceRetinaConfig { 21 | public: 22 | SyszuxFaceRetina() = default; 23 | SyszuxFaceRetina(std::string path, std::string device = "cpu"); 24 | SyszuxFaceRetina(std::vector&& buffer, std::string device = "cpu"); 25 | SyszuxFaceRetina(const SyszuxFaceRetina&) = default; 26 | SyszuxFaceRetina& operator=(const SyszuxFaceRetina&) = default; 27 | SyszuxFaceRetina(SyszuxFaceRetina&&) = default; 28 | SyszuxFaceRetina& operator=(SyszuxFaceRetina&&) = default; 29 | virtual ~SyszuxFaceRetina() = default; 30 | std::optional, std::vector>>> process(cv::Mat frame); 31 | }; 32 | } //namespace deepvac 33 | -------------------------------------------------------------------------------- /modules/include/syszux_face_retina_config.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | #include "syszux_priorbox.h" 9 | #include "syszux_nms.h" 10 | #include "syszux_align_face.h" 11 | #include "syszux_verify_landmark.h" 12 | #include "syszux_img2tensor.h" 13 | 14 | namespace deepvac{ 15 | class SyszuxFaceRetinaConfig { 16 | public: 17 | SyszuxFaceRetinaConfig() = default; 18 | SyszuxFaceRetinaConfig(const SyszuxFaceRetinaConfig&) = default; 19 | SyszuxFaceRetinaConfig& operator=(const SyszuxFaceRetinaConfig&) = default; 20 | SyszuxFaceRetinaConfig(SyszuxFaceRetinaConfig&&) = default; 21 | SyszuxFaceRetinaConfig& operator=(SyszuxFaceRetinaConfig&&) = default; 22 | virtual ~SyszuxFaceRetinaConfig() = default; 23 | void initParameter(std::string device); 24 | void setTopK(int top_k); 25 | void setKeepTopK(int keep_top_k); 26 | void setConfThreshold(float confidence_threshold); 27 | void setNMSThreshold(float nms_threshold); 28 | void setMaxHW(int max_hw); 29 | void setGapThreshold(float gap_threshold); 30 | 31 | protected: 32 | gemfield_org::PriorBox prior_box_; 33 | gemfield_org::AlignFace align_face_; 34 | int top_k_; 35 | int keep_top_k_; 36 | float nms_threshold_; 37 | float confidence_threshold_; 38 | int max_hw_; 39 | torch::Tensor variances_tensor_; 40 | int last_h_; 41 | int last_w_; 42 | torch::Tensor last_prior_; 43 | torch::Tensor last_box_scale_; 44 | torch::Tensor last_lmk_scale_; 45 | float gap_threshold_; 46 | }; 47 | } //namespace deepvac 48 | -------------------------------------------------------------------------------- /modules/include/syszux_face_retina_nv.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | 9 | #include 10 | #include 11 | #include "syszux_img2tensor.h" 12 | #include "syszux_tensorrt_buffers.h" 13 | #include "deepvac_nv.h" 14 | #include "syszux_face_retina_config.h" 15 | 16 | namespace deepvac{ 17 | class SyszuxFaceRetinaNV : public DeepvacNV, public SyszuxFaceRetinaConfig { 18 | public: 19 | SyszuxFaceRetinaNV(std::string path, std::string device = "cpu"); 20 | SyszuxFaceRetinaNV(std::vector&& buffer, std::string device = "cpu"); 21 | SyszuxFaceRetinaNV(const SyszuxFaceRetinaNV&) = delete; 22 | SyszuxFaceRetinaNV& operator=(const SyszuxFaceRetinaNV&) = delete; 23 | SyszuxFaceRetinaNV(SyszuxFaceRetinaNV&&) = default; 24 | SyszuxFaceRetinaNV& operator=(SyszuxFaceRetinaNV&&) = default; 25 | virtual ~SyszuxFaceRetinaNV() = default; 26 | virtual std::optional, std::vector>>> process(cv::Mat frame); 27 | }; 28 | 29 | } //namespace deepvac 30 | -------------------------------------------------------------------------------- /modules/include/syszux_ocr_db.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | 9 | #include 10 | #include 11 | #include "opencv2/opencv.hpp" 12 | #include "syszux_img2tensor.h" 13 | #include "syszux_glab.h" 14 | #include "syszux_clipper.h" 15 | #include 16 | #include 17 | 18 | namespace deepvac { 19 | class SyszuxOcrDB : public Deepvac { 20 | public: 21 | using Deepvac::Deepvac; 22 | SyszuxOcrDB(const SyszuxOcrDB&) = default; 23 | SyszuxOcrDB& operator=(const SyszuxOcrDB&) = default; 24 | SyszuxOcrDB(SyszuxOcrDB&&) = default; 25 | SyszuxOcrDB& operator=(SyszuxOcrDB&&) = default; 26 | virtual ~SyszuxOcrDB() = default; 27 | public: 28 | std::optional, std::vector>>> process(cv::Mat frame); 29 | void setLongSize(int long_size); 30 | void setCropGap(int crop_gap); 31 | void setTextMinArea(int text_min_area); 32 | void setTextMinSize(int text_min_size); 33 | void setTextMeanScore(float text_mean_score); 34 | void setTextThresh(float text_thresh); 35 | void setUnclipRatio(float unclip_ratio); 36 | void setGlab(bool is_glab); 37 | void setExtend(bool is_extend); 38 | void setUnclip(bool is_unclip); 39 | void setPolygonScore(bool is_polygon_score); 40 | private: 41 | cv::Mat cropRect(cv::Mat &img, cv::RotatedRect &rotated_rects); 42 | float boxScoreFast(cv::Mat &pred, cv::RotatedRect &rect); 43 | float polygonScoreAcc(cv::Mat &pred, std::vector &contour); 44 | std::optional unClip(cv::RotatedRect &rect); 45 | void getContourArea(const std::vector> &box, float &distance); 46 | private: 47 | int long_size_{1280}; 48 | int crop_gap_{10}; 49 | int text_min_area_{300}; 50 | int text_min_size_{3}; 51 | float text_mean_score_{0.5}; 52 | float text_thresh_{0.3}; 53 | float unclip_ratio_{2.0}; 54 | bool is_glab_{false}; 55 | bool is_extend_{false}; 56 | bool is_unclip_{false}; 57 | bool is_polygon_score_{false}; 58 | }; 59 | } //namespace 60 | -------------------------------------------------------------------------------- /modules/include/syszux_ocr_pse.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | 9 | #include 10 | #include 11 | #include "opencv2/opencv.hpp" 12 | #include "syszux_img2tensor.h" 13 | #include 14 | #include 15 | 16 | namespace deepvac { 17 | class SyszuxOcrPse : public Deepvac { 18 | public: 19 | using Deepvac::Deepvac; 20 | SyszuxOcrPse(const SyszuxOcrPse&) = default; 21 | SyszuxOcrPse& operator=(const SyszuxOcrPse&) = default; 22 | SyszuxOcrPse(SyszuxOcrPse&&) = default; 23 | SyszuxOcrPse& operator=(SyszuxOcrPse&&) = default; 24 | virtual ~SyszuxOcrPse() = default; 25 | public: 26 | std::optional, std::vector>>> process(cv::Mat frame); 27 | void set(int long_size, int gap, int text_min_area, float text_mean_score); 28 | void setGlab(bool is_glab); 29 | void setExtend(bool is_extend); 30 | private: 31 | std::vector> merge(std::vector> rects); 32 | void getKernals(torch::Tensor input_data, std::vector &kernals); 33 | void growingTextLine(std::vector &kernals, std::vector> &text_line, float min_area); 34 | std::vector> adaptorPse(torch::Tensor input_data, float min_area); 35 | bool isMerge(std::vector rect1, std::vector rect2); 36 | std::vector> mergeBox(std::vector> rects); 37 | cv::Mat cropRect(cv::Mat &img, cv::RotatedRect &rotated_rects); 38 | private: 39 | int long_size_{1280}; 40 | int crop_gap_{10}; 41 | int text_min_area_{300}; 42 | float text_mean_score_{0.93}; 43 | bool is_glab_{false}; 44 | bool is_extend_{false}; 45 | }; 46 | } //namespace 47 | -------------------------------------------------------------------------------- /modules/include/syszux_seg_esp.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | 9 | #include "deepvac.h" 10 | #include "syszux_img2tensor.h" 11 | 12 | namespace deepvac{ 13 | class SyszuxSegEsp : public Deepvac{ 14 | public: 15 | using Deepvac::Deepvac; 16 | SyszuxSegEsp(const SyszuxSegEsp&) = default; 17 | SyszuxSegEsp& operator=(const SyszuxSegEsp&) = default; 18 | SyszuxSegEsp(SyszuxSegEsp&&) = default; 19 | SyszuxSegEsp& operator=(SyszuxSegEsp&&) = default; 20 | virtual ~SyszuxSegEsp() = default; 21 | public: 22 | void set(std::vector image_size); 23 | std::optional process(cv::Mat frame); 24 | private: 25 | std::vector image_size_; 26 | }; 27 | } //namespace deepvac 28 | -------------------------------------------------------------------------------- /modules/src/syszux_cls_mobile.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | #include "syszux_cls_mobile.h" 7 | 8 | namespace deepvac{ 9 | std::optional> SyszuxClsMobile::process(cv::Mat frame){ 10 | auto input_tensor_opt = gemfield_org::cvMat2Tensor(std::move(frame), gemfield_org::NORMALIZE_1_1, gemfield_org::NO_MEAN_STD, device_); 11 | 12 | if(!input_tensor_opt){ 13 | return std::nullopt; 14 | } 15 | auto input_tensor = input_tensor_opt.value(); 16 | 17 | auto pred = forward(input_tensor); 18 | auto softmaxs = pred.softmax(1); 19 | std::tuple max_res = torch::max(softmaxs, 1); 20 | auto max_probability = std::get<0>(max_res).item(); 21 | auto index = std::get<1>(max_res).item(); 22 | 23 | std::pair result(index, max_probability); 24 | return result; 25 | } 26 | 27 | std::optional>> SyszuxClsMobile::process(std::vector frames){ 28 | auto input_tensor_opt = gemfield_org::cvMat2Tensor(frames, gemfield_org::NORMALIZE_1_1, gemfield_org::NO_MEAN_STD, device_); 29 | 30 | if(!input_tensor_opt){ 31 | return std::nullopt; 32 | } 33 | auto input_tensor = input_tensor_opt.value(); 34 | 35 | auto pred = forward(input_tensor); 36 | auto softmaxs = pred.softmax(1); 37 | std::vector> results; 38 | for (int i=0; i max_res = torch::max(softmaxs[i], 0); 40 | auto max_probability = std::get<0>(max_res).item(); 41 | auto index = std::get<1>(max_res).item(); 42 | std::pair result(index, max_probability); 43 | results.push_back(result); 44 | } 45 | return results; 46 | } 47 | }//namespace 48 | -------------------------------------------------------------------------------- /modules/src/syszux_cls_resnet.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | #include "syszux_cls_resnet.h" 7 | #include "gemfield.h" 8 | 9 | namespace deepvac{ 10 | void SyszuxClsResnet::set(std::vector input_size) { 11 | input_size_ = input_size; 12 | } 13 | 14 | std::optional> SyszuxClsResnet::process(cv::Mat frame){ 15 | cv::Mat input_frame = frame.clone(); 16 | cv::cvtColor(input_frame, input_frame, cv::COLOR_BGR2RGB); 17 | cv::resize(input_frame, input_frame, cv::Size(input_size_[0], input_size_[1])); 18 | 19 | auto input_tensor_opt = gemfield_org::cvMat2Tensor(input_frame, gemfield_org::NORMALIZE0_1, gemfield_org::MEAN_STD_FROM_IMAGENET, device_); 20 | if(!input_tensor_opt){ 21 | return std::nullopt; 22 | } 23 | auto input_tensor = input_tensor_opt.value(); 24 | 25 | auto output = forward(input_tensor); 26 | 27 | auto softmaxs = output.softmax(1); 28 | std::tuple max_res = torch::max(softmaxs, 1); 29 | 30 | auto max_probability = std::get<0>(max_res).item(); 31 | auto index = std::get<1>(max_res).item(); 32 | 33 | std::pair result(index, max_probability); 34 | return result; 35 | } 36 | 37 | at::Tensor SyszuxClsResnet::process(at::Tensor& t){ 38 | return forward(t); 39 | } 40 | 41 | } //namespace deepvac 42 | -------------------------------------------------------------------------------- /modules/src/syszux_detect_yolo.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | #include "syszux_detect_yolo.h" 7 | namespace deepvac{ 8 | void SyszuxDetectYolo::set(int input_size, float iou_thresh, float score_thresh) { 9 | input_size_ = input_size; 10 | iou_thresh_ = iou_thresh; 11 | score_thresh_ = score_thresh; 12 | } 13 | 14 | torch::Tensor SyszuxDetectYolo::postProcess(torch::Tensor& preds) { 15 | std::vector index = torch::where(preds.select(2, 4) > score_thresh_); 16 | auto pred = preds.index_select(1, index[1])[0]; 17 | if (pred.sizes()[0] == 0) { 18 | return torch::zeros({0, 6}); 19 | } 20 | 21 | pred.slice(1, 5, pred.sizes()[1]) = pred.slice(1, 5, pred.sizes()[1]) * pred.slice(1, 4, 5); 22 | pred.select(1, 0) = pred.select(1, 0) - pred.select(1, 2).div(2); 23 | pred.select(1, 1) = pred.select(1, 1) - pred.select(1, 3).div(2); 24 | pred.select(1, 2) = pred.select(1, 2) + pred.select(1, 0); 25 | pred.select(1, 3) = pred.select(1, 3) + pred.select(1, 1); 26 | 27 | std::tuple max_classes = torch::max(pred.slice(1, 5, pred.sizes()[1]), 1); 28 | 29 | auto max_conf = std::get<0>(max_classes).unsqueeze(1); 30 | auto max_index = std::get<1>(max_classes).unsqueeze(1); 31 | 32 | pred = torch::cat({pred.slice(1, 0, 4), max_conf, max_index}, 1); 33 | if (pred.sizes()[0] == 0) { 34 | return torch::zeros({0, 6}); 35 | } 36 | 37 | pred = torch::index_select(pred, 0, torch::nonzero(max_conf.view(-1) > score_thresh_).select(1, 0)); 38 | 39 | int max_side = 4096; 40 | auto class_offset = pred.slice(1, 5, 6) * max_side; 41 | auto boxes = pred.slice(1, 0, 4) + class_offset; 42 | auto scores = pred.slice(1, 4, 5); 43 | auto dets = torch::cat({boxes, scores}, 1); 44 | auto keep = gemfield_org::nms(dets, iou_thresh_); 45 | pred = pred.index({keep}); 46 | 47 | return pred; 48 | 49 | } 50 | 51 | std::optional>>> SyszuxDetectYolo::process(cv::Mat frame){ 52 | cv::Mat input_img = frame.clone(); 53 | 54 | int h = input_img.rows; 55 | int w = input_img.cols; 56 | float r = std::min((float)input_size_ / (float)h, (float)input_size_ / (float)w); 57 | 58 | int w_new = (int)(std::round((float)w * r)); 59 | int h_new = (int)(std::round((float)h * r)); 60 | 61 | float dw = (float)((int)(input_size_ - w_new) % 32) / 2.0; 62 | float dh = (float)((int)(input_size_ - h_new) % 32) / 2.0; 63 | 64 | if (h != h_new and w != w_new) { 65 | cv::resize(input_img, input_img, cv::Size(w_new, h_new), cv::INTER_LINEAR); 66 | } 67 | 68 | int top = (int)(std::round(dh - 0.1)); 69 | int bottom = (int)(std::round(dh + 0.1)); 70 | int left = (int)(std::round(dw - 0.1)); 71 | int right = (int)(std::round(dw + 0.1)); 72 | 73 | cv::copyMakeBorder(input_img, input_img, top, bottom, left, right, cv::BORDER_CONSTANT, {114, 114, 114}); 74 | cv::cvtColor(input_img, input_img, cv::COLOR_BGR2RGB); 75 | 76 | auto input_tensor_opt = gemfield_org::cvMat2Tensor(std::move(input_img), gemfield_org::NORMALIZE0_1, gemfield_org::NO_MEAN_STD, device_); 77 | 78 | if(!input_tensor_opt){ 79 | return std::nullopt; 80 | } 81 | auto input_tensor = input_tensor_opt.value(); 82 | 83 | at::Tensor preds = forward(input_tensor); 84 | 85 | auto pred = postProcess(preds); 86 | 87 | std::vector>> results; 88 | 89 | if (pred.sizes()[0] == 0) { 90 | return results; 91 | } 92 | 93 | float ori_w = float(frame.cols); 94 | float ori_h = float(frame.rows); 95 | for (int i=0; i() - (float)left)/r; // x padding 98 | float y1 = (p[1].item() - (float)top)/r; // y padding 99 | float x2 = (p[2].item() - (float)left)/r; // x padding 100 | float y2 = (p[3].item() - (float)top)/r; // y padding 101 | 102 | x1 = std::max(0.0f, std::min(x1, ori_w)); 103 | y1 = std::max(0.0f, std::min(y1, ori_h)); 104 | x2 = std::max(0.0f, std::min(x2, ori_w)); 105 | y2 = std::max(0.0f, std::min(y2, ori_h)); 106 | std::vector bbox_and_score = {x1, y1, x2, y2, p[4].item()}; 107 | std::pair result(int(p[5].item()), bbox_and_score); 108 | results.push_back(result); 109 | } 110 | return results; 111 | } 112 | }//namespace 113 | -------------------------------------------------------------------------------- /modules/src/syszux_face_id_nv.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "NvInfer.h" 7 | #include "gemfield.h" 8 | #include "syszux_face_id_nv.h" 9 | 10 | namespace deepvac { 11 | SyszuxFaceIdNV::SyszuxFaceIdNV(std::string path, std::string device):DeepvacNV(path, device) { 12 | setBinding(2); 13 | initBinding(); 14 | } 15 | 16 | SyszuxFaceIdNV::SyszuxFaceIdNV(std::vector&& buffer, std::string device):DeepvacNV(std::move(buffer), device){ 17 | setBinding(2); 18 | initBinding(); 19 | } 20 | 21 | void SyszuxFaceIdNV::initBinding() { 22 | datas_[0].hostBuffer.resize({1, 3, 112, 112}); 23 | datas_[0].deviceBuffer.resize({1, 3, 112, 112}); 24 | datas_[1].hostBuffer.resize({1, 512}); 25 | datas_[1].deviceBuffer.resize({1, 512}); 26 | } 27 | 28 | 29 | void SyszuxFaceIdNV::loadDB(std::string path){ 30 | try{ 31 | std::ifstream inputstream(path + ".id"); 32 | std::copy(std::istream_iterator(inputstream),std::istream_iterator(),std::back_inserter(id_vec_)); 33 | torch::load(db_,path + ".db", device_); 34 | }catch(...){ 35 | throw std::runtime_error("invalid parameter path sepcified."); 36 | } 37 | //db_ = db_.to(getDevice()); 38 | std::stringstream feature_size; 39 | feature_size << db_.sizes(); 40 | std::string msg = gemfield_org::format("%s: %s | %s : %d", "load db size: ", feature_size.str().c_str(), "load name vector size: ", id_vec_.size()); 41 | GEMFIELD_I(msg.c_str()); 42 | } 43 | 44 | int SyszuxFaceIdNV::size(){ 45 | return id_vec_.size(); 46 | } 47 | 48 | int SyszuxFaceIdNV::add(cv::Mat& frame, std::string name){ 49 | cv::Mat dst; 50 | frame.convertTo(dst, CV_32F, 1.0 / 127.5, -1.0); 51 | float* input_data = (float*)dst.data; 52 | float* hostDataBuffer = static_cast(datas_[0].hostBuffer.data()); 53 | for (int c = 0; c < 3; ++c) { 54 | for(int j = 0, volChl=112*112; j < volChl; ++j) { 55 | hostDataBuffer[c*volChl + j] = input_data[j*3 + c]; 56 | } 57 | } 58 | cudaMemcpy(datas_[0].deviceBuffer.data(), datas_[0].hostBuffer.data(), datas_[0].hostBuffer.nbBytes(), cudaMemcpyHostToDevice); 59 | std::vector predicitonBindings = {datas_[0].deviceBuffer.data(), datas_[1].deviceBuffer.data()}; 60 | auto predict = forward(predicitonBindings.data()); 61 | cudaMemcpy(datas_[1].hostBuffer.data(), datas_[1].deviceBuffer.data(), datas_[1].deviceBuffer.nbBytes(), cudaMemcpyDeviceToHost); 62 | auto emb = torch::from_blob(datas_[1].hostBuffer.data(), {1, 512}).to(device_); 63 | db2commit_vec_.push_back(emb); 64 | id2commit_vec_.push_back(name); 65 | return id2commit_vec_.size() - 1; 66 | } 67 | 68 | int SyszuxFaceIdNV::cachedSize(){ 69 | return id2commit_vec_.size(); 70 | } 71 | 72 | void SyszuxFaceIdNV::commit(std::string path_prefix){ 73 | torch::Tensor db = torch::cat(db2commit_vec_, 0); 74 | std::stringstream feature_size; 75 | feature_size << db.sizes(); 76 | std::string msg = gemfield_org::format("commit feature size: %s | commit name vector size: %d", "", feature_size.str().c_str(), id2commit_vec_.size() ); 77 | GEMFIELD_I(msg.c_str()); 78 | 79 | std::ofstream name_output(path_prefix + ".id"); 80 | for (auto &name : id2commit_vec_){ 81 | name_output << name << "\n"; 82 | } 83 | torch::save(db , path_prefix + ".db"); 84 | db2commit_vec_.clear(); 85 | id2commit_vec_.clear(); 86 | } 87 | 88 | 89 | 90 | std::optional>> SyszuxFaceIdNV::process(std::vector& frames) { 91 | if (frames.size()==0) { 92 | return std::nullopt; 93 | } 94 | 95 | std::vector embs_vec; 96 | cv::Mat dst; 97 | for(auto& frame : frames) { 98 | frame.convertTo(dst, CV_32F, 1.0 / 127.5, -1.0); 99 | float* input_data = (float*)dst.data; 100 | float* hostDataBuffer = static_cast(datas_[0].hostBuffer.data()); 101 | for (int c = 0; c < 3; ++c) { 102 | for(int j = 0, volChl=112*112; j < volChl; ++j) { 103 | hostDataBuffer[c*volChl + j] = input_data[j*3 + c]; 104 | } 105 | } 106 | cudaMemcpy(datas_[0].deviceBuffer.data(), datas_[0].hostBuffer.data(), datas_[0].hostBuffer.nbBytes(), cudaMemcpyHostToDevice); 107 | std::vector predicitonBindings = {datas_[0].deviceBuffer.data(), datas_[1].deviceBuffer.data()}; 108 | auto predict = forward(predicitonBindings.data()); 109 | cudaMemcpy(datas_[1].hostBuffer.data(), datas_[1].deviceBuffer.data(), datas_[1].deviceBuffer.nbBytes(), cudaMemcpyDeviceToHost); 110 | auto emb = torch::from_blob(datas_[1].hostBuffer.data(), {1, 512}).to(device_); 111 | embs_vec.emplace_back(std::move(emb)); 112 | } 113 | 114 | std::vector> results; 115 | for (auto embs : embs_vec) { 116 | for (int i=0; i(); 120 | float min_distance = torch::min(distance).item(); 121 | GEMFIELD_DI2("predict: ", id_vec_.at(min_index).c_str()); 122 | auto tmp_tuple = std::make_tuple(min_index, id_vec_.at(min_index), min_distance); 123 | results.push_back(tmp_tuple); 124 | } 125 | } 126 | return results; 127 | 128 | } 129 | 130 | }//namespace 131 | -------------------------------------------------------------------------------- /modules/src/syszux_face_retina.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "syszux_face_retina.h" 12 | #include "gemfield.h" 13 | 14 | namespace deepvac{ 15 | 16 | SyszuxFaceRetina::SyszuxFaceRetina(std::string path, std::string device):Deepvac(path, device){ 17 | initParameter(device); 18 | } 19 | 20 | SyszuxFaceRetina::SyszuxFaceRetina(std::vector&& buffer, std::string device):Deepvac(std::move(buffer), device){ 21 | initParameter(device); 22 | } 23 | 24 | std::optional, std::vector>>> SyszuxFaceRetina::process(cv::Mat frame){ 25 | GEMFIELD_SI; 26 | //prepare input 27 | int h = frame.rows; 28 | int w = frame.cols; 29 | int c = frame.channels(); 30 | int max_edge = std::max(h, w); 31 | if(max_edge > max_hw_){ 32 | cv::resize(frame, frame, cv::Size(int(w*max_hw_/max_edge), int(h*max_hw_/max_edge))); 33 | h = frame.rows; 34 | w = frame.cols; 35 | } 36 | //gemfield, prepare output 37 | if ( std::abs(h-last_h_)<=gap_threshold_*last_h_ and std::abs(w-last_w_)<=gap_threshold_*last_w_) { 38 | if ( h!=last_h_ or w!=last_w_ ) { 39 | cv::resize(frame, frame, cv::Size(last_w_, last_h_)); 40 | } 41 | } else { 42 | last_w_ = w; 43 | last_h_ = h; 44 | 45 | last_prior_ = prior_box_.forward({h, w}); 46 | last_prior_ = last_prior_.to(device_); 47 | 48 | last_box_scale_ = torch::tensor({w, h, w, h}); 49 | last_box_scale_ = last_box_scale_.to(device_); 50 | 51 | last_lmk_scale_ = torch::tensor({w, h, w, h, w, h, w, h, w, h}); 52 | last_lmk_scale_ = last_lmk_scale_.to(device_); 53 | } 54 | 55 | auto input_tensor_opt = gemfield_org::cvMat2Tensor(std::move(frame), gemfield_org::NO_NORMALIZE, gemfield_org::MEAN_STD_FROM_FACE, device_); 56 | 57 | if(!input_tensor_opt){ 58 | return std::nullopt; 59 | } 60 | auto input_tensor = input_tensor_opt.value(); 61 | //forward 62 | auto output = forward>(input_tensor); 63 | //Nx4 //Nx2 //Nx10 64 | auto loc = output[0].toTensor(); 65 | auto forward_conf = output[1].toTensor(); 66 | auto landms = output[2].toTensor(); 67 | 68 | loc = loc.squeeze(0); 69 | forward_conf = forward_conf.squeeze(0); 70 | landms = landms.squeeze(0); 71 | 72 | float resize = 1.; 73 | 74 | //gemfield 75 | torch::Tensor boxes = gemfield_org::getDecodeBox(last_prior_, variances_tensor_, loc); 76 | boxes = torch::div(torch::mul(boxes, last_box_scale_), resize); 77 | 78 | gemfield_org::decodeLandmark(last_prior_, variances_tensor_, landms); 79 | landms = torch::div(torch::mul(landms, last_lmk_scale_), resize); 80 | 81 | torch::Tensor scores = forward_conf.slice(1, 1, 2); 82 | std::vector index = torch::where(scores>confidence_threshold_); 83 | if (index[0].size(0) == 0) { 84 | return std::nullopt; 85 | } 86 | 87 | boxes = boxes.index({index[0]}); 88 | landms = landms.index({index[0]}); 89 | scores = scores.index({index[0]}); 90 | 91 | std::tuple sort_ret = torch::sort(scores, 0, 1); 92 | torch::Tensor idx = std::get<1>(sort_ret).squeeze(1); 93 | idx = idx.slice(0, 0, top_k_); 94 | 95 | boxes = boxes.index({idx}); 96 | landms = landms.index({idx}); 97 | scores = scores.index({idx}); 98 | 99 | torch::Tensor dets = torch::cat({boxes, scores}, 1); 100 | torch::Tensor keep; 101 | keep = gemfield_org::nms(dets, nms_threshold_); 102 | 103 | // keep top-K faster NMS 104 | keep = keep.slice(0, 0, keep_top_k_); 105 | dets = dets.index({keep}); 106 | landms = landms.index({keep}); 107 | if(dets.size(0) == 0){ 108 | return std::nullopt; 109 | } 110 | 111 | std::string msg = gemfield_org::format("detected %d faces", dets.size(0)); 112 | GEMFIELD_I(msg.c_str()); 113 | 114 | if(dets.size(0) != landms.size(0)){ 115 | std::string msg = gemfield_org::format("dets len mismatched landms len: %d vs %d", dets.size(0), landms.size(0)); 116 | GEMFIELD_E(msg.c_str()); 117 | return std::nullopt; 118 | } 119 | 120 | landms = landms.to(torch::kCPU); 121 | cv::Mat landms_mat(landms.size(0), landms.size(1), CV_32F); 122 | std::memcpy((void *) landms_mat.data, landms.data_ptr(), torch::elementSize(torch::kF32) * landms.numel()); 123 | dets = dets.to(torch::kCPU); 124 | cv::Mat dets_mat(dets.size(0), dets.size(1), CV_32F); 125 | std::memcpy((void *) dets_mat.data, dets.data_ptr(), torch::elementSize(torch::kF32) * dets.numel()); 126 | 127 | std::vector, std::vector>> faces_info; 128 | for(int i=0; i bbox_vec(bbox.begin(), bbox.end()); 133 | dst_img.convertTo(dst_img, CV_32FC3); 134 | faces_info.emplace_back(std::tuple(dst_img, bbox_vec, dst_points)); 135 | } 136 | 137 | return faces_info; 138 | } 139 | } //namespace deepvac 140 | -------------------------------------------------------------------------------- /modules/src/syszux_face_retina_config.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "syszux_face_retina_config.h" 12 | #include "gemfield.h" 13 | 14 | namespace deepvac{ 15 | 16 | 17 | void SyszuxFaceRetinaConfig::initParameter(std::string device){ 18 | variances_tensor_ = torch::tensor({0.1, 0.2}).to(device); 19 | prior_box_ = gemfield_org::PriorBox({{16,32},{64,128},{256,512}}, {8,16,32}); 20 | setTopK(50); 21 | setKeepTopK(50); 22 | setConfThreshold(0.4); 23 | setNMSThreshold(0.4); 24 | setMaxHW(2000); 25 | setGapThreshold(0.1); 26 | 27 | last_w_ = 0; 28 | last_h_ = 0; 29 | last_prior_ = torch::ones({1, 4}); 30 | last_box_scale_ = torch::ones({1, 4}); 31 | last_lmk_scale_ = torch::ones({1, 10}); 32 | } 33 | 34 | void SyszuxFaceRetinaConfig::setTopK(int top_k){ 35 | top_k_ = top_k; 36 | } 37 | 38 | void SyszuxFaceRetinaConfig::setKeepTopK(int keep_top_k){ 39 | keep_top_k_ = keep_top_k; 40 | } 41 | 42 | void SyszuxFaceRetinaConfig::setConfThreshold(float confidence_threshold){ 43 | confidence_threshold_ = confidence_threshold; 44 | } 45 | 46 | void SyszuxFaceRetinaConfig::setNMSThreshold(float nms_threshold){ 47 | nms_threshold_ = nms_threshold; 48 | } 49 | 50 | void SyszuxFaceRetinaConfig::setMaxHW(int max_hw){ 51 | max_hw_ = max_hw; 52 | } 53 | 54 | void SyszuxFaceRetinaConfig::setGapThreshold(float gap_threshold){ 55 | gap_threshold_ = gap_threshold; 56 | } 57 | 58 | } //namespace deepvac 59 | -------------------------------------------------------------------------------- /modules/src/syszux_face_retina_nv.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "NvInfer.h" 12 | #include "syszux_face_retina_nv.h" 13 | #include "gemfield.h" 14 | 15 | namespace deepvac{ 16 | SyszuxFaceRetinaNV::SyszuxFaceRetinaNV(std::string path, std::string device):DeepvacNV(path, device) { 17 | initParameter(device); 18 | setBinding(trt_module_->getNbBindings()); 19 | } 20 | 21 | SyszuxFaceRetinaNV::SyszuxFaceRetinaNV(std::vector&& buffer, std::string device):DeepvacNV(std::move(buffer), device){ 22 | initParameter(device); 23 | setBinding(trt_module_->getNbBindings()); 24 | } 25 | 26 | std::optional, std::vector>>> SyszuxFaceRetinaNV::process(cv::Mat frame){ 27 | GEMFIELD_SI; 28 | //prepare input 29 | int h = frame.rows; 30 | int w = frame.cols; 31 | int c = frame.channels(); 32 | int max_edge = std::max(h, w); 33 | if(max_edge > max_hw_){ 34 | cv::resize(frame, frame, cv::Size(int(w*max_hw_/max_edge), int(h*max_hw_/max_edge))); 35 | h = frame.rows; 36 | w = frame.cols; 37 | } 38 | 39 | if ( std::abs(h-last_h_)<=gap_threshold_*last_h_ and std::abs(w-last_w_)<=gap_threshold_*last_w_) { 40 | if ( h!=last_h_ or w!=last_w_ ) { 41 | cv::resize(frame, frame, cv::Size(last_w_, last_h_)); 42 | } 43 | } else { 44 | last_w_ = w; 45 | last_h_ = h; 46 | 47 | last_prior_ = prior_box_.forward({h, w}); 48 | last_prior_ = last_prior_.to(device_); 49 | 50 | last_box_scale_ = torch::tensor({w, h, w, h}); 51 | last_box_scale_ = last_box_scale_.to(device_); 52 | 53 | last_lmk_scale_ = torch::tensor({w, h, w, h, w, h, w, h, w, h}); 54 | last_lmk_scale_ = last_lmk_scale_.to(device_); 55 | } 56 | 57 | auto input_tensor_opt = gemfield_org::cvMat2Tensor(std::move(frame), gemfield_org::NO_NORMALIZE, gemfield_org::MEAN_STD_FROM_FACE, device_); 58 | if(!input_tensor_opt){ 59 | return std::nullopt; 60 | } 61 | 62 | auto predicitonBindings = prepareInputOutput(input_tensor_opt.value()); 63 | auto predict = forward(predicitonBindings.data()); 64 | torch::Tensor loc, landms, forward_conf; 65 | for(int i = 1; i < 4; ++i) { 66 | auto channel = datas_[i].deviceBuffer.shape()[2]; 67 | if(4 == channel) { 68 | loc = datas_[i].deviceBuffer.toTensor(); 69 | } else if(2 == channel) { 70 | forward_conf = datas_[i].deviceBuffer.toTensor(); 71 | } else if(10 == channel) { 72 | landms = datas_[i].deviceBuffer.toTensor(); 73 | } else { 74 | GEMFIELD_E("face detect model error, invalid output dims"); 75 | } 76 | } 77 | 78 | loc = loc.squeeze(0); 79 | forward_conf = forward_conf.squeeze(0); 80 | landms = landms.squeeze(0); 81 | float resize = 1.; 82 | 83 | //gemfield 84 | torch::Tensor boxes = gemfield_org::getDecodeBox(last_prior_, variances_tensor_, loc); 85 | boxes = torch::div(torch::mul(boxes, last_box_scale_), resize); 86 | 87 | gemfield_org::decodeLandmark(last_prior_, variances_tensor_, landms); 88 | landms = torch::div(torch::mul(landms, last_lmk_scale_), resize); 89 | 90 | torch::Tensor scores = forward_conf.slice(1, 1, 2); 91 | std::vector index = torch::where(scores>confidence_threshold_); 92 | if (index[0].size(0) == 0) { 93 | return std::nullopt; 94 | } 95 | 96 | boxes = boxes.index({index[0]}); 97 | landms = landms.index({index[0]}); 98 | scores = scores.index({index[0]}); 99 | 100 | std::tuple sort_ret = torch::sort(scores, 0, 1); 101 | torch::Tensor idx = std::get<1>(sort_ret).squeeze(1); 102 | idx = idx.slice(0, 0, top_k_); 103 | 104 | boxes = boxes.index({idx}); 105 | landms = landms.index({idx}); 106 | scores = scores.index({idx}); 107 | 108 | torch::Tensor dets = torch::cat({boxes, scores}, 1); 109 | torch::Tensor keep; 110 | keep = gemfield_org::nms(dets, nms_threshold_); 111 | 112 | // keep top-K faster NMS 113 | keep = keep.slice(0, 0, keep_top_k_); 114 | dets = dets.index({keep}); 115 | landms = landms.index({keep}); 116 | if(dets.size(0) == 0){ 117 | return std::nullopt; 118 | } 119 | 120 | std::string msg = gemfield_org::format("detected %d faces", dets.size(0)); 121 | GEMFIELD_I(msg.c_str()); 122 | 123 | if(dets.size(0) != landms.size(0)){ 124 | std::string msg = gemfield_org::format("dets len mismatched landms len: %d vs %d", dets.size(0), landms.size(0)); 125 | GEMFIELD_E(msg.c_str()); 126 | return std::nullopt; 127 | } 128 | 129 | landms = landms.to(torch::kCPU); 130 | cv::Mat landms_mat(landms.size(0), landms.size(1), CV_32F); 131 | std::memcpy((void *) landms_mat.data, landms.data_ptr(), torch::elementSize(torch::kF32) * landms.numel()); 132 | dets = dets.to(torch::kCPU); 133 | cv::Mat dets_mat(dets.size(0), dets.size(1), CV_32F); 134 | std::memcpy((void *) dets_mat.data, dets.data_ptr(), torch::elementSize(torch::kF32) * dets.numel()); 135 | 136 | std::vector, std::vector>> faces_info; 137 | for(int i=0; i bbox_vec(bbox.begin(), bbox.end()); 142 | dst_img.convertTo(dst_img, CV_32FC3); 143 | faces_info.emplace_back(std::tuple(dst_img, bbox_vec, dst_points)); 144 | } 145 | 146 | return faces_info; 147 | } 148 | 149 | } //namespace deepvac 150 | -------------------------------------------------------------------------------- /modules/src/syszux_ocr_db.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include "syszux_ocr_db.h" 8 | 9 | namespace deepvac { 10 | 11 | void SyszuxOcrDB::setLongSize(int long_size){ 12 | long_size_ = long_size; 13 | } 14 | 15 | void SyszuxOcrDB::setCropGap(int crop_gap){ 16 | crop_gap_ = crop_gap; 17 | } 18 | 19 | void SyszuxOcrDB::setTextMinArea(int text_min_area){ 20 | text_min_area_ = text_min_area; 21 | } 22 | 23 | void SyszuxOcrDB::setTextMinSize(int text_min_size){ 24 | text_min_size_ = text_min_size; 25 | } 26 | 27 | void SyszuxOcrDB::setTextMeanScore(float text_mean_score){ 28 | text_mean_score_ = text_mean_score; 29 | } 30 | 31 | void SyszuxOcrDB::setTextThresh(float text_thresh){ 32 | text_thresh_ = text_thresh; 33 | } 34 | 35 | void SyszuxOcrDB::setUnclipRatio(float unclip_ratio){ 36 | unclip_ratio_ = unclip_ratio; 37 | } 38 | 39 | void SyszuxOcrDB::setGlab(bool is_glab){ 40 | is_glab_ = is_glab; 41 | } 42 | 43 | void SyszuxOcrDB::setExtend(bool is_extend){ 44 | is_extend_ = is_extend; 45 | } 46 | 47 | void SyszuxOcrDB::setUnclip(bool is_unclip){ 48 | is_unclip_ = is_unclip; 49 | } 50 | 51 | void SyszuxOcrDB::setPolygonScore(bool is_polygon_score){ 52 | is_polygon_score_ = is_polygon_score; 53 | } 54 | 55 | cv::Mat SyszuxOcrDB::cropRect(cv::Mat &img, cv::RotatedRect &rotated_rects) { 56 | cv::Point2f center = rotated_rects.center; 57 | cv::Size2f size = rotated_rects.size; 58 | float angle = rotated_rects.angle; 59 | cv::Point center_i; 60 | cv::Size size_i; 61 | center_i.x = int(center.x); 62 | center_i.y = int(center.y); 63 | size_i.width = int(size.width); 64 | size_i.height = int(size.height); 65 | 66 | if (size_i.width < size_i.height) { 67 | angle += 90.; 68 | int temp = size_i.width; 69 | size_i.width = size_i.height; 70 | size_i.height = temp; 71 | } 72 | auto M = cv::getRotationMatrix2D(center_i, angle, 1); 73 | cv::Mat img_rot, img_crop; 74 | cv::warpAffine(img, img_rot, M, img.size(), cv::INTER_CUBIC); 75 | cv::getRectSubPix(img_rot, size_i, center_i, img_crop); 76 | return img_crop; 77 | } 78 | 79 | float SyszuxOcrDB::polygonScoreAcc(cv::Mat &pred, std::vector &contour){ 80 | int width = pred.cols; 81 | int height = pred.rows; 82 | std::vector box_x; 83 | std::vector box_y; 84 | for(int i=0; i(0, 0), vertex.at(1, 0), vertex.at(2, 0), vertex.at(3, 0)}; 121 | float box_y[4] = {vertex.at(0, 1), vertex.at(1, 1), vertex.at(2, 1), vertex.at(3, 1)}; 122 | 123 | int xmin = std::clamp(int(std::floor(*(std::min_element(box_x, box_x + 4)))), 0, width - 1); 124 | int xmax = std::clamp(int(std::ceil(*(std::max_element(box_x, box_x + 4)))), 0, width - 1); 125 | int ymin = std::clamp(int(std::floor(*(std::min_element(box_y, box_y + 4)))), 0, height - 1); 126 | int ymax = std::clamp(int(std::ceil(*(std::max_element(box_y, box_y + 4)))), 0, height - 1); 127 | 128 | cv::Mat mask; 129 | mask = cv::Mat::zeros(ymax - ymin + 1, xmax - xmin + 1, CV_8UC1); 130 | 131 | cv::Point rook_point[4]; 132 | rook_point[0] = cv::Point(int(box_x[0]) - xmin, int(box_y[0]) - ymin); 133 | rook_point[1] = cv::Point(int(box_x[1]) - xmin, int(box_y[1]) - ymin); 134 | rook_point[2] = cv::Point(int(box_x[2]) - xmin, int(box_y[2]) - ymin); 135 | rook_point[3] = cv::Point(int(box_x[3]) - xmin, int(box_y[3]) - ymin); 136 | const cv::Point *ppt[1] = {rook_point}; 137 | int npt[] = {4}; 138 | cv::fillPoly(mask, ppt, npt, 1, cv::Scalar(1)); 139 | 140 | cv::Mat croppedImg; 141 | pred(cv::Rect(xmin, ymin, xmax - xmin + 1, ymax - ymin + 1)).copyTo(croppedImg); 142 | float score = cv::mean(croppedImg, mask)[0]; 143 | return score; 144 | } 145 | 146 | void SyszuxOcrDB::getContourArea(const std::vector> &box, float &distance){ 147 | int pts_num = 4; 148 | float area = 0.0f; 149 | float dist = 0.0f; 150 | for (int i = 0; i < pts_num; i++) { 151 | area += box[i][0] * box[(i + 1) % pts_num][1] - box[i][1] * box[(i + 1) % pts_num][0]; 152 | dist += sqrtf(std::pow(box[i][0]-box[(i + 1)%pts_num][0], 2) + std::pow(box[i][1] - box[(i + 1) % pts_num][1], 2)); 153 | } 154 | area = fabs(float(area / 2.0)); 155 | distance = area * unclip_ratio_ / dist; 156 | } 157 | 158 | std::optional SyszuxOcrDB::unClip(cv::RotatedRect &rect){ 159 | cv::Mat vertex; 160 | cv::boxPoints(rect, vertex); 161 | 162 | std::vector> box; 163 | for (int i = 0; i < vertex.rows; ++i) { 164 | std::vector tmp; 165 | for (int j = 0; j < vertex.cols; ++j) { 166 | tmp.push_back(vertex.at(i, j)); 167 | } 168 | box.push_back(tmp); 169 | } 170 | 171 | float distance = 1.0; 172 | getContourArea(box, distance); 173 | 174 | ClipperLib::ClipperOffset offset; 175 | ClipperLib::Path p; 176 | p << ClipperLib::IntPoint(int(box[0][0]), int(box[0][1])) 177 | << ClipperLib::IntPoint(int(box[1][0]), int(box[1][1])) 178 | << ClipperLib::IntPoint(int(box[2][0]), int(box[2][1])) 179 | << ClipperLib::IntPoint(int(box[3][0]), int(box[3][1])); 180 | offset.AddPath(p, ClipperLib::jtRound, ClipperLib::etClosedPolygon); 181 | 182 | ClipperLib::Paths soln; 183 | offset.Execute(soln, distance); 184 | std::vector points; 185 | 186 | for (int j = 0; j < soln.size(); j++) { 187 | for (int i = 0; i < soln[soln.size() - 1].size(); i++) { 188 | points.emplace_back(soln[j][i].X, soln[j][i].Y); 189 | } 190 | } 191 | if (points.size() <= 0){ 192 | return std::nullopt; 193 | } 194 | cv::RotatedRect res; 195 | res = cv::minAreaRect(points); 196 | return res; 197 | } 198 | 199 | std::optional< std::pair, std::vector>> > SyszuxOcrDB::process(cv::Mat img){ 200 | GEMFIELD_SI; 201 | //prepare input 202 | std::vector crop_imgs; 203 | std::vector> rects; 204 | cv::Mat img_ori = img.clone(); 205 | cv::Mat resize_img, rgb_img; 206 | cv::cvtColor(img, rgb_img, cv::COLOR_BGR2RGB); 207 | float scale1 = long_size_ * 1.0 / std::max(img.rows, img.cols); 208 | cv::resize(rgb_img, resize_img, cv::Size(), scale1, scale1); 209 | 210 | auto input_tensor_opt = gemfield_org::cvMat2Tensor(resize_img, gemfield_org::NORMALIZE0_1, gemfield_org::MEAN_STD_FROM_IMAGENET, device_); 211 | 212 | if(!input_tensor_opt){ 213 | return std::nullopt; 214 | } 215 | auto input_tensor = input_tensor_opt.value(); 216 | 217 | // forward 218 | auto outputs = forward(input_tensor); 219 | 220 | // binarize probability map 221 | outputs = outputs.squeeze(); 222 | torch::Tensor pred = outputs.select(0, 0).to(torch::kCPU).toType(torch::kFloat); 223 | torch::Tensor segmentation = (pred>text_thresh_); 224 | segmentation = segmentation.mul(255).clamp(0, 255).toType(torch::kU8); 225 | 226 | // binary map and pre convert to cvMat 227 | cv::Mat image_binary(segmentation.size(0), segmentation.size(1), CV_8UC1); 228 | std::memcpy((void*)image_binary.data, segmentation.data_ptr(), torch::elementSize(torch::kU8) * segmentation.numel()); 229 | cv::Mat pred_Mat(pred.size(0), pred.size(1), CV_32FC1); 230 | std::memcpy((void*)pred_Mat.data, pred.data_ptr(), torch::elementSize(torch::kFloat) * pred.numel()); 231 | 232 | // binary cvMat find contours 233 | std::vector> contours; 234 | cv::findContours(image_binary, contours, cv::RETR_LIST, cv::CHAIN_APPROX_SIMPLE); 235 | 236 | // extract text box to db_detect_out 237 | std::vector scale2 = {(float)(img.cols * 1.0 / pred.size(1)), (float)(img.rows * 1.0 / pred.size(0))}; 238 | std::vector db_detect_out; 239 | for(int i=0; i= rect.size.height){ 267 | rect.size.width = rect.size.width + rect.size.height * (unclip_ratio_ - 1); 268 | rect.size.height = rect.size.height * unclip_ratio_; 269 | } else { 270 | rect.size.height = rect.size.height + rect.size.width * (unclip_ratio_ - 1); 271 | rect.size.width = rect.size.width * unclip_ratio_; 272 | } 273 | db_detect_out.push_back(rect); 274 | } 275 | 276 | if(db_detect_out.size()<=0) 277 | return std::nullopt; 278 | 279 | std::vector result; 280 | if(is_glab_){ 281 | deepvac::DeepvacOcrFrame ocr_frame(img_ori, db_detect_out); 282 | auto glab_out_opt = ocr_frame(); 283 | 284 | if (!glab_out_opt) { 285 | return std::nullopt; 286 | } 287 | std::vector glab_result = glab_out_opt.value(); 288 | for(int i=0; i= box.size.height){ 299 | box.size.width += 2*crop_gap_; 300 | } else{ 301 | box.size.height += 2*crop_gap_; 302 | } 303 | } 304 | cv::Mat img_crop = cropRect(img_ori, box); 305 | crop_imgs.push_back(img_crop); 306 | 307 | std::vector rect_; 308 | cv::Mat crop_box; 309 | cv::boxPoints(box, crop_box); 310 | for (int row=0; row(row, col))); 313 | } 314 | } 315 | rects.push_back(rect_); 316 | } 317 | std::pair, std::vector>> crop_imgs_and_rects(crop_imgs, rects); 318 | return crop_imgs_and_rects; 319 | } 320 | 321 | }//namespace 322 | -------------------------------------------------------------------------------- /modules/src/syszux_ocr_pse.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include "opencv2/opencv.hpp" 8 | #include "syszux_ocr_pse.h" 9 | #include "syszux_glab.h" 10 | 11 | namespace deepvac { 12 | 13 | void SyszuxOcrPse::set(int long_size, int crop_gap, int text_min_area, float text_mean_score) { 14 | long_size_ = long_size; 15 | crop_gap_ = crop_gap; 16 | text_min_area_ = text_min_area; 17 | text_mean_score_ = text_mean_score; 18 | } 19 | 20 | void SyszuxOcrPse::setGlab(bool is_glab){ 21 | is_glab_ = is_glab; 22 | } 23 | 24 | void SyszuxOcrPse::setExtend(bool is_extend){ 25 | is_extend_ = is_extend; 26 | } 27 | 28 | cv::Mat SyszuxOcrPse::cropRect(cv::Mat &img, cv::RotatedRect &rotated_rects) { 29 | cv::Point2f center = rotated_rects.center; 30 | cv::Size2f size = rotated_rects.size; 31 | float angle = rotated_rects.angle; 32 | cv::Point center_i; 33 | cv::Size size_i; 34 | center_i.x = int(center.x); 35 | center_i.y = int(center.y); 36 | size_i.width = int(size.width); 37 | size_i.height = int(size.height); 38 | 39 | if (size_i.width < size_i.height) { 40 | angle += 90.; 41 | int temp = size_i.width; 42 | size_i.width = size_i.height; 43 | size_i.height = temp; 44 | } 45 | auto M = cv::getRotationMatrix2D(center_i, angle, 1); 46 | cv::Mat img_rot, img_crop; 47 | cv::warpAffine(img, img_rot, M, img.size(), cv::INTER_CUBIC); 48 | cv::getRectSubPix(img_rot, size_i, center_i, img_crop); 49 | return img_crop; 50 | } 51 | 52 | std::optional< std::pair, std::vector>> > SyszuxOcrPse::process(cv::Mat img) 53 | { 54 | GEMFIELD_SI; 55 | //prepare input 56 | std::vector crop_imgs; 57 | std::vector> rects; 58 | cv::Mat img_ori = img.clone(); 59 | cv::Mat resize_img, rgb_img; 60 | cv::cvtColor(img, rgb_img, cv::COLOR_BGR2RGB); 61 | float scale1 = long_size_ * 1.0 / std::max(img.rows, img.cols); 62 | cv::resize(rgb_img, resize_img, cv::Size(), scale1, scale1); 63 | 64 | auto input_tensor_opt = gemfield_org::cvMat2Tensor(resize_img, gemfield_org::NORMALIZE0_1, gemfield_org::MEAN_STD_FROM_IMAGENET, device_); 65 | 66 | if(!input_tensor_opt){ 67 | return std::nullopt; 68 | } 69 | auto input_tensor = input_tensor_opt.value(); 70 | 71 | //prepare forward 72 | auto outputs = forward(input_tensor); 73 | 74 | //prepare output 75 | outputs = outputs.to(device_); 76 | outputs = outputs.squeeze(); 77 | auto scores = torch::sigmoid(outputs.select(0, 0)); 78 | outputs = torch::sign(outputs.sub_(1.0)); 79 | outputs = outputs.add_(1).div_(2); 80 | auto text = outputs.select(0, 0); 81 | 82 | // kernel_num can be 3 or 7 83 | int kernel_num = 7; 84 | auto kernels = outputs.slice(0, 0, kernel_num) * text; 85 | kernels = kernels.toType(torch::kU8); 86 | 87 | float min_area = 10.0; 88 | 89 | auto pred = adaptorPse(kernels, min_area); 90 | std::vector scale2 = {(float)(img.cols * 1.0 / pred[0].size()), (float)(img.rows * 1.0 / pred.size())}; 91 | torch::Tensor label = torch::randn({(int)pred.size(), (int)pred[0].size()}); 92 | for (int i=0; i() + 1; 96 | 97 | std::vector pse_detect_out; 98 | for(int i=1; i(); 111 | if (score_mean < text_mean_score_){ 112 | continue; 113 | } 114 | 115 | points = points.toType(torch::kFloat); 116 | points = points.to(torch::kCPU); 117 | cv::Mat points_mat(points.size(0), points.size(1), CV_32FC1); 118 | std::memcpy((void *) points_mat.data, points.data_ptr(), torch::elementSize(torch::kFloat) * points.numel()); 119 | cv::RotatedRect rect = cv::minAreaRect(points_mat); 120 | 121 | rect.center.x = rect.center.x * scale2[0]; 122 | rect.center.y = rect.center.y * scale2[1]; 123 | rect.size.width = rect.size.width * scale2[0]; 124 | rect.size.height = rect.size.height * scale2[1]; 125 | pse_detect_out.push_back(rect); 126 | } 127 | 128 | std::vector result; 129 | if(is_glab_){ 130 | deepvac::DeepvacOcrFrame ocr_frame(img_ori, pse_detect_out); 131 | auto glab_out_opt = ocr_frame(); 132 | 133 | if (!glab_out_opt) { 134 | return std::nullopt; 135 | } 136 | std::vector glab_result = glab_out_opt.value(); 137 | for(int i=0; i= box.size.height){ 148 | box.size.width += 2*crop_gap_; 149 | } else{ 150 | box.size.height += 2*crop_gap_; 151 | } 152 | } 153 | cv::Mat img_crop = cropRect(img_ori, box); 154 | crop_imgs.push_back(img_crop); 155 | 156 | std::vector rect_; 157 | cv::Mat crop_box; 158 | cv::boxPoints(box, crop_box); 159 | for (int row=0; row(row, col))); 162 | } 163 | } 164 | rects.push_back(rect_); 165 | } 166 | std::pair, std::vector>> crop_imgs_and_rects(crop_imgs, rects); 167 | return crop_imgs_and_rects; 168 | } 169 | 170 | std::vector> SyszuxOcrPse::merge(std::vector> rects) { 171 | std::vector> keep; 172 | for (int i=0; i> SyszuxOcrPse::mergeBox(std::vector> rects) { 193 | std::vector> keep = rects; 194 | int length = keep.size(); 195 | while (true) { 196 | keep = merge(keep); 197 | if (keep.size() == length) { 198 | break; 199 | } else { 200 | length = keep.size(); 201 | } 202 | } 203 | return keep; 204 | } 205 | 206 | bool SyszuxOcrPse::isMerge(std::vector rect1, std::vector rect2) { 207 | float x1_min = rect1[0]; 208 | float y1_min = rect1[1]; 209 | float x1_max = rect1[2]; 210 | float y1_max = rect1[3]; 211 | float x2_min = rect2[0]; 212 | float y2_min = rect2[1]; 213 | float x2_max = rect2[2]; 214 | float y2_max = rect2[3]; 215 | 216 | 217 | if (y1_max <= y2_min || y1_min >= y2_max) { 218 | return false; 219 | } 220 | 221 | float y[4] = {y1_min, y1_max, y2_min, y2_max}; 222 | std::sort(y, y + 4); 223 | float x_thre = 2 * (y[3] - y[0]); 224 | if ((y[2]-y[1])/(y[3]-y[0]) < 0.7) { 225 | return false; 226 | } 227 | if ( (((x1_min - x2_max)>=0) && ((x1_min - x2_max)>x_thre)) || (((x2_min - x1_max)>=0) && ((x2_min - x1_max)>x_thre)) ) { 228 | return false; 229 | } 230 | return true; 231 | } 232 | 233 | void SyszuxOcrPse::getKernals(torch::Tensor input_data, std::vector &kernals) { 234 | for (int i = 0; i < input_data.size(0); ++i) { 235 | cv::Mat kernal(input_data[i].size(0), input_data[i].size(1), CV_8UC1); 236 | std::memcpy((void *) kernal.data, input_data[i].data_ptr(), sizeof(torch::kU8) * input_data[i].numel()); 237 | kernals.emplace_back(kernal); 238 | } 239 | } 240 | 241 | void SyszuxOcrPse::growingTextLine(std::vector &kernals, std::vector> &text_line, float min_area) { 242 | cv::Mat label_mat; 243 | int label_num = connectedComponents(kernals[kernals.size() - 1], label_mat, 4); 244 | 245 | int area[label_num + 1]; 246 | memset(area, 0, sizeof(area)); 247 | for (int x = 0; x < label_mat.rows; ++x) { 248 | for (int y = 0; y < label_mat.cols; ++y) { 249 | int label = label_mat.at(x, y); 250 | if (label == 0) continue; 251 | area[label] += 1; 252 | } 253 | } 254 | 255 | std::queue queue, next_queue; 256 | for (int x = 0; x < label_mat.rows; ++x) { 257 | std::vector row(label_mat.cols); 258 | for (int y = 0; y < label_mat.cols; ++y) { 259 | int label = label_mat.at(x, y); 260 | 261 | if (label == 0) continue; 262 | if (area[label] < min_area) continue; 263 | 264 | cv::Point point(x, y); 265 | queue.push(point); 266 | row[y] = label; 267 | } 268 | text_line.emplace_back(row); 269 | } 270 | 271 | int dx[] = {-1, 1, 0, 0}; 272 | int dy[] = {0, 0, -1, 1}; 273 | 274 | for (int kernal_id = kernals.size() - 2; kernal_id >= 0; --kernal_id) { 275 | while (!queue.empty()) { 276 | cv::Point point = queue.front(); queue.pop(); 277 | int x = point.x; 278 | int y = point.y; 279 | int label = text_line[x][y]; 280 | 281 | bool is_edge = true; 282 | for (int d = 0; d < 4; ++d) { 283 | int tmp_x = x + dx[d]; 284 | int tmp_y = y + dy[d]; 285 | 286 | if (tmp_x < 0 || tmp_x >= (int)text_line.size()) continue; 287 | if (tmp_y < 0 || tmp_y >= (int)text_line[1].size()) continue; 288 | if (kernals[kernal_id].at(tmp_x, tmp_y) == 0) continue; 289 | if (text_line[tmp_x][tmp_y] > 0) continue; 290 | 291 | cv::Point point(tmp_x, tmp_y); 292 | queue.push(point); 293 | text_line[tmp_x][tmp_y] = label; 294 | is_edge = false; 295 | } 296 | 297 | if (is_edge) { 298 | next_queue.push(point); 299 | } 300 | } 301 | swap(queue, next_queue); 302 | } 303 | } 304 | 305 | std::vector> SyszuxOcrPse::adaptorPse(torch::Tensor input_data, float min_area) { 306 | std::vector kernals; 307 | input_data = input_data.to(torch::kCPU); 308 | getKernals(input_data, kernals); 309 | 310 | std::vector> text_line; 311 | growingTextLine(kernals, text_line, min_area); 312 | return text_line; 313 | } 314 | }//namespace 315 | -------------------------------------------------------------------------------- /modules/src/syszux_seg_esp.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include "syszux_seg_esp.h" 8 | 9 | namespace deepvac{ 10 | 11 | void SyszuxSegEsp::set(std::vector image_size) { 12 | image_size_ = image_size; 13 | } 14 | 15 | std::optional SyszuxSegEsp::process(cv::Mat frame){ 16 | int h = frame.rows; 17 | int w = frame.cols; 18 | int c = frame.channels(); 19 | cv::Mat resize_img; 20 | cv::cvtColor(frame, resize_img, cv::COLOR_BGR2RGB); 21 | cv::resize(resize_img, resize_img, cv::Size(image_size_[0], image_size_[1]), cv::INTER_LINEAR); 22 | 23 | auto input_tensor_opt = gemfield_org::cvMat2Tensor(resize_img, gemfield_org::NORMALIZE0_1, gemfield_org::MEAN_STD_FROM_IMAGENET, device_); 24 | 25 | if(!input_tensor_opt){ 26 | return std::nullopt; 27 | } 28 | auto input_tensor = input_tensor_opt.value(); 29 | //forward 30 | auto output = forward(input_tensor); 31 | 32 | output = output.squeeze(); 33 | auto out = output.argmax(0).cpu().to(torch::kByte); 34 | 35 | cv::Mat mask(out.size(0), out.size(1), CV_8UC1); 36 | std::memcpy((void *) mask.data, out.data_ptr(), torch::elementSize(torch::kU8) * out.numel()); 37 | 38 | cv::resize(mask, mask, cv::Size(w, h), cv::INTER_LINEAR); 39 | 40 | return mask; 41 | } 42 | } //namespace deepvac 43 | -------------------------------------------------------------------------------- /python/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 Gemfield 2 | # This file is part of libdeepvac, licensed under the GPLv3 (the "License") 3 | # You may not use this file except in compliance with the License. 4 | 5 | file(GLOB CPP4PYTHON_LIST src/*.cpp) 6 | message(STATUS "found CPP4PYTHON_LIST: " ${CPP4PYTHON_LIST}) 7 | 8 | file(GLOB HEADER_LIST include/*.h) 9 | add_syszux_headers(${HEADER_LIST}) 10 | 11 | if(NOT CPP4PYTHON_LIST) 12 | message(STATUS "No cpp found in python dir and return.") 13 | return() 14 | endif() 15 | 16 | find_package(pybind11 REQUIRED) 17 | pybind11_add_module(pydeepvac ${CPP4PYTHON_LIST}) 18 | 19 | find_library(TORCH_PYTHON_LIBRARY torch_python PATHS "${TORCH_INSTALL_PREFIX}/lib") 20 | 21 | message(STATUS "TORCH_PYTHON_LIBRARY: ${TORCH_PYTHON_LIBRARY}") 22 | 23 | target_link_libraries( pydeepvac "${TORCH_LIBRARIES}" ${TORCH_PYTHON_LIBRARY} ${OpenCV_LIBS} deepvac) 24 | 25 | target_include_directories(pydeepvac PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) 26 | 27 | execute_process( 28 | COMMAND "${PYTHON_EXECUTABLE}" -c "from distutils import sysconfig as sc;print(sc.get_python_lib())" 29 | OUTPUT_VARIABLE PYTHON_SITE 30 | OUTPUT_STRIP_TRAILING_WHITESPACE) 31 | 32 | message(STATUS "PYTHON_SITE: ${PYTHON_SITE}") 33 | 34 | install(TARGETS pydeepvac 35 | COMPONENT python 36 | LIBRARY DESTINATION ${PYTHON_SITE}) -------------------------------------------------------------------------------- /python/include/syszux_type_caster.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | /* 听到项目上传来的好消息，放诗一首： 8 | * 剑外忽传收蓟北，初闻涕泪满衣裳。 9 | * 却看妻子愁何在，漫卷诗书喜欲狂。 10 | * 白日放歌须纵酒，青春作伴好还乡。 11 | * 即从巴峡穿巫峡，便下襄阳向洛阳。 12 | */ 13 | #pragma once 14 | 15 | #include 16 | #include 17 | /* 18 | #define CV_CN_MAX 512 19 | #define CV_CN_SHIFT 3 20 | #define CV_DEPTH_MAX (1 << CV_CN_SHIFT) 21 | 22 | #define CV_MAT_DEPTH_MASK (CV_DEPTH_MAX - 1) 23 | #define CV_MAT_DEPTH(flags) ((flags) & CV_MAT_DEPTH_MASK) 24 | 25 | #define CV_MAKETYPE(depth,cn) (CV_MAT_DEPTH(depth) + (((cn)-1) << CV_CN_SHIFT)) 26 | #define CV_MAKE_TYPE CV_MAKETYPE 27 | 28 | #define CV_8UC1 CV_MAKETYPE(CV_8U,1) 29 | #define CV_8UC2 CV_MAKETYPE(CV_8U,2) 30 | #define CV_8UC3 CV_MAKETYPE(CV_8U,3) 31 | #define CV_8UC4 CV_MAKETYPE(CV_8U,4) 32 | #define CV_8UC(n) CV_MAKETYPE(CV_8U,(n)) 33 | 34 | #define CV_8SC1 CV_MAKETYPE(CV_8S,1) 35 | #define CV_8SC2 CV_MAKETYPE(CV_8S,2) 36 | #define CV_8SC3 CV_MAKETYPE(CV_8S,3) 37 | #define CV_8SC4 CV_MAKETYPE(CV_8S,4) 38 | #define CV_8SC(n) CV_MAKETYPE(CV_8S,(n)) 39 | 40 | #define CV_16UC1 CV_MAKETYPE(CV_16U,1) 41 | #define CV_16UC2 CV_MAKETYPE(CV_16U,2) 42 | #define CV_16UC3 CV_MAKETYPE(CV_16U,3) 43 | #define CV_16UC4 CV_MAKETYPE(CV_16U,4) 44 | #define CV_16UC(n) CV_MAKETYPE(CV_16U,(n)) 45 | 46 | #define CV_16SC1 CV_MAKETYPE(CV_16S,1) 47 | #define CV_16SC2 CV_MAKETYPE(CV_16S,2) 48 | #define CV_16SC3 CV_MAKETYPE(CV_16S,3) 49 | #define CV_16SC4 CV_MAKETYPE(CV_16S,4) 50 | #define CV_16SC(n) CV_MAKETYPE(CV_16S,(n)) 51 | 52 | #define CV_32SC1 CV_MAKETYPE(CV_32S,1) 53 | #define CV_32SC2 CV_MAKETYPE(CV_32S,2) 54 | #define CV_32SC3 CV_MAKETYPE(CV_32S,3) 55 | #define CV_32SC4 CV_MAKETYPE(CV_32S,4) 56 | #define CV_32SC(n) CV_MAKETYPE(CV_32S,(n)) 57 | 58 | #define CV_32FC1 CV_MAKETYPE(CV_32F,1) 59 | #define CV_32FC2 CV_MAKETYPE(CV_32F,2) 60 | #define CV_32FC3 CV_MAKETYPE(CV_32F,3) 61 | #define CV_32FC4 CV_MAKETYPE(CV_32F,4) 62 | #define CV_32FC(n) CV_MAKETYPE(CV_32F,(n)) 63 | 64 | #define CV_64FC1 CV_MAKETYPE(CV_64F,1) 65 | #define CV_64FC2 CV_MAKETYPE(CV_64F,2) 66 | #define CV_64FC3 CV_MAKETYPE(CV_64F,3) 67 | #define CV_64FC4 CV_MAKETYPE(CV_64F,4) 68 | #define CV_64FC(n) CV_MAKETYPE(CV_64F,(n)) 69 | 70 | #define CV_16FC1 CV_MAKETYPE(CV_16F,1) 71 | #define CV_16FC2 CV_MAKETYPE(CV_16F,2) 72 | #define CV_16FC3 CV_MAKETYPE(CV_16F,3) 73 | #define CV_16FC4 CV_MAKETYPE(CV_16F,4) 74 | #define CV_16FC(n) CV_MAKETYPE(CV_16F,(n)) 75 | */ 76 | 77 | namespace pybind11 { 78 | namespace detail { 79 | 80 | //based on https://github.com/pybind/pybind11/issues/538 81 | template <> 82 | struct type_caster { 83 | //This macro establishes the name 'inty' in function signatures 84 | //and declares a local variable 'value' of type inty. 85 | PYBIND11_TYPE_CASTER(cv::Mat, _("numpy.ndarray")); 86 | 87 | bool load(handle src, bool) { 88 | if (!isinstance(src)){ 89 | throw std::runtime_error("parameter type not array."); 90 | return false; 91 | } 92 | array src_array = reinterpret_borrow(src); 93 | auto src_array_info = src_array.request(); 94 | //The number of dimensions the memory represents as an n-dimensional array. 95 | //If it is 0, buf points to a single item representing a scalar. 96 | //In this case, shape, strides and suboffsets MUST be NULL. Gemfield 97 | int ndims = src_array_info.ndim; 98 | if(ndims != 3){ 99 | throw std::runtime_error("only support 3d ndarray."); 100 | return false; 101 | } 102 | 103 | decltype(CV_32F) dtype; 104 | //#define CV_8UC(n) CV_MAKETYPE(CV_8U,(n)) 105 | if(src_array_info.format == format_descriptor::format()){ 106 | dtype = CV_32FC(ndims); 107 | }else if (src_array_info.format == format_descriptor::format()){ 108 | dtype = CV_64FC(ndims); 109 | }else if (src_array_info.format == format_descriptor::format()){ 110 | dtype = CV_8UC(ndims); 111 | }else if (src_array_info.format == format_descriptor::format()){ 112 | dtype = CV_32SC(ndims); 113 | }else{ 114 | throw std::runtime_error("Only support float,double,uchar,int."); 115 | return false; 116 | } 117 | 118 | //shape[0] * ... * shape[ndim-1] * itemsize MUST be equal to len. Gemfield 119 | int h = src_array_info.shape[0]; 120 | int w = src_array_info.shape[1]; 121 | 122 | value = cv::Mat(h, w, dtype, src_array_info.ptr, cv::Mat::AUTO_STEP); 123 | return true; 124 | } 125 | //convert an inty instance into a Python object. 126 | //The second and third arguments are used to indicate the return value policy and parent object 127 | // (for ``return_value_policy::reference_internal``) and are generally ignored by implicit casters. 128 | static handle cast(const cv::Mat &m, return_value_policy, handle defval) { 129 | std::string format = format_descriptor::format(); 130 | size_t elemsize = sizeof(unsigned char); 131 | int mat_w = m.cols; 132 | int mat_h = m.rows; 133 | int mat_c = m.channels(); 134 | auto type = m.type(); 135 | auto depth = m.depth(); 136 | int dim = (depth == type)? 2 : 3; 137 | 138 | //enum{CV_8U=0,CV_8S=1,CV_16U=2,CV_16S=3,CV_32S=4,CV_32F=5,CV_64F=6,CV_16F=7} 139 | switch(depth) { 140 | case CV_8U: 141 | format = format_descriptor::format(); 142 | elemsize = sizeof(unsigned char); 143 | break; 144 | case CV_32S: 145 | format = format_descriptor::format(); 146 | elemsize = sizeof(int); 147 | break; 148 | case CV_32F: 149 | format = format_descriptor::format(); 150 | elemsize = sizeof(float); 151 | break; 152 | default: 153 | throw std::runtime_error("Unsupported type"); 154 | } 155 | 156 | std::vector bufferdim; 157 | std::vector strides; 158 | if (dim == 2) { 159 | bufferdim = {(size_t) mat_h, (size_t) mat_w}; 160 | strides = {elemsize * (size_t) mat_w, elemsize}; 161 | } else if (dim == 3) { 162 | bufferdim = {(size_t) mat_h, (size_t) mat_w, (size_t) 3}; 163 | strides = {(size_t) elemsize * mat_w * 3, (size_t) elemsize * 3, (size_t) elemsize}; 164 | } else{ 165 | throw std::runtime_error("Unsupported dimension."); 166 | } 167 | return array(buffer_info( 168 | m.data, /* Pointer to buffer */ 169 | elemsize, /* Size of one scalar */ 170 | format, /* Python struct-style format descriptor */ 171 | dim, /* Number of dimensions */ 172 | bufferdim, /* Buffer dimensions */ 173 | strides /* Strides (in bytes) for each index */ 174 | )).release(); 175 | } 176 | }; 177 | }//namespace detail 178 | }//namespace pybind11 179 | 180 | 181 | 182 | -------------------------------------------------------------------------------- /python/src/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DeepVAC/libdeepvac/f6d2669882714fc90e6ec499733311772d205479/python/src/README.md -------------------------------------------------------------------------------- /utils/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 Gemfield 2 | # This file is part of libdeepvac, licensed under the GPLv3 (the "License") 3 | # You may not use this file except in compliance with the License. 4 | 5 | file(GLOB UTILS_SRC_LIST src/*.cpp) 6 | add_syszux_sources(${UTILS_SRC_LIST}) 7 | message(STATUS "found UTILS_SRC_LIST: " ${UTILS_SRC_LIST}) 8 | 9 | file(GLOB HEADER_LIST include/*.h) 10 | add_syszux_headers(${HEADER_LIST}) 11 | 12 | add_header_dir(${CMAKE_CURRENT_SOURCE_DIR}/include) -------------------------------------------------------------------------------- /utils/include/gemfield.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 gemfield 3 | * This file is part of libgemfield.so (https://github.com/civilnet/gemfield). 4 | * Licensed under the GPLv3 (the "License") 5 | * You may not use this file except in compliance with the License. 6 | */ 7 | #ifndef _GEMFIELD_H_ 8 | #define _GEMFIELD_H_ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | namespace gemfield_org{ 19 | enum LOG_LEVEL{ 20 | STACK_INFO = 0, 21 | DETAIL_INFO = 1, 22 | INFO = 2, 23 | WARNING = 5, 24 | ERROR = 6 25 | }; 26 | } 27 | namespace gemfield_org{ 28 | const LOG_LEVEL global_log_level = DETAIL_INFO; 29 | class LogFromFile{ 30 | public: 31 | LogFromFile(){ 32 | std::cout<<"GEMFIELD INITIALIZATION ONLY ONCE!"<> k >> v)) { 42 | break; 43 | } 44 | if(k == "LOGLEVEL"){ 45 | log_level_ = static_cast(v); 46 | break; 47 | } 48 | } 49 | } catch(...){ 50 | std::cout<<"Warning: read log configuration failed."< __attribute__((weak)) gemfield_counter_map; 91 | namespace gemfield_org{ 92 | template 93 | std::string format( const std::string& format, Args ... args ){ 94 | size_t size = snprintf( nullptr, 0, format.c_str(), args ... ) + 1; 95 | std::unique_ptr buf( new char[ size ] ); 96 | snprintf( buf.get(), size, format.c_str(), args ... ); 97 | return std::string( buf.get(), buf.get() + size - 1 ); 98 | } 99 | 100 | class Gemfield{ 101 | public: 102 | Gemfield(std::initializer_list src, LOG_LEVEL level):level_(level){ 103 | if(level_ < getLogLevel()){ 104 | return; 105 | } 106 | std::stringstream ss; 107 | ss << "["< lock(gemfield_lock); 129 | 130 | std::stringstream ss; 131 | ss << std::this_thread::get_id(); 132 | uint64_t current_tid = std::stoull(ss.str()); 133 | 134 | if(c == '+'){ 135 | ++gemfield_counter; 136 | } 137 | for(int i=0; i< gemfield_counter; i++){ 138 | std::cout< log_token = {{STACK_INFO,""},{DETAIL_INFO," | DETAIL_INFO | "},{INFO," | INFO | "},{WARNING," | WARNING | "},{ERROR," | ERROR | "}}; 141 | std::cout< 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | #include 9 | #include 10 | #include "opencv2/opencv.hpp" 11 | 12 | namespace gemfield_org{ 13 | class AlignFace 14 | { 15 | public: 16 | AlignFace(); 17 | ~AlignFace() = default; 18 | std::tuple> operator() (cv::Mat& frame, cv::Mat& facial_5pts); 19 | private: 20 | cv::Mat warpAndCrop(cv::Mat& src_img, cv::Mat& facial_5pts); 21 | cv::Mat getAffineTransform(cv::Mat& facial_5pts); 22 | cv::Mat findNonereflectiveSimilarity(cv::Mat& facial_5pts, cv::Mat& ref_facial_5pts); 23 | cv::Mat tformfwd(cv::Mat& trans, cv::Mat facial_5pts); 24 | std::vector pointsTransform(const std::vector& points, const cv::Mat& matrix); 25 | private: 26 | cv::Mat ref_facial_5pts_; 27 | cv::Mat crop_size_; 28 | }; 29 | }//namespace gemfield_org 30 | -------------------------------------------------------------------------------- /utils/include/syszux_base64.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace gemfield_org { 7 | std::vector unbase64(const std::string& base64_data); 8 | std::string base64(const unsigned char* bin, size_t len, int lineLenght = -1); 9 | 10 | }// namespace 11 | -------------------------------------------------------------------------------- /utils/include/syszux_decrypt.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include "gemfield.h" 20 | 21 | namespace gemfield_org{ 22 | 23 | inline std::vector readFileToVectorByte(const std::string& file_path){ 24 | std::ifstream syszux_file(file_path, std::ios::binary | std::ios::ate); 25 | std::vector vector_bytes; 26 | 27 | if (!syszux_file.eof() && !syszux_file.fail()) { 28 | syszux_file.seekg(0, std::ios_base::end); 29 | std::streampos file_size = syszux_file.tellg(); 30 | vector_bytes.resize(file_size); 31 | syszux_file.seekg(0, std::ios_base::beg); 32 | syszux_file.read((char*)&vector_bytes[0], file_size); 33 | } else { 34 | std::string msg = gemfield_org::format("Can't read file at: %s",file_path); 35 | GEMFIELD_E(msg.c_str()); 36 | throw std::runtime_error(msg); 37 | } 38 | return vector_bytes; 39 | } 40 | 41 | inline std::string md5(std::string& data){ 42 | unsigned char md[MD5_DIGEST_LENGTH]; 43 | unsigned long data_len = data.size(); 44 | MD5(reinterpret_cast(data.c_str()), data_len, md); 45 | 46 | char result[MD5_DIGEST_LENGTH * 2 + 1]; 47 | memset(result, 0, sizeof result); 48 | for (int i = 0; i < MD5_DIGEST_LENGTH; i++){ 49 | sprintf(&result[i*2], "%02x", md[i]); 50 | } 51 | 52 | return std::string(result); 53 | } 54 | 55 | inline std::string md5(const char* data){ 56 | std::string tmp(data); 57 | return md5(tmp); 58 | } 59 | 60 | using DeepvacKeyBytes = std::array; 61 | class SyszuxDecrypt{ 62 | public: 63 | SyszuxDecrypt(){ 64 | ctx_ = EVP_CIPHER_CTX_new(); 65 | GEMFIELD_I("construct SyszuxDecrypt succeeded."); 66 | }; 67 | SyszuxDecrypt(const SyszuxDecrypt&) = delete; 68 | SyszuxDecrypt& operator=(const SyszuxDecrypt&) = delete; 69 | SyszuxDecrypt(SyszuxDecrypt&&) = default; 70 | SyszuxDecrypt& operator=(SyszuxDecrypt&&) = default; 71 | SyszuxDecrypt(std::string key):key_(key){ 72 | ctx_ = EVP_CIPHER_CTX_new(); 73 | GEMFIELD_I("construct SyszuxDecrypt succeeded."); 74 | } 75 | virtual ~SyszuxDecrypt(){ 76 | EVP_CIPHER_CTX_cleanup(ctx_); 77 | EVP_CIPHER_CTX_free(ctx_); 78 | GEMFIELD_I("destruct SyszuxDecrypt succeeded."); 79 | } 80 | 81 | std::vector de(const std::string file_path, const std::string k) { 82 | GEMFIELD_SI; 83 | key_ = k; 84 | DeepvacKeyBytes key = calculateSha256(key_); 85 | auto pt_bytes = readFileToVectorByte(file_path); 86 | 87 | GEMFIELD_I2("SYSZUX_DE version: ", OPENSSL_VERSION_TEXT); 88 | if (pt_bytes.size() <= AES_BLOCK_SIZE) { 89 | std::string msg = gemfield_org::format("Input encrypted content size = %d is too small for AES CBC decryption.", file_path); 90 | GEMFIELD_E(msg.c_str()); 91 | throw std::runtime_error(msg); 92 | } 93 | 94 | const std::vector iv(pt_bytes.begin(), pt_bytes.begin() + AES_BLOCK_SIZE); 95 | 96 | EVP_CIPHER_CTX_init(ctx_); 97 | status_ = EVP_DecryptInit_ex(ctx_, EVP_aes_256_cbc(), nullptr, key.data(), iv.data()); 98 | isThrowThisMsg("[SYSZUX_DE] INIT Error"); 99 | 100 | int p_len = (int)pt_bytes.size(); 101 | int f_len = 0; 102 | std::vector result_bytes(p_len); 103 | status_ = EVP_DecryptUpdate(ctx_, result_bytes.data(), &p_len, pt_bytes.data() + AES_BLOCK_SIZE, (int)pt_bytes.size() - AES_BLOCK_SIZE); 104 | isThrowThisMsg("[SYSZUX_DE] UPDATE Error"); 105 | 106 | status_ = EVP_DecryptFinal_ex(ctx_, result_bytes.data() + p_len, &f_len); 107 | isThrowThisMsg("[SYSZUX_DE] FINAL Error"); 108 | 109 | result_bytes.resize(p_len + f_len); 110 | return result_bytes; 111 | } 112 | 113 | private: 114 | DeepvacKeyBytes calculateSha256(std::string key) { 115 | DeepvacKeyBytes key_bytes; 116 | SHA256_CTX sha256; 117 | SHA256_Init(&sha256); 118 | SHA256_Update(&sha256, key.c_str(), key.size()); 119 | SHA256_Final(key_bytes.data(), &sha256); 120 | return key_bytes; 121 | } 122 | void isThrowThisMsg(std::string msg){ 123 | if(status_ != 0){ 124 | return; 125 | } 126 | ERR_print_errors_fp(stderr); 127 | GEMFIELD_E(msg.c_str()); 128 | throw std::runtime_error(msg); 129 | } 130 | 131 | private: 132 | std::string key_; 133 | EVP_CIPHER_CTX* ctx_; 134 | int status_{1}; 135 | }; 136 | } 137 | -------------------------------------------------------------------------------- /utils/include/syszux_glab.h: -------------------------------------------------------------------------------- 1 | /* 2 | * * Copyright (c) 2020 Gemfield 3 | * * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * * You may not use this file except in compliance with the License. 5 | * */ 6 | 7 | #pragma once 8 | // #include "syszux_ocr_pse.h" 9 | #include "opencv2/opencv.hpp" 10 | 11 | namespace deepvac { 12 | 13 | class AggressiveBox { 14 | public: 15 | AggressiveBox(cv::RotatedRect rect, std::vector shape, float real_angle, bool credit_by_score=true, bool credit_by_shape=true); 16 | void init4Points(); 17 | void scaleBox(); 18 | cv::RotatedRect getRect(); 19 | std::pair>> ratio(cv::RotatedRect rect); 20 | void addCandidateBox2Merge(AggressiveBox rect, int rect_id, float merge_ratio, std::vector> contex); 21 | void sortCandidateBox(); 22 | bool isMerge(AggressiveBox rect); 23 | void merge(std::tuple>> rect); 24 | std::vector mergeLeftOrUpElseRightOrBottom(std::vector>>> candidate_box_list_left_up_or_right_down); 25 | std::vector mergeRects(); 26 | private: 27 | float min_h_ratio_ = 0.75; 28 | float max_h_ratio_ = 1.3; 29 | float box_min_wh_ratio_ = 2.0; 30 | 31 | cv::RotatedRect rect_; 32 | cv::RotatedRect ori_rect_; 33 | cv::RotatedRect rect2reg_; 34 | 35 | std::vector shape_; 36 | bool credit_by_score_; 37 | bool credit_by_shape_; 38 | std::vector>>> candidate_box_list_left_up_; 39 | std::vector>>> candidate_box_list_right_down_; 40 | float real_angle_; 41 | std::string scaleAxis_; 42 | public: 43 | std::vector most_left_xy_; 44 | std::vector most_right_xy_; 45 | std::vector most_top_xy_; 46 | std::vector most_bottom_xy_; 47 | bool valid_ = true; 48 | }; 49 | 50 | class DeepvacOcrFrame { 51 | public: 52 | DeepvacOcrFrame(cv::Mat img, std::vector rect_list, bool is_oneway=false); 53 | void sortBoxByRatio(); 54 | int initDominantAngle(); 55 | AggressiveBox creatAggressiveBox(cv::RotatedRect rect); 56 | void aggressive4mergePeer(AggressiveBox& aggressive_rect, int offset); 57 | std::optional> operator() (); 58 | private: 59 | float merge_ratio_ = 0.7; 60 | float similar_box_ratio_ = 0.95; 61 | float credit_shape_ratio_ = 2.0; 62 | std::vector shape_; 63 | float median_angle = 0; 64 | std::vector rect_list_; 65 | bool is_oneway_; 66 | std::vector aggressive_box_list_; 67 | std::vector real_angle_list_; 68 | int total_box_num_; 69 | int similar_box_num_; 70 | float median_angle_; 71 | }; 72 | 73 | } //namespace deepvac 74 | -------------------------------------------------------------------------------- /utils/include/syszux_img2tensor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | #include 9 | #include 10 | #include 11 | 12 | namespace gemfield_org{ 13 | enum NORMALIZE_TYPE { 14 | NO_NORMALIZE, 15 | NORMALIZE0_1, 16 | NORMALIZE_1_1 17 | }; 18 | enum MEAN_STD_TYPE { 19 | NO_MEAN_STD, 20 | MEAN_STD_FROM_IMAGENET, 21 | MEAN_STD_FROM_FACE 22 | }; 23 | std::optional img2CvMat(std::string img_path, bool is_rgb=false); 24 | void normalizeTensor(at::Tensor& t, NORMALIZE_TYPE normalize, MEAN_STD_TYPE mean_std, std::string device="cpu"); 25 | std::optional cvMat2Tensor(cv::Mat& frame, NORMALIZE_TYPE normalize=NO_NORMALIZE, MEAN_STD_TYPE mean_std=NO_MEAN_STD, std::string device="cpu"); 26 | std::optional cvMat2Tensor(std::vector& frames, NORMALIZE_TYPE normalize=NO_NORMALIZE, MEAN_STD_TYPE mean_std=NO_MEAN_STD,std::string device="cpu"); 27 | std::optional cvMat2Tensor(cv::Mat&& tmp_frame, NORMALIZE_TYPE normalize=NO_NORMALIZE, MEAN_STD_TYPE mean_std=NO_MEAN_STD,std::string device="cpu"); 28 | std::optional img2Tensor(std::string img_path, bool is_rgb=false, NORMALIZE_TYPE normalize=NO_NORMALIZE, MEAN_STD_TYPE mean_std=NO_MEAN_STD,std::string device="cpu"); 29 | } 30 | -------------------------------------------------------------------------------- /utils/include/syszux_nms.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | 9 | #include 10 | 11 | namespace gemfield_org{ 12 | torch::Tensor nms(torch::Tensor& dets, float threshold); 13 | } -------------------------------------------------------------------------------- /utils/include/syszux_priorbox.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | 9 | #include 10 | #include 11 | 12 | namespace gemfield_org{ 13 | class PriorBox{ 14 | public: 15 | PriorBox() = default; 16 | PriorBox(std::vector>&& min_sizes, std::vector&& steps, bool clip = false): 17 | min_sizes_(min_sizes),steps_(steps),clip_(clip){} 18 | ~PriorBox() = default; 19 | at::Tensor forward(std::vector& img_size); 20 | at::Tensor forward(std::vector&& img_size); 21 | 22 | private: 23 | std::vector> min_sizes_; 24 | std::vector steps_; 25 | bool clip_; 26 | }; 27 | } 28 | -------------------------------------------------------------------------------- /utils/include/syszux_stream_buffer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | 9 | #include 10 | #include 11 | 12 | class SyszuxStreamBuffer : public std::streambuf 13 | { 14 | public: 15 | SyszuxStreamBuffer(const uint8_t *begin, const size_t size); 16 | SyszuxStreamBuffer() = delete; 17 | SyszuxStreamBuffer(const SyszuxStreamBuffer&) = delete; 18 | SyszuxStreamBuffer& operator=(const SyszuxStreamBuffer&) = delete; 19 | SyszuxStreamBuffer(SyszuxStreamBuffer&&) = default; 20 | SyszuxStreamBuffer& operator=(SyszuxStreamBuffer&&) = default; 21 | 22 | private: 23 | int_type underflow(); 24 | int_type uflow(); 25 | int_type pbackfail(int_type ch); 26 | std::streamsize showmanyc(); 27 | std::streampos seekoff ( std::streamoff off, std::ios_base::seekdir way,std::ios_base::openmode which = std::ios_base::in | std::ios_base::out ); 28 | std::streampos seekpos ( std::streampos sp,std::ios_base::openmode which = std::ios_base::in | std::ios_base::out); 29 | 30 | private: 31 | const uint8_t * const begin_; 32 | const uint8_t * const end_; 33 | const uint8_t * current_; 34 | }; -------------------------------------------------------------------------------- /utils/include/syszux_tensorrt_buffers.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace gemfield_org{ 10 | 11 | enum class DataType : int32_t { 12 | kFLOAT = 0, 13 | kHALF = 1, 14 | kINT8 = 2, 15 | kINT32 = 3, 16 | kBOOL = 4 17 | }; 18 | 19 | enum class DeviceType { 20 | CPU, 21 | CUDA 22 | }; 23 | 24 | inline unsigned int getElementSize(DataType t){ 25 | switch (t){ 26 | case DataType::kINT32: return 4; 27 | case DataType::kFLOAT: return 4; 28 | case DataType::kHALF: return 2; 29 | case DataType::kBOOL: 30 | case DataType::kINT8: return 1; 31 | } 32 | throw std::runtime_error("Invalid DataType."); 33 | return 0; 34 | } 35 | 36 | inline int64_t volume(const std::vector& d){ 37 | return std::accumulate(d.begin(), d.end(), 1, std::multiplies()); 38 | } 39 | 40 | template 41 | class GenericBuffer 42 | { 43 | public: 44 | GenericBuffer(DataType type = DataType::kFLOAT) 45 | : size_(0), type_(type), buffer_(nullptr), cleanup_(true) {} 46 | 47 | GenericBuffer(void* buffer, const std::vector& shape, const DataType& type): size_(volume(shape)), shape_(shape), type_(type), 48 | buffer_(buffer), cleanup_(false) {} 49 | 50 | GenericBuffer(GenericBuffer&& buf) 51 | : size_(buf.size_), shape_(std::move(buf.shape_)), type_(buf.type_), buffer_(buf.buffer_), cleanup_(buf.cleanup_) { 52 | buf.reset(); 53 | } 54 | 55 | GenericBuffer& operator=(GenericBuffer&& buf) { 56 | if (this != &buf) { 57 | if(cleanup_) { 58 | free_fn_(buffer_); 59 | } 60 | size_ = buf.size_; 61 | shape_ = std::move(buf.shape_); 62 | type_ = buf.type_; 63 | buffer_ = buf.buffer_; 64 | cleanup_ = buf.cleanup_; 65 | buf.reset(); 66 | } 67 | return *this; 68 | } 69 | 70 | void* data() { 71 | return buffer_; 72 | } 73 | 74 | const void* data() const { 75 | return buffer_; 76 | } 77 | 78 | size_t size() const { 79 | return size_; 80 | } 81 | 82 | size_t nbBytes() const { 83 | return this->size() * getElementSize(type_); 84 | } 85 | 86 | std::vector shape() const { 87 | return shape_; 88 | } 89 | 90 | void resize(const std::vector& shape) { 91 | shape_ = shape; 92 | auto new_size = volume(shape_); 93 | 94 | if(size_ == new_size) { 95 | return; 96 | } 97 | if(cleanup_) { 98 | free_fn_(buffer_); 99 | } 100 | if(!alloc_fn_(&buffer_, new_size*getElementSize(type_))) { 101 | throw std::bad_alloc{}; 102 | } 103 | size_ = new_size; 104 | cleanup_ = true; 105 | } 106 | 107 | void fromTensor(const at::Tensor& tensor) { 108 | auto contiguous_tensor = tensor.contiguous(); 109 | auto* input_data = static_cast(contiguous_tensor.to("cpu").data_ptr()); 110 | auto sizes = contiguous_tensor.sizes(); 111 | std::vector shape; 112 | std::copy(sizes.begin(), sizes.end(), std::back_inserter(shape)); 113 | resize(shape); 114 | if(Device == DeviceType::CPU) { 115 | memcpy(buffer_, input_data, nbBytes()); 116 | } else if(Device == DeviceType::CUDA) { 117 | cudaMemcpy(buffer_, input_data, nbBytes(), cudaMemcpyHostToDevice); 118 | } 119 | } 120 | 121 | at::Tensor toTensor() { 122 | return torch::from_blob(buffer_, shape_, Device == DeviceType::CPU ? torch::kCPU : torch::kCUDA); 123 | } 124 | 125 | ~GenericBuffer() { 126 | if(cleanup_) { 127 | free_fn_(buffer_); 128 | } 129 | } 130 | 131 | private: 132 | void reset() { 133 | size_ = 0; 134 | buffer_ = nullptr; 135 | shape_.clear(); 136 | } 137 | 138 | private: 139 | size_t size_; 140 | std::vector shape_; 141 | DataType type_; 142 | void* buffer_; 143 | AllocFunc alloc_fn_; 144 | FreeFunc free_fn_; 145 | bool cleanup_; 146 | }; 147 | 148 | class DeviceAllocator 149 | { 150 | public: 151 | bool operator()(void** ptr, size_t size) const { 152 | return cudaMalloc(ptr, size) == cudaSuccess; 153 | } 154 | }; 155 | 156 | class DeviceFree 157 | { 158 | public: 159 | void operator()(void* ptr) const{ 160 | cudaFree(ptr); 161 | } 162 | }; 163 | 164 | class HostAllocator 165 | { 166 | public: 167 | bool operator()(void** ptr, size_t size) const{ 168 | *ptr = malloc(size); 169 | return *ptr != nullptr; 170 | } 171 | }; 172 | 173 | class HostFree 174 | { 175 | public: 176 | void operator()(void* ptr) const{ 177 | free(ptr); 178 | } 179 | }; 180 | 181 | using DeviceBuffer = GenericBuffer; 182 | using HostBuffer = GenericBuffer; 183 | 184 | class ManagedBuffer 185 | { 186 | public: 187 | DeviceBuffer deviceBuffer; 188 | HostBuffer hostBuffer; 189 | }; 190 | 191 | } 192 | -------------------------------------------------------------------------------- /utils/include/syszux_verify_landmark.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #pragma once 8 | #include "opencv2/opencv.hpp" 9 | #include 10 | #include 11 | 12 | namespace gemfield_org{ 13 | bool isValidLandmark(std::vector bbox, std::vector landmark, int min_face_size=24); 14 | torch::Tensor getDecodeBox(torch::Tensor& prior, torch::Tensor& variances, torch::Tensor& loc); 15 | void decodeLandmark(torch::Tensor& prior, torch::Tensor& variances, torch::Tensor& landms); 16 | } 17 | -------------------------------------------------------------------------------- /utils/src/syszux_align_face.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include "gemfield.h" 8 | #include "syszux_align_face.h" 9 | #include 10 | 11 | namespace gemfield_org{ 12 | AlignFace::AlignFace() 13 | { 14 | double REFERENCE_FACIAL_POINTS_112x112[5][2] = { 15 | {38.29459953, 51.69630051}, 16 | {73.53179932, 51.50139999}, 17 | {56.02519989, 71.73660278}, 18 | {41.54930115, 92.3655014}, 19 | {70.72990036, 92.20410156} 20 | }; 21 | ref_facial_5pts_ = cv::Mat(5, 2, CV_64F, &REFERENCE_FACIAL_POINTS_112x112).clone(); 22 | 23 | double CROP_SIZE[2] = {112,112}; 24 | crop_size_ = cv::Mat(1,2,CV_64F,&CROP_SIZE).clone(); 25 | } 26 | 27 | std::tuple> AlignFace::operator() (cv::Mat& frame, cv::Mat& facial_5pts) 28 | { 29 | cv::Mat x, y; 30 | cv::Mat facial; 31 | std::vector points; 32 | facial_5pts.convertTo(facial_5pts, CV_64F); 33 | //x1,y1,x2,y2... somebody familiar with cv::Mat, please refactor this code snippet. 34 | for(int i=0; i(i); 36 | double coord_y = facial_5pts.at(i+1); 37 | x.push_back(coord_x); 38 | y.push_back(coord_y); 39 | points.emplace_back(cv::Point2f(coord_x, coord_y)); 40 | } 41 | 42 | cv::hconcat(x, y, facial); 43 | cv::Mat tfm = getAffineTransform(facial); 44 | 45 | cv::Mat point_matrix = tfm.t(); 46 | std::vector dst_points = pointsTransform(points, point_matrix); 47 | cv::Mat img_matrix = tfm.colRange(0, 2).t(); 48 | cv::Mat dst_img = warpAndCrop(frame, img_matrix); 49 | 50 | return std::tuple(dst_img, dst_points); 51 | } 52 | 53 | cv::Mat AlignFace::warpAndCrop(cv::Mat& src_img, cv::Mat& matrix) 54 | { 55 | cv::Mat dst_img; 56 | cv::warpAffine(src_img, dst_img, matrix, cv::Size(int(crop_size_.at(0)), int(crop_size_.at(1)))); 57 | return dst_img; 58 | } 59 | 60 | std::vector AlignFace::pointsTransform(const std::vector& points, const cv::Mat& matrix) { 61 | std::vector transformed_points; 62 | cv::perspectiveTransform(points, transformed_points, matrix); 63 | 64 | std::vector dst_points; 65 | for(auto p : transformed_points) { 66 | dst_points.emplace_back(p.x); 67 | dst_points.emplace_back(p.y); 68 | } 69 | return dst_points; 70 | } 71 | 72 | cv::Mat AlignFace::getAffineTransform(cv::Mat& facial_5pts) 73 | { 74 | if(ref_facial_5pts_.rows != facial_5pts.rows){ 75 | std::string msg = gemfield_org::format("ref_facial_5pts_ and facial_5pts must have the same shape: %d: vs %d", int(ref_facial_5pts_.rows), int(facial_5pts.rows)); 76 | GEMFIELD_E(msg.c_str()); 77 | throw std::runtime_error(msg); 78 | } 79 | 80 | cv::Mat trans1 = findNonereflectiveSimilarity(facial_5pts, ref_facial_5pts_); 81 | 82 | cv::Mat xyR = ref_facial_5pts_.clone(); 83 | xyR.col(0) = -1 * xyR.col(0); 84 | 85 | cv::Mat trans2r = findNonereflectiveSimilarity(facial_5pts, xyR); 86 | 87 | cv::Mat TreflectY = cv::Mat::eye(3, 3, CV_64F); 88 | TreflectY.at(0, 0) = -1.; 89 | 90 | cv::Mat trans2 = trans2r * TreflectY; 91 | 92 | cv::Mat xy1 = tformfwd(trans1, facial_5pts); 93 | double norm1 = cv::norm(xy1, ref_facial_5pts_); 94 | cv::Mat xy2 = tformfwd(trans2, facial_5pts); 95 | double norm2 = cv::norm(xy2, ref_facial_5pts_); 96 | 97 | cv::Mat trans; 98 | if(norm1 <= norm2){ 99 | trans = trans1; 100 | }else{ 101 | trans = trans2; 102 | } 103 | //return trans.colRange(0, 2).t(); 104 | return trans; 105 | } 106 | 107 | cv::Mat AlignFace::findNonereflectiveSimilarity(cv::Mat& facial_5pts, cv::Mat& ref_facial_5pts) 108 | { 109 | int K = 2; 110 | int M = ref_facial_5pts.rows; 111 | cv::Mat x = ref_facial_5pts.col(0); 112 | cv::Mat y = ref_facial_5pts.col(1); 113 | 114 | cv::Mat temp1, temp2, X; 115 | std::vector matrices1 = {x, y, cv::Mat::ones(M, 1, CV_64F), cv::Mat::zeros(M, 1, CV_64F)}; 116 | std::vector matrices2 = {y, -x, cv::Mat::zeros(M, 1, CV_64F), cv::Mat::ones(M, 1, CV_64F)}; 117 | 118 | cv::hconcat(matrices1, temp1); 119 | cv::hconcat(matrices2, temp2); 120 | cv::vconcat(temp1, temp2, X); 121 | 122 | cv::Mat U; 123 | cv::Mat u = facial_5pts.col(0); 124 | cv::Mat v = facial_5pts.col(1); 125 | cv::vconcat(u, v, U); 126 | 127 | 128 | U.convertTo(U, CV_32F); 129 | X.convertTo(X, CV_32F); 130 | auto U_tensor = torch::from_blob(U.data,{U.rows, U.cols}); 131 | auto X_tensor = torch::from_blob(X.data,{X.rows, X.cols}); 132 | auto res = torch::lstsq(U_tensor, X_tensor); 133 | torch::Tensor r = std::get<0>(res); 134 | 135 | double sc = r[0].item(); 136 | double ss = r[1].item(); 137 | double tx = r[2].item(); 138 | double ty = r[3].item(); 139 | 140 | double Tinv_vec[3][3] = {{sc, -ss, 0.},{ss, sc, 0.},{tx, ty, 1.}}; 141 | cv::Mat trans_inv = cv::Mat(3, 3, CV_64F, Tinv_vec); 142 | cv::Mat trans = trans_inv.inv(); 143 | 144 | std::vector tmp_vec = {0, 0, 1}; 145 | cv::Mat tmp(tmp_vec); 146 | tmp = tmp.t(); 147 | trans.col(2) = tmp; 148 | return trans; 149 | } 150 | 151 | cv::Mat AlignFace::tformfwd(cv::Mat& trans, cv::Mat facial_5pts) 152 | { 153 | cv::hconcat(facial_5pts, cv::Mat::ones(facial_5pts.rows, 1, CV_64F), facial_5pts); 154 | cv::Mat res = facial_5pts * trans; 155 | return res.colRange(0, 2); 156 | } 157 | }//namespace 158 | -------------------------------------------------------------------------------- /utils/src/syszux_base64.cpp: -------------------------------------------------------------------------------- 1 | #include "syszux_base64.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | namespace gemfield_org { 8 | using namespace std; 9 | 10 | const static char* b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; 11 | // maps A=>0,B=>1.. 12 | const static unsigned char unb64[] = { 13 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 10 14 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20 15 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 30 16 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40 17 | 0, 0, 0, 62, 0, 0, 0, 63, 52, 53, // 50 18 | 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, // 60 19 | 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, // 70 20 | 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, // 80 21 | 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, // 90 22 | 25, 0, 0, 0, 0, 0, 0, 26, 27, 28, // 100 23 | 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, // 110 24 | 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, // 120 25 | 49, 50, 51, 0, 0, 0, 0, 0, 0, 0, // 130 26 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 140 27 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 150 28 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 160 29 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 170 30 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 180 31 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 190 32 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 200 33 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 210 34 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 220 35 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 230 36 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 240 37 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 250 38 | 0, 0, 0, 0, 0, 0, 39 | }; // This array has 255 elements 40 | 41 | static void add_CR_if_needed(string& encodeBuffer, int lineLenght) { 42 | if (lineLenght > 0 && ((encodeBuffer.size() + 1) % lineLenght) == 0) { 43 | encodeBuffer += '\n'; 44 | } 45 | } 46 | 47 | string base64(const unsigned char* bin, size_t len, int lineLenght) { 48 | int rc = 0; // result counter 49 | int byteNo; // I need this after the loop 50 | 51 | int modulusLen = len % 3; 52 | int pad = ((modulusLen & 1) << 1) + ((modulusLen & 2) >> 1); // 2 gives 1 and 1 gives 2, but 0 gives 0. 53 | 54 | const size_t flen = 4 * (len + pad) / 3; 55 | size_t totalLength = flen; 56 | if (lineLenght > 0) { 57 | totalLength += ((int)flen / lineLenght) + 3; 58 | } 59 | 60 | string encodeBuffer; 61 | encodeBuffer.reserve(totalLength); 62 | 63 | for (byteNo = 0; byteNo <= len - 3; byteNo += 3) { 64 | unsigned char BYTE0 = bin[byteNo]; 65 | unsigned char BYTE1 = bin[byteNo + 1]; 66 | unsigned char BYTE2 = bin[byteNo + 2]; 67 | 68 | add_CR_if_needed(encodeBuffer, lineLenght); 69 | encodeBuffer += b64[BYTE0 >> 2]; 70 | add_CR_if_needed(encodeBuffer, lineLenght); 71 | encodeBuffer += b64[((0x3 & BYTE0) << 4) + (BYTE1 >> 4)]; 72 | add_CR_if_needed(encodeBuffer, lineLenght); 73 | encodeBuffer += b64[((0x0f & BYTE1) << 2) + (BYTE2 >> 6)]; 74 | add_CR_if_needed(encodeBuffer, lineLenght); 75 | encodeBuffer += b64[0x3f & BYTE2]; 76 | } 77 | 78 | if (pad == 2) { 79 | add_CR_if_needed(encodeBuffer, lineLenght); 80 | encodeBuffer += b64[bin[byteNo] >> 2]; 81 | add_CR_if_needed(encodeBuffer, lineLenght); 82 | encodeBuffer += b64[(0x3 & bin[byteNo]) << 4]; 83 | add_CR_if_needed(encodeBuffer, lineLenght); 84 | encodeBuffer += '='; 85 | add_CR_if_needed(encodeBuffer, lineLenght); 86 | encodeBuffer += '='; 87 | } else if (pad == 1) { 88 | add_CR_if_needed(encodeBuffer, lineLenght); 89 | encodeBuffer += b64[bin[byteNo] >> 2]; 90 | add_CR_if_needed(encodeBuffer, lineLenght); 91 | encodeBuffer += b64[((0x3 & bin[byteNo]) << 4) + (bin[byteNo + 1] >> 4)]; 92 | add_CR_if_needed(encodeBuffer, lineLenght); 93 | encodeBuffer += b64[(0x0f & bin[byteNo + 1]) << 2]; 94 | add_CR_if_needed(encodeBuffer, lineLenght); 95 | encodeBuffer += '='; 96 | } 97 | if (lineLenght > 0 && encodeBuffer[encodeBuffer.length() - 1] != '\n') { 98 | encodeBuffer += '\n'; 99 | } 100 | return encodeBuffer; 101 | } 102 | 103 | std::vector unbase64(const std::string& base64_data) { 104 | string tmp_str(base64_data); 105 | tmp_str.erase(std::remove(tmp_str.begin(), tmp_str.end(), '\n'), tmp_str.end()); 106 | const unsigned char* safeAsciiPtr = (const unsigned char*)tmp_str.c_str(); 107 | std::vector bin; 108 | int cb = 0; 109 | int charNo; 110 | int pad = 0; 111 | size_t len = tmp_str.size(); 112 | 113 | if (len < 2) { // 2 accesses below would be OOB. 114 | // catch empty string, return NULL as result. 115 | puts("ERROR: You passed an invalid base64 string (too short). You get NULL back."); 116 | return bin; 117 | } 118 | if (safeAsciiPtr[len - 1] == '=') ++pad; 119 | if (safeAsciiPtr[len - 2] == '=') ++pad; 120 | 121 | size_t flen = 3 * len / 4 - pad; 122 | bin.reserve(flen); 123 | 124 | for (charNo = 0; charNo <= len - 4 - pad; charNo += 4) { 125 | int A = unb64[safeAsciiPtr[charNo]]; 126 | int B = unb64[safeAsciiPtr[charNo + 1]]; 127 | int C = unb64[safeAsciiPtr[charNo + 2]]; 128 | int D = unb64[safeAsciiPtr[charNo + 3]]; 129 | 130 | bin.push_back((A << 2) | (B >> 4)); 131 | bin.push_back((B << 4) | (C >> 2)); 132 | bin.push_back((C << 6) | (D)); 133 | } 134 | 135 | if (pad == 1) { 136 | int A = unb64[safeAsciiPtr[charNo]]; 137 | int B = unb64[safeAsciiPtr[charNo + 1]]; 138 | int C = unb64[safeAsciiPtr[charNo + 2]]; 139 | bin.push_back((A << 2) | (B >> 4)); 140 | bin.push_back((B << 4) | (C >> 2)); 141 | } else if (pad == 2) { 142 | int A = unb64[safeAsciiPtr[charNo]]; 143 | int B = unb64[safeAsciiPtr[charNo + 1]]; 144 | bin.push_back((A << 2) | (B >> 4)); 145 | } 146 | return bin; 147 | } 148 | }//namespace 149 | -------------------------------------------------------------------------------- /utils/src/syszux_img2tensor.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | #include "syszux_img2tensor.h" 7 | #include "gemfield.h" 8 | 9 | namespace gemfield_org{ 10 | std::optional img2CvMat(std::string img_path, bool is_rgb){ 11 | cv::Mat frame; 12 | try{ 13 | frame = cv::imread(img_path); 14 | }catch(std::exception& e){ 15 | GEMFIELD_E2("Error to read img: ", img_path.c_str()); 16 | return std::nullopt; 17 | } 18 | if (frame.rows == 0 or frame.cols == 0){ 19 | GEMFIELD_E2("illegal img: ", img_path.c_str()); 20 | return std::nullopt; 21 | } 22 | if(is_rgb){ 23 | cv::cvtColor(frame, frame, cv::COLOR_BGR2RGB); 24 | } 25 | return frame; 26 | } 27 | 28 | std::optional cvMat2Tensor(cv::Mat&& tmp_frame, NORMALIZE_TYPE normalize, MEAN_STD_TYPE mean_std,std::string device){ 29 | cv::Mat frame = tmp_frame.clone(); 30 | auto t_opt = cvMat2Tensor(frame, normalize, mean_std); 31 | if (!t_opt) { 32 | return std::nullopt; 33 | } 34 | auto t = t_opt.value(); 35 | return t.clone(); 36 | } 37 | 38 | void normalizeTensor(at::Tensor& t, NORMALIZE_TYPE normalize, MEAN_STD_TYPE mean_std, std::string device){ 39 | t = t.to(device).permute({0, 3, 1, 2}).to(at::kFloat); 40 | switch(normalize) { 41 | case NO_NORMALIZE: 42 | break; 43 | case NORMALIZE0_1: 44 | t = t.div(255); 45 | break; 46 | case NORMALIZE_1_1: 47 | t = t.div(127.5).sub(1); 48 | break; 49 | default: 50 | throw std::runtime_error("invalid selection of normalize."); 51 | } 52 | switch(mean_std){ 53 | case NO_MEAN_STD : 54 | break; 55 | case MEAN_STD_FROM_IMAGENET : 56 | t[0][0] = t[0][0].sub_(0.485).div_(0.229); 57 | t[0][1] = t[0][1].sub_(0.456).div_(0.224); 58 | t[0][2] = t[0][2].sub_(0.406).div_(0.225); 59 | break; 60 | case MEAN_STD_FROM_FACE : 61 | t[0][0] = t[0][0].sub_(104); 62 | t[0][1] = t[0][1].sub_(117); 63 | t[0][2] = t[0][2].sub_(123); 64 | break; 65 | default: 66 | throw std::runtime_error("invalid selection of mean_std."); 67 | } 68 | } 69 | std::optional cvMat2Tensor(cv::Mat& frame, NORMALIZE_TYPE normalize, MEAN_STD_TYPE mean_std,std::string device){ 70 | if (frame.rows == 0 or frame.cols == 0){ 71 | GEMFIELD_E("illegal img: wrong rows or cols."); 72 | return std::nullopt; 73 | } 74 | //auto options = torch::TensorOptions().dtype(torch::kFloat32).layout(torch::kStrided).device(device); bug2fix!!! 75 | auto options = torch::TensorOptions().dtype(torch::kUInt8); 76 | auto input_tensor = torch::from_blob(frame.data, {1, frame.rows, frame.cols, 3}, options); 77 | 78 | normalizeTensor(input_tensor, normalize, mean_std, device); 79 | return input_tensor; 80 | } 81 | 82 | std::optional cvMat2Tensor(std::vector& frames, NORMALIZE_TYPE normalize, MEAN_STD_TYPE mean_std,std::string device){ 83 | if (frames.size() == 0) { 84 | GEMFIELD_E("illegal vector: wrong size (size = 0)."); 85 | return std::nullopt; 86 | } 87 | int h = frames[0].rows; 88 | int w = frames[0].cols; 89 | for (auto frame : frames) { 90 | if (frame.rows == 0 or frame.cols == 0){ 91 | GEMFIELD_E("illegal img: wrong rows or cols."); 92 | return std::nullopt; 93 | } 94 | if (frame.rows != h or frame.cols != w){ 95 | GEMFIELD_E("illegal img: rows or cols are not consistent."); 96 | return std::nullopt; 97 | } 98 | } 99 | cv::Mat batch_image; 100 | cv::vconcat(frames, batch_image); 101 | 102 | auto options = torch::TensorOptions().dtype(torch::kUInt8);//.device() 103 | // auto options = torch::TensorOptions().dtype(torch::kFloat32).layout(torch::kStrided).device(torch::kCUDA, 1) 104 | auto input_tensor = torch::from_blob(batch_image.data, {(int)frames.size(), h, w, 3}, options); 105 | normalizeTensor(input_tensor, normalize, mean_std, device); 106 | return input_tensor.clone(); 107 | } 108 | 109 | std::optional img2Tensor(std::string img_path, bool is_rgb, NORMALIZE_TYPE normalize, MEAN_STD_TYPE mean_std,std::string device){ 110 | auto frame_opt = img2CvMat(img_path, is_rgb); 111 | if(!frame_opt){ 112 | return std::nullopt; 113 | } 114 | return cvMat2Tensor(std::move(frame_opt.value()), normalize, mean_std, device); 115 | } 116 | }//namespace 117 | 118 | 119 | -------------------------------------------------------------------------------- /utils/src/syszux_nms.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include "syszux_nms.h" 8 | 9 | namespace gemfield_org{ 10 | torch::Tensor nms(torch::Tensor& dets, float threshold) 11 | { 12 | //1 represent column 13 | torch::Tensor all_box_x1 = dets.select(1, 0); 14 | torch::Tensor all_box_y1 = dets.select(1, 1); 15 | torch::Tensor all_box_x2 = dets.select(1, 2); 16 | torch::Tensor all_box_y2 = dets.select(1, 3); 17 | torch::Tensor all_box_scores = dets.select(1, 4); 18 | 19 | int keep_index = 0; 20 | torch::Tensor keep = torch::zeros({all_box_scores.size(0)}).to(torch::kLong).to(all_box_scores.device()); 21 | torch::Tensor all_box_areas = (all_box_x2 - all_box_x1 + 1) * (all_box_y2 - all_box_y1 + 1); 22 | 23 | std::tuple sorted_score_and_index = torch::sort(all_box_scores, 0, 1); 24 | torch::Tensor sorted_index = std::get<1>(sorted_score_and_index); 25 | 26 | //torch::Tensor second2last_box_x1, second2last_box_y1, second2last_box_x2, second2last_box_y2, second2last_box_w, second2last_box_h; 27 | while(sorted_index.numel() > 0){ 28 | auto top_score_index = sorted_index[0]; 29 | keep[keep_index] = top_score_index; 30 | 31 | keep_index += 1; 32 | 33 | if(sorted_index.size(0)==1){ 34 | break; 35 | } 36 | 37 | auto second2last_index = sorted_index.slice(0, 1, sorted_index.size(0)); 38 | auto second2last_box_x1 = all_box_x1.index_select(0, second2last_index); 39 | auto second2last_box_y1 = all_box_y1.index_select(0, second2last_index); 40 | auto second2last_box_x2 = all_box_x2.index_select(0, second2last_index); 41 | auto second2last_box_y2 = all_box_y2.index_select(0, second2last_index); 42 | 43 | auto second2last_intersection_x1 = second2last_box_x1.clamp(all_box_x1[top_score_index].item().toFloat(), INT_MAX*1.0); 44 | auto second2last_intersection_y1 = second2last_box_y1.clamp(all_box_y1[top_score_index].item().toFloat(), INT_MAX*1.0); 45 | auto second2last_intersection_x2 = second2last_box_x2.clamp(INT_MIN*1.0, all_box_x2[top_score_index].item().toFloat()); 46 | auto second2last_intersection_y2 = second2last_box_y2.clamp(INT_MIN*1.0, all_box_y2[top_score_index].item().toFloat()); 47 | 48 | auto second2last_intersection_w = second2last_intersection_x2 - second2last_intersection_x1 + 1; 49 | auto second2last_intersection_h = second2last_intersection_y2 - second2last_intersection_y1 + 1; 50 | //change negative w,h to 0 51 | second2last_intersection_w = second2last_intersection_w.clamp(0., INT_MAX*1.0); 52 | second2last_intersection_h = second2last_intersection_h.clamp(0., INT_MAX*1.0); 53 | 54 | torch::Tensor intersection_areas_between_top_and_others = second2last_intersection_w * second2last_intersection_h; 55 | 56 | torch::Tensor second2last_box_areas = all_box_areas.index_select(0, second2last_index); 57 | 58 | torch::Tensor union_areas_between_top_and_others = (second2last_box_areas - intersection_areas_between_top_and_others) + all_box_areas[top_score_index]; 59 | torch::Tensor iou_between_top_and_others = intersection_areas_between_top_and_others * 1.0 / union_areas_between_top_and_others; 60 | torch::Tensor index_whether_swallowed_by_top = iou_between_top_and_others <= threshold; 61 | auto sorted_index_should_left = torch::nonzero(index_whether_swallowed_by_top).squeeze(); 62 | sorted_index = second2last_index.index_select(0, sorted_index_should_left); 63 | } 64 | return keep.slice(0, 0, keep_index); 65 | } 66 | }//nms 67 | -------------------------------------------------------------------------------- /utils/src/syszux_priorbox.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include "syszux_priorbox.h" 8 | #include 9 | 10 | namespace gemfield_org{ 11 | at::Tensor PriorBox::forward(std::vector&& img_size){ 12 | std::vector t = std::move(img_size); 13 | return this->forward(t); 14 | } 15 | at::Tensor PriorBox::forward(std::vector& img_size){ 16 | std::vector anchors; 17 | std::vector> feature_maps; 18 | 19 | for(int i=0; i(std::ceil(1.0 * img_size[0]/steps_[i])), static_cast(std::ceil(1.0 * img_size[1]/steps_[i])) }); 21 | } 22 | 23 | for(int i=0; i> product_vec; 25 | int fh = feature_maps[i][0]; 26 | int fw = feature_maps[i][1]; 27 | for(int h = 0; h < fh; h++){ 28 | for(int w = 0; w < fw; w++){ 29 | for(auto min_size : min_sizes_[i]){ 30 | float s_kx = 1.0 * min_size / img_size[1]; 31 | float s_ky = 1.0 * min_size / img_size[0]; 32 | float dense_cx = (w + 0.5) * steps_[i] / img_size[1]; 33 | float dense_cy = (h + 0.5) * steps_[i] / img_size[0]; 34 | anchors.insert(anchors.end(), {dense_cx, dense_cy, s_kx, s_ky}); 35 | } 36 | } 37 | } 38 | } 39 | at::Tensor output = at::tensor(anchors).view({-1,4}); 40 | if(clip_){ 41 | output = torch::clamp(output, 1, 0); 42 | } 43 | return output; 44 | } 45 | }//namespace 46 | -------------------------------------------------------------------------------- /utils/src/syszux_stream_buffer.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | #include 7 | #include "syszux_stream_buffer.h" 8 | 9 | SyszuxStreamBuffer::SyszuxStreamBuffer(const uint8_t *begin, const size_t size) :begin_(begin),end_(begin + size),current_(begin_) 10 | { 11 | assert(std::less_equal()(begin_, end_)); 12 | } 13 | 14 | SyszuxStreamBuffer::int_type SyszuxStreamBuffer::underflow() 15 | { 16 | if (current_ == end_) 17 | return traits_type::eof(); 18 | 19 | return traits_type::to_int_type(*current_); 20 | } 21 | 22 | SyszuxStreamBuffer::int_type SyszuxStreamBuffer::uflow() 23 | { 24 | if (current_ == end_) 25 | return traits_type::eof(); 26 | 27 | return traits_type::to_int_type(*current_++); 28 | } 29 | 30 | SyszuxStreamBuffer::int_type SyszuxStreamBuffer::pbackfail(int_type ch) 31 | { 32 | if (current_ == begin_ || (ch != traits_type::eof() && ch != current_[-1])) 33 | return traits_type::eof(); 34 | 35 | return traits_type::to_int_type(*--current_); 36 | } 37 | 38 | std::streamsize SyszuxStreamBuffer::showmanyc() 39 | { 40 | assert(std::less_equal()(current_, end_)); 41 | return end_ - current_; 42 | } 43 | 44 | std::streampos SyszuxStreamBuffer::seekoff ( std::streamoff off, std::ios_base::seekdir way, std::ios_base::openmode which ) 45 | { 46 | if (way == std::ios_base::beg){ 47 | current_ = begin_ + off; 48 | }else if (way == std::ios_base::cur){ 49 | current_ += off; 50 | }else if (way == std::ios_base::end){ 51 | current_ = end_ + off; 52 | } 53 | 54 | if (current_ < begin_ || current_ > end_){ 55 | return -1; 56 | } 57 | 58 | return current_ - begin_; 59 | } 60 | 61 | std::streampos SyszuxStreamBuffer::seekpos ( std::streampos sp, std::ios_base::openmode which ) 62 | { 63 | current_ = begin_ + sp; 64 | 65 | if (current_ < begin_ || current_ > end_){ 66 | return -1; 67 | } 68 | 69 | return current_ - begin_; 70 | } 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /utils/src/syszux_verify_landmark.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020 Gemfield 3 | * This file is part of libdeepvac, licensed under the GPLv3 (the "License") 4 | * You may not use this file except in compliance with the License. 5 | */ 6 | 7 | #include "syszux_verify_landmark.h" 8 | 9 | namespace gemfield_org{ 10 | bool isValidLandmark(std::vector bbox, std::vector landmark, int min_face_size){ 11 | auto width = bbox[2]-bbox[0]; 12 | auto height = bbox[3]-bbox[1]; 13 | if (width < min_face_size || height < min_face_size){ 14 | return false; 15 | } 16 | // landmark x1,y1,x2,y2,...,x5,y5 17 | auto leye_x = landmark[0]; 18 | auto reye_x = landmark[2]; 19 | auto nose_x = landmark[4]; 20 | auto lmouth_x = landmark[6]; 21 | auto rmouth_x = landmark[8]; 22 | 23 | auto lx_eye = nose_x - leye_x; 24 | if(lx_eye <= 0){ 25 | return false; 26 | } 27 | auto rx_eye = reye_x - nose_x; 28 | if(rx_eye <= 0){ 29 | return false; 30 | } 31 | auto eyex = lx_eye / rx_eye; 32 | auto lx_mouth = nose_x - lmouth_x; 33 | if(lx_mouth <= 0){ 34 | return false; 35 | } 36 | auto rx_mouth = rmouth_x - nose_x; 37 | if(rx_mouth <= 0){ 38 | return false; 39 | } 40 | auto mouthx = lx_mouth / rx_mouth; 41 | if((eyex<0.6 || eyex>1.4) || (mouthx<0.5 || mouthx>3)){ 42 | return false; 43 | } 44 | return true; 45 | } 46 | torch::Tensor getDecodeBox(torch::Tensor& prior, torch::Tensor& variances, torch::Tensor& loc) 47 | { 48 | torch::Tensor boxes; 49 | boxes = torch::cat({ torch::add(prior.slice(1, 0, 2), torch::mul(torch::mul(loc.slice(1, 0, 2), variances[0]), prior.slice(1, 2, 4))), 50 | torch::mul(prior.slice(1, 2, 4), torch::exp(torch::mul(loc.slice(1, 2, 4), variances[1])))}, 1); 51 | boxes.slice(1, 0, 2) = torch::sub(boxes.slice(1, 0, 2), torch::div(boxes.slice(1, 2, 4), 2)); 52 | boxes.slice(1, 2, 4) = torch::add(boxes.slice(1, 2, 4), boxes.slice(1, 0, 2)); 53 | return boxes; 54 | } 55 | 56 | void decodeLandmark(torch::Tensor& prior, torch::Tensor& variances, torch::Tensor& landms) 57 | { 58 | landms = torch::cat({ 59 | torch::add(prior.slice(1, 0, 2), torch::mul(torch::mul(landms.slice(1, 0, 2), variances[0]), prior.slice(1, 2, 4))), 60 | torch::add(prior.slice(1, 0, 2), torch::mul(torch::mul(landms.slice(1, 2, 4), variances[0]), prior.slice(1, 2, 4))), 61 | torch::add(prior.slice(1, 0, 2), torch::mul(torch::mul(landms.slice(1, 4, 6), variances[0]), prior.slice(1, 2, 4))), 62 | torch::add(prior.slice(1, 0, 2), torch::mul(torch::mul(landms.slice(1, 6, 8), variances[0]), prior.slice(1, 2, 4))), 63 | torch::add(prior.slice(1, 0, 2), torch::mul(torch::mul(landms.slice(1, 8, 10), variances[0]), prior.slice(1, 2, 4))) 64 | }, 1); 65 | } 66 | }//namespace 67 | --------------------------------------------------------------------------------