├── .github ├── ISSUE_TEMPLATE │ ├── issue-report.md │ └── others.md └── workflows │ ├── ci_android.yml │ ├── ci_arm.yml │ ├── ci_ubuntu.yml │ └── ci_windows.yml ├── .gitignore ├── .gitmodules ├── 00_doc ├── class_diagram.drawio ├── class_diagram.png ├── logo.png ├── overview.drawio └── overview.png ├── 01_script ├── build_run_linux.sh └── build_run_windows.ps1 ├── LICENSE ├── NOTICE.md ├── README.md ├── inference_helper ├── CMakeLists.txt ├── inference_helper.cpp ├── inference_helper.h ├── inference_helper_armnn.cpp ├── inference_helper_armnn.h ├── inference_helper_libtorch.cpp ├── inference_helper_libtorch.h ├── inference_helper_log.h ├── inference_helper_mnn.cpp ├── inference_helper_mnn.h ├── inference_helper_ncnn.cpp ├── inference_helper_ncnn.h ├── inference_helper_nnabla.cpp ├── inference_helper_nnabla.h ├── inference_helper_onnx_runtime.cpp ├── inference_helper_onnx_runtime.h ├── inference_helper_opencv.cpp ├── inference_helper_opencv.h ├── inference_helper_sample.cpp ├── inference_helper_sample.h ├── inference_helper_snpe.cpp ├── inference_helper_snpe.h ├── inference_helper_tensorflow.cpp ├── inference_helper_tensorflow.h ├── inference_helper_tensorflow_lite.cpp ├── inference_helper_tensorflow_lite.h ├── inference_helper_tensorrt.cpp ├── inference_helper_tensorrt.h ├── snpe │ ├── CreateUserBuffer.cpp │ ├── CreateUserBuffer.hpp │ ├── NOTICE.txt │ ├── Util.cpp │ ├── Util.hpp │ ├── udlExample.cpp │ └── udlExample.hpp └── tensorrt │ ├── BatchStream.h │ ├── EntropyCalibrator.h │ ├── ErrorRecorder.h │ ├── calibration │ ├── batchPrepare.py │ ├── sample_org │ │ ├── 000000000139.jpg │ │ ├── 000000000285.jpg │ │ ├── 000000000632.jpg │ │ ├── 000000000724.jpg │ │ ├── 000000000776.jpg │ │ ├── 000000000785.jpg │ │ ├── 000000000802.jpg │ │ ├── 000000000872.jpg │ │ ├── 000000000885.jpg │ │ ├── 000000001000.jpg │ │ ├── 000000001268.jpg │ │ ├── 000000001296.jpg │ │ ├── 000000001353.jpg │ │ ├── 000000001425.jpg │ │ ├── 000000001490.jpg │ │ ├── 000000001503.jpg │ │ ├── 000000001532.jpg │ │ ├── 000000001584.jpg │ │ ├── 000000001675.jpg │ │ └── 000000001761.jpg │ └── sample_ppm │ │ ├── .gitattributes │ │ ├── 000000000139.ppm │ │ ├── 000000000285.ppm │ │ ├── 000000000632.ppm │ │ ├── 000000000724.ppm │ │ ├── 000000000776.ppm │ │ ├── 000000000785.ppm │ │ ├── 000000000802.ppm │ │ ├── 000000000872.ppm │ │ ├── 000000000885.ppm │ │ ├── 000000001000.ppm │ │ ├── 000000001268.ppm │ │ ├── 000000001296.ppm │ │ ├── 000000001353.ppm │ │ ├── 000000001425.ppm │ │ ├── 000000001490.ppm │ │ ├── 000000001503.ppm │ │ ├── 000000001532.ppm │ │ ├── 000000001584.ppm │ │ ├── 000000001675.ppm │ │ ├── 000000001761.ppm │ │ └── list.txt │ ├── common.h │ ├── logger.cpp │ ├── logger.h │ └── logging.h └── third_party ├── cmakes ├── armnn.cmake ├── libtorch.cmake ├── mnn.cmake ├── ncnn.cmake ├── nnabla.cmake ├── onnx_runtime.cmake ├── sample.cmake ├── snpe.cmake ├── tensorflow.cmake ├── tflite.cmake ├── tflite_edgetpu.cmake ├── tflite_edgetpu_pipeline.cmake └── tflite_gpu.cmake └── download_prebuilt_libraries.sh /.github/ISSUE_TEMPLATE/issue-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Issue report 3 | about: issue, bug, question 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Environment (Hardware) 11 | - Hardware: Device, CPU, GPU, etc. 12 | - Software: OS, Compiler, etc. 13 | (Please include version information) 14 | 15 | ## Issue Details 16 | A clear and concise description of what the issue is. 17 | 18 | ## How to Reproduce 19 | Steps to reproduce the behavior. Please include your cmake command. 20 | 21 | ## Error Log 22 | ``` 23 | error log 24 | ``` 25 | 26 | ## Additional Information 27 | Add any other context about the problem here. 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/others.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Others 3 | about: other topics 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/workflows/ci_android.yml: -------------------------------------------------------------------------------- 1 | name: CI Android 2 | 3 | on: 4 | push: 5 | tags: 'v*' 6 | branches: [ master ] 7 | pull_request: 8 | branches: [ master ] 9 | workflow_dispatch: 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-20.04 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Get project source code 18 | run: | 19 | git clone https://github.com/iwatake2222/InferenceHelper_Sample 20 | cd InferenceHelper_Sample 21 | rm -rf InferenceHelper 22 | mv * ../../. 23 | cd ../../ 24 | sh ./InferenceHelper/third_party/download_prebuilt_libraries.sh 1 25 | 26 | - name: Install Requirements 27 | run: | 28 | cd ../ 29 | sudo apt update 30 | sudo apt install -y g++ git cmake wget unzip vulkan-utils libvulkan1 libvulkan-dev 31 | 32 | ### Android NDK ### 33 | wget https://dl.google.com/android/repository/android-ndk-r23b-linux.zip 34 | unzip android-ndk-r23b-linux.zip 35 | export ANDROID_NDK_HOME=`pwd`/android-ndk-r23b 36 | 37 | ### Prepare OpenCV For Android, and don't use rtti to avoid build error in ncnn ### 38 | wget https://github.com/opencv/opencv/releases/download/4.5.4/opencv-4.5.4-android-sdk.zip 39 | unzip opencv-4.5.4-android-sdk.zip 40 | mv OpenCV-android-sdk/sdk ViewAndroid/. 41 | sed -i s/"#define HAVE_OPENCV_FLANN"//g ViewAndroid/sdk/native/jni/include/opencv2/opencv_modules.hpp 42 | 43 | ### Delete setting to specify Inference Helper Type ### 44 | sed -i "/INFERENCE_HELPER_ENABLE/d" ViewAndroid/app/src/main/cpp/CMakeLists.txt 45 | 46 | 47 | - name: Build 48 | run: | 49 | cd ../ 50 | export ANDROID_NDK_HOME=`pwd`/android-ndk-r23b 51 | 52 | cd ViewAndroid/app/src/main/cpp 53 | mkdir -p build && cd build 54 | echo "[CI Building] INFERENCE_HELPER_ENABLE_OPENCV" 55 | cmake .. -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_HOME}/build/cmake/android.toolchain.cmake -DANDROID_ABI="arm64-v8a" -DANDROID_PLATFORM=android-24 -DINFERENCE_HELPER_ENABLE_OPENCV=ON && make -j4 && rm -rf * 56 | echo "[CI Building] INFERENCE_HELPER_ENABLE_TFLITE" 57 | cmake .. -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_HOME}/build/cmake/android.toolchain.cmake -DANDROID_ABI="arm64-v8a" -DANDROID_PLATFORM=android-24 -DINFERENCE_HELPER_ENABLE_TFLITE=ON && make -j4 && rm -rf * 58 | echo "[CI Building] INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_XNNPACK" 59 | cmake .. -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_HOME}/build/cmake/android.toolchain.cmake -DANDROID_ABI="arm64-v8a" -DANDROID_PLATFORM=android-24 -DINFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_XNNPACK=ON && make -j4 && rm -rf * 60 | echo "[CI Building] INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_GPU" 61 | cmake .. -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_HOME}/build/cmake/android.toolchain.cmake -DANDROID_ABI="arm64-v8a" -DANDROID_PLATFORM=android-24 -DINFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_GPU=ON && make -j4 && rm -rf * 62 | echo "[CI Building] INFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_NNAPI" 63 | cmake .. -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_HOME}/build/cmake/android.toolchain.cmake -DANDROID_ABI="arm64-v8a" -DANDROID_PLATFORM=android-24 -DINFERENCE_HELPER_ENABLE_TFLITE_DELEGATE_NNAPI=ON && make -j4 && rm -rf * 64 | echo "[CI Building] INFERENCE_HELPER_ENABLE_NCNN" 65 | cmake .. -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_HOME}/build/cmake/android.toolchain.cmake -DANDROID_ABI="arm64-v8a" -DANDROID_PLATFORM=android-24 -DINFERENCE_HELPER_ENABLE_NCNN=ON && make -j4 && rm -rf * 66 | echo "[CI Building] INFERENCE_HELPER_ENABLE_MNN" 67 | cmake .. -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_HOME}/build/cmake/android.toolchain.cmake -DANDROID_ABI="arm64-v8a" -DANDROID_PLATFORM=android-24 -DINFERENCE_HELPER_ENABLE_MNN=ON && make -j4 && rm -rf * 68 | echo "[CI Building] INFERENCE_HELPER_ENABLE_ONNX_RUNTIME" 69 | cmake .. -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_HOME}/build/cmake/android.toolchain.cmake -DANDROID_ABI="arm64-v8a" -DANDROID_PLATFORM=android-24 -DINFERENCE_HELPER_ENABLE_ONNX_RUNTIME=ON && make -j4 && rm -rf * 70 | -------------------------------------------------------------------------------- /.github/workflows/ci_arm.yml: -------------------------------------------------------------------------------- 1 | name: CI Arm 2 | 3 | on: 4 | push: 5 | tags: 'v*' 6 | branches: [ master ] 7 | pull_request: 8 | branches: [ master ] 9 | workflow_dispatch: 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-18.04 14 | strategy: 15 | matrix: 16 | include: 17 | - arch: armv7 18 | distro: ubuntu18.04 19 | artifact_name: time_inference_armv7.txt 20 | - arch: aarch64 21 | distro: ubuntu20.04 22 | artifact_name: time_inference_aarch64.txt 23 | 24 | steps: 25 | - uses: actions/checkout@v2 26 | - name: Get project source code 27 | run: | 28 | git clone https://github.com/iwatake2222/InferenceHelper_Sample 29 | cd InferenceHelper_Sample 30 | rm -rf InferenceHelper 31 | mv * ../../. 32 | cd ../../ 33 | sh ./InferenceHelper/third_party/download_prebuilt_libraries.sh 1 34 | sh ./download_resource.sh 35 | 36 | - name: Build and Run 37 | uses: uraimo/run-on-arch-action@v2.0.5 38 | with: 39 | arch: ${{ matrix.arch }} 40 | distro: ${{ matrix.distro }} 41 | githubToken: ${{ github.token }} 42 | shell: /bin/sh 43 | dockerRunArgs: | 44 | --volume ${PWD}/../:/InferenceHelper_Sample 45 | install: | 46 | apt-get update -q -y 47 | apt-get install -q -y git g++ cmake 48 | apt-get install -q -y libopencv-dev 49 | apt-get install -q -y vulkan-utils libvulkan1 libvulkan-dev 50 | run: | 51 | cd /InferenceHelper_Sample 52 | echo "inference time" > time_inference_linux.txt 53 | case "${{ matrix.arch }}" in 54 | armv7) 55 | sh ./01_script/build_run_linux.sh TFLITE 56 | # sh ./01_script/build_run_linux.sh TFLITE_DELEGATE_XNNPACK 57 | sh ./01_script/build_run_linux.sh TFLITE_DELEGATE_EDGETPU 1 58 | # sh ./01_script/build_run_linux.sh OPENCV 59 | # sh ./01_script/build_run_linux.sh NCNN 60 | # sh ./01_script/build_run_linux.sh MNN 61 | # sh ./01_script/build_run_linux.sh ONNX_RUNTIME 62 | # sh ./01_script/build_run_linux.sh LIBTORCH 63 | # sh ./01_script/build_run_linux.sh TENSORFLOW 64 | ;; 65 | aarch64) 66 | sh ./01_script/build_run_linux.sh TFLITE 67 | sh ./01_script/build_run_linux.sh TFLITE_DELEGATE_XNNPACK 68 | sh ./01_script/build_run_linux.sh TFLITE_DELEGATE_EDGETPU 1 69 | sh ./01_script/build_run_linux.sh OPENCV 70 | # sh ./01_script/build_run_linux.sh NCNN 71 | sh ./01_script/build_run_linux.sh MNN 72 | sh ./01_script/build_run_linux.sh ARMNN 73 | # sh ./01_script/build_run_linux.sh NNABLA 74 | sh ./01_script/build_run_linux.sh ONNX_RUNTIME 75 | # sh ./01_script/build_run_linux.sh LIBTORCH 76 | # sh ./01_script/build_run_linux.sh TENSORFLOW 77 | ;; 78 | esac 79 | mv ./time_inference_linux.txt "${{ matrix.artifact_name }}" 80 | - name: Move artifacts (because relative paths seems not allowed) 81 | run: mv ../"${{ matrix.artifact_name }}" "${{ matrix.artifact_name }}" 82 | - name: Upload Artifacts 83 | uses: actions/upload-artifact@v2 84 | with: 85 | name: ${{ matrix.artifact_name }} 86 | path: ${{ matrix.artifact_name }} 87 | -------------------------------------------------------------------------------- /.github/workflows/ci_ubuntu.yml: -------------------------------------------------------------------------------- 1 | name: CI Ubuntu 2 | 3 | on: 4 | push: 5 | tags: 'v*' 6 | branches: [ master ] 7 | pull_request: 8 | branches: [ master ] 9 | workflow_dispatch: 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-20.04 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Get project source code 18 | run: | 19 | git clone https://github.com/iwatake2222/InferenceHelper_Sample 20 | cd InferenceHelper_Sample 21 | rm -rf InferenceHelper 22 | mv * ../../. 23 | cd ../../ 24 | sh ./InferenceHelper/third_party/download_prebuilt_libraries.sh 1 25 | sh ./download_resource.sh 26 | 27 | - name: Install Requirements 28 | run: | 29 | sudo apt update 30 | sudo apt install -y g++ git cmake wget unzip vulkan-utils libvulkan1 libvulkan-dev 31 | 32 | # OpenCV for INFERENCE_HELPER_ENABLE_OPENCV 33 | sudo apt install -y libopencv-dev 34 | 35 | # Vulkan for INFERENCE_HELPER_ENABLE_NCNN 36 | wget https://sdk.lunarg.com/sdk/download/latest/linux/vulkan-sdk.tar.gz 37 | tar xzvf vulkan-sdk.tar.gz 38 | export VULKAN_SDK=$(pwd)/1.2.198.1/x86_64 39 | 40 | - name: Build and Run 41 | run: | 42 | export VULKAN_SDK=$(pwd)/1.2.198.1/x86_64 43 | cd ../ 44 | 45 | # Build and Run 46 | echo "inference time" > time_inference_linux.txt 47 | sh ./01_script/build_run_linux.sh TFLITE 48 | sh ./01_script/build_run_linux.sh TFLITE_DELEGATE_XNNPACK 49 | sh ./01_script/build_run_linux.sh TFLITE_DELEGATE_EDGETPU 1 50 | sh ./01_script/build_run_linux.sh OPENCV 51 | sh ./01_script/build_run_linux.sh NCNN 52 | sh ./01_script/build_run_linux.sh MNN 53 | sh ./01_script/build_run_linux.sh ARMNN 54 | # sh ./01_script/build_run_linux.sh NNABLA 55 | sh ./01_script/build_run_linux.sh ONNX_RUNTIME 56 | sh ./01_script/build_run_linux.sh LIBTORCH 57 | sh ./01_script/build_run_linux.sh TENSORFLOW 58 | mv time_inference_linux.txt InferenceHelper/. 59 | 60 | - name: Upload Artifacts 61 | uses: actions/upload-artifact@v2 62 | with: 63 | name: time_inference_linux.txt 64 | path: time_inference_linux.txt 65 | -------------------------------------------------------------------------------- /.github/workflows/ci_windows.yml: -------------------------------------------------------------------------------- 1 | name: CI Windows 2 | 3 | on: 4 | push: 5 | tags: 'v*' 6 | branches: [ master ] 7 | pull_request: 8 | branches: [ master ] 9 | workflow_dispatch: 10 | 11 | jobs: 12 | build: 13 | runs-on: windows-2019 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Get project source code 18 | run: | 19 | git clone https://github.com/iwatake2222/InferenceHelper_Sample 20 | cd InferenceHelper_Sample 21 | Remove-Item -Recurse -Force InferenceHelper 22 | mv * ../../. 23 | cd ../../ 24 | sh ./InferenceHelper/third_party/download_prebuilt_libraries.sh 1 25 | sh ./download_resource.sh 26 | 27 | - name: setup-msbuild 28 | uses: microsoft/setup-msbuild@v1.1 29 | with: 30 | vs-version: '[16.0,16.20)' # Make sure to use Visual Studio 2019 31 | 32 | - name: Install Requirements 33 | run: | 34 | # OpenCV for INFERENCE_HELPER_ENABLE_OPENCV 35 | choco install opencv -Version 4.5.4 36 | 37 | # Vulkan for INFERENCE_HELPER_ENABLE_NCNN 38 | Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.2.189.0/windows/VulkanSDK-1.2.189.0-Installer.exe?Human=true -OutFile VulkanSDK.exe 39 | $installer = Start-Process -FilePath VulkanSDK.exe -Wait -PassThru -ArgumentList @("/S"); 40 | $installer.WaitForExit(); 41 | 42 | - name: Build and Run 43 | shell: powershell 44 | run: | 45 | # Setup OpenCV for INFERENCE_HELPER_ENABLE_OPENCV 46 | $env:OPENCV_DIR="C:/tools/opencv/build/x64/vc15/lib" 47 | $env:Path+=";C:/tools/opencv/build/x64/vc15/bin" 48 | 49 | # Setup Vulkan for INFERENCE_HELPER_ENABLE_NCNN 50 | $env:VULKAN_SDK="C:/VulkanSDK/1.2.189.0" 51 | $env:Path+=";C:/VulkanSDK/1.2.189.0/Bin" 52 | 53 | # Build and Run 54 | cd ../ 55 | if($?) { echo "inference time" > time_inference_windows.txt } 56 | if($?) { ./01_script/build_run_windows.ps1 TFLITE } 57 | if($?) { ./01_script/build_run_windows.ps1 TFLITE_DELEGATE_XNNPACK } 58 | if($?) { ./01_script/build_run_windows.ps1 TFLITE_DELEGATE_EDGETPU -BUILD_ONLY } 59 | if($?) { ./01_script/build_run_windows.ps1 OPENCV } 60 | if($?) { ./01_script/build_run_windows.ps1 NCNN -BUILD_ONLY } 61 | if($?) { ./01_script/build_run_windows.ps1 MNN } 62 | if($?) { ./01_script/build_run_windows.ps1 NNABLA } 63 | if($?) { ./01_script/build_run_windows.ps1 ONNX_RUNTIME } 64 | if($?) { ./01_script/build_run_windows.ps1 LIBTORCH } 65 | if($?) { ./01_script/build_run_windows.ps1 TENSORFLOW } 66 | mv time_inference_windows.txt InferenceHelper/. 67 | exit $LASTEXITCODE 68 | 69 | - name: Upload Artifacts 70 | uses: actions/upload-artifact@v2 71 | with: 72 | name: time_inference_windows.txt 73 | path: time_inference_windows.txt 74 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | build/ 3 | ThirdParty/ 4 | third_party/ 5 | resource/ 6 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "ThirdParty/tensorflow"] 2 | path = third_party/tensorflow 3 | url = https://github.com/tensorflow/tensorflow 4 | [submodule "third_party/tensorflow_deps/abseil-cpp"] 5 | path = third_party/tensorflow_deps/abseil-cpp 6 | url = https://github.com/abseil/abseil-cpp 7 | [submodule "third_party/tensorflow_deps/flatbuffers"] 8 | path = third_party/tensorflow_deps/flatbuffers 9 | url = https://github.com/google/flatbuffers 10 | -------------------------------------------------------------------------------- /00_doc/class_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/00_doc/class_diagram.png -------------------------------------------------------------------------------- /00_doc/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/00_doc/logo.png -------------------------------------------------------------------------------- /00_doc/overview.drawio: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /00_doc/overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/00_doc/overview.png -------------------------------------------------------------------------------- /01_script/build_run_linux.sh: -------------------------------------------------------------------------------- 1 | # docker create -v /mnt/c/iwatake/devel:/root/devel -v /etc/localtime:/etc/localtime:ro -it --name=ubuntu20 ubuntu:20.04 2 | # docker start ubuntu20 3 | # docker exec -it ubuntu20 bash 4 | 5 | # Check if sudo needed 6 | sudo 7 | if [ "$?" -le 10 ] 8 | then 9 | L_SUDO=sudo 10 | else 11 | L_SUDO= 12 | fi 13 | 14 | set -e 15 | 16 | FRAMEWORK_NAME=${1:-"MNN"} 17 | BUILD_ONLY=${2:-0} 18 | LOG_HEADER="[CI_LINUX_${FRAMEWORK_NAME}]" 19 | echo "${LOG_HEADER} Start" 20 | 21 | # ${L_SUDO} apt update 22 | # ${L_SUDO} apt install -y g++ git cmake wget unzip vulkan-utils libvulkan1 libvulkan-dev 23 | 24 | 25 | echo "${LOG_HEADER} Build Start" 26 | cd pj_cls_mobilenet_v2_wo_opencv 27 | rm -rf build 28 | mkdir build && cd build 29 | cmake .. -DINFERENCE_HELPER_ENABLE_${FRAMEWORK_NAME}=on 30 | make -j4 31 | echo "${LOG_HEADER} Build End" 32 | 33 | if [ ${BUILD_ONLY} -ne 0 ]; then 34 | exit 0 35 | fi 36 | 37 | echo "${LOG_HEADER} Run Start" 38 | ./main 39 | # if [ ${?} -ne 0 ]; then 40 | # echo "${LOG_HEADER} Run Error" 41 | # exit 1 42 | # fi 43 | echo "${LOG_HEADER} Run End" 44 | 45 | 46 | echo "$FRAMEWORK_NAME" >> ../../time_inference_linux.txt 47 | cat time_inference.txt >> ../../time_inference_linux.txt 48 | 49 | echo "${LOG_HEADER} End" 50 | -------------------------------------------------------------------------------- /01_script/build_run_windows.ps1: -------------------------------------------------------------------------------- 1 | # Run on Visual Studio 2019 Developer PowerShell 2 | # You may need the following command before executing this script 3 | # Set-ExecutionPolicy Unrestricted -Scope Process 4 | 5 | Param( 6 | [string]$FRAMEWORK_NAME = "MNN", 7 | [switch]$BUILD_ONLY 8 | ) 9 | $LOG_HEADER = "[CI_WINDOWS_${FRAMEWORK_NAME}]" 10 | echo "${LOG_HEADER} Start" 11 | 12 | echo "${LOG_HEADER} Build Start" 13 | if(Test-Path build) { 14 | del -R build 15 | } 16 | mkdir build 17 | cd build 18 | cmake -DINFERENCE_HELPER_ENABLE_"$FRAMEWORK_NAME"=on ../pj_cls_mobilenet_v2_wo_opencv 19 | MSBuild -m:4 ./main.sln /p:Configuration=Release 20 | if(!($?)) { 21 | echo "${LOG_HEADER} Build Error" 22 | cd .. 23 | exit -1 24 | } 25 | echo "${LOG_HEADER} Build End" 26 | 27 | if($BUILD_ONLY) { 28 | cd .. 29 | exit 0 30 | } 31 | 32 | 33 | echo "${LOG_HEADER} Run Start" 34 | ./Release/main.exe 35 | if(!($?)) { 36 | echo "${LOG_HEADER} Run Error" 37 | cd .. 38 | exit -1 39 | } 40 | echo "${LOG_HEADER} Run End" 41 | 42 | cd .. 43 | echo "$FRAMEWORK_NAME" >> time_inference_windows.txt 44 | cat build/time_inference.txt >> time_inference_windows.txt 45 | 46 | echo "${LOG_HEADER} End" 47 | 48 | exit 0 49 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /inference_helper/inference_helper.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2021 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef INFERENCE_HELPER_ 16 | #define INFERENCE_HELPER_ 17 | 18 | /* for general */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | class TensorInfo { 27 | public: 28 | enum { 29 | kTensorTypeNone, 30 | kTensorTypeUint8, 31 | kTensorTypeInt8, 32 | kTensorTypeFp32, 33 | kTensorTypeInt32, 34 | kTensorTypeInt64, 35 | }; 36 | 37 | public: 38 | TensorInfo() 39 | : name("") 40 | , id(-1) 41 | , tensor_type(kTensorTypeNone) 42 | , is_nchw(true) 43 | {} 44 | ~TensorInfo() {} 45 | 46 | int32_t GetElementNum() const 47 | { 48 | int32_t element_num = 1; 49 | for (const auto& dim : tensor_dims) { 50 | element_num *= dim; 51 | } 52 | return element_num; 53 | } 54 | 55 | int32_t GetBatch() const 56 | { 57 | if (tensor_dims.size() <= 0) return -1; 58 | return tensor_dims[0]; 59 | } 60 | 61 | int32_t GetChannel() const 62 | { 63 | if (is_nchw) { 64 | if (tensor_dims.size() <= 1) return -1; 65 | return tensor_dims[1]; 66 | } else { 67 | if (tensor_dims.size() <= 3) return -1; 68 | return tensor_dims[3]; 69 | } 70 | } 71 | 72 | int32_t GetHeight() const 73 | { 74 | if (is_nchw) { 75 | if (tensor_dims.size() <= 2) return -1; 76 | return tensor_dims[2]; 77 | } else { 78 | if (tensor_dims.size() <= 1) return -1; 79 | return tensor_dims[1]; 80 | } 81 | } 82 | 83 | int32_t GetWidth() const 84 | { 85 | if (is_nchw) { 86 | if (tensor_dims.size() <= 3) return -1; 87 | return tensor_dims[3]; 88 | } else { 89 | if (tensor_dims.size() <= 2) return -1; 90 | return tensor_dims[2]; 91 | } 92 | } 93 | 94 | public: 95 | std::string name; // [In] Set the name_ of tensor 96 | int32_t id; // [Out] Do not modify (Used in InferenceHelper) 97 | int32_t tensor_type; // [In] The type of tensor (e.g. kTensorTypeFp32) 98 | std::vector tensor_dims; // InputTensorInfo: [In] The dimentions of tensor. (If empty at initialize, the size is updated from model info.) 99 | // OutputTensorInfo: [Out] The dimentions of tensor is set from model information 100 | bool is_nchw; // [IN] NCHW or NHWC 101 | }; 102 | 103 | class InputTensorInfo : public TensorInfo { 104 | public: 105 | enum { 106 | kDataTypeImage, 107 | kDataTypeBlobNhwc, // data_ which already finished preprocess(color conversion, resize, normalize_, etc.) 108 | kDataTypeBlobNchw, 109 | }; 110 | 111 | public: 112 | InputTensorInfo() 113 | : data(nullptr) 114 | , data_type(kDataTypeImage) 115 | , image_info({ -1, -1, -1, -1, -1, -1, -1, true, false }) 116 | , normalize({ 0.0f, 0.0f, 0.0f, 1.0f, 1.0f, 1.0f }) 117 | {} 118 | 119 | InputTensorInfo(std::string name_, int32_t tensor_type_, bool is_nchw_ = true) 120 | : InputTensorInfo() 121 | { 122 | name = name_; 123 | tensor_type = tensor_type_; 124 | is_nchw = is_nchw_; 125 | } 126 | 127 | ~InputTensorInfo() {} 128 | 129 | public: 130 | void* data; // [In] Set the pointer to image/blob 131 | int32_t data_type; // [In] Set the type of data_ (e.g. kDataTypeImage) 132 | 133 | struct { 134 | int32_t width; 135 | int32_t height; 136 | int32_t channel; 137 | int32_t crop_x; 138 | int32_t crop_y; 139 | int32_t crop_width; 140 | int32_t crop_height; 141 | bool is_bgr; // used when channel == 3 (true: BGR, false: RGB) 142 | bool swap_color; 143 | } image_info; // [In] used when data_type_ == kDataTypeImage 144 | 145 | struct { 146 | float mean[3]; 147 | float norm[3]; 148 | } normalize; // [In] used when data_type_ == kDataTypeImage 149 | }; 150 | 151 | 152 | class OutputTensorInfo : public TensorInfo { 153 | public: 154 | OutputTensorInfo() 155 | : data(nullptr) 156 | , quant({ 1.0f, 0 }) 157 | , data_fp32_(nullptr) 158 | {} 159 | 160 | OutputTensorInfo(std::string name_, int32_t tensor_type_, bool is_nchw_ = true) 161 | : OutputTensorInfo() 162 | { 163 | name = name_; 164 | tensor_type = tensor_type_; 165 | is_nchw = is_nchw_; 166 | } 167 | 168 | ~OutputTensorInfo() { 169 | if (data_fp32_ != nullptr) { 170 | delete[] data_fp32_; 171 | } 172 | } 173 | 174 | float* GetDataAsFloat() { /* Returned pointer should be with const, but returning pointer without const is convenient to create cv::Mat */ 175 | if (tensor_type == kTensorTypeUint8 || tensor_type == kTensorTypeInt8) { 176 | if (data_fp32_ == nullptr) { 177 | data_fp32_ = new float[GetElementNum()]; 178 | } 179 | if (tensor_type == kTensorTypeUint8) { 180 | #pragma omp parallel 181 | for (int32_t i = 0; i < GetElementNum(); i++) { 182 | const uint8_t* val_uint8 = static_cast(data); 183 | float val_float = (val_uint8[i] - quant.zero_point) * quant.scale; 184 | data_fp32_[i] = val_float; 185 | } 186 | } else { 187 | #pragma omp parallel 188 | for (int32_t i = 0; i < GetElementNum(); i++) { 189 | const int8_t* val_int8 = static_cast(data); 190 | float val_float = (val_int8[i] - quant.zero_point) * quant.scale; 191 | data_fp32_[i] = val_float; 192 | } 193 | } 194 | return data_fp32_; 195 | } else if (tensor_type == kTensorTypeFp32) { 196 | return static_cast(data); 197 | } else { 198 | return nullptr; 199 | } 200 | } 201 | 202 | public: 203 | void* data; // [Out] Pointer to the output data_ 204 | struct { 205 | float scale; 206 | int32_t zero_point; 207 | } quant; // [Out] Parameters for dequantization (convert uint8 to float) 208 | 209 | private: 210 | float* data_fp32_; 211 | }; 212 | 213 | 214 | namespace cv { 215 | class Mat; 216 | }; 217 | 218 | class InferenceHelper { 219 | public: 220 | enum { 221 | kRetOk = 0, 222 | kRetErr = -1, 223 | }; 224 | 225 | typedef enum { 226 | kOpencv, 227 | kOpencvGpu, 228 | kTensorflowLite, 229 | kTensorflowLiteXnnpack, 230 | kTensorflowLiteGpu, 231 | kTensorflowLiteEdgetpu, 232 | kTensorflowLiteNnapi, 233 | kTensorrt, 234 | kNcnn, 235 | kNcnnVulkan, 236 | kMnn, 237 | kSnpe, 238 | kArmnn, 239 | kNnabla, 240 | kNnablaCuda, 241 | kOnnxRuntime, 242 | kOnnxRuntimeCuda, 243 | kLibtorch, 244 | kLibtorchCuda, 245 | kTensorflow, 246 | kTensorflowGpu, 247 | kSample, 248 | } HelperType; 249 | 250 | public: 251 | static InferenceHelper* Create(const HelperType helper_type); 252 | static void PreProcessByOpenCV(const InputTensorInfo& input_tensor_info, bool is_nchw, cv::Mat& img_blob); // use this if the selected inference engine doesn't support pre-process 253 | 254 | public: 255 | virtual ~InferenceHelper() {} 256 | virtual int32_t SetNumThreads(const int32_t num_threads) = 0; 257 | virtual int32_t SetCustomOps(const std::vector>& custom_ops) = 0; 258 | virtual int32_t Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) = 0; 259 | virtual int32_t Finalize(void) = 0; 260 | virtual int32_t PreProcess(const std::vector& input_tensor_info_list) = 0; 261 | virtual int32_t Process(std::vector& output_tensor_info_list) = 0; 262 | 263 | protected: 264 | void ConvertNormalizeParameters(InputTensorInfo& tensor_info); 265 | 266 | void PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, float* dst); 267 | void PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, uint8_t* dst); 268 | void PreProcessImage(int32_t num_thread, const InputTensorInfo& input_tensor_info, int8_t* dst); 269 | 270 | template 271 | void PreProcessBlob(int32_t num_thread, const InputTensorInfo& input_tensor_info, T *dst); 272 | 273 | protected: 274 | HelperType helper_type_; 275 | }; 276 | 277 | #endif 278 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_armnn.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2021 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef INFERENCE_HELPER_ARMNN_ 16 | #define INFERENCE_HELPER_ARMNN_ 17 | 18 | /* for general */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | /* for My modules */ 27 | #include "inference_helper.h" 28 | 29 | class ArmnnWrapper; 30 | 31 | class InferenceHelperArmnn : public InferenceHelper { 32 | public: 33 | InferenceHelperArmnn(); 34 | ~InferenceHelperArmnn() override = default; 35 | int32_t SetNumThreads(const int32_t num_threads) override; 36 | int32_t SetCustomOps(const std::vector>& custom_ops) override; 37 | int32_t Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) override; 38 | int32_t Finalize(void) override; 39 | int32_t PreProcess(const std::vector& input_tensor_info_list) override; 40 | int32_t Process(std::vector& output_tensor_info_list) override; 41 | 42 | private: 43 | int32_t num_threads_; 44 | std::unique_ptr armnn_wrapper_; 45 | 46 | }; 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_libtorch.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2022 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | /*** Include ***/ 16 | /* for general */ 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #ifdef _WIN32 29 | #include 30 | #endif 31 | 32 | /* for LibTorch */ 33 | //#include 34 | #include // One-stop header. 35 | #include 36 | #include 37 | #include 38 | 39 | /* for My modules */ 40 | #include "inference_helper_log.h" 41 | #include "inference_helper_libtorch.h" 42 | 43 | /*** Macro ***/ 44 | #define TAG "InferenceHelperLibtorch" 45 | #define PRINT(...) INFERENCE_HELPER_LOG_PRINT(TAG, __VA_ARGS__) 46 | #define PRINT_E(...) INFERENCE_HELPER_LOG_PRINT_E(TAG, __VA_ARGS__) 47 | 48 | 49 | /*** Function ***/ 50 | InferenceHelperLibtorch::InferenceHelperLibtorch() 51 | { 52 | num_threads_ = 1; 53 | } 54 | 55 | InferenceHelperLibtorch::~InferenceHelperLibtorch() 56 | { 57 | } 58 | 59 | int32_t InferenceHelperLibtorch::SetNumThreads(const int32_t num_threads) 60 | { 61 | num_threads_ = num_threads; 62 | torch::set_num_threads(num_threads_); 63 | return kRetOk; 64 | } 65 | 66 | int32_t InferenceHelperLibtorch::SetCustomOps(const std::vector>& custom_ops) 67 | { 68 | PRINT("[WARNING] This method is not supported\n"); 69 | return kRetOk; 70 | } 71 | 72 | int32_t InferenceHelperLibtorch::Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) 73 | { 74 | /*** Note 75 | * Do not analyze model information. 76 | * The order of model inputs/ontputs must be the same as that of input_tensor_info_list/output_tensor_info_list 77 | */ 78 | 79 | /*** Check CUDA ***/ 80 | if (torch::cuda::is_available()) { 81 | PRINT("CUDA is available\n"); 82 | if (helper_type_ == InferenceHelper::kLibtorchCuda) { 83 | device_type_ = torch::kCUDA; 84 | } else { 85 | device_type_ = torch::kCPU; 86 | } 87 | } else { 88 | PRINT("CUDA is not available\n"); 89 | if (helper_type_ == InferenceHelper::kLibtorchCuda) { 90 | PRINT("[WARNING] kLibtorchCuda is selected, but CUDA is not available\n"); 91 | device_type_ = torch::kCPU; 92 | } else { 93 | device_type_ = torch::kCPU; 94 | } 95 | } 96 | 97 | /*** Load model ***/ 98 | try { 99 | module_ = torch::jit::load(model_filename); 100 | } 101 | catch (const c10::Error& e) { 102 | PRINT_E("[ERROR] Unable to load model %s: %s\n", model_filename.c_str(), e.what()); 103 | return kRetErr; 104 | } 105 | module_.to(device_type_); 106 | module_.eval(); 107 | 108 | /*** Convert normalize parameter to speed up ***/ 109 | for (auto& input_tensor_info : input_tensor_info_list) { 110 | ConvertNormalizeParameters(input_tensor_info); 111 | } 112 | 113 | return kRetOk; 114 | }; 115 | 116 | int32_t InferenceHelperLibtorch::Finalize(void) 117 | { 118 | return kRetOk; 119 | } 120 | 121 | 122 | int32_t InferenceHelperLibtorch::PreProcess(const std::vector& input_tensor_info_list) 123 | { 124 | /*** Allocate input tensor every frame ***/ 125 | /* We need this only for the first time for kCPU, but tensor device changes for kCUDA. So, We need to reallocate it */ 126 | /* Todo: there may be a way to reuse allocated GPU memory */ 127 | input_tensor_list_.clear(); 128 | 129 | /*** Normalize input data and store the converted data into the input tensor buffer ***/ 130 | for (size_t input_tensor_index = 0; input_tensor_index < input_tensor_info_list.size(); input_tensor_index++) { 131 | const auto& input_tensor_info = input_tensor_info_list[input_tensor_index]; 132 | const int32_t img_width = input_tensor_info.GetWidth(); 133 | const int32_t img_height = input_tensor_info.GetHeight(); 134 | const int32_t img_channel = input_tensor_info.GetChannel(); 135 | 136 | torch::TensorOptions tensor_options; 137 | if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeFp32) { 138 | tensor_options = torch::TensorOptions().dtype(torch::kFloat32).requires_grad(false); 139 | } 140 | std::vector sizes; 141 | for (auto v : input_tensor_info.tensor_dims) { 142 | sizes.push_back(v); 143 | } 144 | torch::Tensor input_tensor = torch::zeros(sizes, tensor_options); 145 | 146 | 147 | if (input_tensor_info.data_type == InputTensorInfo::kDataTypeImage) { 148 | if ((input_tensor_info.image_info.width != input_tensor_info.image_info.crop_width) || (input_tensor_info.image_info.height != input_tensor_info.image_info.crop_height)) { 149 | PRINT_E("Crop is not supported\n"); 150 | return kRetErr; 151 | } 152 | if ((input_tensor_info.image_info.crop_width != img_width) || (input_tensor_info.image_info.crop_height != img_height)) { 153 | PRINT_E("Resize is not supported\n"); 154 | return kRetErr; 155 | } 156 | if (input_tensor_info.image_info.channel != img_channel) { 157 | PRINT_E("Color conversion is not supported\n"); 158 | return kRetErr; 159 | } 160 | 161 | /* Normalize image */ 162 | if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeFp32) { 163 | float* dst = (float*)(input_tensor.data_ptr()); 164 | PreProcessImage(num_threads_, input_tensor_info, dst); 165 | } else if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeUint8) { 166 | uint8_t* dst = (uint8_t*)(input_tensor.data_ptr()); 167 | PreProcessImage(num_threads_, input_tensor_info, dst); 168 | } else if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeInt8) { 169 | int8_t* dst = (int8_t*)(input_tensor.data_ptr()); 170 | PreProcessImage(num_threads_, input_tensor_info, dst); 171 | } else { 172 | PRINT_E("Unsupported tensor_type (%d)\n", input_tensor_info.tensor_type); 173 | return kRetErr; 174 | } 175 | } else if ((input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNhwc) || (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNchw)) { 176 | if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeFp32) { 177 | float* dst = (float*)(input_tensor.data_ptr()); 178 | PreProcessBlob(num_threads_, input_tensor_info, dst); 179 | } else if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeUint8 || input_tensor_info.tensor_type == TensorInfo::kTensorTypeInt8) { 180 | uint8_t* dst = (uint8_t*)(input_tensor.data_ptr()); 181 | PreProcessBlob(num_threads_, input_tensor_info, dst); 182 | } else if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeInt32) { 183 | int32_t* dst = (int32_t*)(input_tensor.data_ptr()); 184 | PreProcessBlob(num_threads_, input_tensor_info, dst); 185 | } else { 186 | PRINT_E("Unsupported tensor_type (%d)\n", input_tensor_info.tensor_type); 187 | return kRetErr; 188 | } 189 | } else { 190 | PRINT_E("Unsupported data_type (%d)\n", input_tensor_info.data_type); 191 | return kRetErr; 192 | } 193 | 194 | input_tensor_list_.push_back(input_tensor.to(device_type_)); 195 | } 196 | 197 | return kRetOk; 198 | } 199 | 200 | int32_t InferenceHelperLibtorch::Process(std::vector& output_tensor_info_list) 201 | { 202 | /*** Inference ***/ 203 | torch::jit::IValue outputs; 204 | try { 205 | outputs = module_.forward(input_tensor_list_); 206 | } catch (std::exception& e) { 207 | PRINT("Error at forward: %s\n", e.what()); 208 | } 209 | 210 | /*** Extract output tensor data and save them to output_tensor_list_ ***/ 211 | output_tensor_list_.clear(); 212 | if (outputs.isTensor()) { 213 | torch::Tensor output_tensor = outputs.toTensor().to(torch::kCPU); 214 | output_tensor_list_.emplace_back(output_tensor); 215 | //std::cout << output_tensor << std::endl; 216 | } else if (outputs.isTuple()) { 217 | PRINT("Multiple output is not tested\n"); 218 | const auto& output_tuple = outputs.toTuple()->elements(); 219 | for (const auto& o : output_tuple) { 220 | torch::Tensor output_tensor = o.toTensor().to(torch::kCPU); 221 | output_tensor_list_.emplace_back(output_tensor); 222 | } 223 | // } else if (outputs.isTensorList()) { 224 | // PRINT("Multiple output is not tested\n"); 225 | // const auto& output_list = outputs.toTensorList(); 226 | // for (const auto& o : output_list) { 227 | // torch::Tensor output_tensor = o; 228 | // output_tensor = output_tensor.to(torch::kCPU); 229 | // output_tensor_list_.emplace_back(output_tensor); 230 | // } 231 | } else { 232 | PRINT_E("Invalid output format\n"); 233 | return kRetErr; 234 | } 235 | 236 | /*** Set output data for caller ***/ 237 | if (output_tensor_list_.size() != output_tensor_info_list.size()) { 238 | PRINT_E("The num of output tensors doesn't match. Model has %zu output, but code expects %zu\n", output_tensor_list_.size(), output_tensor_info_list.size()); 239 | } 240 | 241 | for (size_t i = 0; i < output_tensor_list_.size(); i++) { 242 | const auto& output_tensor = output_tensor_list_[i]; 243 | auto& tensor_info = output_tensor_info_list[i]; 244 | int32_t ndim = output_tensor.dim(); 245 | tensor_info.tensor_dims.clear(); 246 | for (int idim = 0; idim < ndim; idim++) { 247 | tensor_info.tensor_dims.push_back(output_tensor.size(idim)); 248 | } 249 | tensor_info.data = output_tensor.data_ptr(); 250 | } 251 | 252 | return kRetOk; 253 | } 254 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_libtorch.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2022 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef INFERENCE_HELPER_LIBTORCH_ 16 | #define INFERENCE_HELPER_LIBTORCH_ 17 | 18 | /* for general */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | /* for LibTorch */ 27 | //#include 28 | #include // One-stop header. 29 | 30 | /* for My modules */ 31 | #include "inference_helper.h" 32 | 33 | class InferenceHelperLibtorch : public InferenceHelper { 34 | public: 35 | InferenceHelperLibtorch(); 36 | ~InferenceHelperLibtorch() override; 37 | int32_t SetNumThreads(const int32_t num_threads) override; 38 | int32_t SetCustomOps(const std::vector>& custom_ops) override; 39 | int32_t Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) override; 40 | int32_t Finalize(void) override; 41 | int32_t PreProcess(const std::vector& input_tensor_info_list) override; 42 | int32_t Process(std::vector& output_tensor_info_list) override; 43 | 44 | private: 45 | int32_t num_threads_; 46 | 47 | torch::jit::script::Module module_; 48 | torch::DeviceType device_type_; 49 | std::vector input_tensor_list_; 50 | std::vector output_tensor_list_; 51 | 52 | }; 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_log.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2021 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef INFERENCE_HELPER_LOG_ 16 | #define INFERENCE_HELPER_LOG_ 17 | 18 | /* for general */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | 26 | #if defined(ANDROID) || defined(__ANDROID__) 27 | #define CV_COLOR_IS_RGB 28 | #include 29 | #define INFERENCE_HELPER_LOG_NDK_TAG "MyApp_NDK" 30 | #define INFERENCE_HELPER_LOG_PRINT_(...) __android_log_print(ANDROID_LOG_INFO, INFERENCE_HELPER_LOG_NDK_TAG, __VA_ARGS__) 31 | #else 32 | #define INFERENCE_HELPER_LOG_PRINT_(...) printf(__VA_ARGS__) 33 | #endif 34 | 35 | #define INFERENCE_HELPER_LOG_PRINT(INFERENCE_HELPER_LOG_PRINT_TAG, ...) do { \ 36 | INFERENCE_HELPER_LOG_PRINT_("[" INFERENCE_HELPER_LOG_PRINT_TAG "][%d] ", __LINE__); \ 37 | INFERENCE_HELPER_LOG_PRINT_(__VA_ARGS__); \ 38 | } while(0); 39 | 40 | #define INFERENCE_HELPER_LOG_PRINT_E(INFERENCE_HELPER_LOG_PRINT_TAG, ...) do { \ 41 | INFERENCE_HELPER_LOG_PRINT_("[ERR: " INFERENCE_HELPER_LOG_PRINT_TAG "][%d] ", __LINE__); \ 42 | INFERENCE_HELPER_LOG_PRINT_(__VA_ARGS__); \ 43 | } while(0); 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_mnn.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2021 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef INFERENCE_HELPER_MNN_ 16 | #define INFERENCE_HELPER_MNN_ 17 | 18 | /* for general */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | /* for MNN */ 27 | #include 28 | #include 29 | #include 30 | 31 | /* for My modules */ 32 | #include "inference_helper.h" 33 | 34 | class InferenceHelperMnn : public InferenceHelper { 35 | public: 36 | InferenceHelperMnn(); 37 | ~InferenceHelperMnn() override; 38 | int32_t SetNumThreads(const int32_t num_threads) override; 39 | int32_t SetCustomOps(const std::vector>& custom_ops) override; 40 | int32_t Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) override; 41 | int32_t Finalize(void) override; 42 | int32_t PreProcess(const std::vector& input_tensor_info_list) override; 43 | int32_t Process(std::vector& output_tensor_info_list) override; 44 | 45 | private: 46 | std::unique_ptr net_; 47 | MNN::Session* session_; 48 | std::vector> out_mat_list_; 49 | int32_t num_threads_; 50 | }; 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_ncnn.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2021 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | /*** Include ***/ 16 | /* for general */ 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | /* for ncnn */ 28 | #include "net.h" 29 | 30 | /* for My modules */ 31 | #include "inference_helper_log.h" 32 | #include "inference_helper_ncnn.h" 33 | 34 | /*** Macro ***/ 35 | #define TAG "InferenceHelperNcnn" 36 | #define PRINT(...) INFERENCE_HELPER_LOG_PRINT(TAG, __VA_ARGS__) 37 | #define PRINT_E(...) INFERENCE_HELPER_LOG_PRINT_E(TAG, __VA_ARGS__) 38 | 39 | /*** Function ***/ 40 | /* Reference: https://github.com/Tencent/ncnn/blob/master/examples/yolox.cpp */ 41 | class YoloV5Focus : public ncnn::Layer 42 | { 43 | public: 44 | YoloV5Focus() 45 | { 46 | one_blob_only = true; 47 | } 48 | 49 | virtual int forward(const ncnn::Mat& bottom_blob, ncnn::Mat& top_blob, const ncnn::Option& opt) const 50 | { 51 | int w = bottom_blob.w; 52 | int h = bottom_blob.h; 53 | int channels = bottom_blob.c; 54 | 55 | int outw = w / 2; 56 | int outh = h / 2; 57 | int outc = channels * 4; 58 | 59 | top_blob.create(outw, outh, outc, 4u, 1, opt.blob_allocator); 60 | if (top_blob.empty()) 61 | return -100; 62 | 63 | #pragma omp parallel for num_threads(opt.num_threads) 64 | for (int p = 0; p < outc; p++) 65 | { 66 | const float* ptr = bottom_blob.channel(p % channels).row((p / channels) % 2) + ((p / channels) / 2); 67 | float* outptr = top_blob.channel(p); 68 | 69 | for (int i = 0; i < outh; i++) 70 | { 71 | for (int j = 0; j < outw; j++) 72 | { 73 | *outptr = *ptr; 74 | 75 | outptr += 1; 76 | ptr += 2; 77 | } 78 | 79 | ptr += w; 80 | } 81 | } 82 | 83 | return 0; 84 | } 85 | }; 86 | DEFINE_LAYER_CREATOR(YoloV5Focus) 87 | 88 | InferenceHelperNcnn::InferenceHelperNcnn() 89 | { 90 | custom_ops_.clear(); 91 | custom_ops_.push_back(std::pair("YoloV5Focus", (const void*)YoloV5Focus_layer_creator)); 92 | num_threads_ = 1; 93 | } 94 | 95 | InferenceHelperNcnn::~InferenceHelperNcnn() 96 | { 97 | } 98 | 99 | int32_t InferenceHelperNcnn::SetNumThreads(const int32_t num_threads) 100 | { 101 | num_threads_ = num_threads; 102 | return kRetOk; 103 | } 104 | 105 | int32_t InferenceHelperNcnn::SetCustomOps(const std::vector>& custom_ops) 106 | { 107 | for (auto op : custom_ops) { 108 | custom_ops_.push_back(op); 109 | } 110 | return kRetOk; 111 | } 112 | 113 | int32_t InferenceHelperNcnn::Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) 114 | { 115 | /*** Create network ***/ 116 | net_.reset(new ncnn::Net()); 117 | net_->opt.use_fp16_arithmetic = true; 118 | net_->opt.use_fp16_packed = true; 119 | net_->opt.use_fp16_storage = true; 120 | if (helper_type_ == kNcnnVulkan) { 121 | net_->opt.use_vulkan_compute = 1; 122 | } 123 | 124 | for (auto op : custom_ops_) { 125 | net_->register_custom_layer(op.first, (ncnn::layer_creator_func)(op.second)); 126 | } 127 | 128 | std::string bin_filename = model_filename; 129 | if (model_filename.find(".param") == std::string::npos) { 130 | PRINT_E("Invalid model param filename (%s)\n", model_filename.c_str()); 131 | return kRetErr; 132 | } 133 | bin_filename = bin_filename.replace(bin_filename.find(".param"), std::string(".param").length(), ".bin\0"); 134 | if (net_->load_param(model_filename.c_str()) != 0) { 135 | PRINT_E("Failed to load model param file (%s)\n", model_filename.c_str()); 136 | return kRetErr; 137 | } 138 | if (net_->load_model(bin_filename.c_str()) != 0) { 139 | PRINT_E("Failed to load model bin file (%s)\n", bin_filename.c_str()); 140 | return kRetErr; 141 | } 142 | 143 | /* Convert normalize parameter to speed up */ 144 | for (auto& input_tensor_info : input_tensor_info_list) { 145 | ConvertNormalizeParameters(input_tensor_info); 146 | } 147 | 148 | /* Check if tensor info is set */ 149 | for (const auto& input_tensor_info : input_tensor_info_list) { 150 | for (const auto& dim : input_tensor_info.tensor_dims) { 151 | if (dim <= 0) { 152 | PRINT_E("Invalid tensor size\n"); 153 | return kRetErr; 154 | } 155 | } 156 | } 157 | //for (const auto& output_tensor_info : output_tensor_info_list) { 158 | // for (const auto& dim : output_tensor_info.tensor_dims) { 159 | // if (dim <= 0) { 160 | // PRINT_E("Invalid tensor size\n"); 161 | // return kRetErr; 162 | // } 163 | // } 164 | //} 165 | 166 | return kRetOk; 167 | }; 168 | 169 | 170 | int32_t InferenceHelperNcnn::Finalize(void) 171 | { 172 | net_.reset(); 173 | in_mat_list_.clear(); 174 | out_mat_list_.clear(); 175 | if (helper_type_ == kNcnnVulkan) { 176 | ncnn::destroy_gpu_instance(); 177 | } 178 | return kRetErr; 179 | } 180 | 181 | int32_t InferenceHelperNcnn::PreProcess(const std::vector& input_tensor_info_list) 182 | { 183 | in_mat_list_.clear(); 184 | for (const auto& input_tensor_info : input_tensor_info_list) { 185 | ncnn::Mat ncnn_mat; 186 | if (input_tensor_info.data_type == InputTensorInfo::kDataTypeImage) { 187 | /* Crop */ 188 | if ((input_tensor_info.image_info.width != input_tensor_info.image_info.crop_width) || (input_tensor_info.image_info.height != input_tensor_info.image_info.crop_height)) { 189 | PRINT_E("Crop is not supported\n"); 190 | return kRetErr; 191 | } 192 | /* Convert color type */ 193 | int32_t pixel_type = 0; 194 | if ((input_tensor_info.image_info.channel == 3) && (input_tensor_info.GetChannel() == 3)) { 195 | pixel_type = (input_tensor_info.image_info.is_bgr) ? ncnn::Mat::PIXEL_BGR : ncnn::Mat::PIXEL_RGB; 196 | if (input_tensor_info.image_info.swap_color) { 197 | pixel_type = (input_tensor_info.image_info.is_bgr) ? ncnn::Mat::PIXEL_BGR2RGB : ncnn::Mat::PIXEL_RGB2BGR; 198 | } 199 | } else if ((input_tensor_info.image_info.channel == 1) && (input_tensor_info.GetChannel() == 1)) { 200 | pixel_type = ncnn::Mat::PIXEL_GRAY; 201 | } else if ((input_tensor_info.image_info.channel == 3) && (input_tensor_info.GetChannel() == 1)) { 202 | pixel_type = (input_tensor_info.image_info.is_bgr) ? ncnn::Mat::PIXEL_BGR2GRAY : ncnn::Mat::PIXEL_RGB2GRAY; 203 | } else if ((input_tensor_info.image_info.channel == 1) && (input_tensor_info.GetChannel() == 3)) { 204 | pixel_type = ncnn::Mat::PIXEL_GRAY2RGB; 205 | } else { 206 | PRINT_E("Unsupported color conversion (%d, %d)\n", input_tensor_info.image_info.channel, input_tensor_info.GetChannel()); 207 | return kRetErr; 208 | } 209 | 210 | if (input_tensor_info.image_info.crop_width == input_tensor_info.GetWidth() && input_tensor_info.image_info.crop_height == input_tensor_info.GetHeight()) { 211 | /* Convert to blob */ 212 | ncnn_mat = ncnn::Mat::from_pixels((uint8_t*)input_tensor_info.data, pixel_type, input_tensor_info.image_info.width, input_tensor_info.image_info.height); 213 | } else { 214 | /* Convert to blob with resize */ 215 | ncnn_mat = ncnn::Mat::from_pixels_resize((uint8_t*)input_tensor_info.data, pixel_type, input_tensor_info.image_info.width, input_tensor_info.image_info.height, input_tensor_info.GetWidth(), input_tensor_info.GetHeight()); 216 | } 217 | /* Normalize image */ 218 | ncnn_mat.substract_mean_normalize(input_tensor_info.normalize.mean, input_tensor_info.normalize.norm); 219 | } else if (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNhwc) { 220 | PRINT_E("[ToDo] Unsupported data type (%d)\n", input_tensor_info.data_type); 221 | ncnn_mat = ncnn::Mat::from_pixels((uint8_t*)input_tensor_info.data, input_tensor_info.GetChannel() == 3 ? ncnn::Mat::PIXEL_RGB : ncnn::Mat::PIXEL_GRAY, input_tensor_info.GetWidth(), input_tensor_info.GetHeight()); 222 | } else if (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNchw) { 223 | ncnn_mat = ncnn::Mat(input_tensor_info.GetWidth(), input_tensor_info.GetHeight(), input_tensor_info.GetChannel(), input_tensor_info.data); 224 | } else { 225 | PRINT_E("Unsupported data type (%d)\n", input_tensor_info.data_type); 226 | return kRetErr; 227 | } 228 | in_mat_list_.push_back(std::pair(input_tensor_info.name, ncnn_mat)); 229 | } 230 | return kRetOk; 231 | } 232 | 233 | int32_t InferenceHelperNcnn::Process(std::vector& output_tensor_info_list) 234 | { 235 | ncnn::Extractor ex = net_->create_extractor(); 236 | ex.set_light_mode(true); 237 | ex.set_num_threads(num_threads_); 238 | for (const auto& inputMat : in_mat_list_) { 239 | if (ex.input(inputMat.first.c_str(), inputMat.second) != 0) { 240 | PRINT_E("Input mat error (%s)\n", inputMat.first.c_str()); 241 | return kRetErr; 242 | } 243 | } 244 | 245 | out_mat_list_.clear(); 246 | for (auto& output_tensor_info : output_tensor_info_list) { 247 | ncnn::Mat ncnn_out; 248 | if (ex.extract(output_tensor_info.name.c_str(), ncnn_out) != 0) { 249 | PRINT_E("Output mat error (%s)\n", output_tensor_info.name.c_str()); 250 | return kRetErr; 251 | } 252 | out_mat_list_.push_back(ncnn_out); // store ncnn mat in member variable so that data keep exist 253 | output_tensor_info.data = ncnn_out.data; 254 | output_tensor_info.tensor_dims.clear(); 255 | output_tensor_info.tensor_dims.push_back(1); 256 | output_tensor_info.tensor_dims.push_back(ncnn_out.c); 257 | output_tensor_info.tensor_dims.push_back(ncnn_out.h); 258 | output_tensor_info.tensor_dims.push_back(ncnn_out.w); 259 | } 260 | 261 | return kRetOk; 262 | } 263 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_ncnn.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2021 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef INFERENCE_HELPER_NCNN_ 16 | #define INFERENCE_HELPER_NCNN_ 17 | 18 | /* for general */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | /* for ncnn */ 27 | #include "net.h" 28 | 29 | /* for My modules */ 30 | #include "inference_helper.h" 31 | 32 | class InferenceHelperNcnn : public InferenceHelper { 33 | public: 34 | InferenceHelperNcnn(); 35 | ~InferenceHelperNcnn() override; 36 | int32_t SetNumThreads(const int32_t num_threads) override; 37 | int32_t SetCustomOps(const std::vector>& custom_ops) override; 38 | int32_t Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) override; 39 | int32_t Finalize(void) override; 40 | int32_t PreProcess(const std::vector& input_tensor_info_list) override; 41 | int32_t Process(std::vector& output_tensor_info_list) override; 42 | 43 | private: 44 | std::unique_ptr net_; 45 | std::vector> in_mat_list_; // 46 | std::vector out_mat_list_; 47 | int32_t num_threads_; 48 | std::vector> custom_ops_; 49 | }; 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_nnabla.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2021 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef INFERENCE_HELPER_NNABLA_ 16 | #define INFERENCE_HELPER_NNABLA_ 17 | 18 | /* for general */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | /* for nnabla */ 27 | namespace nbla { 28 | class Variable; 29 | class Context; 30 | namespace utils { 31 | namespace nnp { 32 | class Nnp; 33 | class Executor; 34 | } 35 | } 36 | } 37 | 38 | /* for My modules */ 39 | #include "inference_helper.h" 40 | 41 | class InferenceHelperNnabla : public InferenceHelper { 42 | public: 43 | InferenceHelperNnabla(); 44 | ~InferenceHelperNnabla() override; 45 | int32_t SetNumThreads(const int32_t num_threads) override; 46 | int32_t SetCustomOps(const std::vector>& custom_ops) override; 47 | int32_t Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) override; 48 | int32_t Finalize(void) override; 49 | int32_t PreProcess(const std::vector& input_tensor_info_list) override; 50 | int32_t Process(std::vector& output_tensor_info_list) override; 51 | 52 | private: 53 | void DisplayModelInfo(); 54 | int32_t CheckTensorInfo(TensorInfo& tensor_info, const std::shared_ptr variable); 55 | int32_t AllocateBuffers(std::vector& input_tensor_info_list, std::vector& output_tensor_info_list); 56 | std::shared_ptr GetInputVariable(int32_t index); 57 | std::shared_ptr GetOutputVariable(int32_t index); 58 | 59 | private: 60 | int32_t num_threads_; 61 | std::shared_ptr ctx_cpu_; 62 | std::shared_ptr ctx_gpu_; 63 | std::shared_ptr nnp_; 64 | std::shared_ptr executor_; 65 | }; 66 | 67 | #endif 68 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_onnx_runtime.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2022 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef INFERENCE_HELPER_ONNX_RUNTIME_ 16 | #define INFERENCE_HELPER_ONNX_RUNTIME_ 17 | 18 | /* for general */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | /* for ONNX Runtime */ 27 | #include 28 | 29 | /* for My modules */ 30 | #include "inference_helper.h" 31 | 32 | class InferenceHelperOnnxRuntime : public InferenceHelper { 33 | public: 34 | InferenceHelperOnnxRuntime(); 35 | ~InferenceHelperOnnxRuntime() override; 36 | int32_t SetNumThreads(const int32_t num_threads) override; 37 | int32_t SetCustomOps(const std::vector>& custom_ops) override; 38 | int32_t Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) override; 39 | int32_t Finalize(void) override; 40 | int32_t PreProcess(const std::vector& input_tensor_info_list) override; 41 | int32_t Process(std::vector& output_tensor_info_list) override; 42 | 43 | private: 44 | int32_t AllocateTensor(bool is_input, size_t index, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list); 45 | 46 | private: 47 | int32_t num_threads_; 48 | 49 | Ort::Session session_{ nullptr }; 50 | Ort::Env env_{ ORT_LOGGING_LEVEL_WARNING, "Default" }; 51 | std::vector input_name_list_; 52 | std::vector output_name_list_; 53 | std::vector input_tensor_list_; 54 | std::vector output_tensor_list_; 55 | std::vector> input_buffer_list_; 56 | std::vector> output_buffer_list_; 57 | }; 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_opencv.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2021 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef INFERENCE_HELPER_OPENCV_ 16 | #define INFERENCE_HELPER_OPENCV_ 17 | 18 | /* for general */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | /* for OpenCV */ 26 | #include 27 | #include 28 | 29 | /* for My modules */ 30 | #include "inference_helper.h" 31 | 32 | class InferenceHelperOpenCV : public InferenceHelper { 33 | public: 34 | InferenceHelperOpenCV(); 35 | ~InferenceHelperOpenCV() override; 36 | int32_t SetNumThreads(const int32_t num_threads) override; 37 | int32_t SetCustomOps(const std::vector>& custom_ops) override; 38 | int32_t Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) override; 39 | int32_t Finalize(void) override; 40 | int32_t PreProcess(const std::vector& input_tensor_info_list) override; 41 | int32_t Process(std::vector& output_tensor_info_list) override; 42 | 43 | private: 44 | cv::dnn::Net net_; 45 | std::vector in_mat_list_; 46 | std::vector out_mat_list_; // store data as member variable so that an user can refer the results 47 | }; 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_sample.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2022 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | /*** Include ***/ 16 | /* for general */ 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #ifdef _WIN32 29 | #include 30 | #endif 31 | 32 | /* for OOO */ 33 | 34 | /* for My modules */ 35 | #include "inference_helper_log.h" 36 | #include "inference_helper_sample.h" 37 | 38 | /*** Macro ***/ 39 | #define TAG "InferenceHelperSample" 40 | #define PRINT(...) INFERENCE_HELPER_LOG_PRINT(TAG, __VA_ARGS__) 41 | #define PRINT_E(...) INFERENCE_HELPER_LOG_PRINT_E(TAG, __VA_ARGS__) 42 | 43 | 44 | /*** Function ***/ 45 | InferenceHelperSample::InferenceHelperSample() 46 | { 47 | num_threads_ = 1; 48 | } 49 | 50 | InferenceHelperSample::~InferenceHelperSample() 51 | { 52 | } 53 | 54 | int32_t InferenceHelperSample::SetNumThreads(const int32_t num_threads) 55 | { 56 | num_threads_ = num_threads; 57 | return kRetOk; 58 | } 59 | 60 | int32_t InferenceHelperSample::SetCustomOps(const std::vector>& custom_ops) 61 | { 62 | PRINT("[WARNING] This method is not supported\n"); 63 | return kRetOk; 64 | } 65 | 66 | int32_t InferenceHelperSample::Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) 67 | { 68 | 69 | /*** Convert normalize parameter to speed up ***/ 70 | for (auto& input_tensor_info : input_tensor_info_list) { 71 | ConvertNormalizeParameters(input_tensor_info); 72 | } 73 | 74 | return kRetOk; 75 | }; 76 | 77 | int32_t InferenceHelperSample::Finalize(void) 78 | { 79 | return kRetOk; 80 | } 81 | 82 | int32_t InferenceHelperSample::PreProcess(const std::vector& input_tensor_info_list) 83 | { 84 | return kRetOk; 85 | } 86 | 87 | int32_t InferenceHelperSample::Process(std::vector& output_tensor_info_list) 88 | { 89 | return kRetOk; 90 | } 91 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_sample.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2022 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef INFERENCE_HELPER_SAMPLE_ 16 | #define INFERENCE_HELPER_SAMPLE_ 17 | 18 | /* for general */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | /* for My modules */ 27 | #include "inference_helper.h" 28 | 29 | class InferenceHelperSample : public InferenceHelper { 30 | public: 31 | InferenceHelperSample(); 32 | ~InferenceHelperSample() override; 33 | int32_t SetNumThreads(const int32_t num_threads) override; 34 | int32_t SetCustomOps(const std::vector>& custom_ops) override; 35 | int32_t Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) override; 36 | int32_t Finalize(void) override; 37 | int32_t PreProcess(const std::vector& input_tensor_info_list) override; 38 | int32_t Process(std::vector& output_tensor_info_list) override; 39 | 40 | private: 41 | int32_t num_threads_; 42 | }; 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_snpe.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2021 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef INFERENCE_HELPER_SNPE_ 16 | #define INFERENCE_HELPER_SNPE_ 17 | 18 | /* for general */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | /* for SNPE */ 28 | 29 | /* for My modules */ 30 | #include "inference_helper.h" 31 | 32 | namespace zdl { namespace SNPE { class SNPE; } } 33 | namespace zdl { namespace DlSystem { class IUserBuffer; } } 34 | namespace zdl { namespace DlSystem { class UserBufferMap; } } 35 | 36 | class InferenceHelperSnpe : public InferenceHelper { 37 | private: 38 | enum { UNKNOWN, USERBUFFER_FLOAT, USERBUFFER_TF8, ITENSOR, USERBUFFER_TF16 }; 39 | enum { CPUBUFFER, GLBUFFER }; 40 | 41 | public: 42 | InferenceHelperSnpe(); 43 | ~InferenceHelperSnpe() override; 44 | int32_t SetNumThreads(const int32_t num_threads) override; 45 | int32_t SetCustomOps(const std::vector>& custom_ops) override; 46 | int32_t Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) override; 47 | int32_t Finalize(void) override; 48 | int32_t PreProcess(const std::vector& input_tensor_info_list) override; 49 | int32_t Process(std::vector& output_tensor_info_list) override; 50 | 51 | private: 52 | std::unique_ptr CreateSnpe(const std::string& model_filename, bool use_user_supplied_buffers); 53 | int32_t GetTensorInfo(std::unique_ptr const& snpe, const std::string& name, std::vector& dims); 54 | int32_t GetAllTensorInfo(std::unique_ptr const& snpe, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list); 55 | 56 | private: 57 | int32_t num_threads_; 58 | std::unique_ptr snpe_; 59 | std::unique_ptr input_map_; 60 | std::unique_ptr output_map_; 61 | std::vector > snpe_user_input_buffers_; 62 | std::vector > snpe_user_output_buffers_; 63 | std::unordered_map > application_input_buffers_; 64 | std::unordered_map > application_output_buffers_; 65 | }; 66 | 67 | #endif 68 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_tensorflow.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2022 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | /*** Include ***/ 16 | /* for general */ 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #ifdef _WIN32 29 | #include 30 | #endif 31 | 32 | /* for TensorFlow */ 33 | #include 34 | 35 | /* for My modules */ 36 | #include "inference_helper_log.h" 37 | #include "inference_helper_tensorflow.h" 38 | 39 | /*** Macro ***/ 40 | #define TAG "InferenceHelperTensorflow" 41 | #define PRINT(...) INFERENCE_HELPER_LOG_PRINT(TAG, __VA_ARGS__) 42 | #define PRINT_E(...) INFERENCE_HELPER_LOG_PRINT_E(TAG, __VA_ARGS__) 43 | 44 | 45 | /*** Function ***/ 46 | InferenceHelperTensorflow::InferenceHelperTensorflow() 47 | { 48 | num_threads_ = 1; 49 | 50 | session_ = nullptr; 51 | graph_ = TF_NewGraph(); 52 | } 53 | 54 | InferenceHelperTensorflow::~InferenceHelperTensorflow() 55 | { 56 | } 57 | 58 | int32_t InferenceHelperTensorflow::SetNumThreads(const int32_t num_threads) 59 | { 60 | num_threads_ = num_threads; 61 | return kRetOk; 62 | } 63 | 64 | int32_t InferenceHelperTensorflow::SetCustomOps(const std::vector>& custom_ops) 65 | { 66 | PRINT("[WARNING] This method is not supported\n"); 67 | return kRetOk; 68 | } 69 | 70 | static std::string GetOpName(const std::string& model_filename) 71 | { 72 | std::string name; 73 | for (const auto c : model_filename) { 74 | if (c == ':') break; 75 | name.push_back(c); 76 | } 77 | return name; 78 | } 79 | 80 | static int32_t GetOpIndex(const std::string& model_filename) 81 | { 82 | bool is_index_start = false; 83 | std::string index_str; 84 | for (const auto c : model_filename) { 85 | if (c == ':') { 86 | is_index_start = true; 87 | continue; 88 | } 89 | if (is_index_start) { 90 | index_str.push_back(c); 91 | } 92 | } 93 | return std::stoi(index_str); 94 | } 95 | 96 | int32_t InferenceHelperTensorflow::Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) 97 | { 98 | /*** Load model ***/ 99 | TF_Status* status = TF_NewStatus(); 100 | TF_SessionOptions* session_options = TF_NewSessionOptions(); 101 | const char* tags = "serve"; 102 | int32_t ntags = 1; 103 | session_ = TF_LoadSessionFromSavedModel(session_options, nullptr, model_filename.c_str(), &tags, ntags, graph_, nullptr, status); 104 | TF_DeleteSessionOptions(session_options); 105 | TF_Code status_code = TF_GetCode(status); 106 | TF_DeleteStatus(status); 107 | if (status_code != TF_OK) { 108 | PRINT_E("Unable to load model: %d, %s, %s\n", status_code, model_filename.c_str(), tags); 109 | return kRetErr; 110 | } 111 | 112 | /*** Display graph ***/ 113 | //size_t pos = 0; 114 | //TF_Operation* oper; 115 | //printf("--- graph info ---\n"); 116 | //while ((oper = TF_GraphNextOperation(graph, &pos)) != nullptr) { 117 | // printf("%s\n", TF_OperationName(oper)); 118 | //} 119 | //printf("--- graph info ---\n"); 120 | 121 | /*** Allocate tensors ***/ 122 | int32_t id_input = 0; 123 | for (auto& input_tensor_info : input_tensor_info_list) { 124 | input_tensor_info.id = id_input++; 125 | TF_Output op = { TF_GraphOperationByName(graph_, GetOpName(input_tensor_info.name).c_str()), GetOpIndex(input_tensor_info.name) }; 126 | if (op.oper == nullptr) { 127 | PRINT_E("Can't find input tensor name: %s\n", input_tensor_info.name.c_str()); 128 | return kRetErr; 129 | } 130 | input_op_list_.emplace_back(op); 131 | 132 | std::vector dims; 133 | for (const auto& dim : input_tensor_info.tensor_dims) { 134 | dims.push_back(dim); 135 | } 136 | TF_Tensor* input_tensor = TF_AllocateTensor(TF_FLOAT, dims.data(), static_cast(dims.size()), input_tensor_info.GetElementNum() * sizeof(float)); 137 | input_tensor_list_.emplace_back(input_tensor); 138 | } 139 | 140 | for (auto& output_tensor_info : output_tensor_info_list) { 141 | TF_Output op = { TF_GraphOperationByName(graph_, GetOpName(output_tensor_info.name).c_str()), GetOpIndex(output_tensor_info.name) }; 142 | if (op.oper == nullptr) { 143 | PRINT_E("Can't find output tensor name: %s\n", output_tensor_info.name.c_str()); 144 | return kRetErr; 145 | } 146 | output_op_list_.emplace_back(op); 147 | output_tensor_list_.emplace_back(nullptr); 148 | } 149 | 150 | /*** Convert normalize parameter to speed up ***/ 151 | for (auto& input_tensor_info : input_tensor_info_list) { 152 | ConvertNormalizeParameters(input_tensor_info); 153 | } 154 | 155 | return kRetOk; 156 | }; 157 | 158 | int32_t InferenceHelperTensorflow::Finalize(void) 159 | { 160 | for (auto& tensor : input_tensor_list_) { 161 | TF_DeleteTensor(tensor); 162 | } 163 | for (auto& tensor : output_tensor_list_) { 164 | TF_DeleteTensor(tensor); 165 | } 166 | TF_DeleteGraph(graph_); 167 | TF_Status* status = TF_NewStatus(); 168 | TF_CloseSession(session_, status); 169 | TF_DeleteSession(session_, status); 170 | TF_DeleteStatus(status); 171 | 172 | input_op_list_.clear(); 173 | output_op_list_.clear(); 174 | input_tensor_list_.clear(); 175 | output_tensor_list_.clear(); 176 | return kRetOk; 177 | } 178 | 179 | int32_t InferenceHelperTensorflow::PreProcess(const std::vector& input_tensor_info_list) 180 | { 181 | for (const auto& input_tensor_info : input_tensor_info_list) { 182 | const int32_t img_width = input_tensor_info.GetWidth(); 183 | const int32_t img_height = input_tensor_info.GetHeight(); 184 | const int32_t img_channel = input_tensor_info.GetChannel(); 185 | if (input_tensor_info.data_type == InputTensorInfo::kDataTypeImage) { 186 | if ((input_tensor_info.image_info.width != input_tensor_info.image_info.crop_width) || (input_tensor_info.image_info.height != input_tensor_info.image_info.crop_height)) { 187 | PRINT_E("Crop is not supported\n"); 188 | return kRetErr; 189 | } 190 | if ((input_tensor_info.image_info.crop_width != img_width) || (input_tensor_info.image_info.crop_height != img_height)) { 191 | PRINT_E("Resize is not supported\n"); 192 | return kRetErr; 193 | } 194 | if (input_tensor_info.image_info.channel != img_channel) { 195 | PRINT_E("Color conversion is not supported\n"); 196 | return kRetErr; 197 | } 198 | 199 | /* Normalize image */ 200 | if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeFp32) { 201 | float* dst = static_cast(TF_TensorData(input_tensor_list_[input_tensor_info.id])); 202 | PreProcessImage(num_threads_, input_tensor_info, dst); 203 | } else if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeUint8) { 204 | uint8_t* dst = static_cast(TF_TensorData(input_tensor_list_[input_tensor_info.id])); 205 | PreProcessImage(num_threads_, input_tensor_info, dst); 206 | } else if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeInt8) { 207 | int8_t* dst = static_cast(TF_TensorData(input_tensor_list_[input_tensor_info.id])); 208 | PreProcessImage(num_threads_, input_tensor_info, dst); 209 | } else { 210 | PRINT_E("Unsupported tensor_type (%d)\n", input_tensor_info.tensor_type); 211 | return kRetErr; 212 | } 213 | } else if ((input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNhwc) || (input_tensor_info.data_type == InputTensorInfo::kDataTypeBlobNchw)) { 214 | if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeFp32) { 215 | float* dst = static_cast(TF_TensorData(input_tensor_list_[input_tensor_info.id])); 216 | PreProcessBlob(num_threads_, input_tensor_info, dst); 217 | } else if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeUint8 || input_tensor_info.tensor_type == TensorInfo::kTensorTypeInt8) { 218 | uint8_t* dst = static_cast(TF_TensorData(input_tensor_list_[input_tensor_info.id])); 219 | PreProcessBlob(num_threads_, input_tensor_info, dst); 220 | } else if (input_tensor_info.tensor_type == TensorInfo::kTensorTypeInt32) { 221 | int32_t* dst = static_cast(TF_TensorData(input_tensor_list_[input_tensor_info.id])); 222 | PreProcessBlob(num_threads_, input_tensor_info, dst); 223 | } else { 224 | PRINT_E("Unsupported tensor_type (%d)\n", input_tensor_info.tensor_type); 225 | return kRetErr; 226 | } 227 | } else { 228 | PRINT_E("Unsupported data_type (%d)\n", input_tensor_info.data_type); 229 | return kRetErr; 230 | } 231 | } 232 | return kRetOk; 233 | } 234 | 235 | int32_t InferenceHelperTensorflow::Process(std::vector& output_tensor_info_list) 236 | { 237 | /*** Delete previous result ***/ 238 | for (auto& output_tensor : output_tensor_list_) { 239 | TF_DeleteTensor(output_tensor); 240 | } 241 | 242 | /*** Run session ***/ 243 | TF_Code status_code = TF_ABORTED; 244 | TF_Status* status = TF_NewStatus(); 245 | try { 246 | TF_SessionRun(session_, nullptr, 247 | &input_op_list_[0], &input_tensor_list_[0], static_cast(input_op_list_.size()), 248 | &output_op_list_[0], &output_tensor_list_[0], static_cast(output_op_list_.size()), 249 | nullptr, 0, nullptr, status 250 | ); 251 | status_code = TF_GetCode(status); 252 | } catch (std::exception& e) { 253 | PRINT_E("Exception during run session: %s\n", e.what()); 254 | TF_DeleteStatus(status); 255 | return kRetErr; 256 | } 257 | TF_DeleteStatus(status); 258 | if (status_code != TF_OK) { 259 | PRINT_E("Error run session: %d\n", status_code); 260 | return kRetErr; 261 | } 262 | 263 | /*** Get result ***/ 264 | for (size_t i = 0; i < output_tensor_info_list.size(); i++) { 265 | auto& output_tensor_info = output_tensor_info_list[i]; 266 | auto& output_tensor = output_tensor_list_[i]; 267 | 268 | /* Get output tensor type */ 269 | TF_DataType data_type = TF_TensorType(output_tensor); 270 | switch (data_type) { 271 | case TF_FLOAT: 272 | output_tensor_info.tensor_type = TensorInfo::kTensorTypeFp32; 273 | break; 274 | case TF_INT32: 275 | output_tensor_info.tensor_type = TensorInfo::kTensorTypeInt32; 276 | break; 277 | case TF_INT8: 278 | output_tensor_info.tensor_type = TensorInfo::kTensorTypeInt8; 279 | break; 280 | case TF_UINT8: 281 | output_tensor_info.tensor_type = TensorInfo::kTensorTypeUint8; 282 | break; 283 | default: 284 | PRINT_E("Unsupported data type: %d\n", data_type); 285 | return kRetErr; 286 | } 287 | 288 | /* Get output tensor dims */ 289 | output_tensor_info.tensor_dims.clear(); 290 | int32_t ndims = TF_NumDims(output_tensor); 291 | for (int32_t index_dim = 0; index_dim < ndims; index_dim++) { 292 | int64_t dim = TF_Dim(output_tensor, index_dim); 293 | output_tensor_info.tensor_dims.push_back(static_cast(dim)); 294 | } 295 | 296 | /* Get output tensor data */ 297 | output_tensor_info.data = TF_TensorData(output_tensor); 298 | } 299 | 300 | return kRetOk; 301 | } 302 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_tensorflow.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2022 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef INFERENCE_HELPER_TENSORFLOW_ 16 | #define INFERENCE_HELPER_TENSORFLOW_ 17 | 18 | /* for general */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | /* for TensorFLow */ 27 | #include 28 | 29 | /* for My modules */ 30 | #include "inference_helper.h" 31 | 32 | class InferenceHelperTensorflow : public InferenceHelper { 33 | public: 34 | InferenceHelperTensorflow(); 35 | ~InferenceHelperTensorflow() override; 36 | int32_t SetNumThreads(const int32_t num_threads) override; 37 | int32_t SetCustomOps(const std::vector>& custom_ops) override; 38 | int32_t Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) override; 39 | int32_t Finalize(void) override; 40 | int32_t PreProcess(const std::vector& input_tensor_info_list) override; 41 | int32_t Process(std::vector& output_tensor_info_list) override; 42 | 43 | private: 44 | int32_t num_threads_; 45 | TF_Session* session_; 46 | TF_Graph* graph_; 47 | std::vector input_op_list_; 48 | std::vector output_op_list_; 49 | std::vector input_tensor_list_; 50 | std::vector output_tensor_list_; 51 | }; 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_tensorflow_lite.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2021 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef INFERENCE_HELPER_TENSORFLOW_LITE_ 16 | #define INFERENCE_HELPER_TENSORFLOW_LITE_ 17 | 18 | /* for general */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | /* for Tensorflow Lite */ 27 | #include 28 | #include 29 | #include 30 | 31 | /* for My modules */ 32 | #include "inference_helper.h" 33 | 34 | class InferenceHelperTensorflowLite : public InferenceHelper { 35 | public: 36 | InferenceHelperTensorflowLite(); 37 | ~InferenceHelperTensorflowLite() override; 38 | int32_t SetNumThreads(const int32_t num_threads) override; 39 | int32_t SetCustomOps(const std::vector>& custom_ops) override; 40 | int32_t Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) override; 41 | int32_t Finalize(void) override; 42 | int32_t PreProcess(const std::vector& input_tensor_info_list) override; 43 | int32_t Process(std::vector& output_tensor_info_list) override; 44 | 45 | private: 46 | int32_t GetInputTensorInfo(InputTensorInfo& tensor_info); 47 | int32_t GetOutputTensorInfo(OutputTensorInfo& tensor_info); 48 | void DisplayModelInfo(const tflite::Interpreter& interpreter); 49 | 50 | int32_t SetBufferToTensor(int32_t index, void *data); 51 | 52 | 53 | private: 54 | std::vector model_buffer_; 55 | std::unique_ptr model_; 56 | std::unique_ptr resolver_; 57 | std::unique_ptr interpreter_; 58 | TfLiteDelegate* delegate_; 59 | 60 | int32_t num_threads_; 61 | }; 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /inference_helper/inference_helper_tensorrt.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2021 iwatake2222 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | #ifndef INFERENCE_HELPER_TENSORRT_ 16 | #define INFERENCE_HELPER_TENSORRT_ 17 | 18 | /* for general */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | /* for My modules */ 27 | #include "inference_helper.h" 28 | 29 | namespace nvinfer1 { 30 | class IRuntime; 31 | class ICudaEngine; 32 | class IExecutionContext; 33 | } 34 | 35 | class InferenceHelperTensorRt : public InferenceHelper { 36 | public: 37 | InferenceHelperTensorRt(); 38 | ~InferenceHelperTensorRt() override; 39 | int32_t SetNumThreads(const int32_t num_threads) override; 40 | int32_t SetCustomOps(const std::vector>& custom_ops) override; 41 | int32_t Initialize(const std::string& model_filename, std::vector& input_tensor_info_list, std::vector& output_tensor_info_list) override; 42 | int32_t Finalize(void) override; 43 | int32_t PreProcess(const std::vector& input_tensor_info_list) override; 44 | int32_t Process(std::vector& output_tensor_info_list) override; 45 | void SetDlaCore(int32_t dla_core) { 46 | dla_core_ = dla_core; 47 | } 48 | 49 | private: 50 | int32_t AllocateBuffers(std::vector& input_tensor_info_list, std::vector& output_tensor_info_list); 51 | 52 | private: 53 | int32_t num_threads_; 54 | int32_t dla_core_; 55 | std::unique_ptr runtime_; 56 | std::unique_ptr engine_; 57 | std::unique_ptr context_; 58 | std::vector> buffer_list_cpu_; // pointer and size (can be overwritten by user) 59 | std::vector> buffer_list_cpu_reserved_; // pointer and size (fixed in initialization) 60 | std::vector buffer_list_gpu_; 61 | }; 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /inference_helper/snpe/CreateUserBuffer.cpp: -------------------------------------------------------------------------------- 1 | //============================================================================== 2 | // 3 | // Copyright (c) 2017-2020 Qualcomm Technologies, Inc. 4 | // All Rights Reserved. 5 | // Confidential and Proprietary - Qualcomm Technologies, Inc. 6 | // 7 | //============================================================================== 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "CreateUserBuffer.hpp" 17 | #include "Util.hpp" 18 | 19 | #include "SNPE/SNPE.hpp" 20 | #include "SNPE/SNPEFactory.hpp" 21 | #include "DlSystem/StringList.hpp" 22 | #include "DlSystem/TensorShape.hpp" 23 | #include "DlSystem/IUserBuffer.hpp" 24 | #include "DlSystem/IUserBufferFactory.hpp" 25 | #include "DlSystem/UserBufferMap.hpp" 26 | 27 | void createUserBuffer(zdl::DlSystem::UserBufferMap& userBufferMap, 28 | std::unordered_map>& applicationBuffers, 29 | std::vector>& snpeUserBackedBuffers, 30 | std::unique_ptr& snpe, 31 | const char * name, 32 | const bool isTfNBuffer, 33 | int bitWidth) 34 | { 35 | // get attributes of buffer by name 36 | auto bufferAttributesOpt = snpe->getInputOutputBufferAttributes(name); 37 | if (!bufferAttributesOpt) throw std::runtime_error(std::string("Error obtaining attributes for input tensor ") + name); 38 | 39 | // calculate the size of buffer required by the input tensor 40 | const zdl::DlSystem::TensorShape& bufferShape = (*bufferAttributesOpt)->getDims(); 41 | 42 | size_t bufferElementSize = 0; 43 | if (isTfNBuffer) { 44 | bufferElementSize = bitWidth / 8; 45 | } 46 | else { 47 | bufferElementSize = sizeof(float); 48 | } 49 | 50 | // Calculate the stride based on buffer strides. 51 | // Note: Strides = Number of bytes to advance to the next element in each dimension. 52 | // For example, if a float tensor of dimension 2x4x3 is tightly packed in a buffer of 96 bytes, then the strides would be (48,12,4) 53 | // Note: Buffer stride is usually known and does not need to be calculated. 54 | std::vector strides(bufferShape.rank()); 55 | strides[strides.size() - 1] = bufferElementSize; 56 | size_t stride = strides[strides.size() - 1]; 57 | for (size_t i = bufferShape.rank() - 1; i > 0; i--) 58 | { 59 | (bufferShape[i] == 0) ? stride *= getResizableDim() : stride *= bufferShape[i]; 60 | strides[i-1] = stride; 61 | } 62 | 63 | size_t bufSize = calcSizeFromDims(bufferShape.getDimensions(), bufferShape.rank(), bufferElementSize); 64 | 65 | // set the buffer encoding type 66 | std::unique_ptr userBufferEncoding; 67 | if (isTfNBuffer) 68 | { 69 | userBufferEncoding = std::unique_ptr(new zdl::DlSystem::UserBufferEncodingTfN(0,1.0, bitWidth)); 70 | } 71 | else 72 | { 73 | userBufferEncoding = std::unique_ptr(new zdl::DlSystem::UserBufferEncodingFloat()); 74 | } 75 | 76 | // create user-backed storage to load input data onto it 77 | applicationBuffers.emplace(name, std::vector(bufSize)); 78 | 79 | // create SNPE user buffer from the user-backed buffer 80 | zdl::DlSystem::IUserBufferFactory& ubFactory = zdl::SNPE::SNPEFactory::getUserBufferFactory(); 81 | snpeUserBackedBuffers.push_back(ubFactory.createUserBuffer(applicationBuffers.at(name).data(), 82 | bufSize, 83 | strides, 84 | userBufferEncoding.get())); 85 | if (snpeUserBackedBuffers.back() == nullptr) 86 | { 87 | std::cerr << "Error while creating user buffer." << std::endl; 88 | } 89 | // add the user-backed buffer to the inputMap, which is later on fed to the network for execution 90 | userBufferMap.add(name, snpeUserBackedBuffers.back().get()); 91 | } 92 | 93 | void createInputBufferMap(zdl::DlSystem::UserBufferMap& inputMap, 94 | std::unordered_map>& applicationBuffers, 95 | std::vector>& snpeUserBackedBuffers, 96 | std::unique_ptr& snpe, 97 | bool isTfNBuffer, 98 | int bitWidth) 99 | { 100 | // get input tensor names of the network that need to be populated 101 | const auto& inputNamesOpt = snpe->getInputTensorNames(); 102 | if (!inputNamesOpt) throw std::runtime_error("Error obtaining input tensor names"); 103 | const zdl::DlSystem::StringList& inputNames = *inputNamesOpt; 104 | assert(inputNames.size() > 0); 105 | 106 | // create SNPE user buffers for each application storage buffer 107 | for (const char *name : inputNames) { 108 | createUserBuffer(inputMap, applicationBuffers, snpeUserBackedBuffers, snpe, name, isTfNBuffer, bitWidth); 109 | } 110 | } 111 | 112 | void createOutputBufferMap(zdl::DlSystem::UserBufferMap& outputMap, 113 | std::unordered_map>& applicationBuffers, 114 | std::vector>& snpeUserBackedBuffers, 115 | std::unique_ptr& snpe, 116 | bool isTfNBuffer, 117 | int bitWidth) 118 | { 119 | // get input tensor names of the network that need to be populated 120 | const auto& outputNamesOpt = snpe->getOutputTensorNames(); 121 | if (!outputNamesOpt) throw std::runtime_error("Error obtaining output tensor names"); 122 | const zdl::DlSystem::StringList& outputNames = *outputNamesOpt; 123 | 124 | // create SNPE user buffers for each application storage buffer 125 | for (const char *name : outputNames) { 126 | createUserBuffer(outputMap, applicationBuffers, snpeUserBackedBuffers, snpe, name, isTfNBuffer, bitWidth); 127 | } 128 | } 129 | 130 | void createUserBuffer(zdl::DlSystem::UserBufferMap& userBufferMap, 131 | std::unordered_map& applicationBuffers, 132 | std::vector>& snpeUserBackedBuffers, 133 | std::unique_ptr& snpe, 134 | const char * name) 135 | { 136 | // get attributes of buffer by name 137 | auto bufferAttributesOpt = snpe->getInputOutputBufferAttributes(name); 138 | if (!bufferAttributesOpt) throw std::runtime_error(std::string("Error obtaining attributes for input tensor ") + name); 139 | // calculate the size of buffer required by the input tensor 140 | const zdl::DlSystem::TensorShape& bufferShape = (*bufferAttributesOpt)->getDims(); 141 | 142 | // calculate stride based on buffer strides 143 | // Note: Strides = Number of bytes to advance to the next element in each dimension. 144 | // For example, if a float tensor of dimension 2x4x3 is tightly packed in a buffer of 96 bytes, then the strides would be (48,12,4) 145 | std::vector strides(bufferShape.rank()); 146 | strides[strides.size() - 1] = sizeof(float); 147 | size_t stride = strides[strides.size() - 1]; 148 | for (size_t i = bufferShape.rank() - 1; i > 0; i--) 149 | { 150 | stride *= bufferShape[i]; 151 | strides[i-1] = stride; 152 | } 153 | 154 | const size_t bufferElementSize = (*bufferAttributesOpt)->getElementSize(); 155 | size_t bufSize = calcSizeFromDims(bufferShape.getDimensions(), bufferShape.rank(), bufferElementSize); 156 | 157 | // set the buffer encoding type 158 | zdl::DlSystem::UserBufferEncodingFloat userBufferEncodingFloat; 159 | zdl::DlSystem::UserBufferSourceGLBuffer userBufferSourceGLBuffer; 160 | 161 | // create user-backed storage to load input data onto it 162 | applicationBuffers.emplace(name, GLuint(1)); 163 | 164 | // create SNPE user buffer from the user-backed buffer 165 | zdl::DlSystem::IUserBufferFactory& ubFactory = zdl::SNPE::SNPEFactory::getUserBufferFactory(); 166 | snpeUserBackedBuffers.push_back(ubFactory.createUserBuffer(&applicationBuffers.at(name), 167 | bufSize, 168 | strides, 169 | &userBufferEncodingFloat, 170 | &userBufferSourceGLBuffer)); 171 | 172 | // add the user-backed buffer to the inputMap, which is later on fed to the network for execution 173 | userBufferMap.add(name, snpeUserBackedBuffers.back().get()); 174 | } 175 | 176 | void createInputBufferMap(zdl::DlSystem::UserBufferMap& inputMap, 177 | std::unordered_map& applicationBuffers, 178 | std::vector>& snpeUserBackedBuffers, 179 | std::unique_ptr& snpe) 180 | { 181 | // get input tensor names of the network that need to be populated 182 | const auto& inputNamesOpt = snpe->getInputTensorNames(); 183 | if (!inputNamesOpt) throw std::runtime_error("Error obtaining input tensor names"); 184 | const zdl::DlSystem::StringList& inputNames = *inputNamesOpt; 185 | assert(inputNames.size() > 0); 186 | 187 | // create SNPE user buffers for each application storage buffer 188 | for (const char *name : inputNames) { 189 | createUserBuffer(inputMap, applicationBuffers, snpeUserBackedBuffers, snpe, name); 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /inference_helper/snpe/CreateUserBuffer.hpp: -------------------------------------------------------------------------------- 1 | //============================================================================== 2 | // 3 | // Copyright (c) 2017-2020 Qualcomm Technologies, Inc. 4 | // All Rights Reserved. 5 | // Confidential and Proprietary - Qualcomm Technologies, Inc. 6 | // 7 | //============================================================================== 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include "SNPE/SNPE.hpp" 14 | #include "DlSystem/IUserBuffer.hpp" 15 | #include "DlSystem/UserBufferMap.hpp" 16 | 17 | typedef unsigned int GLuint; 18 | 19 | // Helper function to fill a single entry of the UserBufferMap with the given user-backed buffer 20 | void createUserBuffer(zdl::DlSystem::UserBufferMap& userBufferMap, 21 | std::unordered_map>& applicationBuffers, 22 | std::vector>& snpeUserBackedBuffers, 23 | std::unique_ptr& snpe, 24 | const char * name, 25 | const bool isTfNBuffer, 26 | int bitWidth); 27 | 28 | // Create a UserBufferMap of the SNPE network inputs 29 | void createInputBufferMap(zdl::DlSystem::UserBufferMap& inputMap, 30 | std::unordered_map>& applicationBuffers, 31 | std::vector>& snpeUserBackedBuffers, 32 | std::unique_ptr& snpe, 33 | const bool isTfNBuffer, 34 | int bitWidth); 35 | 36 | // Create a UserBufferMap of the SNPE network outputs 37 | void createOutputBufferMap(zdl::DlSystem::UserBufferMap& outputMap, 38 | std::unordered_map>& applicationBuffers, 39 | std::vector>& snpeUserBackedBuffers, 40 | std::unique_ptr& snpe, 41 | const bool isTfNBuffer, 42 | int bitWidth); 43 | 44 | void createUserBuffer(zdl::DlSystem::UserBufferMap& userBufferMap, 45 | std::unordered_map& applicationBuffers, 46 | std::vector>& snpeUserBackedBuffers, 47 | std::unique_ptr& snpe, 48 | const char * name); 49 | 50 | void createInputBufferMap(zdl::DlSystem::UserBufferMap& inputMap, 51 | std::unordered_map& applicationBuffers, 52 | std::vector>& snpeUserBackedBuffers, 53 | std::unique_ptr& snpe); 54 | -------------------------------------------------------------------------------- /inference_helper/snpe/NOTICE.txt: -------------------------------------------------------------------------------- 1 | The source code files in this directory are retrieved from snpe-1.50.0.2622\examples\NativeCpp\SampleCode\jni 2 | -------------------------------------------------------------------------------- /inference_helper/snpe/Util.cpp: -------------------------------------------------------------------------------- 1 | //============================================================================== 2 | // 3 | // Copyright (c) 2017-2020 Qualcomm Technologies, Inc. 4 | // All Rights Reserved. 5 | // Confidential and Proprietary - Qualcomm Technologies, Inc. 6 | // 7 | //============================================================================== 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "Util.hpp" 21 | 22 | #include "DlSystem/ITensorFactory.hpp" 23 | #include "DlSystem/TensorShape.hpp" 24 | 25 | size_t resizable_dim; 26 | 27 | size_t calcSizeFromDims(const zdl::DlSystem::Dimension *dims, size_t rank, size_t elementSize ) 28 | { 29 | if (rank == 0) return 0; 30 | size_t size = elementSize; 31 | while (rank--) { 32 | (*dims == 0) ? size *= resizable_dim : size *= *dims; 33 | dims++; 34 | } 35 | return size; 36 | } 37 | 38 | 39 | bool EnsureDirectory(const std::string& dir) 40 | { 41 | auto i = dir.find_last_of('/'); 42 | std::string prefix = dir.substr(0, i); 43 | 44 | if (dir.empty() || dir == "." || dir == "..") 45 | { 46 | return true; 47 | } 48 | 49 | if (i != std::string::npos && !EnsureDirectory(prefix)) 50 | { 51 | return false; 52 | } 53 | 54 | int rc = mkdir(dir.c_str(), S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); 55 | if (rc == -1 && errno != EEXIST) 56 | { 57 | return false; 58 | } 59 | else 60 | { 61 | struct stat st; 62 | if (stat(dir.c_str(), &st) == -1) 63 | { 64 | return false; 65 | } 66 | 67 | return S_ISDIR(st.st_mode); 68 | } 69 | } 70 | 71 | std::vector loadFloatDataFile(const std::string& inputFile) 72 | { 73 | std::vector vec; 74 | loadByteDataFile(inputFile, vec); 75 | return vec; 76 | } 77 | 78 | std::vector loadByteDataFile(const std::string& inputFile) 79 | { 80 | std::vector vec; 81 | loadByteDataFile(inputFile, vec); 82 | return vec; 83 | } 84 | 85 | template 86 | bool loadByteDataFile(const std::string& inputFile, std::vector& loadVector) 87 | { 88 | std::ifstream in(inputFile, std::ifstream::binary); 89 | if (!in.is_open() || !in.good()) 90 | { 91 | std::cerr << "Failed to open input file: " << inputFile << "\n"; 92 | } 93 | 94 | in.seekg(0, in.end); 95 | size_t length = in.tellg(); 96 | in.seekg(0, in.beg); 97 | 98 | if (length % sizeof(T) != 0) { 99 | std::cerr << "Size of input file should be divisible by sizeof(dtype).\n"; 100 | return false; 101 | } 102 | 103 | if (loadVector.size() == 0) { 104 | loadVector.resize(length / sizeof(T)); 105 | } else if (loadVector.size() < length / sizeof(T)) { 106 | std::cerr << "Vector is not large enough to hold data of input file: " << inputFile << "\n"; 107 | loadVector.resize(length / sizeof(T)); 108 | } 109 | 110 | if (!in.read(reinterpret_cast(&loadVector[0]), length)) 111 | { 112 | std::cerr << "Failed to read the contents of: " << inputFile << "\n"; 113 | } 114 | return true; 115 | } 116 | 117 | std::vector loadByteDataFileBatched(const std::string& inputFile) 118 | { 119 | std::vector vec; 120 | size_t offset=0; 121 | loadByteDataFileBatched(inputFile, vec, offset); 122 | return vec; 123 | } 124 | 125 | template 126 | bool loadByteDataFileBatched(const std::string& inputFile, std::vector& loadVector, size_t offset) 127 | { 128 | std::ifstream in(inputFile, std::ifstream::binary); 129 | if (!in.is_open() || !in.good()) 130 | { 131 | std::cerr << "Failed to open input file: " << inputFile << "\n"; 132 | } 133 | 134 | in.seekg(0, in.end); 135 | size_t length = in.tellg(); 136 | in.seekg(0, in.beg); 137 | 138 | if (length % sizeof(T) != 0) { 139 | std::cerr << "Size of input file should be divisible by sizeof(dtype).\n"; 140 | return false; 141 | } 142 | 143 | if (loadVector.size() == 0) { 144 | loadVector.resize(length / sizeof(T)); 145 | } else if (loadVector.size() < length / sizeof(T)) { 146 | std::cerr << "Vector is not large enough to hold data of input file: " << inputFile << "\n"; 147 | } 148 | 149 | loadVector.resize( (offset+1) * length / sizeof(T) ); 150 | 151 | if (!in.read( reinterpret_cast (&loadVector[offset * length/ sizeof(T) ]), length) ) 152 | { 153 | std::cerr << "Failed to read the contents of: " << inputFile << "\n"; 154 | } 155 | return true; 156 | } 157 | 158 | void TfNToFloat(float *out, 159 | uint8_t *in, 160 | const unsigned char stepEquivalentTo0, 161 | const float quantizedStepSize, 162 | size_t numElement, 163 | int bitWidth) 164 | { 165 | for (size_t i = 0; i < numElement; ++i) { 166 | if (8 == bitWidth) { 167 | double quantizedValue = static_cast (in[i]); 168 | double stepEqTo0 = static_cast (stepEquivalentTo0); 169 | out[i] = static_cast ((quantizedValue - stepEqTo0) * quantizedStepSize); 170 | } 171 | else if (16 == bitWidth) { 172 | uint16_t *temp = (uint16_t *)in; 173 | double quantizedValue = static_cast (temp[i]); 174 | double stepEqTo0 = static_cast (stepEquivalentTo0); 175 | out[i] = static_cast ((quantizedValue - stepEqTo0) * quantizedStepSize); 176 | } 177 | } 178 | } 179 | 180 | bool FloatToTfN(uint8_t* out, 181 | unsigned char& stepEquivalentTo0, 182 | float& quantizedStepSize, 183 | float* in, 184 | size_t numElement, 185 | int bitWidth) 186 | { 187 | float trueMin = std::numeric_limits ::max(); 188 | float trueMax = std::numeric_limits ::min(); 189 | 190 | for (size_t i = 0; i < numElement; ++i) { 191 | trueMin = fmin(trueMin, in[i]); 192 | trueMax = fmax(trueMax, in[i]); 193 | } 194 | 195 | double encodingMin; 196 | double encodingMax; 197 | double stepCloseTo0; 198 | double trueBitWidthMax = pow(2, bitWidth) -1; 199 | 200 | if (trueMin > 0.0f) { 201 | stepCloseTo0 = 0.0; 202 | encodingMin = 0.0; 203 | encodingMax = trueMax; 204 | } else if (trueMax < 0.0f) { 205 | stepCloseTo0 = trueBitWidthMax; 206 | encodingMin = trueMin; 207 | encodingMax = 0.0; 208 | } else { 209 | double trueStepSize = static_cast (trueMax - trueMin) / trueBitWidthMax; 210 | stepCloseTo0 = -trueMin / trueStepSize; 211 | if (stepCloseTo0==round(stepCloseTo0)) { 212 | // 0.0 is exactly representable 213 | encodingMin = trueMin; 214 | encodingMax = trueMax; 215 | } else { 216 | stepCloseTo0 = round(stepCloseTo0); 217 | encodingMin = (0.0 - stepCloseTo0) * trueStepSize; 218 | encodingMax = (trueBitWidthMax - stepCloseTo0) * trueStepSize; 219 | } 220 | } 221 | 222 | const double minEncodingRange = 0.01; 223 | double encodingRange = encodingMax - encodingMin; 224 | quantizedStepSize = encodingRange / trueBitWidthMax; 225 | stepEquivalentTo0 = static_cast (round(stepCloseTo0)); 226 | 227 | if (encodingRange < minEncodingRange) { 228 | std::cerr << "Expect the encoding range to be larger than " << minEncodingRange << "\n" 229 | << "Got: " << encodingRange << "\n"; 230 | return false; 231 | } else { 232 | for (size_t i = 0; i < numElement; ++i) { 233 | int quantizedValue = round(trueBitWidthMax * (in[i] - encodingMin) / encodingRange); 234 | 235 | if (quantizedValue < 0) 236 | quantizedValue = 0; 237 | else if (quantizedValue > (int)trueBitWidthMax) 238 | quantizedValue = (int)trueBitWidthMax; 239 | 240 | if(bitWidth == 8){ 241 | out[i] = static_cast (quantizedValue); 242 | } 243 | else if(bitWidth == 16){ 244 | uint16_t *temp = (uint16_t *)out; 245 | temp[i] = static_cast (quantizedValue); 246 | } 247 | } 248 | } 249 | return true; 250 | } 251 | 252 | bool loadByteDataFileBatchedTfN(const std::string& inputFile, std::vector& loadVector, size_t offset, 253 | unsigned char& stepEquivalentTo0, float& quantizedStepSize, int bitWidth) 254 | { 255 | std::ifstream in(inputFile, std::ifstream::binary); 256 | std::vector inVector; 257 | if (!in.is_open() || !in.good()) 258 | { 259 | std::cerr << "Failed to open input file: " << inputFile << "\n"; 260 | } 261 | 262 | in.seekg(0, in.end); 263 | size_t length = in.tellg(); 264 | in.seekg(0, in.beg); 265 | 266 | if (loadVector.size() == 0) { 267 | loadVector.resize(length / sizeof(uint8_t)); 268 | } else if (loadVector.size() < length/sizeof(float)) { 269 | std::cerr << "Vector is not large enough to hold data of input file: " << inputFile << "\n"; 270 | } 271 | 272 | inVector.resize(length / sizeof(float)); 273 | if (!in.read( reinterpret_cast (&inVector[0]), length) ) 274 | { 275 | std::cerr << "Failed to read the contents of: " << inputFile << "\n"; 276 | } 277 | int elementSize = bitWidth / 8; 278 | size_t dataStartPos = (offset * length * elementSize) / sizeof(float); 279 | if(!FloatToTfN(&loadVector[dataStartPos], stepEquivalentTo0, quantizedStepSize, inVector.data(), inVector.size(), bitWidth)) 280 | { 281 | return false; 282 | } 283 | return true; 284 | } 285 | 286 | bool loadByteDataFileBatchedTf8(const std::string& inputFile, std::vector& loadVector, size_t offset) 287 | { 288 | std::ifstream in(inputFile, std::ifstream::binary); 289 | std::vector inVector; 290 | if (!in.is_open() || !in.good()) 291 | { 292 | std::cerr << "Failed to open input file: " << inputFile << "\n"; 293 | } 294 | 295 | in.seekg(0, in.end); 296 | size_t length = in.tellg(); 297 | in.seekg(0, in.beg); 298 | 299 | if (loadVector.size() == 0) { 300 | loadVector.resize(length / sizeof(uint8_t)); 301 | } else if (loadVector.size() < length/sizeof(float)) { 302 | std::cerr << "Vector is not large enough to hold data of input file: " << inputFile << "\n"; 303 | } 304 | 305 | inVector.resize((offset+1) * length / sizeof(uint8_t)); 306 | if (!in.read( reinterpret_cast (&inVector[offset * length/ sizeof(uint8_t) ]), length) ) 307 | { 308 | std::cerr << "Failed to read the contents of: " << inputFile << "\n"; 309 | } 310 | 311 | unsigned char stepEquivalentTo0; 312 | float quantizedStepSize; 313 | if(!FloatToTfN(loadVector.data(), stepEquivalentTo0, quantizedStepSize, inVector.data(), loadVector.size(), 8)) 314 | { 315 | return false; 316 | } 317 | return true; 318 | } 319 | 320 | bool SaveITensorBatched(const std::string& path, const zdl::DlSystem::ITensor* tensor, size_t batchIndex, size_t batchChunk) 321 | { 322 | if(batchChunk == 0) 323 | batchChunk = tensor->getSize(); 324 | // Create the directory path if it does not exist 325 | auto idx = path.find_last_of('/'); 326 | if (idx != std::string::npos) 327 | { 328 | std::string dir = path.substr(0, idx); 329 | if (!EnsureDirectory(dir)) 330 | { 331 | std::cerr << "Failed to create output directory: " << dir << ": " 332 | << std::strerror(errno) << "\n"; 333 | return false; 334 | } 335 | } 336 | 337 | std::ofstream os(path, std::ofstream::binary); 338 | if (!os) 339 | { 340 | std::cerr << "Failed to open output file for writing: " << path << "\n"; 341 | return false; 342 | } 343 | 344 | for ( auto it = tensor->cbegin() + batchIndex * batchChunk; it != tensor->cbegin() + (batchIndex+1) * batchChunk; ++it ) 345 | { 346 | float f = *it; 347 | if (!os.write(reinterpret_cast(&f), sizeof(float))) 348 | { 349 | std::cerr << "Failed to write data to: " << path << "\n"; 350 | return false; 351 | } 352 | } 353 | return true; 354 | } 355 | 356 | bool SaveUserBufferBatched(const std::string& path, const std::vector& buffer, size_t batchIndex, size_t batchChunk) 357 | { 358 | if(batchChunk == 0) 359 | batchChunk = buffer.size(); 360 | // Create the directory path if it does not exist 361 | auto idx = path.find_last_of('/'); 362 | if (idx != std::string::npos) 363 | { 364 | std::string dir = path.substr(0, idx); 365 | if (!EnsureDirectory(dir)) 366 | { 367 | std::cerr << "Failed to create output directory: " << dir << ": " 368 | << std::strerror(errno) << "\n"; 369 | return false; 370 | } 371 | } 372 | 373 | std::ofstream os(path, std::ofstream::binary); 374 | if (!os) 375 | { 376 | std::cerr << "Failed to open output file for writing: " << path << "\n"; 377 | return false; 378 | } 379 | 380 | for ( auto it = buffer.begin() + batchIndex * batchChunk; it != buffer.begin() + (batchIndex+1) * batchChunk; ++it ) 381 | { 382 | uint8_t u = *it; 383 | if(!os.write((char*)(&u), sizeof(uint8_t))) 384 | { 385 | std::cerr << "Failed to write data to: " << path << "\n"; 386 | return false; 387 | } 388 | } 389 | return true; 390 | } 391 | 392 | void setResizableDim(size_t resizableDim) 393 | { 394 | resizable_dim = resizableDim; 395 | } 396 | 397 | size_t getResizableDim() 398 | { 399 | return resizable_dim; 400 | } 401 | -------------------------------------------------------------------------------- /inference_helper/snpe/Util.hpp: -------------------------------------------------------------------------------- 1 | //============================================================================== 2 | // 3 | // Copyright (c) 2017-2020 Qualcomm Technologies, Inc. 4 | // All Rights Reserved. 5 | // Confidential and Proprietary - Qualcomm Technologies, Inc. 6 | // 7 | //============================================================================== 8 | 9 | #ifndef UTIL_H 10 | #define UTIL_H 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | #include "DlSystem/ITensorFactory.hpp" 17 | #include "DlSystem/TensorShape.hpp" 18 | 19 | template Container& split(Container& result, const typename Container::value_type & s, typename Container::value_type::value_type delimiter ) 20 | { 21 | result.clear(); 22 | std::istringstream ss( s ); 23 | while (!ss.eof()) 24 | { 25 | typename Container::value_type field; 26 | getline( ss, field, delimiter ); 27 | if (field.empty()) continue; 28 | result.push_back( field ); 29 | } 30 | return result; 31 | } 32 | 33 | size_t calcSizeFromDims(const zdl::DlSystem::Dimension *dims, size_t rank, size_t elementSize); 34 | 35 | std::vector loadFloatDataFile(const std::string& inputFile); 36 | std::vector loadByteDataFile(const std::string& inputFile); 37 | template bool loadByteDataFile(const std::string& inputFile, std::vector& loadVector); 38 | 39 | std::vector loadByteDataFileBatched(const std::string& inputFile); 40 | template bool loadByteDataFileBatched(const std::string& inputFile, std::vector& loadVector, size_t offset); 41 | bool loadByteDataFileBatchedTf8(const std::string& inputFile, std::vector& loadVector, size_t offset); 42 | bool loadByteDataFileBatchedTfN(const std::string& inputFile, std::vector& loadVector, size_t offset, 43 | unsigned char& stepEquivalentTo0, float& quantizedStepSize, int bitWidth); 44 | 45 | bool SaveITensorBatched(const std::string& path, const zdl::DlSystem::ITensor* tensor, size_t batchIndex=0, size_t batchChunk=0); 46 | bool SaveUserBufferBatched(const std::string& path, const std::vector& buffer, size_t batchIndex=0, size_t batchChunk=0); 47 | bool EnsureDirectory(const std::string& dir); 48 | 49 | void TfNToFloat(float *out, uint8_t *in, const unsigned char stepEquivalentTo0, const float quantizedStepSize, size_t numElement, int bitWidth); 50 | bool FloatToTfN(uint8_t* out, unsigned char& stepEquivalentTo0, float& quantizedStepSize, float* in, size_t numElement, int bitWidth); 51 | 52 | void setResizableDim(size_t resizableDim); 53 | size_t getResizableDim(); 54 | 55 | #endif 56 | 57 | -------------------------------------------------------------------------------- /inference_helper/snpe/udlExample.cpp: -------------------------------------------------------------------------------- 1 | //============================================================================== 2 | // 3 | // Copyright (c) 2016-2020 Qualcomm Technologies, Inc. 4 | // All Rights Reserved. 5 | // Confidential and Proprietary - Qualcomm Technologies, Inc. 6 | // 7 | //============================================================================== 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "udlExample.hpp" 19 | 20 | #define DEADBEAF_PTR (void*)(static_cast(0xdeadbeaf)) 21 | 22 | namespace { 23 | 24 | std::size_t getSizeByDim(const std::vector& dim) { 25 | return std::accumulate(std::begin(dim), 26 | std::end(dim), 27 | 1, 28 | std::multiplies()); 29 | } 30 | 31 | void printArray(const size_t* start, const size_t* end) { 32 | (void)std::for_each(start, end, [](const size_t i){ 33 | std::cout << i << " "; 34 | }); 35 | 36 | } 37 | 38 | bool isPassthrough(const std::string& type) { 39 | std::string lower; 40 | (void)std::transform(std::begin(type), std::end(type), 41 | std::back_inserter(lower), [](const char c){ return std::tolower(c); }); 42 | if (lower != std::string("passthrough")) { 43 | std::cerr << "isPassthrough expecting type passthrough got " << 44 | type << std::endl; 45 | return false; 46 | } 47 | return true; 48 | } 49 | 50 | } // ns 51 | 52 | namespace UdlExample { 53 | 54 | zdl::DlSystem::IUDL* MyUDLFactory(void* cookie, const zdl::DlSystem::UDLContext* c) { 55 | if (!c) return nullptr; 56 | if (cookie != DEADBEAF_PTR) { 57 | std::cerr << "MyUDLFactory cookie should be 0xdeadbeaf" << std::endl; 58 | return nullptr; 59 | } 60 | 61 | if (!isPassthrough(c->getType())) { 62 | std::cerr << "MyUDLFactory expecting Passthrough layer, got " << c->getType() << std::endl; 63 | return nullptr; 64 | } 65 | return new UdlPassthrough(*c); 66 | } 67 | 68 | bool UdlPassthrough::setup(void* cookie, 69 | size_t insz, const size_t* indim[], const size_t indimsz[], 70 | size_t outsz, const size_t* outdim[], const size_t outdimsz[]) { 71 | if (cookie != DEADBEAF_PTR) { 72 | std::cerr << "UdlPassthrough::setup() cookie should be 0xdeadbeaf" << std::endl; 73 | return false; 74 | } 75 | 76 | // FIXME we need to use proper logging here not using streams 77 | std::cout << "UdlPassthrough::setup() of name " << m_Context.getName() << 78 | " and of type " << m_Context.getType() << std::endl; 79 | 80 | if (!isPassthrough(m_Context.getType())) { 81 | std::cerr << "UdlPassthrough::setup() expecting passthrough layer type got " << 82 | m_Context.getType() << std::endl; 83 | return false; 84 | } 85 | 86 | std::cout << " input array size " << insz << std::endl; 87 | std::cout << " output array size " << outsz << std::endl; 88 | 89 | // print the input/output dims 90 | std::cout << "UdlPassthrough::setup() input dims\n"; 91 | (void)std::copy(indimsz, indimsz + insz, std::ostream_iterator (std::cout, " ")); 92 | std::cout << std::endl; 93 | size_t idx = 0; 94 | (void)std::for_each(indim, indim + insz, [&idx, &indimsz](const size_t* arr){ 95 | std::cout << "["; 96 | printArray(arr, arr + indimsz[idx]); 97 | std::cout << "]"; 98 | ++idx; 99 | }); 100 | std::cout << std::endl; 101 | std::cout << "UdlPassthrough::setup() output dims\n"; 102 | (void)std::copy(outdimsz, outdimsz + insz, std::ostream_iterator (std::cout, " ")); 103 | std::cout << std::endl; 104 | idx = 0; 105 | (void)std::for_each(outdim, outdim + outsz, [&idx, &outdimsz](const size_t* arr){ 106 | std::cout << "["; 107 | printArray(arr, arr + outdimsz[idx]); 108 | std::cout << "]"; 109 | ++idx; 110 | }); 111 | std::cout << std::endl; 112 | 113 | if (insz != outsz) { 114 | std::cerr << "UdlPassthrough::setup() not the same number of dim, in:" << 115 | insz << " != : " << outsz << std::endl; 116 | return false; 117 | } 118 | m_Insz = insz; 119 | size_t cnt = insz; 120 | 121 | // If the user want to refer to the indim[] and outdim[], 122 | // he/she needs to make a copy of this arrays. 123 | // After setup, these arrays are destroyes, so you cannot cache it as is 124 | m_OutSzDim.reserve(cnt); 125 | while (cnt-- > 0) { 126 | // compute dims and compare. keep the output dim 127 | const size_t *indims = indim[cnt]; 128 | const size_t inszdim = getSizeByDim(std::vector(indims, indims + indimsz[cnt])); 129 | const size_t *outdims = outdim[cnt]; // insz == outsz 130 | m_OutSzDim[cnt] = getSizeByDim(std::vector(outdims, outdims + outdimsz[cnt])); 131 | 132 | std::cout << "UdlPassthrough::setup() input size for index " << cnt 133 | << " is dim: " << inszdim << ", output: " << m_OutSzDim[cnt] <(blob)) << std::endl; 152 | return true; 153 | } 154 | 155 | void UdlPassthrough::close(void* cookie) noexcept { 156 | if (cookie != DEADBEAF_PTR) { 157 | std::cerr << "UdlPassthrough::close() cookie should be 0xdeadbeaf" << std::endl; 158 | } 159 | std::cout << "UdlPassthrough::close()" << std::endl; 160 | delete this; 161 | } 162 | 163 | bool UdlPassthrough::execute(void *cookie, const float **input, float **output) { 164 | if (cookie != DEADBEAF_PTR) { 165 | std::cerr << "UdlPassthrough::execute() cookie should be 0xdeadbeaf" << std::endl; 166 | return false; 167 | } 168 | std::cout << "UdlPassthrough::execute() number of I/Os is:" << m_Insz << std::endl; 169 | // 0...m_OutSzDim --> going backwards 170 | // m_OutSzDim is assumed to be != 0 171 | size_t cnt = m_Insz; 172 | while (cnt-- > 0) { 173 | std::cout << std::dec; 174 | std::cout << "UdlPassthrough::execute() running index " << cnt << std::endl; 175 | size_t dim = sizeof(float) * m_OutSzDim[cnt]; 176 | std::cout << "UdlPassthrough::execute() dims (a*b*c*...) is:" << m_OutSzDim[cnt] << std::endl; 177 | std::cout << "UdlPassthrough::execute() dim(total number of bytes) is:" << dim << std::endl; 178 | 179 | const float *i = input[cnt]; 180 | float *o = output[cnt]; 181 | if (!i || !o) { 182 | std::cerr << "Input or output cannot be 0" << std::endl; 183 | return false; 184 | } 185 | std::cout << "input: 0x" << std::hex << std::setw(8) << std::setfill('0') << i << std::endl; 186 | std::cout << "output: 0x" << std::hex << std::setw(8) << std::setfill('0') << o << std::endl; 187 | std::memcpy(o, i, dim); 188 | } 189 | return true; 190 | } 191 | 192 | } // ns batchrun 193 | -------------------------------------------------------------------------------- /inference_helper/snpe/udlExample.hpp: -------------------------------------------------------------------------------- 1 | //============================================================================== 2 | // 3 | // Copyright (c) 2016-2019 Qualcomm Technologies, Inc. 4 | // All Rights Reserved. 5 | // Confidential and Proprietary - Qualcomm Technologies, Inc. 6 | // 7 | //============================================================================== 8 | 9 | #ifndef UDL_EXAMPLE_HPP 10 | #define UDL_EXAMPLE_HPP 11 | 12 | // Include IUDL using the prefix 13 | #include "DlSystem/IUDL.hpp" 14 | #include "DlSystem/UDLContext.hpp" 15 | 16 | namespace UdlExample{ 17 | 18 | zdl::DlSystem::IUDL* MyUDLFactory(void*, const zdl::DlSystem::UDLContext*); 19 | 20 | // Example of Passthrough layer that looks like this: 21 | // layer { 22 | // name: "Passthrough1" 23 | // type: "Passthrough" 24 | // bottom: "norm1" 25 | // top: "norm1" 26 | // passthrough_param { 27 | // blob_count: 1 28 | // } 29 | // } 30 | class UdlPassthrough final : public zdl::DlSystem::IUDL { 31 | public: 32 | UdlPassthrough(const UdlPassthrough&) = delete; 33 | UdlPassthrough& operator=(const UdlPassthrough&) = delete; 34 | 35 | /** 36 | * @brief UDLContext by value but it has move operation 37 | */ 38 | UdlPassthrough(zdl::DlSystem::UDLContext context) : m_Context(context) {} 39 | 40 | /** 41 | * @brief Setup User's environment. 42 | * This is being called by DnnRunTime framework 43 | * to let the user opportunity to setup anything 44 | * which is needed for running user defined layers 45 | * @return true on success, false otherwise 46 | */ 47 | virtual bool setup(void *cookie, 48 | size_t insz, const size_t **indim, const size_t *indimsz, 49 | size_t outsz, const size_t **outdim, const size_t *outdimsz) override; 50 | 51 | /** 52 | * Close the instance. Invoked by DnnRunTime to let 53 | * the user the opportunity to close handels etc... 54 | */ 55 | virtual void close(void *cookie) noexcept override; 56 | 57 | /** 58 | * Execute the user defined layer 59 | * will contain the return value/output tensor 60 | */ 61 | virtual bool execute(void *cookie, const float **input, float **output) override; 62 | private: 63 | zdl::DlSystem::UDLContext m_Context; 64 | // this is a*b*c*...*n 65 | std::vector m_OutSzDim; 66 | // cache the insz/outsz of the incoming 67 | size_t m_Insz = 0; 68 | // No need for this since in passthrough its all the same 69 | // size_t m_Outsz = 0; 70 | }; 71 | 72 | } // ns sample 73 | 74 | #endif // UDL_EXAMPLE_HPP 75 | -------------------------------------------------------------------------------- /inference_helper/tensorrt/EntropyCalibrator.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef ENTROPY_CALIBRATOR_H 18 | #define ENTROPY_CALIBRATOR_H 19 | 20 | #include "BatchStream.h" 21 | #include "NvInfer.h" 22 | 23 | //! \class EntropyCalibratorImpl 24 | //! 25 | //! \brief Implements common functionality for Entropy calibrators. 26 | //! 27 | template 28 | class EntropyCalibratorImpl 29 | { 30 | public: 31 | EntropyCalibratorImpl( 32 | TBatchStream stream, int firstBatch, std::string networkName, const char* inputBlobName, bool readCache = true) 33 | : mStream{stream} 34 | , mCalibrationTableName("CalibrationTable" + networkName) 35 | , mInputBlobName(inputBlobName) 36 | , mReadCache(readCache) 37 | { 38 | nvinfer1::Dims dims = mStream.getDims(); 39 | mInputCount = samplesCommon::volume(dims); 40 | CHECK(cudaMalloc(&mDeviceInput, mInputCount * sizeof(float))); 41 | mStream.reset(firstBatch); 42 | } 43 | 44 | virtual ~EntropyCalibratorImpl() 45 | { 46 | CHECK(cudaFree(mDeviceInput)); 47 | } 48 | 49 | int getBatchSize() const noexcept 50 | { 51 | return mStream.getBatchSize(); 52 | } 53 | 54 | bool getBatch(void* bindings[], const char* names[], int nbBindings) noexcept 55 | { 56 | if (!mStream.next()) 57 | { 58 | return false; 59 | } 60 | CHECK(cudaMemcpy(mDeviceInput, mStream.getBatch(), mInputCount * sizeof(float), cudaMemcpyHostToDevice)); 61 | ASSERT(!strcmp(names[0], mInputBlobName)); 62 | bindings[0] = mDeviceInput; 63 | return true; 64 | } 65 | 66 | const void* readCalibrationCache(size_t& length) noexcept 67 | { 68 | mCalibrationCache.clear(); 69 | std::ifstream input(mCalibrationTableName, std::ios::binary); 70 | input >> std::noskipws; 71 | if (mReadCache && input.good()) 72 | { 73 | std::copy(std::istream_iterator(input), std::istream_iterator(), 74 | std::back_inserter(mCalibrationCache)); 75 | } 76 | length = mCalibrationCache.size(); 77 | return length ? mCalibrationCache.data() : nullptr; 78 | } 79 | 80 | void writeCalibrationCache(const void* cache, size_t length) noexcept 81 | { 82 | std::ofstream output(mCalibrationTableName, std::ios::binary); 83 | output.write(reinterpret_cast(cache), length); 84 | } 85 | 86 | private: 87 | TBatchStream mStream; 88 | size_t mInputCount; 89 | std::string mCalibrationTableName; 90 | const char* mInputBlobName; 91 | bool mReadCache{true}; 92 | void* mDeviceInput{nullptr}; 93 | std::vector mCalibrationCache; 94 | }; 95 | 96 | //! \class Int8EntropyCalibrator2 97 | //! 98 | //! \brief Implements Entropy calibrator 2. 99 | //! CalibrationAlgoType is kENTROPY_CALIBRATION_2. 100 | //! 101 | template 102 | class Int8EntropyCalibrator2 : public IInt8EntropyCalibrator2 103 | { 104 | public: 105 | Int8EntropyCalibrator2( 106 | TBatchStream stream, int firstBatch, const char* networkName, const char* inputBlobName, bool readCache = true) 107 | : mImpl(stream, firstBatch, networkName, inputBlobName, readCache) 108 | { 109 | } 110 | 111 | int getBatchSize() const noexcept override 112 | { 113 | return mImpl.getBatchSize(); 114 | } 115 | 116 | bool getBatch(void* bindings[], const char* names[], int nbBindings) noexcept override 117 | { 118 | return mImpl.getBatch(bindings, names, nbBindings); 119 | } 120 | 121 | const void* readCalibrationCache(size_t& length) noexcept override 122 | { 123 | return mImpl.readCalibrationCache(length); 124 | } 125 | 126 | void writeCalibrationCache(const void* cache, size_t length) noexcept override 127 | { 128 | mImpl.writeCalibrationCache(cache, length); 129 | } 130 | 131 | private: 132 | EntropyCalibratorImpl mImpl; 133 | }; 134 | 135 | #endif // ENTROPY_CALIBRATOR_H 136 | -------------------------------------------------------------------------------- /inference_helper/tensorrt/ErrorRecorder.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef ERROR_RECORDER_H 18 | #define ERROR_RECORDER_H 19 | #include "NvInferRuntimeCommon.h" 20 | #include "logger.h" 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | using namespace nvinfer1; 27 | //! 28 | //! A simple implementation of the IErrorRecorder interface for 29 | //! use by samples. This interface also can be used as a reference 30 | //! implementation. 31 | //! The sample Error recorder is based on a vector that pairs the error 32 | //! code and the error string into a single element. It also uses 33 | //! standard mutex's and atomics in order to make sure that the code 34 | //! works in a multi-threaded environment. 35 | //! 36 | class SampleErrorRecorder : public IErrorRecorder 37 | { 38 | using errorPair = std::pair; 39 | using errorStack = std::vector; 40 | 41 | public: 42 | SampleErrorRecorder() = default; 43 | 44 | virtual ~SampleErrorRecorder() noexcept {} 45 | int32_t getNbErrors() const noexcept final 46 | { 47 | return mErrorStack.size(); 48 | } 49 | ErrorCode getErrorCode(int32_t errorIdx) const noexcept final 50 | { 51 | return invalidIndexCheck(errorIdx) ? ErrorCode::kINVALID_ARGUMENT : (*this)[errorIdx].first; 52 | }; 53 | IErrorRecorder::ErrorDesc getErrorDesc(int32_t errorIdx) const noexcept final 54 | { 55 | return invalidIndexCheck(errorIdx) ? "errorIdx out of range." : (*this)[errorIdx].second.c_str(); 56 | } 57 | // This class can never overflow since we have dynamic resize via std::vector usage. 58 | bool hasOverflowed() const noexcept final 59 | { 60 | return false; 61 | } 62 | 63 | // Empty the errorStack. 64 | void clear() noexcept final 65 | { 66 | try 67 | { 68 | // grab a lock so that there is no addition while clearing. 69 | std::lock_guard guard(mStackLock); 70 | mErrorStack.clear(); 71 | } 72 | catch (const std::exception& e) 73 | { 74 | sample::gLogFatal << "Internal Error: " << e.what() << std::endl; 75 | } 76 | }; 77 | 78 | //! Simple helper function that 79 | bool empty() const noexcept 80 | { 81 | return mErrorStack.empty(); 82 | } 83 | 84 | bool reportError(ErrorCode val, IErrorRecorder::ErrorDesc desc) noexcept final 85 | { 86 | try 87 | { 88 | std::lock_guard guard(mStackLock); 89 | sample::gLogError << "Error[" << static_cast(val) << "]: " << desc << std::endl; 90 | mErrorStack.push_back(errorPair(val, desc)); 91 | } 92 | catch (const std::exception& e) 93 | { 94 | sample::gLogFatal << "Internal Error: " << e.what() << std::endl; 95 | } 96 | // All errors are considered fatal. 97 | return true; 98 | } 99 | 100 | // Atomically increment or decrement the ref counter. 101 | IErrorRecorder::RefCount incRefCount() noexcept final 102 | { 103 | return ++mRefCount; 104 | } 105 | IErrorRecorder::RefCount decRefCount() noexcept final 106 | { 107 | return --mRefCount; 108 | } 109 | 110 | private: 111 | // Simple helper functions. 112 | const errorPair& operator[](size_t index) const noexcept 113 | { 114 | return mErrorStack[index]; 115 | } 116 | 117 | bool invalidIndexCheck(int32_t index) const noexcept 118 | { 119 | // By converting signed to unsigned, we only need a single check since 120 | // negative numbers turn into large positive greater than the size. 121 | size_t sIndex = index; 122 | return sIndex >= mErrorStack.size(); 123 | } 124 | 125 | // Mutex to hold when locking mErrorStack. 126 | std::mutex mStackLock; 127 | 128 | // Reference count of the class. Destruction of the class when mRefCount 129 | // is not zero causes undefined behavior. 130 | std::atomic mRefCount{0}; 131 | 132 | // The error stack that holds the errors recorded by TensorRT. 133 | errorStack mErrorStack; 134 | 135 | }; // class SampleErrorRecorder 136 | #endif // ERROR_RECORDER_H 137 | -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/batchPrepare.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # Editor: 18 | # iwatake (2020/09/06) 19 | 20 | 21 | 22 | import argparse 23 | import numpy as np 24 | import sys 25 | import os 26 | import glob 27 | import shutil 28 | import struct 29 | from random import shuffle 30 | 31 | try: 32 | from PIL import Image 33 | except ImportError as err: 34 | raise ImportError("""ERROR: Failed to import module ({}) 35 | Please make sure you have Pillow installed. 36 | For installation instructions, see: 37 | http://pillow.readthedocs.io/en/stable/installation.html""".format(err)) 38 | 39 | 40 | height = 224 41 | width = 224 42 | 43 | parser = argparse.ArgumentParser() 44 | parser.add_argument('--inDir', required=True, help='Input directory') 45 | parser.add_argument('--outDir', required=True, help='Output directory') 46 | 47 | args = parser.parse_args() 48 | CALIBRATION_DATASET_LOC = args.inDir + '/*.jpg' 49 | 50 | 51 | # images to test 52 | img_file_list = [] 53 | print("Location of dataset = " + CALIBRATION_DATASET_LOC) 54 | img_file_list = glob.glob(CALIBRATION_DATASET_LOC) 55 | print("Total number of images = " + str(len(img_file_list))) 56 | 57 | # output 58 | outDir = args.outDir 59 | 60 | # prepare output 61 | if not os.path.exists(outDir): 62 | os.makedirs(outDir) 63 | 64 | # Convert and output images (use \n (do not use \r\n)) 65 | with open(outDir + "/list.txt", "wb") as f: 66 | for img_file in img_file_list: 67 | img = Image.open(img_file) 68 | img_resize = img.resize((width, height)) 69 | basename_without_ext = os.path.splitext(os.path.basename(img_file))[0] 70 | img_resize.save(outDir+"/" + basename_without_ext + ".ppm") 71 | f.write((basename_without_ext + "\n").encode('utf-8')) 72 | -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000000139.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000000139.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000000285.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000000285.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000000632.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000000632.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000000724.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000000724.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000000776.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000000776.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000000785.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000000785.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000000802.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000000802.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000000872.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000000872.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000000885.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000000885.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000001000.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000001000.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000001268.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000001268.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000001296.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000001296.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000001353.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000001353.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000001425.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000001425.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000001490.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000001490.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000001503.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000001503.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000001532.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000001532.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000001584.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000001584.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000001675.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000001675.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_org/000000001761.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_org/000000001761.jpg -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/.gitattributes: -------------------------------------------------------------------------------- 1 | *.sh text eol=lf 2 | *.txt text eol=lf 3 | -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000000139.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000000139.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000000285.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000000285.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000000632.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000000632.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000000724.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000000724.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000000776.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000000776.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000000785.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000000785.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000000802.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000000802.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000000872.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000000872.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000000885.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000000885.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000001000.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000001000.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000001268.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000001268.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000001296.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000001296.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000001353.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000001353.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000001425.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000001425.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000001490.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000001490.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000001503.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000001503.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000001532.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000001532.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000001584.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000001584.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000001675.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000001675.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/000000001761.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iwatake2222/InferenceHelper/7c975cf9d32016aa6bfceabf6922fe6d808dc5eb/inference_helper/tensorrt/calibration/sample_ppm/000000001761.ppm -------------------------------------------------------------------------------- /inference_helper/tensorrt/calibration/sample_ppm/list.txt: -------------------------------------------------------------------------------- 1 | 000000000139 2 | 000000000285 3 | 000000000632 4 | 000000000724 5 | 000000000776 6 | 000000000785 7 | 000000000802 8 | 000000000872 9 | 000000000885 10 | 000000001000 11 | 000000001268 12 | 000000001296 13 | 000000001353 14 | 000000001425 15 | 000000001490 16 | 000000001503 17 | 000000001532 18 | 000000001584 19 | 000000001675 20 | 000000001761 21 | -------------------------------------------------------------------------------- /inference_helper/tensorrt/logger.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "logger.h" 18 | #include "ErrorRecorder.h" 19 | #include "logging.h" 20 | 21 | SampleErrorRecorder gRecorder; 22 | namespace sample 23 | { 24 | Logger gLogger{Logger::Severity::kINFO}; 25 | LogStreamConsumer gLogVerbose{LOG_VERBOSE(gLogger)}; 26 | LogStreamConsumer gLogInfo{LOG_INFO(gLogger)}; 27 | LogStreamConsumer gLogWarning{LOG_WARN(gLogger)}; 28 | LogStreamConsumer gLogError{LOG_ERROR(gLogger)}; 29 | LogStreamConsumer gLogFatal{LOG_FATAL(gLogger)}; 30 | 31 | void setReportableSeverity(Logger::Severity severity) 32 | { 33 | gLogger.setReportableSeverity(severity); 34 | gLogVerbose.setReportableSeverity(severity); 35 | gLogInfo.setReportableSeverity(severity); 36 | gLogWarning.setReportableSeverity(severity); 37 | gLogError.setReportableSeverity(severity); 38 | gLogFatal.setReportableSeverity(severity); 39 | } 40 | } // namespace sample 41 | -------------------------------------------------------------------------------- /inference_helper/tensorrt/logger.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef LOGGER_H 18 | #define LOGGER_H 19 | 20 | #include "logging.h" 21 | 22 | class SampleErrorRecorder; 23 | extern SampleErrorRecorder gRecorder; 24 | namespace sample 25 | { 26 | extern Logger gLogger; 27 | extern LogStreamConsumer gLogVerbose; 28 | extern LogStreamConsumer gLogInfo; 29 | extern LogStreamConsumer gLogWarning; 30 | extern LogStreamConsumer gLogError; 31 | extern LogStreamConsumer gLogFatal; 32 | 33 | void setReportableSeverity(Logger::Severity severity); 34 | } // namespace sample 35 | 36 | #endif // LOGGER_H 37 | -------------------------------------------------------------------------------- /third_party/cmakes/armnn.cmake: -------------------------------------------------------------------------------- 1 | if(DEFINED ANDROID_ABI) 2 | message(FATAL_ERROR "[ARMNN] todo") 3 | elseif(${BUILD_SYSTEM} STREQUAL "x64_linux") 4 | set(ARMNN_LIB_PATH ${CMAKE_CURRENT_LIST_DIR}/../armnn_prebuilt/ubuntu/lib/) 5 | set(ARMNN_PROTOBUF_LIB_PATH ${CMAKE_CURRENT_LIST_DIR}/../armnn_prebuilt/protobuf_lib/ubuntu/) 6 | set(ARMNN_INC_PATH ${CMAKE_CURRENT_LIST_DIR}/../armnn_prebuilt/ubuntu/include/) 7 | elseif(${BUILD_SYSTEM} STREQUAL "aarch64") 8 | set(ARMNN_LIB_PATH ${CMAKE_CURRENT_LIST_DIR}/../armnn_prebuilt/aarch64/lib/) 9 | set(ARMNN_PROTOBUF_LIB_PATH ${CMAKE_CURRENT_LIST_DIR}/../armnn_prebuilt/protobuf_lib/aarch64/) 10 | set(ARMNN_INC_PATH ${CMAKE_CURRENT_LIST_DIR}/../armnn_prebuilt/aarch64/include/) 11 | else() 12 | message(FATAL_ERROR "[ARMNN] unsupported platform") 13 | endif() 14 | 15 | function(ADD_SO_LIB SO_LIB_NAME SO_FILE_PATH) 16 | add_library(${SO_LIB_NAME} SHARED IMPORTED GLOBAL) 17 | set_target_properties( 18 | ${SO_LIB_NAME} 19 | PROPERTIES IMPORTED_LOCATION 20 | ${SO_FILE_PATH} 21 | ) 22 | set(ARMNN_LIB ${ARMNN_LIB} ${SO_LIB_NAME} PARENT_SCOPE) 23 | endfunction() 24 | 25 | set(ARMNN_LIB ) 26 | ADD_SO_LIB("ARMNN" ${ARMNN_LIB_PATH}/libarmnn.so) 27 | ADD_SO_LIB("ARMNN_ONNX_PARSER" ${ARMNN_LIB_PATH}/libarmnnOnnxParser.so) 28 | ADD_SO_LIB("ARMNN_TFLITE_PARSER" ${ARMNN_LIB_PATH}/libarmnnTfLiteParser.so) 29 | ADD_SO_LIB("PROTOBUF_LITE" ${ARMNN_PROTOBUF_LIB_PATH}/libprotobuf-lite.so) 30 | ADD_SO_LIB("PROTOBUF" ${ARMNN_PROTOBUF_LIB_PATH}/libprotobuf.so) 31 | ADD_SO_LIB("PROTOC" ${ARMNN_PROTOBUF_LIB_PATH}/libprotoc.so) 32 | set(ARMNN_LIB ${ARMNN_LIB} pthread) 33 | set(ARMNN_INC ${ARMNN_INC_PATH}) 34 | 35 | # workaround: libprotobuf.so.23 is not found 36 | file(COPY ${ARMNN_PROTOBUF_LIB_PATH}/libprotobuf.so.23 DESTINATION ${CMAKE_BINARY_DIR}) 37 | file(COPY ${ARMNN_PROTOBUF_LIB_PATH}/libprotobuf.so.23.0.0 DESTINATION ${CMAKE_BINARY_DIR}) 38 | 39 | ### dump 40 | # set(Armnn_DIR ${CMAKE_CURRENT_LIST_DIR}/install/lib/cmake/armnn) 41 | # find_package(Armnn REQUIRED) 42 | # target_link_libraries(${LibraryName} ${ARMNN_LIBRARIES}) 43 | # target_link_libraries(${LibraryName} pthread) 44 | -------------------------------------------------------------------------------- /third_party/cmakes/libtorch.cmake: -------------------------------------------------------------------------------- 1 | if(DEFINED ANDROID_ABI) 2 | set(LIBTORCH_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../libtorch_prebuilt/android") 3 | elseif(MSVC_VERSION) 4 | if(INFERENCE_HELPER_ENABLE_LIBTORCH_CUDA) 5 | set(LIBTORCH_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../libtorch_prebuilt/win-x64-gpu") 6 | else() 7 | set(LIBTORCH_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../libtorch_prebuilt/win-x64") 8 | endif() 9 | else() 10 | if(${BUILD_SYSTEM} STREQUAL "armv7") 11 | message(FATAL_ERROR "[LIBTORCH] unsupported platform") 12 | elseif(${BUILD_SYSTEM} STREQUAL "aarch64") 13 | message(FATAL_ERROR "[LIBTORCH] unsupported platform") 14 | else() 15 | if(INFERENCE_HELPER_ENABLE_LIBTORCH_CUDA) 16 | ### For some reasons, LibTorch+CUDA library doesn't work(it's missing some symbols?), and LibTorch+CPU library works with CUDA option 17 | # set(LIBTORCH_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../libtorch_prebuilt/linux-x64-gpu") 18 | set(LIBTORCH_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../libtorch_prebuilt/linux-x64") 19 | else() 20 | set(LIBTORCH_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../libtorch_prebuilt/linux-x64") 21 | endif() 22 | endif() 23 | endif() 24 | 25 | if(DEFINED ANDROID_ABI) 26 | message(FATAL_ERROR "[libtorch] unsupported platform") 27 | add_library(LIBTORCH SHARED IMPORTED GLOBAL) 28 | set_target_properties( 29 | LIBTORCH 30 | PROPERTIES IMPORTED_LOCATION 31 | ${LIBTORCH_ROOTDIR}/jni/${ANDROID_ABI}/libpytorch_vision_jni.so 32 | ) 33 | set(LIBTORCH_LIB LIBTORCH) 34 | set(LIBTORCH_INC "${LIBTORCH_ROOTDIR}/headers") 35 | else() 36 | set(TORCH_CUDA_ARCH_LIST Auto) 37 | find_package(Caffe2 REQUIRED PATHS "${LIBTORCH_ROOTDIR}") 38 | find_package(Torch REQUIRED PATHS "${LIBTORCH_ROOTDIR}") 39 | 40 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}") 41 | set(LIBTORCH_LIB "${TORCH_LIBRARIES}") 42 | set(LIBTORCH_INC ) 43 | 44 | if(MSVC) 45 | file(GLOB TORCH_DLLS "${TORCH_INSTALL_PREFIX}/lib/*.dll") 46 | add_custom_command(TARGET InferenceHelper 47 | POST_BUILD 48 | COMMAND ${CMAKE_COMMAND} -E copy_if_different 49 | ${TORCH_DLLS} 50 | ${CMAKE_BINARY_DIR}) 51 | endif(MSVC) 52 | endif() 53 | -------------------------------------------------------------------------------- /third_party/cmakes/mnn.cmake: -------------------------------------------------------------------------------- 1 | set(mnn_use_vulkan OFF) 2 | if(DEFINED ANDROID_ABI) # Always tries to use Vulkan on Android 3 | set(mnn_use_vulkan ON) 4 | endif() 5 | if(mnn_use_vulkan) 6 | set(mnn_suffix_vulkan "-vulkan") 7 | else() 8 | set(mnn_suffix_vulkan "") 9 | endif() 10 | 11 | if(DEFINED ANDROID_ABI) 12 | set(mnn_DIR ${CMAKE_CURRENT_LIST_DIR}/../mnn_prebuilt/android${mnn_suffix_vulkan}/${ANDROID_ABI}) 13 | elseif(MSVC_VERSION) 14 | set(mnn_DIR ${CMAKE_CURRENT_LIST_DIR}/../mnn_prebuilt/windows-vs2019${mnn_suffix_vulkan}) 15 | file(COPY ${mnn_DIR}/lib/MNN.dll DESTINATION ${CMAKE_BINARY_DIR}) 16 | file(COPY ${mnn_DIR}/lib/debug/MNN.dll DESTINATION ${CMAKE_BINARY_DIR}/Debug) 17 | else() 18 | if(${BUILD_SYSTEM} STREQUAL "armv7") 19 | set(mnn_DIR ${CMAKE_CURRENT_LIST_DIR}/../mnn_prebuilt/armv7${mnn_suffix_vulkan}) 20 | elseif(${BUILD_SYSTEM} STREQUAL "aarch64") 21 | set(mnn_DIR ${CMAKE_CURRENT_LIST_DIR}/../mnn_prebuilt/aarch64${mnn_suffix_vulkan}) 22 | else() 23 | set(mnn_DIR ${CMAKE_CURRENT_LIST_DIR}/../mnn_prebuilt/ubuntu${mnn_suffix_vulkan}) 24 | endif() 25 | endif() 26 | 27 | if(MSVC_VERSION) 28 | set(MNN_LIB 29 | $<$:${mnn_DIR}/lib/debug/MNN.lib> 30 | $<$:${mnn_DIR}/lib/MNN.lib> 31 | $<$:${mnn_DIR}/lib/MNN.lib> 32 | $<$:${mnn_DIR}/lib/MNN.lib> 33 | ) 34 | # set_target_properties(mnn PROPERTIES IMPORTED_LOCATION ${mnn_DIR}/lib/MNN.lib) 35 | else() 36 | add_library(mnn SHARED IMPORTED GLOBAL) 37 | set_target_properties(mnn PROPERTIES IMPORTED_LOCATION ${mnn_DIR}/lib/libMNN.so) 38 | if(mnn_use_vulkan) 39 | find_package(Vulkan REQUIRED) 40 | message(STATUS "Vulkan Include = ${Vulkan_INCLUDE_DIR} , Vulkan Lib = ${Vulkan_LIBRARY}") 41 | add_library(mnn_vulkan SHARED IMPORTED GLOBAL) 42 | set_target_properties(mnn_vulkan PROPERTIES IMPORTED_LOCATION ${mnn_DIR}/lib/libMNN_Vulkan.so) 43 | set(MNN_LIB mnn mnn_vulkan ${Vulkan_LIBRARY}) 44 | else() 45 | set(MNN_LIB mnn) 46 | endif() 47 | endif() 48 | set(MNN_INC ${mnn_DIR}/include) 49 | -------------------------------------------------------------------------------- /third_party/cmakes/ncnn.cmake: -------------------------------------------------------------------------------- 1 | if(MSVC_VERSION) 2 | # It's better to use DLL for MSVC so that I can use Debug mode 3 | set(ncnn_use_shared ON) 4 | else() 5 | set(ncnn_use_shared OFF) 6 | endif() 7 | if(ncnn_use_shared) 8 | set(ncnn_suffix_shared "-shared") 9 | else() 10 | set(ncnn_suffix_shared "") 11 | endif() 12 | 13 | if(DEFINED ANDROID_ABI) 14 | set(ncnn_DIR ${CMAKE_CURRENT_LIST_DIR}/../ncnn_prebuilt/android-vulkan/${ANDROID_ABI}/lib/cmake/ncnn/) 15 | elseif(MSVC_VERSION) 16 | set(ncnn_DIR ${CMAKE_CURRENT_LIST_DIR}/../ncnn_prebuilt/windows-vs2019${ncnn_suffix_shared}/x64/lib/cmake/ncnn/) 17 | if(ncnn_use_shared) 18 | file(COPY ${ncnn_DIR}/../../../bin/ncnn.dll DESTINATION ${CMAKE_BINARY_DIR}) 19 | endif() 20 | else() 21 | if(${BUILD_SYSTEM} STREQUAL "armv7") 22 | set(ncnn_DIR ${CMAKE_CURRENT_LIST_DIR}/../ncnn_prebuilt/armv7/lib/cmake/ncnn/) 23 | elseif(${BUILD_SYSTEM} STREQUAL "aarch64") 24 | set(ncnn_DIR ${CMAKE_CURRENT_LIST_DIR}/../ncnn_prebuilt/aarch64/lib/cmake/ncnn/) 25 | else() 26 | set(ncnn_DIR ${CMAKE_CURRENT_LIST_DIR}/../ncnn_prebuilt/ubuntu-2004${ncnn_suffix_shared}/lib/cmake/ncnn/) 27 | endif() 28 | endif() 29 | 30 | # message(FATAL_ERROR "ncnn_DIR = ${ncnn_DIR}") 31 | # set(CMAKE_PREFIX_PATH "${ncnn_DIR}/lib/cmake/ncnn/") 32 | # find_package(ncnn REQUIRED PATHS "${ncnn_DIR}/lib/cmake/ncnn/") 33 | find_package(ncnn REQUIRED) 34 | set(NCNN_LIB ncnn) 35 | set(NCNN_INC "") # no need to set 36 | -------------------------------------------------------------------------------- /third_party/cmakes/nnabla.cmake: -------------------------------------------------------------------------------- 1 | if(DEFINED ANDROID_ABI) 2 | # set(NNABLA_LIB ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/android/${ANDROID_ABI}/libtensorflowlite.so) 3 | # add_library(NNABLA SHARED IMPORTED GLOBAL) 4 | # set_target_properties( 5 | # NNABLA 6 | # PROPERTIES IMPORTED_LOCATION 7 | # ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/android/${ANDROID_ABI}/libtensorflowlite.so 8 | # ) 9 | # set(NNABLA_LIB NNABLA) 10 | elseif(MSVC_VERSION) 11 | set(NNABLA_INC ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/windows-vs2019/include) 12 | set(NNABLA_LIB 13 | ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/windows-vs2019/lib/nnabla.lib 14 | ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/windows-vs2019/lib/nnabla_utils.lib 15 | ) 16 | file(COPY ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/windows-vs2019/bin/nnabla.dll DESTINATION ${CMAKE_BINARY_DIR}) 17 | file(COPY ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/windows-vs2019/bin/nnabla_utils.dll DESTINATION ${CMAKE_BINARY_DIR}) 18 | file(COPY ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/windows-vs2019/bin/archive.dll DESTINATION ${CMAKE_BINARY_DIR}) 19 | if(INFERENCE_HELPER_ENABLE_NNABLA_CUDA) 20 | set(NNABLA_LIB ${NNABLA_LIB} ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/windows-vs2019/lib/nnabla_cuda_110_8.lib) 21 | file(COPY ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/windows-vs2019/bin/nnabla_cuda_110_8.dll DESTINATION ${CMAKE_BINARY_DIR}) 22 | endif() 23 | else() 24 | if(${BUILD_SYSTEM} STREQUAL "x64_linux") 25 | set(NNABLA_INC ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/ubuntu/include) 26 | set(NNABLA_LIB 27 | ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/ubuntu/lib/libnnabla.so 28 | ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/ubuntu/lib/libnnabla_utils.so 29 | ) 30 | if(INFERENCE_HELPER_ENABLE_NNABLA_CUDA) 31 | set(NNABLA_LIB ${NNABLA_LIB} ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/ubuntu/lib/libnnabla_cuda.so) 32 | endif() 33 | elseif(${BUILD_SYSTEM} STREQUAL "armv7") 34 | message(FATAL_ERROR "[nnabla] unsupported platform") 35 | elseif(${BUILD_SYSTEM} STREQUAL "aarch64") 36 | set(NNABLA_INC ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/aarch64/include) 37 | set(NNABLA_LIB 38 | ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/aarch64/lib/libnnabla.so 39 | ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/aarch64/lib/libnnabla_utils.so 40 | ) 41 | if(INFERENCE_HELPER_ENABLE_NNABLA_CUDA) 42 | set(NNABLA_LIB ${NNABLA_LIB} ${CMAKE_CURRENT_LIST_DIR}/../nnabla_prebuilt/aarch64/lib/libnnabla_cuda.so) 43 | endif() 44 | else() 45 | message(FATAL_ERROR "[nnabla] unsupported platform") 46 | endif() 47 | file(COPY ${NNABLA_LIB} DESTINATION ${CMAKE_BINARY_DIR}) 48 | endif() 49 | -------------------------------------------------------------------------------- /third_party/cmakes/onnx_runtime.cmake: -------------------------------------------------------------------------------- 1 | if(DEFINED ANDROID_ABI) 2 | set(ONNX_RUNTIME_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../onnxruntime_prebuilt/android") 3 | elseif(MSVC_VERSION) 4 | if(INFERENCE_HELPER_ENABLE_ONNX_RUNTIME_CUDA) 5 | set(ONNX_RUNTIME_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../onnxruntime_prebuilt/win-x64-gpu") 6 | else() 7 | set(ONNX_RUNTIME_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../onnxruntime_prebuilt/win-x64") 8 | endif() 9 | else() 10 | if(${BUILD_SYSTEM} STREQUAL "armv7") 11 | message(FATAL_ERROR "[onnx_runtime] unsupported platform") 12 | elseif(${BUILD_SYSTEM} STREQUAL "aarch64") 13 | set(ONNX_RUNTIME_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../onnxruntime_prebuilt/linux-aarch64") 14 | else() 15 | if(INFERENCE_HELPER_ENABLE_ONNX_RUNTIME_CUDA) 16 | set(ONNX_RUNTIME_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../onnxruntime_prebuilt/linux-x64-gpu") 17 | else() 18 | set(ONNX_RUNTIME_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../onnxruntime_prebuilt/linux-x64") 19 | endif() 20 | endif() 21 | endif() 22 | 23 | if(DEFINED ANDROID_ABI) 24 | add_library(ONNX_RUNTIME SHARED IMPORTED GLOBAL) 25 | set_target_properties( 26 | ONNX_RUNTIME 27 | PROPERTIES IMPORTED_LOCATION 28 | ${ONNX_RUNTIME_ROOTDIR}/jni/${ANDROID_ABI}/libonnxruntime.so 29 | ) 30 | set(ONNX_RUNTIME_LIB ONNX_RUNTIME) 31 | set(ONNX_RUNTIME_INC "${ONNX_RUNTIME_ROOTDIR}/headers") 32 | else() 33 | if(MSVC_VERSION) 34 | set(ONNX_RUNTIME_LIB ${ONNX_RUNTIME_ROOTDIR}/lib/onnxruntime.lib) 35 | if(INFERENCE_HELPER_ENABLE_ONNX_RUNTIME_CUDA) 36 | set(ONNX_RUNTIME_LIB ${ONNX_RUNTIME_LIB} ${ONNX_RUNTIME_ROOTDIR}/lib/onnxruntime_providers_cuda.lib) 37 | endif() 38 | # Copy dll files into the binary directory 39 | file(GLOB DLL_FILES files "${ONNX_RUNTIME_ROOTDIR}/lib/*.dll") 40 | file(COPY ${DLL_FILES} DESTINATION ${CMAKE_BINARY_DIR}) 41 | # Ensure to use pre-built onnxruntime.dll rather than that in system folder 42 | file(COPY ${DLL_FILES} DESTINATION ${CMAKE_BINARY_DIR}/Debug) 43 | file(COPY ${DLL_FILES} DESTINATION ${CMAKE_BINARY_DIR}/RelWithDebInfo) 44 | file(COPY ${DLL_FILES} DESTINATION ${CMAKE_BINARY_DIR}/Release) 45 | else() 46 | set(ONNX_RUNTIME_LIB ${ONNX_RUNTIME_ROOTDIR}/lib/libonnxruntime.so) 47 | endif() 48 | 49 | set(ONNX_RUNTIME_INC "${ONNX_RUNTIME_ROOTDIR}/include" "${ONNX_RUNTIME_ROOTDIR}/include/onnxruntime/core/session") 50 | endif() 51 | -------------------------------------------------------------------------------- /third_party/cmakes/sample.cmake: -------------------------------------------------------------------------------- 1 | if(DEFINED ANDROID_ABI) 2 | elseif(MSVC_VERSION) 3 | else() 4 | if(${BUILD_SYSTEM} STREQUAL "armv7") 5 | elseif(${BUILD_SYSTEM} STREQUAL "aarch64") 6 | else() 7 | endif() 8 | endif() 9 | 10 | if(DEFINED ANDROID_ABI) 11 | else() 12 | if(MSVC_VERSION) 13 | else() 14 | endif() 15 | endif() 16 | 17 | # set the following variables 18 | set(sample_LIB libsample.so) 19 | set(sample_INC "./") 20 | -------------------------------------------------------------------------------- /third_party/cmakes/snpe.cmake: -------------------------------------------------------------------------------- 1 | set(SNPE_INC 2 | ${CMAKE_CURRENT_LIST_DIR}/../snpe_prebuilt/include/zdl 3 | ) 4 | 5 | if(DEFINED ANDROID_ABI) 6 | # Copy libSNPE.so to app/jniExternalLibs/arm64-v8a 7 | # if(${ANDROID_ABI} STREQUAL "arm64-v8a") 8 | # set(SNPE_LIB ${CMAKE_CURRENT_LIST_DIR}/../snpe_prebuilt/lib/aarch64-android-clang6.0/libSNPE.so) 9 | # elseif(${ANDROID_ABI} STREQUAL "armeabi-v7a") 10 | # set(SNPE_LIB ${CMAKE_CURRENT_LIST_DIR}/../snpe_prebuilt/lib/arm-android-clang6.0/libSNPE.so) 11 | # else() 12 | # message(FATAL_ERROR "other") 13 | # endif() 14 | 15 | add_library(SNPE SHARED IMPORTED GLOBAL) 16 | if(${ANDROID_ABI} STREQUAL "arm64-v8a") 17 | set_target_properties( 18 | SNPE 19 | PROPERTIES IMPORTED_LOCATION 20 | ${CMAKE_CURRENT_LIST_DIR}/../snpe_prebuilt/lib/aarch64-android-clang6.0/libSNPE.so 21 | ) 22 | elseif(${ANDROID_ABI} STREQUAL "armeabi-v7a") 23 | set_target_properties( 24 | SNPE 25 | PROPERTIES IMPORTED_LOCATION 26 | ${CMAKE_CURRENT_LIST_DIR}/../snpe_prebuilt/lib/arm-android-clang6.0/libSNPE.so 27 | ) 28 | else() 29 | message(FATAL_ERROR "other") 30 | endif() 31 | set(SNPE_LIB SNPE) 32 | elseif(MSVC_VERSION) 33 | message(FATAL_ERROR "[SNPE] unsupported platform") 34 | else() 35 | if(${BUILD_SYSTEM} STREQUAL "x64_linux") 36 | set(SNPE_PATH ${CMAKE_CURRENT_LIST_DIR}/../snpe_prebuilt/lib/x86_64-linux-clang/) 37 | elseif(${BUILD_SYSTEM} STREQUAL "armv7") 38 | message(FATAL_ERROR "[SNPE] unsupported platform") 39 | elseif(${BUILD_SYSTEM} STREQUAL "aarch64") 40 | set(SNPE_PATH ${CMAKE_CURRENT_LIST_DIR}/../snpe_prebuilt/lib/aarch64-linux-gcc4.9/) 41 | else() 42 | message(FATAL_ERROR "[SNPE] unsupported platform") 43 | endif() 44 | file(GLOB LIB_FILES ${SNPE_PATH}/*) 45 | file(COPY ${LIB_FILES} DESTINATION ${CMAKE_BINARY_DIR}) 46 | file(GLOB LIB_FILES ${SNPE_PATH}/../dsp/*) 47 | file(COPY ${LIB_FILES} DESTINATION ${CMAKE_BINARY_DIR}) 48 | set(SNPE_LIB ${CMAKE_BINARY_DIR}/libSNPE.so) 49 | endif() 50 | -------------------------------------------------------------------------------- /third_party/cmakes/tensorflow.cmake: -------------------------------------------------------------------------------- 1 | if(DEFINED ANDROID_ABI) 2 | message(FATAL_ERROR "[tensorflow] unsupported platform") 3 | elseif(MSVC_VERSION) 4 | if(INFERENCE_HELPER_ENABLE_TENSORFLOW_GPU) 5 | set(TENSORFLOW_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../tensorflow_prebuilt/win-x64-gpu") 6 | else() 7 | set(TENSORFLOW_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../tensorflow_prebuilt/win-x64") 8 | endif() 9 | else() 10 | if(${BUILD_SYSTEM} STREQUAL "armv7") 11 | message(FATAL_ERROR "[tensorflow] unsupported platform") 12 | elseif(${BUILD_SYSTEM} STREQUAL "aarch64") 13 | message(FATAL_ERROR "[tensorflow] unsupported platform") 14 | else() 15 | if(INFERENCE_HELPER_ENABLE_TENSORFLOW_GPU) 16 | set(TENSORFLOW_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../tensorflow_prebuilt/linux-x64-gpu") 17 | else() 18 | set(TENSORFLOW_ROOTDIR "${CMAKE_CURRENT_LIST_DIR}/../tensorflow_prebuilt/linux-x64") 19 | endif() 20 | endif() 21 | endif() 22 | 23 | if(DEFINED ANDROID_ABI) 24 | message(FATAL_ERROR "[tensorflow] unsupported platform") 25 | else() 26 | if(MSVC_VERSION) 27 | set(TENSORFLOW_LIB ${TENSORFLOW_ROOTDIR}/lib/tensorflow.lib) 28 | # Copy dll files into the binary directory 29 | file(GLOB DLL_FILES files "${TENSORFLOW_ROOTDIR}/lib/*.dll") 30 | file(COPY ${DLL_FILES} DESTINATION ${CMAKE_BINARY_DIR}) 31 | # Ensure to use pre-built tensorflow.dll rather than that in system folder 32 | file(COPY ${DLL_FILES} DESTINATION ${CMAKE_BINARY_DIR}/Debug) 33 | file(COPY ${DLL_FILES} DESTINATION ${CMAKE_BINARY_DIR}/RelWithDebInfo) 34 | file(COPY ${DLL_FILES} DESTINATION ${CMAKE_BINARY_DIR}/Release) 35 | else() 36 | set(TENSORFLOW_LIB ${TENSORFLOW_ROOTDIR}/lib/libtensorflow.so) 37 | endif() 38 | 39 | set(TENSORFLOW_INC "${TENSORFLOW_ROOTDIR}/include") 40 | endif() 41 | -------------------------------------------------------------------------------- /third_party/cmakes/tflite.cmake: -------------------------------------------------------------------------------- 1 | set(TFLITE_INC 2 | ${CMAKE_CURRENT_LIST_DIR}/../tensorflow 3 | ${CMAKE_CURRENT_LIST_DIR}/../tensorflow_deps/abseil-cpp 4 | ${CMAKE_CURRENT_LIST_DIR}/../tensorflow_deps/flatbuffers/include 5 | ) 6 | 7 | if(DEFINED ANDROID_ABI) 8 | # set(TFLITE_LIB ${CMAKE_CURRENT_LIST_DIR}/../tensorflow_prebuilt/android/${ANDROID_ABI}/libtensorflowlite.so) 9 | add_library(TFLITE SHARED IMPORTED GLOBAL) 10 | set_target_properties( 11 | TFLITE 12 | PROPERTIES IMPORTED_LOCATION 13 | ${CMAKE_CURRENT_LIST_DIR}/../tflite_prebuilt/android/${ANDROID_ABI}/libtensorflowlite.so 14 | ) 15 | set(TFLITE_LIB TFLITE) 16 | elseif(MSVC_VERSION) 17 | set(TFLITE_LIB 18 | $<$:${CMAKE_CURRENT_LIST_DIR}/../tflite_prebuilt/windows-vs2019/debug/libtensorflowlite.so.if.lib> 19 | $<$:${CMAKE_CURRENT_LIST_DIR}/../tflite_prebuilt/windows-vs2019/libtensorflowlite.so.if.lib> 20 | $<$:${CMAKE_CURRENT_LIST_DIR}/../tflite_prebuilt/windows-vs2019/libtensorflowlite.so.if.lib> 21 | $<$:${CMAKE_CURRENT_LIST_DIR}/../tflite_prebuilt/windows-vs2019/libtensorflowlite.so.if.lib> 22 | ) 23 | file(COPY ${CMAKE_CURRENT_LIST_DIR}/../tflite_prebuilt/windows-vs2019/libtensorflowlite.so DESTINATION ${CMAKE_BINARY_DIR}) 24 | file(COPY ${CMAKE_CURRENT_LIST_DIR}/../tflite_prebuilt/windows-vs2019/debug/libtensorflowlite.so DESTINATION ${CMAKE_BINARY_DIR}/Debug) 25 | else() 26 | if(${BUILD_SYSTEM} STREQUAL "x64_linux") 27 | set(TFLITE_LIB ${CMAKE_CURRENT_LIST_DIR}/../tflite_prebuilt/ubuntu/libtensorflowlite.so) 28 | elseif(${BUILD_SYSTEM} STREQUAL "armv7") 29 | set(TFLITE_LIB ${CMAKE_CURRENT_LIST_DIR}/../tflite_prebuilt/armv7/libtensorflowlite.so) 30 | elseif(${BUILD_SYSTEM} STREQUAL "aarch64") 31 | set(TFLITE_LIB ${CMAKE_CURRENT_LIST_DIR}/../tflite_prebuilt/aarch64/libtensorflowlite.so) 32 | else() 33 | message(FATAL_ERROR "[tflite] unsupported platform") 34 | endif() 35 | file(COPY ${TFLITE_LIB} DESTINATION ${CMAKE_BINARY_DIR}) 36 | endif() 37 | -------------------------------------------------------------------------------- /third_party/cmakes/tflite_edgetpu.cmake: -------------------------------------------------------------------------------- 1 | set(TFLITE_EDGETPU_INC 2 | ${CMAKE_CURRENT_LIST_DIR}/../edgetpu_prebuilt/include/ 3 | ) 4 | 5 | if(MSVC_VERSION) 6 | set(TFLITE_EDGETPU_LIB 7 | $<$:${CMAKE_CURRENT_LIST_DIR}/../edgetpu_prebuilt/direct/windows-vs2019/debug/edgetpu_direct_all.dll.if.lib> 8 | $<$:${CMAKE_CURRENT_LIST_DIR}/../edgetpu_prebuilt/direct/windows-vs2019/edgetpu_direct_all.dll.if.lib> 9 | $<$:${CMAKE_CURRENT_LIST_DIR}/../edgetpu_prebuilt/direct/windows-vs2019/edgetpu_direct_all.dll.if.lib> 10 | $<$:${CMAKE_CURRENT_LIST_DIR}/../edgetpu_prebuilt/direct/windows-vs2019/edgetpu_direct_all.dll.if.lib> 11 | ) 12 | file(COPY ${CMAKE_CURRENT_LIST_DIR}/../edgetpu_prebuilt/direct/windows-vs2019/edgetpu_direct_all.dll DESTINATION ${CMAKE_BINARY_DIR}) 13 | file(COPY ${CMAKE_CURRENT_LIST_DIR}/../edgetpu_prebuilt/direct/windows-vs2019/debug/edgetpu_direct_all.dll DESTINATION ${CMAKE_BINARY_DIR}/Debug) 14 | file(COPY ${CMAKE_CURRENT_LIST_DIR}/../edgetpu_prebuilt/direct/windows-vs2019/libusb-1.0.dll DESTINATION ${CMAKE_BINARY_DIR}) 15 | else() 16 | if(${BUILD_SYSTEM} STREQUAL "x64_linux") 17 | set(TFLITE_EDGETPU_LIB ${CMAKE_CURRENT_LIST_DIR}/../edgetpu_prebuilt/direct/k8/libedgetpu.so.1.0) 18 | elseif(${BUILD_SYSTEM} STREQUAL "armv7") 19 | set(TFLITE_EDGETPU_LIB ${CMAKE_CURRENT_LIST_DIR}/../edgetpu_prebuilt/direct/armv7a/libedgetpu.so.1.0) 20 | elseif(${BUILD_SYSTEM} STREQUAL "aarch64") 21 | set(TFLITE_EDGETPU_LIB ${CMAKE_CURRENT_LIST_DIR}/../edgetpu_prebuilt/direct/aarch64/libedgetpu.so.1.0) 22 | else() 23 | message(FATAL_ERROR "[tflite_edgetpu] unsupported platform") 24 | endif() 25 | file(COPY ${TFLITE_EDGETPU_LIB} DESTINATION ${CMAKE_BINARY_DIR}) 26 | endif() 27 | -------------------------------------------------------------------------------- /third_party/cmakes/tflite_edgetpu_pipeline.cmake: -------------------------------------------------------------------------------- 1 | # Create sub library for Edge TPU Pipiline 2 | set(TFLITE_EDGETPU_PIPELINE "TFLITE_EDGETPU_PIPELINE") 3 | add_library(${TFLITE_EDGETPU_PIPELINE} SHARED 4 | ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline/allocator.h 5 | ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline/common.h 6 | ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline/utils.cc 7 | ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline/utils.h 8 | ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline/pipelined_model_runner.cc 9 | ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline/pipelined_model_runner.h 10 | ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline/internal/aligned_alloc.h 11 | ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline/internal/default_allocator.h 12 | ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline/internal/memory_pool_allocator.cc 13 | ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline/internal/memory_pool_allocator.h 14 | ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline/internal/segment_runner.cc 15 | ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline/internal/segment_runner.h 16 | ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline/internal/thread_safe_queue.h 17 | ) 18 | 19 | target_include_directories(${TFLITE_EDGETPU_PIPELINE} PUBLIC ${CMAKE_CURRENT_LIST_DIR}/../edgetpu) 20 | target_include_directories(${TFLITE_EDGETPU_PIPELINE} PUBLIC ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/libedgetpu) 21 | target_include_directories(${TFLITE_EDGETPU_PIPELINE} PUBLIC ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline) 22 | target_include_directories(${TFLITE_EDGETPU_PIPELINE} PUBLIC ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline/internal) 23 | target_include_directories(${TFLITE_EDGETPU_PIPELINE} PUBLIC ${CMAKE_CURRENT_LIST_DIR}/../tensorflow) 24 | target_include_directories(${TFLITE_EDGETPU_PIPELINE} PUBLIC ${CMAKE_CURRENT_LIST_DIR}/../tensorflow/tensorflow/lite/tools/make/downloads/flatbuffers/include) 25 | target_include_directories(${TFLITE_EDGETPU_PIPELINE} PUBLIC ${CMAKE_CURRENT_LIST_DIR}/../tensorflow/tensorflow/lite/tools/make/downloads/absl) 26 | 27 | # libraries used in pipeline 28 | ## glog 29 | add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../glog glog) 30 | target_link_libraries(${TFLITE_EDGETPU_PIPELINE} glog) 31 | target_include_directories(${TFLITE_EDGETPU_PIPELINE} PUBLIC ${CMAKE_BINARY_DIR}/glog) 32 | ## absl 33 | set(BUILD_TESTING OFF CACHE BOOL "disable BUILD_TESTING for absl" FORCE) 34 | add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../abseil-cpp absl) 35 | set(ABSL_LIBS absl_synchronization absl_stacktrace absl_symbolize absl_demangle_internal absl_debugging_internal absl_dynamic_annotations absl_time absl_time_zone absl_graphcycles_internal absl_failure_signal_handler absl_malloc_internal absl_base absl_spinlock_wait) 36 | target_link_libraries(${TFLITE_EDGETPU_PIPELINE} ${ABSL_LIBS}) 37 | if(NOT WIN32) 38 | target_link_libraries(${TFLITE_EDGETPU_PIPELINE} atomic) 39 | endif() 40 | 41 | set(TFLITE_EDGETPU_PIPELINE_INC 42 | ${CMAKE_CURRENT_LIST_DIR}/../edgetpu/src/cpp/pipeline 43 | ) 44 | set(TFLITE_EDGETPU_PIPELINE_LIB 45 | ${TFLITE_EDGETPU_PIPELINE} 46 | ) 47 | -------------------------------------------------------------------------------- /third_party/cmakes/tflite_gpu.cmake: -------------------------------------------------------------------------------- 1 | set(TFLITE_GPU_INC 2 | ${CMAKE_CURRENT_LIST_DIR}/../tensorflow 3 | ${CMAKE_CURRENT_LIST_DIR}/../tensorflow_deps/abseil-cpp 4 | ${CMAKE_CURRENT_LIST_DIR}/../tensorflow_deps/flatbuffers/include 5 | ) 6 | 7 | if(DEFINED ANDROID_ABI) 8 | # set(TFLITE_GPU_LIB ${CMAKE_CURRENT_LIST_DIR}/../tensorflow_prebuilt/android/${ANDROID_ABI}/libtensorflowlite_gpu_delegate.so) 9 | add_library(TFLITE_GPU SHARED IMPORTED GLOBAL) 10 | set_target_properties( 11 | TFLITE_GPU 12 | PROPERTIES IMPORTED_LOCATION 13 | ${CMAKE_CURRENT_LIST_DIR}/../tflite_prebuilt/android/${ANDROID_ABI}/libtensorflowlite_gpu_delegate.so 14 | ) 15 | set(TFLITE_GPU_LIB TFLITE_GPU) 16 | else() 17 | message(FATAL_ERROR "[tflite_gpu] unsupported platform") 18 | endif() 19 | -------------------------------------------------------------------------------- /third_party/download_prebuilt_libraries.sh: -------------------------------------------------------------------------------- 1 | IGNORE_GPU=${1:-0} 2 | 3 | move_dir_to_shell_file() { 4 | dir_shell_file=`dirname "$0"` 5 | cd ${dir_shell_file} 6 | } 7 | 8 | download_and_extract_ncnn() { 9 | local url_base="https://github.com/Tencent/ncnn/releases/download/" 10 | local tag="20211208" 11 | local prefix="ncnn-20211208-" 12 | local name=$1 13 | local url="${url_base}${tag}/${prefix}${name}.zip" 14 | echo "Downloading ${url}" 15 | curl -Lo temp.zip ${url} 16 | unzip -o temp.zip 17 | rm -rf ${name} 18 | mv ${prefix}${name} ${name} 19 | } 20 | 21 | download_and_extract() { 22 | local url=$1 23 | echo "Downloading ${url}" 24 | local ext=${url##*.} 25 | if [ `echo ${ext} | grep zip` ]; then 26 | curl -Lo temp.zip ${url} 27 | unzip -o temp.zip 28 | rm temp.zip 29 | else 30 | curl -Lo temp.tgz ${url} 31 | tar xzvf temp.tgz 32 | rm temp.tgz 33 | fi 34 | } 35 | 36 | download_and_extract_onnxruntime() { 37 | local url_base="https://github.com/microsoft/onnxruntime/releases/download/" 38 | local tag="v1.10.0" 39 | local prefix="onnxruntime-" 40 | local suffix="-1.10.0" 41 | local name=$1 42 | local ext=$2 43 | local url="${url_base}${tag}/${prefix}${name}${suffix}.${ext}" 44 | echo "Downloading ${url}" 45 | if [ `echo ${ext} | grep zip` ]; then 46 | curl -Lo temp.zip ${url} 47 | unzip -o temp.zip 48 | rm temp.zip 49 | else 50 | curl -Lo temp.tgz ${url} 51 | tar xzvf temp.tgz 52 | rm temp.tgz 53 | fi 54 | rm -rf ${name} 55 | mv ${prefix}${name}${suffix} ${name} 56 | } 57 | 58 | download_and_extract_onnxruntime_andriod() { 59 | # https://search.maven.org/artifact/com.microsoft.onnxruntime/onnxruntime-mobile/1.10.0/aar 60 | local url="https://search.maven.org/remotecontent?filepath=com/microsoft/onnxruntime/onnxruntime-mobile/1.10.0/onnxruntime-mobile-1.10.0.aar" 61 | echo "Downloading ${url}" 62 | mkdir -p android && cd android 63 | curl -Lo temp.zip ${url} 64 | unzip -o temp.zip 65 | rm temp.zip 66 | cd .. 67 | } 68 | 69 | download_and_extract_libtorch_andriod() { 70 | local url="https://repo1.maven.org/maven2/org/pytorch/pytorch_android_torchvision/1.12/pytorch_android_torchvision-1.12.aar" 71 | echo "Downloading ${url}" 72 | mkdir -p android && cd android 73 | curl -Lo temp.zip ${url} 74 | unzip -o temp.zip 75 | rm temp.zip 76 | cd .. 77 | } 78 | ######################################################################## 79 | 80 | ### cd to the same directory as this shell file ### 81 | move_dir_to_shell_file 82 | 83 | ### Get other projects (dependencies) ### 84 | git submodule update --init --recommend-shallow --depth 1 85 | 86 | ### Download pre-built libraries from https://github.com/iwatake2222/InferenceHelper_Binary ### 87 | download_and_extract "https://github.com/iwatake2222/InferenceHelper_Binary/releases/download/v20220210/armnn_prebuilt_linux_1804.tgz" 88 | # download_and_extract "https://github.com/iwatake2222/InferenceHelper_Binary/releases/download/v20220210/armnn_prebuilt_linux_2004.tgz" 89 | download_and_extract "https://github.com/iwatake2222/InferenceHelper_Binary/releases/download/v20220210/edgetpu_prebuilt_linux_1804.tgz" 90 | # download_and_extract "https://github.com/iwatake2222/InferenceHelper_Binary/releases/download/v20220210/edgetpu_prebuilt_linux_2004.tgz" 91 | download_and_extract "https://github.com/iwatake2222/InferenceHelper_Binary/releases/download/v20220210/edgetpu_prebuilt_windows.zip" 92 | download_and_extract "https://github.com/iwatake2222/InferenceHelper_Binary/releases/download/v20220210/mnn_prebuilt_linux_1804.tgz" 93 | # download_and_extract "https://github.com/iwatake2222/InferenceHelper_Binary/releases/download/v20220210/mnn_prebuilt_linux_2004.tgz" 94 | download_and_extract "https://github.com/iwatake2222/InferenceHelper_Binary/releases/download/v20220210/mnn_prebuilt_windows.zip" 95 | download_and_extract "https://github.com/iwatake2222/InferenceHelper_Binary/releases/download/v20220210/tflite_prebuilt_linux_1804.tgz" 96 | # download_and_extract "https://github.com/iwatake2222/InferenceHelper_Binary/releases/download/v20220210/tflite_prebuilt_linux_2004.tgz" 97 | download_and_extract "https://github.com/iwatake2222/InferenceHelper_Binary/releases/download/v20220210/tflite_prebuilt_windows.zip" 98 | 99 | ### Download ncnn pre-built libraries from https://github.com/Tencent/ncnn ### 100 | mkdir -p ncnn_prebuilt && cd ncnn_prebuilt 101 | download_and_extract_ncnn "android-vulkan" 102 | download_and_extract_ncnn "android" 103 | download_and_extract_ncnn "ubuntu-2004-shared" 104 | download_and_extract_ncnn "ubuntu-2004" 105 | download_and_extract_ncnn "windows-vs2019-shared" 106 | download_and_extract_ncnn "windows-vs2019" 107 | cd .. 108 | 109 | 110 | ### Download NNabla pre-built libraries from https://nnabla.org/cpplib 111 | mkdir -p nnabla_prebuilt/windows-vs2019 && mkdir -p nnabla_prebuilt/aarch64 && cd nnabla_prebuilt 112 | curl -L https://nnabla.org/cpplib/1.25.0/nnabla-cpplib-1.25.0-win64.zip -o temp.zip 113 | unzip -o temp.zip 114 | rm temp.zip 115 | mv nnabla-cpplib-1.25.0-win64/* windows-vs2019/. 116 | 117 | if [ ${IGNORE_GPU} -eq 0 ]; then 118 | curl -L https://nnabla.org/cpplib/1.25.0/nnabla-cpplib-cuda_110_8-1.25.0-win64.zip -o temp.zip 119 | unzip -o temp.zip 120 | rm temp.zip 121 | cp -r nnabla-cpplib-cuda_110_8-1.25.0-win64/* windows-vs2019/. 122 | fi 123 | 124 | curl -L https://github.com/libarchive/libarchive/releases/download/v3.5.2/libarchive-v3.5.2-win64.zip -o temp.zip 125 | unzip -o temp.zip 126 | mv libarchive/bin/archive.dll windows-vs2019/bin/. 127 | 128 | curl -L https://nnabla.org/cpplib/1.25.0/nnabla-cpplib-1.25.0-Linux_aarch64.zip -o temp.zip 129 | unzip -o temp.zip 130 | rm temp.zip 131 | mv nnabla-cpplib-1.25.0-Linux_aarch64/* aarch64/. 132 | cd .. 133 | 134 | 135 | ### Download ONNX Runtime pre-built libraries from https://github.com/microsoft/onnxruntime ### 136 | mkdir -p onnxruntime_prebuilt && cd onnxruntime_prebuilt 137 | download_and_extract_onnxruntime "win-x64" "zip" 138 | download_and_extract_onnxruntime "linux-x64" "tgz" 139 | download_and_extract_onnxruntime "linux-aarch64" "tgz" 140 | download_and_extract_onnxruntime_andriod 141 | if [ ${IGNORE_GPU} -eq 0 ]; then 142 | download_and_extract_onnxruntime "win-x64-gpu" "zip" 143 | download_and_extract_onnxruntime "linux-x64-gpu" "tgz" 144 | fi 145 | cd .. 146 | 147 | 148 | ### Download LibTorch pre-built libraries from https://pytorch.org/get-started/locally/ ### 149 | mkdir -p libtorch_prebuilt && cd libtorch_prebuilt 150 | download_and_extract https://download.pytorch.org/libtorch/cpu/libtorch-win-shared-with-deps-1.11.0%2Bcpu.zip 151 | # download_and_extract https://download.pytorch.org/libtorch/cpu/libtorch-win-shared-with-deps-debug-1.11.0%2Bcpu.zip 152 | mv libtorch win-x64 153 | download_and_extract https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-1.11.0%2Bcpu.zip 154 | mv libtorch linux-x64 155 | download_and_extract_libtorch_andriod 156 | if [ ${IGNORE_GPU} -eq 0 ]; then 157 | download_and_extract https://download.pytorch.org/libtorch/cu113/libtorch-win-shared-with-deps-1.11.0%2Bcu113.zip 158 | # download_and_extract https://download.pytorch.org/libtorch/cu113/libtorch-win-shared-with-deps-debug-1.11.0%2Bcu113.zip 159 | mv libtorch win-x64-gpu 160 | download_and_extract https://download.pytorch.org/libtorch/cu113/libtorch-cxx11-abi-shared-with-deps-1.11.0%2Bcu113.zip 161 | mv libtorch linux-x64-gpu 162 | fi 163 | cd .. 164 | 165 | 166 | ### Download TensorFlow pre-built libraries from https://www.tensorflow.org/install/lang_c ### 167 | mkdir -p tensorflow_prebuilt && cd tensorflow_prebuilt 168 | mkdir -p win-x64 && cd win-x64 169 | download_and_extract https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-windows-x86_64-2.7.0.zip 170 | cd .. && mkdir -p linux-x64 && cd linux-x64 171 | download_and_extract https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-2.7.0.tar.gz 172 | if [ ${IGNORE_GPU} -eq 0 ]; then 173 | cd .. && mkdir -p win-x64-gpu && cd win-x64-gpu 174 | download_and_extract https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-windows-x86_64-2.7.0.zip 175 | cd .. && mkdir -p linux-x64-gpu && cd linux-x64-gpu 176 | download_and_extract https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-2.7.0.tar.gz 177 | fi 178 | cd .. 179 | --------------------------------------------------------------------------------