├── CMakeLists.txt ├── LICENSE ├── README.md ├── RELEASE_NOTES.md ├── SCR-1.5.1.txt ├── common ├── common_utils.sh ├── cpp │ ├── README.md │ ├── include │ │ ├── common.hpp │ │ ├── gst_pipeline_imx.hpp │ │ ├── gst_source_imx.hpp │ │ ├── gst_video_imx.hpp │ │ ├── gst_video_post_process.hpp │ │ ├── imx_devices.hpp │ │ ├── logging.hpp │ │ ├── model_infos.hpp │ │ ├── nn_decoder.hpp │ │ └── tensor_custom_data_generator.hpp │ └── src │ │ ├── gst_pipeline_imx.cpp │ │ ├── gst_source_imx.cpp │ │ ├── gst_video_imx.cpp │ │ ├── gst_video_post_process.cpp │ │ ├── model_infos.cpp │ │ ├── nn_decoder.cpp │ │ └── tensor_custom_data_generator.cpp └── python │ └── imxpy │ ├── __init__.py │ ├── common_utils.py │ └── imx_dev.py ├── downloads ├── README.md ├── compile_models.sh ├── download.ipynb └── requirements.txt ├── licenses └── LGPL-2.1-only.txt ├── tasks ├── classification │ ├── README.md │ ├── classification_demo.webp │ ├── classification_utils.sh │ ├── cpp │ │ └── example_classification_mobilenet_v1_tflite.cpp │ └── example_classification_mobilenet_v1_tflite.sh ├── face-processing │ ├── README.md │ ├── common │ │ ├── deepface.py │ │ ├── facedetectpipe.py │ │ ├── facenet.py │ │ ├── facenet_create_embedding.py │ │ ├── facenet_create_embedding_requirements.txt │ │ ├── facenet_db │ │ │ ├── angelina_jolie.npy │ │ │ ├── brad_pitt.npy │ │ │ └── thispersondoesnotexist.npy │ │ └── ultraface.py │ ├── emotion-classification │ │ ├── cpp │ │ │ ├── custom_emotion_decoder.cpp │ │ │ ├── custom_emotion_decoder.hpp │ │ │ └── example_emotion_classification_tflite.cpp │ │ └── example_emotion_classification_tflite.py │ ├── face-detection │ │ ├── cpp │ │ │ ├── custom_face_decoder.cpp │ │ │ ├── custom_face_decoder.hpp │ │ │ └── example_face_detection_tflite.cpp │ │ └── example_face_detection_tflite.py │ ├── face-recogniton │ │ └── example_face_recognition_tflite.py │ ├── face_demo.webp │ ├── main.svg │ ├── secondary.svg │ └── thispersondoesnotexist.jpeg ├── mixed-demos │ ├── README.md │ ├── cpp │ │ ├── custom_face_and_pose_decoder.cpp │ │ ├── custom_face_and_pose_decoder.hpp │ │ ├── example_classification_and_detection_tflite.cpp │ │ ├── example_double_classification_tflite.cpp │ │ ├── example_emotion_and_detection_tflite.cpp │ │ └── example_face_and_pose_detection_tflite.cpp │ └── mixed_demo.webp ├── monocular-depth-estimation │ ├── README.md │ ├── cpp │ │ ├── custom_depth_decoder.cpp │ │ ├── custom_depth_decoder.hpp │ │ └── example_depth_midas_v2_tflite.cpp │ ├── depth_demo.webp │ ├── export_midas-v2_to_TensorFlow.sh │ └── requirements.txt ├── object-detection │ ├── README.md │ ├── README_postprocess_yolov4_tiny.md │ ├── cpp │ │ └── example_detection_mobilenet_ssd_v2_tflite.cpp │ ├── detection_demo.webp │ ├── detection_utils.sh │ ├── example_detection_mobilenet_ssd_v2_tflite.sh │ ├── example_detection_yolo_v4_tiny_tflite.sh │ ├── postprocess_yolov4_tiny.py │ └── yolov4-tiny_export_model.py ├── pose-estimation │ ├── README.md │ ├── cpp │ │ ├── custom_pose_decoder.cpp │ │ ├── custom_pose_decoder.hpp │ │ └── example_pose_movenet_tflite.cpp │ ├── example_pose_movenet_tflite.py │ └── pose_demo.webp └── semantic-segmentation │ ├── README.md │ ├── cpp │ └── example_segmentation_deeplab_v3_tflite.cpp │ ├── example_segmentation_deeplab_v3_tflite.sh │ ├── segmentation_demo.webp │ └── segmentation_utils.sh └── tools ├── setup_environment.sh └── upload.sh /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2023-2024 NXP 2 | # SPDX-License-Identifier: BSD-3-Clause 3 | 4 | cmake_minimum_required(VERSION 3.0) 5 | project(nxp-nnstreamer-examples) 6 | 7 | # Specify the C++ standard 8 | set(CMAKE_CXX_STANDARD 17) 9 | set(CMAKE_CXX_STANDARD_REQUIRED True) 10 | 11 | # Add GStreamer library 12 | find_package(PkgConfig) 13 | pkg_check_modules(GSTREAMER REQUIRED gstreamer-1.0) 14 | pkg_check_modules(GSTREAMER REQUIRED gstreamer-app-1.0) 15 | include_directories(${GSTREAMER_INCLUDE_DIRS}) 16 | link_directories(${GSTREAMER_LIBRARY_DIRS}) 17 | 18 | # Add Cairo library 19 | pkg_check_modules(CAIRO REQUIRED cairo) 20 | include_directories(${CAIRO_INCLUDE_DIRS}) 21 | link_directories(${CAIRO_LIBRARY_DIRS}) 22 | 23 | #Add Custom library 24 | include_directories(${CMAKE_CURRENT_SOURCE_DIR}/common/cpp/include) 25 | file(GLOB all_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/common/cpp/src/*.cpp") 26 | 27 | # Example of object classification (mobilenet_v1) 28 | add_executable( 29 | example_classification_mobilenet_v1_tflite 30 | ${all_SRCS} 31 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/classification/cpp/example_classification_mobilenet_v1_tflite.cpp 32 | ) 33 | target_link_libraries(example_classification_mobilenet_v1_tflite ${GSTREAMER_LIBRARIES} ${CAIRO_LIBRARIES} tensorflow-lite) 34 | set_target_properties(example_classification_mobilenet_v1_tflite PROPERTIES RUNTIME_OUTPUT_DIRECTORY ./classification) 35 | 36 | # Example of object detection (mobilenet_ssd) 37 | add_executable( 38 | example_detection_mobilenet_ssd_v2_tflite 39 | ${all_SRCS} 40 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/object-detection/cpp/example_detection_mobilenet_ssd_v2_tflite.cpp 41 | ) 42 | target_link_libraries(example_detection_mobilenet_ssd_v2_tflite ${GSTREAMER_LIBRARIES} ${CAIRO_LIBRARIES} tensorflow-lite) 43 | set_target_properties(example_detection_mobilenet_ssd_v2_tflite PROPERTIES RUNTIME_OUTPUT_DIRECTORY ./object-detection) 44 | 45 | # Example of object classification (mobilenet_v1) and object detection (mobilenet_ssd) in parallel 46 | add_executable( 47 | example_classification_and_detection_tflite 48 | ${all_SRCS} 49 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/mixed-demos/cpp/example_classification_and_detection_tflite.cpp 50 | ) 51 | target_link_libraries(example_classification_and_detection_tflite ${GSTREAMER_LIBRARIES} ${CAIRO_LIBRARIES} tensorflow-lite) 52 | set_target_properties(example_classification_and_detection_tflite PROPERTIES RUNTIME_OUTPUT_DIRECTORY ./mixed-demos) 53 | 54 | # Example of object segmentation (deeplab_v3) 55 | add_executable( 56 | example_segmentation_deeplab_v3_tflite 57 | ${all_SRCS} 58 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/semantic-segmentation/cpp/example_segmentation_deeplab_v3_tflite.cpp 59 | ) 60 | target_link_libraries(example_segmentation_deeplab_v3_tflite ${GSTREAMER_LIBRARIES} ${CAIRO_LIBRARIES} tensorflow-lite) 61 | set_target_properties(example_segmentation_deeplab_v3_tflite PROPERTIES RUNTIME_OUTPUT_DIRECTORY ./semantic-segmentation) 62 | 63 | # Example of pose detection (PoseNet Lightning) 64 | add_executable( 65 | example_pose_movenet_tflite 66 | ${all_SRCS} 67 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/pose-estimation/cpp/example_pose_movenet_tflite.cpp 68 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/pose-estimation/cpp/custom_pose_decoder.cpp 69 | ) 70 | target_include_directories(example_pose_movenet_tflite PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tasks/pose-estimation/cpp/) 71 | target_link_libraries(example_pose_movenet_tflite ${GSTREAMER_LIBRARIES} ${CAIRO_LIBRARIES} tensorflow-lite) 72 | set_target_properties(example_pose_movenet_tflite PROPERTIES RUNTIME_OUTPUT_DIRECTORY ./pose-estimation) 73 | 74 | # Example of face detection (UltraFace slim) 75 | add_executable( 76 | example_face_detection_tflite 77 | ${all_SRCS} 78 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/face-processing/face-detection/cpp/example_face_detection_tflite.cpp 79 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/face-processing/face-detection/cpp/custom_face_decoder.cpp 80 | ) 81 | target_include_directories(example_face_detection_tflite PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tasks/face-processing/face-detection/cpp/) 82 | target_link_libraries(example_face_detection_tflite ${GSTREAMER_LIBRARIES} ${CAIRO_LIBRARIES} tensorflow-lite) 83 | set_target_properties(example_face_detection_tflite PROPERTIES RUNTIME_OUTPUT_DIRECTORY ./face-processing) 84 | 85 | # Example of face detection (UltraFace slim) and pose detection (PoseNet Lightning) in parallel 86 | add_executable( 87 | example_face_and_pose_detection_tflite 88 | ${all_SRCS} 89 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/mixed-demos/cpp/example_face_and_pose_detection_tflite.cpp 90 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/mixed-demos/cpp/custom_face_and_pose_decoder.cpp) 91 | target_include_directories(example_face_and_pose_detection_tflite PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tasks/mixed-demos/cpp/) 92 | target_link_libraries(example_face_and_pose_detection_tflite ${GSTREAMER_LIBRARIES} ${CAIRO_LIBRARIES} tensorflow-lite) 93 | set_target_properties(example_face_and_pose_detection_tflite PROPERTIES RUNTIME_OUTPUT_DIRECTORY ./mixed-demos) 94 | 95 | # Example of double object classification (mobilenet_v1) 96 | add_executable( 97 | example_double_classification_tflite 98 | ${all_SRCS} 99 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/mixed-demos/cpp/example_double_classification_tflite.cpp 100 | ) 101 | target_link_libraries(example_double_classification_tflite ${GSTREAMER_LIBRARIES} ${CAIRO_LIBRARIES} tensorflow-lite) 102 | set_target_properties(example_double_classification_tflite PROPERTIES RUNTIME_OUTPUT_DIRECTORY ./mixed-demos) 103 | 104 | # Example of emotion classification (UltraFace slim, and Deepface-emotion) 105 | add_executable( 106 | example_emotion_classification_tflite 107 | ${all_SRCS} 108 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/face-processing/emotion-classification/cpp/example_emotion_classification_tflite.cpp 109 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/face-processing/emotion-classification/cpp/custom_emotion_decoder.cpp 110 | ) 111 | target_include_directories(example_emotion_classification_tflite PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tasks/face-processing/emotion-classification/cpp/) 112 | target_link_libraries(example_emotion_classification_tflite ${GSTREAMER_LIBRARIES} ${CAIRO_LIBRARIES} tensorflow-lite) 113 | set_target_properties(example_emotion_classification_tflite PROPERTIES RUNTIME_OUTPUT_DIRECTORY ./face-processing) 114 | 115 | # Example of multiple pipelines (a pipeline with emotion classification, and a pipeline with object detection) 116 | add_executable( 117 | example_emotion_and_detection_tflite 118 | ${all_SRCS} 119 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/mixed-demos/cpp/example_emotion_and_detection_tflite.cpp 120 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/face-processing/emotion-classification/cpp/custom_emotion_decoder.cpp 121 | ) 122 | target_include_directories(example_emotion_and_detection_tflite PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tasks/face-processing/emotion-classification/cpp/) 123 | target_link_libraries(example_emotion_and_detection_tflite ${GSTREAMER_LIBRARIES} ${CAIRO_LIBRARIES} tensorflow-lite) 124 | set_target_properties(example_emotion_and_detection_tflite PROPERTIES RUNTIME_OUTPUT_DIRECTORY ./mixed-demos) 125 | 126 | # Example of depth estimation (midas_v2) 127 | add_executable( 128 | example_depth_midas_v2_tflite 129 | ${all_SRCS} 130 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/monocular-depth-estimation/cpp/example_depth_midas_v2_tflite.cpp 131 | ${CMAKE_CURRENT_SOURCE_DIR}/tasks/monocular-depth-estimation/cpp/custom_depth_decoder.cpp 132 | ) 133 | target_include_directories(example_depth_midas_v2_tflite PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tasks/monocular-depth-estimation/cpp/) 134 | target_link_libraries(example_depth_midas_v2_tflite ${GSTREAMER_LIBRARIES} ${CAIRO_LIBRARIES} tensorflow-lite) 135 | set_target_properties(example_depth_midas_v2_tflite PROPERTIES RUNTIME_OUTPUT_DIRECTORY ./monocular-depth-estimation) 136 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2022-2025 NXP 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 5 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 6 | 7 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 10 | 11 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NXP NNStreamer examples 2 | Purpose of this repository is to provide, for demonstration purpose, functional examples of GStreamer/NNStreamer-based pipelines optimized and validated for some designated NXP i.MX application processors. 3 | 4 | # How to run examples 5 | ## Models and metadata download 6 | Models and metadata files used by examples are not archived in this repository. 7 | Therefore they have to be downloaded over the network, prior to execution of the examples on the target. Download of those files is to be done from the host PC, running Jupyter Notebook [download.ipynb](./downloads). 8 | Refer to [download instruction](./downloads/README.md) for details. 9 | ## Execution on target 10 | ### Python 11 | Once models have been fetched locally on host PC, repository will contain both examples and downloaded artifacts. Thus it can be uploaded to the target board for individual examples execution. Complete repository can either be uploaded from host PC to target using regular `scp` command or only the necessary directories using [upload.sh](./tools/upload.sh) script provided for host: 12 | ```bash 13 | # replace by relevant value 14 | cd /path/to/nxp-nnstreamer-examples 15 | ./tools/upload.sh root@ 16 | ``` 17 | ### C++ with cross-compilation 18 | 1- Fetch the models locally on host PC.
19 | 2- Build a Yocto BSP SDK for the dedicated i.MX platform. Refer to the [imx-manifest](https://github.com/nxp-imx/imx-manifest) to setup the correct building environment, SDK needs to be compiled with bitbake using `imx-image-full` and `populate_sdk` command as followed:
20 | ```bash 21 | bitbake imx-image-full -c populate_sdk 22 | ``` 23 | 3- Once successfully generated, the Yocto BSP SDK environment setup script located in `/path/to/yocto/bld-xwayland/tmp/deploy/sdk/` must be executed.
24 | ```bash 25 | chmod a+x fsl-imx--glibc-x86_64-imx-image-full-armv8--toolchain-.sh 26 | ./fsl-imx--glibc-x86_64-imx-image-full-armv8--toolchain-.sh 27 | ``` 28 | 4- Source the SDK environment:
29 | *NOTE: the SDK is installed by default in /opt/fsl-imx-xwayland//* 30 | ```bash 31 | . /path/to/sdk/environment-setup-armv8-poky-linux 32 | ``` 33 | 5- Compile C++ examples with CMake: 34 | ```bash 35 | cd /path/to/nxp-nnstreamer-examples 36 | mkdir build && cd $_ 37 | cmake .. 38 | make 39 | ``` 40 | 6- Push the required artifacts in its expected folder on the board (the scp command can be used for this purpose):
41 | *NOTE: path to the folder containing the data can be changed in CMakeLists.txt file as well as model and label names in the cpp example source file.* 42 | ```bash 43 | # Send classification example to target, replacing by relevant value 44 | scp ./classification/example_classification_mobilenet_v1_tflite root@ 45 | ``` 46 | C++ examples use a set of high-level classes to create optimized pipelines for NXP boards, which use NXP hardware optimizations. A description of how to use this set of classes can be found [here](./common/cpp/README.md). 47 | 48 | ### Compile models on target for i.MX 93 49 | Quantized TFLite models must be compiled with vela for i.MX 93 Ethos-U NPU. 50 | This must be done directly on the target: 51 | ```bash 52 | cd /path/to/nxp-nnstreamer-examples 53 | ./downloads/compile_models.sh 54 | ``` 55 | Examples can then be run directly on the target. More information on individual examples execution is available in relevant sections. 56 | 57 | # Tasks 58 | Note that examples may not run on all platforms - check table below for platform compatibility. 59 | 60 | Snapshot | Name | Platforms | Features 61 | --- | --- | --- | --- 62 | [![object detection demo](./tasks/object-detection/detection_demo.webp)](./tasks/object-detection/) | [Object Detection](./tasks/object-detection/) | i.MX 8M Plus
i.MX 93
i.MX 95| MobileNet SSD
YOLOv4 Tiny
TFLite
v4l2 camera
gst-launch
[custom python tensor_filter](./tasks/object-detection/postprocess_yolov4_tiny.py) 63 | [![classification demo](./tasks/classification/classification_demo.webp)](./tasks/classification/) | [Classification](./tasks/classification/) | i.MX 8M Plus
i.MX 93
i.MX 95| MobileNet
TFLite
v4l2 camera
gst-launch 64 | [![semantic segmentation demo](./tasks/semantic-segmentation/segmentation_demo.webp)](./tasks/semantic-segmentation/) | [Semantic Segmentation](./tasks/semantic-segmentation/) | i.MX 8M Plus
i.MX 93
i.MX 95| DeepLabV3
TFLite
jpeg files slideshow
gst-launch 65 | [![pose estmation demo](./tasks/pose-estimation/pose_demo.webp)](./tasks/pose-estimation/) | [Pose Estimation](./tasks/pose-estimation/) |i.MX 8M Plus
i.MX 93
i.MX 95| MoveNet
TFLite
video file decoding (i.MX 8M Plus only)
v4l2 camera
gst-launch
python 66 | [![face processing demo](./tasks/face-processing/face_demo.webp)](./tasks/face-processing/) | [Face Processing](./tasks/face-processing/) | i.MX 8M Plus
i.MX 93
i.MX 95| UltraFace
FaceNet512
Deepface-emotion
TFLite
v4l2 camera
python 67 | [![monocular depth estimation demo](./tasks/monocular-depth-estimation/depth_demo.webp)](./tasks/monocular-depth-estimation/) | [Monocular Depth Estimation](./tasks/monocular-depth-estimation/) | i.MX 8M Plus
i.MX 93
i.MX 95| MiDaS v2
TFLite
v4l2 camera
C++
custom C++ decoding 68 | [![mixed demo](./tasks/mixed-demos/mixed_demo.webp)](./tasks/mixed-demos/) | [Mixed Demos](./tasks/mixed-demos/) | i.MX 8M Plus
i.MX 93
i.MX 95| MobileNet SSD
MobileNet
MoveNet
UltraFace
Deepface-emotion
TFLite
v4l2 camera
C++
custom C++ decoding 69 | 70 | *Images and video used have been released under Creative Commons CC0 1.0 license or belong to Public Domain* -------------------------------------------------------------------------------- /RELEASE_NOTES.md: -------------------------------------------------------------------------------- 1 | # Release Notes 2 | 3 | ## [Release v1.5.1](https://github.com/nxp-imx/nxp-nnstreamer-examples/tree/v1.5.1) 4 | 5 | ### Updates 6 | 7 | | Name | Platforms | Models | Language | Backend | 8 | |-----------------------|-------------|-------------|---------------|---------| 9 | | face detection | **i.MX 95** | UltraFace | Python/C++ | **NPU** | 10 | | object detection | **i.MX 95** | Yolov4-tiny | Bash + Python | **CPU** | 11 | | sementic segmentation | **i.MX 95** | DeepLab v3 | Bash | **CPU** | 12 | | pose estimation | **i.MX 95** | MoveNet | Python | **CPU** | 13 | | all existing examples | **i.MX 95** | | C++ | **CPU** | 14 | 15 | ### Other Changes 16 | - Update download procedure to compile models with latest eIQ Toolkit version (1.15) for i.MX 95 Neutron NPU 17 | - Add video saving for i.MX 95 18 | - C++ examples: options for camera resolution and framerate, documentations and pipelines updates 19 | 20 | 21 | ## [Release v1.4.1](https://github.com/nxp-imx/nxp-nnstreamer-examples/tree/v1.4.1) 22 | 23 | ### New Features 24 | 25 | | Name | Platforms | Models | Language | Backend | 26 | |--------------------------------|-------------------------------|---------|----------|-----------------| 27 | | **all existing examples** | i.MX 8M Plus
i.MX 93 | | **C++** | CPU / GPU / NPU | 28 | | **monocular depth estimation** | i.MX 8M Plus
i.MX 93 | MidasV2 | C++ | CPU / NPU | 29 | | **mixed examples** | i.MX 8M Plus
i.MX 93 | | C++ | CPU / GPU / NPU | 30 | 31 | ### Updates 32 | 33 | | Name | Platforms | Models | Language | Backend | 34 | |-----------------------|-------------|----------------------|----------|-----------------| 35 | | sementic segmentation | **i.MX 93** | DeepLab v3 | Bash | CPU / NPU | 36 | | pose estimation | **i.MX 93** | MoveNet | Python | CPU / NPU | 37 | | image classification | **i.MX 95** | MobileNet v1 | Bash/C++ | CPU / GPU / NPU | 38 | | object detection | **i.MX 95** | ssdlite MobileNet v2 | Bash/C++ | CPU / GPU / NPU | 39 | 40 | ### Other Changes 41 | - Accelerated flip for Python examples 42 | - Update and reformat pose estimation pipeline 43 | - Fix Yolov4-tiny pre-processing 44 | 45 | ## [Release v1.3](https://github.com/nxp-imx/nxp-nnstreamer-examples/tree/v1.3) 46 | 47 | ### New Features 48 | 49 | | Name | Platforms | Models | Language | Backend | 50 | |----------------------|---------------------------|--------------|----------|-----------------| 51 | | image classification | i.MX 8M Plus
i.MX 93 | MobileNet v1 | **C++** | CPU / GPU / NPU | 52 | 53 | ### Updates 54 | 55 | | Name | Platforms | Models | Language | Backend | 56 | |-----------------|--------------|---------|----------|---------| 57 | | pose estimation | i.MX 8M Plus | MoveNet | Python | **NPU** | 58 | 59 | ### Other Changes 60 | - DeepViewRT models removed 61 | - Allow background execution for all examples 62 | 63 | ## [Release v1.2](https://github.com/nxp-imx/nxp-nnstreamer-examples/tree/v1.2) 64 | 65 | ### New Features 66 | 67 | | Name | Platforms | Models | Language | Backend | 68 | |----------------------------|---------------------------|------------------------------|----------|---------| 69 | | **emotion classification** | i.MX 8M Plus
i.MX 93 | UltraFace + Deepface-emotion | Python | NPU | 70 | 71 | ### Bug Fix 72 | - Fix Yolov4-tiny bounding box coordinates 73 | 74 | ### Other Changes 75 | - Store on disk result of the OpenVX graph compilation for iMX8MPlus to get the warmup time only once 76 | - Interrupt all Python examples with `esc` or `ctrl + C` 77 | 78 | ## [Release v1.1](https://github.com/nxp-imx/nxp-nnstreamer-examples/tree/v1.1) 79 | 80 | ### New Features 81 | 82 | | Name | Platforms | Models | Language | Backend | 83 | |----------------------|---------------------------|------------------------|---------------|-----------| 84 | | object detection | i.MX 8M Plus
i.MX 93 | **Yolov4 tiny** | Bash + Python | CPU / NPU | 85 | | **face detection** | i.MX 8M Plus
i.MX 93 | UltraFace | Python | NPU | 86 | | **face recognition** | i.MX 8M Plus
i.MX 93 | UltraFace + FaceNet512 | Python | NPU | 87 | 88 | ## [Release v1.0](https://github.com/nxp-imx/nxp-nnstreamer-examples/tree/v1.0) 89 | 90 | ### New Features 91 | 92 | | Name | Platforms | Models | Language | Backend | 93 | |---------------------------|---------------------------|----------------------|----------|-----------------| 94 | | **image classification** | i.MX 8M Plus
i.MX 93 | MobileNet v1 | Bash | CPU / GPU / NPU | 95 | | **object detection** | i.MX 8M Plus
i.MX 93 | ssdlite MobileNet v2 | Bash | CPU / GPU / NPU | 96 | | **sementic segmentation** | i.MX 8M Plus | DeepLab v3 | Bash | CPU / GPU / NPU | 97 | | **pose estimation** | i.MX 8M Plus | MoveNet | Python | CPU | 98 | -------------------------------------------------------------------------------- /SCR-1.5.1.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------------------------------- 2 | NXP Software Content Register 3 | 4 | Package: nxp-nnstreamer-examples 5 | Version: 1.5.1 6 | 7 | Release Location: https://github.com/nxp-imx/nxp-nnstreamer-examples 8 | Type of content: Source code 9 | Outgoing License: BSD-3-Clause 10 | License file: LICENSE 11 | Description and comments: NNStreamer examples for i.MX platforms 12 | Origin: NXP (BSD-3-Clause) 13 | Dependencies during runtime: 14 | GStreamer (LGPL-2.1-only) - https://github.com/GStreamer/gstreamer 15 | NNStreamer (LGPL-2.1-only) - https://github.com/nnstreamer/nnstreamer 16 | 17 | ------------------------------------------------------------------------------- 18 | -------------------------------------------------------------------------------- /common/common_utils.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2022-2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | function error { 7 | local message="$1" 8 | if [ ! -z "${message}" ]; then 9 | echo "${message}" 10 | fi 11 | exit 1 12 | } 13 | 14 | # setup environment 15 | function setup_env { 16 | # detect i.MX in use 17 | local UNAME=$(uname -a) 18 | if [[ "${UNAME}" =~ "imx8mp" ]]; then 19 | IMX="IMX8MP" 20 | # Store on disk .nb files that contains the result of the OpenVX graph compilation 21 | # This feature is only available for iMX8MPlus to get the warmup time only once 22 | export VIV_VX_ENABLE_CACHE_GRAPH_BINARY="1" 23 | export VIV_VX_CACHE_BINARY_GRAPH_DIR=$HOME 24 | fi 25 | if [[ "${UNAME}" =~ "imx93" ]]; then 26 | IMX="IMX93" 27 | fi 28 | if [[ "${UNAME}" =~ "imx95" ]]; then 29 | IMX="IMX95" 30 | fi 31 | if [ -z ${IMX} ]; then 32 | error "platform not supported" 33 | fi 34 | if [ "${IMX}" = "IMX93" ] && [ "${BACKEND}" = "GPU" ]; then 35 | error "Invalid combination ${IMX}/${BACKEND}" 36 | fi 37 | 38 | REQUIRED_CAMERA=${REQUIRED_CAMERA:-1} 39 | if [ "${REQUIRED_CAMERA}" -gt 0 ] ; then 40 | # default camera configuration (can also be overriden by user) 41 | declare -A CAMERA_DEVICE_DEFAULT 42 | CAMERA_DEVICE_DEFAULT[IMX8MP]="/dev/video3" 43 | CAMERA_DEVICE_DEFAULT[IMX93]="/dev/video0" 44 | CAMERA_DEVICE_DEFAULT[IMX95]="/dev/video13" 45 | local CAMERA_DEVICE_DEFAULT_IMX=${CAMERA_DEVICE_DEFAULT[${IMX}]} 46 | 47 | CAMERA_DEVICE="${CAMERA_DEVICE:-${CAMERA_DEVICE_DEFAULT_IMX}}" 48 | if [ ! -c ${CAMERA_DEVICE} ]; then 49 | local MSG="Camera device ${CAMERA_DEVICE} not found." 50 | MSG="$MSG Check device and set CAMERA_DEVICE variable appropriately." 51 | error "$MSG" 52 | fi 53 | 54 | CAMERA_WIDTH="${CAMERA_WIDTH:-640}" 55 | CAMERA_HEIGHT="${CAMERA_HEIGHT:-480}" 56 | CAMERA_FPS="${CAMERA_FPS:-30}" 57 | fi 58 | 59 | # backend default configuration 60 | BACKEND="${BACKEND:-NPU}" 61 | case "${BACKEND}" in 62 | CPU|NPU) 63 | export USE_GPU_INFERENCE=0 ;; 64 | GPU) 65 | export USE_GPU_INFERENCE=1 ;; 66 | *) 67 | error "invalid backend ${BACKEND}" ;; 68 | esac 69 | 70 | # GPU2D API 71 | case "${IMX}" in 72 | IMX8MP) 73 | GPU2D_API="G2D" ;; 74 | IMX95) 75 | GPU2D_API="G2D" ;; 76 | IMX93) 77 | GPU2D_API="PXP" ;; 78 | *) 79 | GPU2D_API="NONE" ;; 80 | esac 81 | 82 | # XXX: i.MX93: mute noisy ethos-u kernel driver 83 | if [ "${IMX}" = "IMX93" ]; then 84 | echo 4 > /proc/sys/kernel/printk 85 | fi 86 | } 87 | 88 | # Create pipeline segment for accelerated video formatting and csc 89 | # $1 video output width 90 | # $2 video output height 91 | # $3 video output format 92 | function accelerated_video_scale_str { 93 | local VIDEO_SCALE 94 | local OUTPUT_WIDTH=$1 95 | local OUTPUT_HEIGHT=$2 96 | local OUTPUT_FORMAT=$3 97 | 98 | if [[ -z "${OUTPUT_FORMAT}" ]] 99 | then 100 | FORMAT="" 101 | else 102 | FORMAT=",format=${OUTPUT_FORMAT}" 103 | fi 104 | 105 | case "${GPU2D_API}" in 106 | G2D) 107 | # g2d-based 108 | VIDEO_SCALE="imxvideoconvert_g2d ! " 109 | 110 | ;; 111 | PXP) 112 | # pxp-based 113 | VIDEO_SCALE="imxvideoconvert_pxp ! " 114 | ;; 115 | *) 116 | # cpu-based 117 | VIDEO_SCALE="videoscale ! videoconvert ! " 118 | ;; 119 | esac 120 | VIDEO_SCALE+="video/x-raw,width=${OUTPUT_WIDTH},height=${OUTPUT_HEIGHT}${FORMAT} ! " 121 | 122 | echo "${VIDEO_SCALE}" 123 | } 124 | 125 | 126 | # Create pipeline segment for accelerated video scaling and conversion to RGB format 127 | # $1 video output width 128 | # $2 video output height 129 | function accelerated_video_scale_rgb_str { 130 | local VIDEO_SCALE_RGB 131 | local OUTPUT_WIDTH=$1 132 | local OUTPUT_HEIGHT=$2 133 | 134 | case "${GPU2D_API}" in 135 | G2D) 136 | # g2d-based 137 | VIDEO_SCALE_RGB=$(accelerated_video_scale_str ${MODEL_WIDTH} ${MODEL_HEIGHT} "RGBA") 138 | VIDEO_SCALE_RGB+="videoconvert ! video/x-raw,format=RGB ! " 139 | ;; 140 | PXP) 141 | # pxp-based 142 | VIDEO_SCALE_RGB=$(accelerated_video_scale_str ${MODEL_WIDTH} ${MODEL_HEIGHT} "BGR") 143 | VIDEO_SCALE_RGB+="videoconvert ! video/x-raw,format=RGB ! " 144 | ;; 145 | *) 146 | # cpu-based 147 | VIDEO_SCALE_RGB=$(accelerated_video_scale_str ${MODEL_WIDTH} ${MODEL_HEIGHT} "RGB") 148 | ;; 149 | esac 150 | 151 | echo "${VIDEO_SCALE_RGB}" 152 | } 153 | 154 | 155 | # accelerated video mixer 156 | # $1 mixer name 157 | # $2 mixer extra parameters 158 | # parameters for compositor not supporting alpha channel (pxp) 159 | # $3 alpha channel pad number 160 | # $4 alpha channel value 161 | # $5 max inputs latency in nanoseconds 162 | function accelerated_video_mixer_str { 163 | local VIDEO_MIX 164 | local MIXER_NAME=$1 165 | local MIXER_ARGS=$2 166 | local ALPHA_CHANNEL_PAD=$3 167 | local ALPHA_CHANNEL_VALUE=$4 168 | local LATENCY=$5 169 | 170 | case "${GPU2D_API}" in 171 | G2D) 172 | # g2d-based 173 | VIDEO_MIXER="imxcompositor_g2d name=${MIXER_NAME} ${MIXER_ARGS}" 174 | ;; 175 | PXP) 176 | # pxp-based 177 | VIDEO_MIXER="\ 178 | imxcompositor_pxp name=${MIXER_NAME} ${MIXER_ARGS} \ 179 | sink_${ALPHA_CHANNEL_PAD}::alpha=${ALPHA_CHANNEL_VALUE} \ 180 | " 181 | ;; 182 | *) 183 | # cpu-based 184 | VIDEO_MIXER="videomixer name=${MIXER_NAME} ${MIXER_ARGS}" 185 | ;; 186 | esac 187 | 188 | # configure model latency 189 | case "${GPU2D_API}" in 190 | G2D|PXP) 191 | if [ -n "${LATENCY}" ] && [ "${LATENCY}" -ne 0 ]; then 192 | VIDEO_MIXER="${VIDEO_MIXER} latency=${LATENCY} min-upstream-latency=${LATENCY} !" 193 | else 194 | VIDEO_MIXER="${VIDEO_MIXER} !" 195 | fi 196 | ;; 197 | *) 198 | VIDEO_MIXER="${VIDEO_MIXER} !" 199 | ;; 200 | esac 201 | 202 | echo "${VIDEO_MIXER}" 203 | } 204 | -------------------------------------------------------------------------------- /common/cpp/include/common.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #ifndef CPP_COMMON_H_ 7 | #define CPP_COMMON_H_ 8 | 9 | #include "gst_pipeline_imx.hpp" 10 | #include "gst_source_imx.hpp" 11 | #include "gst_video_imx.hpp" 12 | #include "gst_video_post_process.hpp" 13 | #include "imx_devices.hpp" 14 | #include "logging.hpp" 15 | #include "model_infos.hpp" 16 | #include "nn_decoder.hpp" 17 | #include "tensor_custom_data_generator.hpp" 18 | 19 | #endif -------------------------------------------------------------------------------- /common/cpp/include/gst_pipeline_imx.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #ifndef CPP_GST_PIPELINE_IMX_H_ 7 | #define CPP_GST_PIPELINE_IMX_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "imx_devices.hpp" 16 | 17 | 18 | /** 19 | * @brief Data structure for app. 20 | */ 21 | typedef struct { 22 | GstElement *gstPipeline; 23 | GstBus *bus; 24 | bool playing; 25 | float FPS; 26 | std::vector filterNames; 27 | } AppData; 28 | 29 | 30 | /** 31 | * @brief GStreamer queue leaky options. 32 | */ 33 | enum class GstQueueLeaky { 34 | no, 35 | upstream, 36 | downstream, 37 | }; 38 | 39 | 40 | /** 41 | * @brief GStreamer queue options. 42 | */ 43 | typedef struct { 44 | std::string queueName = ""; 45 | int maxSizeBuffer = -1; 46 | GstQueueLeaky leakType = GstQueueLeaky::no; 47 | } GstQueueOptions; 48 | 49 | 50 | /** 51 | * @brief Performances options. 52 | */ 53 | typedef struct { 54 | bool freq; 55 | bool temp; 56 | } Performance; 57 | 58 | 59 | /** 60 | * @brief Outline text for Cairo. 61 | */ 62 | void outlineText(cairo_t* cr, 63 | int x, 64 | int y, 65 | std::string txt, 66 | std::string color); 67 | 68 | 69 | /** 70 | * @brief Setup and run GStreamer pipeline. 71 | */ 72 | class GstPipelineImx { 73 | private: 74 | std::string strPipeline; 75 | AppData gApp {}; 76 | static inline bool save = false; 77 | static inline Performance hasPerf = {false, false}; 78 | static inline GMainLoop *loop = g_main_loop_new(NULL, FALSE); 79 | static inline int pipeCount = 0; 80 | static inline int runCount = 0; 81 | static inline std::vector namesVector; 82 | static inline std::vector infVector; 83 | static inline float perfFontSize = 0; 84 | static inline std::string perfColor = ""; 85 | 86 | public: 87 | static gboolean pipePerfCallback(gpointer user_data); 88 | 89 | static gboolean infPerfCallback(gpointer user_data); 90 | 91 | void parse(int argc, char **argv, char *graphPath); 92 | 93 | void run(); 94 | 95 | void freeData(); 96 | 97 | void addBranch(const std::string &teeName, const GstQueueOptions &options); 98 | 99 | static void busCallback(GstBus* bus, 100 | GstMessage* message, 101 | gpointer user_data); 102 | 103 | static gboolean sigintSignalHandler(gpointer user_data); 104 | 105 | void doInParallel(const std::string &teeName); 106 | 107 | void addToPipeline(const std::string &cmd) { strPipeline.append(cmd); } 108 | 109 | AppData getAppData() const { return gApp; } 110 | 111 | void linkToTextOverlay(const std::string &gstName); 112 | 113 | void addTensorSink(const std::string &gstName, const bool &qos=true); 114 | 115 | void setSave(const bool &save); 116 | 117 | GstElement* getElement(const std::string &gstName); 118 | 119 | template 120 | void connectToElementSignal(const std::string &gstName, 121 | F callback, 122 | const std::string &signal, 123 | gpointer data) 124 | { 125 | GstElement *element = getElement(gstName); 126 | if (element) { 127 | u_long handleID = g_signal_connect(element, 128 | signal.c_str(), 129 | G_CALLBACK(callback), 130 | data); 131 | if (handleID <= 0) { 132 | log_error("could not connect %s\n", gstName.c_str()); 133 | exit(-1); 134 | } 135 | } else { 136 | log_error("Could not get %s\n", gstName.c_str()); 137 | exit(-1); 138 | } 139 | } 140 | 141 | void enablePerfDisplay(const bool &frequency, 142 | const bool &temporal, 143 | const float &fontSize, 144 | const std::string &color=""); 145 | 146 | Performance isPerfAvailable() const 147 | { 148 | return hasPerf; 149 | } 150 | 151 | static void perfDrawCallback(GstElement* overlay, 152 | cairo_t* cr, 153 | guint64 timestamp, 154 | guint64 duration, 155 | gpointer user_data); 156 | 157 | void addFilterName(std::string gstName); 158 | }; 159 | #endif -------------------------------------------------------------------------------- /common/cpp/include/gst_source_imx.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #ifndef CPP_GST_SOURCE_IMX_H_ 7 | #define CPP_GST_SOURCE_IMX_H_ 8 | 9 | #include 10 | #include 11 | 12 | #include "gst_video_imx.hpp" 13 | #include "gst_pipeline_imx.hpp" 14 | #include "imx_devices.hpp" 15 | 16 | typedef struct { 17 | std::filesystem::path cameraDevice; 18 | std::string gstName; 19 | int width; 20 | int height; 21 | bool horizontalFlip; 22 | std::string format; 23 | int framerate; 24 | } CameraOptions; 25 | 26 | 27 | /** 28 | * @brief Parent class for the various input source. 29 | */ 30 | class GstSourceImx { 31 | protected: 32 | int width; 33 | int height; 34 | std::string format; 35 | 36 | public: 37 | GstSourceImx(const int &width, const int &height, const std::string &format) 38 | : format(format), width(width), height(height) {} 39 | 40 | int getWidth() const { return width; } 41 | 42 | int getHeight() const { return height; } 43 | }; 44 | 45 | 46 | /** 47 | * @brief Create pipeline segments for camera. 48 | */ 49 | class GstCameraImx : public GstSourceImx { 50 | private: 51 | std::string gstName; 52 | bool flip; 53 | GstVideoImx videoscale{}; 54 | std::filesystem::path device; 55 | int framerate; 56 | 57 | public: 58 | GstCameraImx(CameraOptions &options); 59 | 60 | void addCameraToPipeline(GstPipelineImx &pipeline); 61 | }; 62 | 63 | 64 | /** 65 | * @brief Create pipeline segments for a video. 66 | */ 67 | class GstVideoFileImx : public GstSourceImx { 68 | private: 69 | std::filesystem::path videoPath; 70 | std::string cmdDecoder; 71 | imx::Imx imx{}; 72 | GstVideoImx videoscale{}; 73 | 74 | public: 75 | GstVideoFileImx(const std::filesystem::path &path, 76 | const int &width=-1, 77 | const int &height=-1); 78 | 79 | void addVideoToPipeline(GstPipelineImx &pipeline); 80 | }; 81 | 82 | 83 | /** 84 | * @brief Create pipeline segments for a slideshow. 85 | */ 86 | class GstSlideshowImx : public GstSourceImx { 87 | private: 88 | std::filesystem::path slideshowPath; 89 | GstVideoImx videoscale{}; 90 | 91 | public: 92 | GstSlideshowImx(const std::filesystem::path &path, 93 | const int &width=-1, 94 | const int &height=-1); 95 | 96 | void addSlideshowToPipeline(GstPipelineImx &pipeline); 97 | }; 98 | 99 | 100 | /** 101 | * @brief Create pipeline segments for appsrc. 102 | */ 103 | class GstAppSrcImx : public GstSourceImx { 104 | private: 105 | std::string gstName; 106 | bool isLive; 107 | bool emitSignal; 108 | int maxBuffers; 109 | GstQueueLeaky leakType; 110 | int formatType; 111 | int framerate; 112 | GstVideoImx videoscale{}; 113 | 114 | public: 115 | GstAppSrcImx(const std::string &gstName, 116 | const bool &isLive, 117 | const bool &emitSignal, 118 | const int &maxBuffers, 119 | const GstQueueLeaky &leakType, 120 | const int &formatType, 121 | const int &width, 122 | const int &height, 123 | const std::string &format="", 124 | const int &framerate=30); 125 | 126 | void addAppSrcToPipeline(GstPipelineImx &pipeline); 127 | }; 128 | #endif -------------------------------------------------------------------------------- /common/cpp/include/gst_video_imx.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #ifndef CPP_GST_VIDEO_IMX_H_ 7 | #define CPP_GST_VIDEO_IMX_H_ 8 | 9 | #include "imx_devices.hpp" 10 | #include "gst_pipeline_imx.hpp" 11 | 12 | 13 | /** 14 | * @brief Create pipeline segments for the various accelerators. 15 | */ 16 | class GstVideoImx { 17 | private: 18 | imx::Imx imx{}; 19 | 20 | public: 21 | GstVideoImx() = default; 22 | 23 | void videoTransform(GstPipelineImx &pipeline, 24 | const std::string &format, 25 | const int &width, 26 | const int &height, 27 | const bool &flip, 28 | const bool &aspectRatio=false, 29 | const bool &useCPU=false); 30 | 31 | void videoscaleToRGB(GstPipelineImx &pipeline, 32 | const int &width, 33 | const int &height); 34 | 35 | void videocrop(GstPipelineImx &pipeline, 36 | const std::string &gstName, 37 | const int &width, 38 | const int &height, 39 | const int &top=0, 40 | const int &bottom=0, 41 | const int &left=0, 42 | const int &right=0); 43 | }; 44 | #endif -------------------------------------------------------------------------------- /common/cpp/include/gst_video_post_process.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #ifndef CPP_GST_VIDEO_POST_PROCESS_H_ 7 | #define CPP_GST_VIDEO_POST_PROCESS_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "gst_pipeline_imx.hpp" 15 | 16 | 17 | typedef struct { 18 | std::string gstName; 19 | std::string fontName; 20 | int fontSize; 21 | std::string color; 22 | std::string vAlignment; 23 | std::string hAlignment; 24 | std::string text; 25 | } TextOverlayOptions; 26 | 27 | 28 | typedef struct { 29 | std::string gstName; 30 | bool sync; 31 | int maxBuffers; 32 | bool drop; 33 | bool emitSignals; 34 | } AppSinkOptions; 35 | 36 | 37 | enum class displayPosition { 38 | left, 39 | right, 40 | center 41 | }; 42 | 43 | 44 | typedef struct { 45 | displayPosition position; 46 | int order; 47 | bool keepRatio; 48 | bool transparency; 49 | } compositorInputParams; 50 | 51 | 52 | /** 53 | * @brief Create pipeline segments for video post processing. 54 | */ 55 | class GstVideoPostProcess { 56 | private: 57 | static inline bool cairoNeeded = true; 58 | 59 | public: 60 | GstVideoPostProcess() = default; 61 | 62 | void display(GstPipelineImx &pipeline, 63 | const bool &sync=false); 64 | 65 | void addTextOverlay(GstPipelineImx &pipeline, 66 | const TextOverlayOptions &options); 67 | 68 | void addCairoOverlay(GstPipelineImx &pipeline, const std::string &gstName); 69 | 70 | void saveToVideo(GstPipelineImx &pipeline, 71 | const std::string &format, 72 | const std::filesystem::path &path); 73 | 74 | void addAppSink(GstPipelineImx &pipeline, 75 | const AppSinkOptions &options); 76 | }; 77 | 78 | 79 | /** 80 | * @brief Create pipeline segments for video compositor. 81 | */ 82 | class GstVideoCompositorImx { 83 | private: 84 | imx::Imx imx{}; 85 | std::string gstName; 86 | std::vector compositorInputs; 87 | static inline int sinkNumber = 0; 88 | 89 | public: 90 | GstVideoCompositorImx(const std::string &gstName) 91 | : gstName(gstName) {} 92 | 93 | void addToCompositor(GstPipelineImx &pipeline, 94 | const compositorInputParams &inputParams); 95 | 96 | void addCompositorToPipeline(GstPipelineImx &pipeline, 97 | const int &latency = 0); 98 | }; 99 | #endif -------------------------------------------------------------------------------- /common/cpp/include/imx_devices.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #ifndef CPP_IMX_DEVICES_H_ 7 | #define CPP_IMX_DEVICES_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include "logging.hpp" 14 | 15 | #define NUMBER_OF_SOC 9 16 | #define NUMBER_OF_FEATURE 3 17 | #define SOC_ID_PATH "/sys/devices/soc0/soc_id" 18 | 19 | 20 | namespace imx { 21 | 22 | /**  23 | * @brief i.MX devices enumeration. 24 | */ 25 | enum iMXSocId { 26 | IMX8MQ, 27 | IMX8MM, 28 | IMX8MN, 29 | IMX8MP, 30 | IMX8ULP, 31 | IMX8QM, 32 | IMX8QXP, 33 | IMX93, 34 | IMX95, 35 | UNKNOWN, 36 | }; 37 | 38 | 39 | /** 40 | * @brief i.MX Features enumeration. 41 | */ 42 | enum iMXHwAccelFeature { 43 | GPU2D, 44 | GPU3D, 45 | NPU, 46 | }; 47 | 48 | 49 | /** 50 | * @brief Dictionary for SoC identification. 51 | */ 52 | const std::array socDictionnary = { 53 | "i.MX8MQ", 54 | "i.MX8MM", 55 | "i.MX8MN", 56 | "i.MX8MP", 57 | "i.MX8ULP", 58 | "i.MX8QM", 59 | "i.MX8QXP", 60 | "i.MX93", 61 | "i.MX95", 62 | }; 63 | 64 | 65 | /** 66 | * @brief Array of SoC name. 67 | */ 68 | const std::array socNameArray = { 69 | "i.MX 8M Quad", 70 | "i.MX 8M Mini", 71 | "i.MX 8M Nano", 72 | "i.MX 8M Plus", 73 | "i.MX 8ULP", 74 | "i.MX 8QuadMax", 75 | "i.MX 8QuadXPlus", 76 | "i.MX 93", 77 | "i.MX 95", 78 | }; 79 | 80 | 81 | /** 82 | * @brief Array of SoC features. 83 | */ 84 | const std::array, NUMBER_OF_SOC> socHasFeature = {{ 85 | [IMX8MQ] = {{[GPU2D] = false, [GPU3D] = true, [NPU] = false}}, 86 | [IMX8MM] = {{[GPU2D] = true, [GPU3D] = true, [NPU] = false}}, 87 | [IMX8MN] = {{[GPU2D] = false, [GPU3D] = true, [NPU] = false}}, 88 | [IMX8MP] = {{[GPU2D] = true, [GPU3D] = true, [NPU] = true}}, 89 | [IMX8ULP] = {{[GPU2D] = true, [GPU3D] = true, [NPU] = false}}, 90 | [IMX8QM] = {{[GPU2D] = true, [GPU3D] = true, [NPU] = false}}, 91 | [IMX8QXP] = {{[GPU2D] = true, [GPU3D] = true, [NPU] = false}}, 92 | [IMX93] = {{[GPU2D] = false, [GPU3D] = false, [NPU] = true}}, 93 | [IMX95] = {{[GPU2D] = true, [GPU3D] = true, [NPU] = true}}, 94 | }}; 95 | 96 | 97 | /** 98 | * @brief i.MX Backend enumeration. 99 | */ 100 | enum class Backend { 101 | CPU, 102 | GPU, 103 | NPU, 104 | }; 105 | 106 | 107 | /** 108 | * @brief Discovers some hardware acceleration features of an i.MX SoC (presence of 2D/3D GPU, presence of NPU...) 109 | */ 110 | class Imx { 111 | private: 112 | int soc; 113 | 114 | public: 115 | /** 116 | * @brief Constructor to detect i.MX device. 117 | * 118 | * @throw Generates an error if the machine name is unknown. 119 | */ 120 | Imx() 121 | { 122 | std::ifstream fileSocId(SOC_ID_PATH); 123 | soc = UNKNOWN; 124 | 125 | if(fileSocId) { 126 | std::string socId; 127 | getline(fileSocId, socId); 128 | 129 | for (int i=0;i 10 | #include 11 | 12 | #include "imx_devices.hpp" 13 | #include "gst_video_imx.hpp" 14 | #include "gst_pipeline_imx.hpp" 15 | #include "tensor_custom_data_generator.hpp" 16 | 17 | 18 | /** 19 | * @brief Create pipeline segments for various models. 20 | */ 21 | class ModelInfos { 22 | protected: 23 | int modelWidth; 24 | int modelHeight; 25 | int modelChannel; 26 | std::filesystem::path modelPath; 27 | std::string backend; 28 | std::string framework; 29 | TensorCustomGenerator tensorCustomData; 30 | imx::Imx imx{}; 31 | GstVideoImx videoscale{}; 32 | TensorData tensorData; 33 | 34 | public: 35 | ModelInfos(const std::filesystem::path &path, 36 | const std::string &backend, 37 | const std::string &norm, 38 | const int &numThreads); 39 | 40 | int getModelWidth() const { return modelWidth; } 41 | 42 | int getModelHeight() const { return modelHeight; } 43 | 44 | int getModelChannel() const { return modelChannel; } 45 | 46 | bool isGrayscale() const { return ((modelChannel == 1) ? true : false); } 47 | 48 | bool isRGB() const { return ((modelChannel == 3) ? true : false); } 49 | 50 | void addInferenceToPipeline(GstPipelineImx &pipeline, 51 | const std::string &gstName="", 52 | const std::string &format="RGB"); 53 | 54 | void setTensorFilterConfig(imx::Imx &imx, const int &numThreads); 55 | }; 56 | 57 | 58 | /** 59 | * @brief Create pipeline segments for tensorflow lite model. 60 | */ 61 | class TFliteModelInfos : public ModelInfos { 62 | public: 63 | TFliteModelInfos(const std::filesystem::path &path, 64 | const std::string &backend, 65 | const std::string &norm, 66 | const int &numThreads=std::thread::hardware_concurrency()); 67 | }; 68 | #endif -------------------------------------------------------------------------------- /common/cpp/include/nn_decoder.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #ifndef CPP_NN_DECODER_H_ 7 | #define CPP_NN_DECODER_H_ 8 | 9 | #include 10 | #include 11 | 12 | #include "gst_pipeline_imx.hpp" 13 | #include "tensor_custom_data_generator.hpp" 14 | 15 | 16 | /** 17 | * @brief Enum of available models for bounding_boxes mode. 18 | */ 19 | enum class ModeBoundingBoxes { 20 | yolov5, 21 | mobilenetssd, 22 | mpPalmDetection, 23 | }; 24 | 25 | 26 | /** 27 | * @brief Dimension structure. 28 | */ 29 | typedef struct { 30 | int width; 31 | int height; 32 | } Dimension; 33 | 34 | 35 | /** 36 | * @brief Structure for bounding_boxes mode options. 37 | */ 38 | typedef struct { 39 | ModeBoundingBoxes modelName; 40 | std::filesystem::path labelsPath; 41 | std::string option3; // Option1-dependent values 42 | Dimension outDim; 43 | Dimension inDim; 44 | bool trackResult; 45 | bool logResult; 46 | } BoundingBoxesOptions; 47 | 48 | 49 | /** 50 | * @brief Yolov5 custom options for bounding_boxes mode. 51 | */ 52 | typedef struct { 53 | int scale; 54 | float confidence = -1; //optional 55 | float IOU = -1; //optional 56 | } YoloCustomOptions; 57 | 58 | 59 | /** 60 | * @brief Mobilenetssd custom options for bounding_boxes mode. 61 | */ 62 | typedef struct { 63 | std::filesystem::path boxesPath; 64 | float threshold = -1; //optional 65 | float yScale = -1; //optional 66 | float xScale = -1; //optional 67 | float hScale = -1; //optional 68 | float wScale = -1; //optional 69 | float IOU = -1; //optional 70 | } SSDMobileNetCustomOptions; 71 | 72 | 73 | /** 74 | * @brief Palm detection custom options for bounding_boxes mode. 75 | */ 76 | typedef struct { 77 | float score; 78 | int anchorLayers = -1; //optional 79 | float minScale = -1; //optional 80 | float maxScale = -1; //optional 81 | float xOffset = -1; //optional 82 | float yOffset = -1; //optional 83 | std::string stride = ""; //optional 84 | } PalmDetectionCustomOptions; 85 | 86 | 87 | /** 88 | * @brief Set custom options for bounding_boxes mode between available models. 89 | */ 90 | template 91 | std::string setCustomOptions(T options) { 92 | std::string cmd; 93 | if constexpr(std::is_same_v) { 94 | cmd += std::to_string(options.scale); 95 | if (options.confidence != -1) 96 | cmd += ":" + std::to_string(options.confidence); 97 | if (options.IOU != -1) 98 | cmd += ":" + std::to_string(options.IOU); 99 | } 100 | if constexpr(std::is_same_v) { 101 | cmd += options.boxesPath.string(); 102 | if (options.threshold != -1) 103 | cmd += ":" + std::to_string(options.threshold); 104 | if (options.yScale != -1) 105 | cmd += ":" + std::to_string(options.yScale); 106 | if (options.xScale != -1) 107 | cmd += ":" + std::to_string(options.xScale); 108 | if (options.hScale != -1) 109 | cmd += ":" + std::to_string(options.hScale); 110 | if (options.wScale != -1) 111 | cmd += ":" + std::to_string(options.wScale); 112 | if (options.IOU != -1) 113 | cmd += ":" + std::to_string(options.IOU); 114 | } 115 | if constexpr(std::is_same_v) { 116 | cmd += std::to_string(options.score); 117 | if (options.anchorLayers != -1) 118 | cmd += ":" + std::to_string(options.anchorLayers); 119 | if (options.minScale != -1) 120 | cmd += ":" + std::to_string(options.minScale); 121 | if (options.maxScale != -1) 122 | cmd += ":" + std::to_string(options.maxScale); 123 | if (options.xOffset != -1) 124 | cmd += ":" + std::to_string(options.xOffset); 125 | if (options.yOffset != -1) 126 | cmd += ":" + std::to_string(options.yOffset); 127 | if (options.stride != "") 128 | cmd += ":" + options.stride; 129 | } 130 | return cmd; 131 | } 132 | 133 | 134 | /** 135 | * @brief Enum of available models for image_segment mode. 136 | */ 137 | enum class ModeImageSegment { 138 | tfliteDeeplab, 139 | snpeDeeplab, 140 | snpeDepth, 141 | }; 142 | 143 | 144 | /** 145 | * @brief Structure for image_segment mode options. 146 | */ 147 | typedef struct { 148 | ModeImageSegment modelName; 149 | int numClass = -1; //optional 150 | } ImageSegmentOptions; 151 | 152 | 153 | /** 154 | * @brief Create pipeline segments for NNStreamer decoder. 155 | */ 156 | class NNDecoder { 157 | public: 158 | void addImageSegment(GstPipelineImx &pipeline, 159 | const ImageSegmentOptions &options); 160 | 161 | void addImageLabeling(GstPipelineImx &pipeline, 162 | const std::filesystem::path &labelsPath); 163 | 164 | void addBoundingBoxes(GstPipelineImx &pipeline, 165 | const BoundingBoxesOptions &options); 166 | }; 167 | #endif -------------------------------------------------------------------------------- /common/cpp/include/tensor_custom_data_generator.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #ifndef CPP_TENSOR_CUSTOM_DATA_GENERATOR_H_ 7 | #define CPP_TENSOR_CUSTOM_DATA_GENERATOR_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include "imx_devices.hpp" 13 | 14 | 15 | /** 16 | * @brief Data structure for tensor. 17 | */ 18 | typedef struct { 19 | std::string tensorFilterCustom; 20 | std::string tensorTransform; 21 | } TensorData; 22 | 23 | 24 | /** 25 | * @brief Data structure for labels and boxes directories. 26 | */ 27 | typedef struct { 28 | std::filesystem::path labelsDir; 29 | std::filesystem::path boxesDir; 30 | } DataDir; 31 | 32 | 33 | /** 34 | * @brief Normalization enumeration. 35 | */ 36 | enum class Normalization { 37 | none, 38 | centered, 39 | reduced, 40 | centeredReduced, 41 | castInt32, 42 | castuInt8, 43 | }; 44 | 45 | 46 | /** 47 | * @brief Check if the element exists in the dictionary. 48 | * 49 | * @param element: element we want to check. 50 | * @param dictionary: dictionary we want to check. 51 | * @return the selected element or a default one. 52 | */ 53 | template 54 | T selectFromDictionary(const std::string &element, 55 | std::map dictionary) 56 | { 57 | for (auto &pair : dictionary) { 58 | std::string key = pair.first; 59 | if (element == key) 60 | return dictionary[element]; 61 | } 62 | if constexpr(std::is_same_v) 63 | return T::none; 64 | if constexpr(std::is_same_v) 65 | return T::NPU; 66 | } 67 | 68 | 69 | /** 70 | * @brief Dictionary of Normalization identification. 71 | */ 72 | extern std::map normDictionary; 73 | 74 | 75 | /** 76 | * @brief Create pipeline segments for customized tensor data and 77 | * set up USE_GPU_INFERENCE, an environment variable used 78 | * for the GPU backend on i.MX 8M Plus. 79 | */ 80 | class TensorCustomGenerator { 81 | private: 82 | TensorData tensorData; 83 | 84 | public: 85 | std::string CPU(const int &numThreads); 86 | 87 | std::string vsiGPU(); 88 | 89 | std::string vsiNPU(); 90 | 91 | std::string ethosNPU(); 92 | 93 | std::string neutronNPU(); 94 | 95 | std::string GPU(); 96 | 97 | std::string setTensorTransformConfig(const std::string &norm); 98 | }; 99 | #endif -------------------------------------------------------------------------------- /common/cpp/src/gst_source_imx.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #include "gst_source_imx.hpp" 7 | 8 | /** 9 | * @brief Parameterized constructor. 10 | * 11 | * @param options: structure of camera parameters. 12 | */ 13 | GstCameraImx::GstCameraImx(CameraOptions &options) 14 | : GstSourceImx(options.width, options.height, options.format), 15 | flip(options.horizontalFlip), 16 | gstName(options.gstName), 17 | framerate(options.framerate) 18 | { 19 | if (options.cameraDevice.string().length() != 0) { 20 | device = options.cameraDevice; 21 | } else { 22 | imx::Imx imx{}; 23 | switch (imx.socId()) 24 | { 25 | case imx::IMX8MP: 26 | device = "/dev/video3"; 27 | break; 28 | 29 | case imx::IMX93: 30 | device = "/dev/video0"; 31 | break; 32 | 33 | case imx::IMX95: 34 | device = "/dev/video13"; 35 | break; 36 | 37 | default: 38 | log_error("Select a camera device using -c argument option\n"); 39 | break; 40 | } 41 | } 42 | } 43 | 44 | 45 | /** 46 | * @brief Create pipeline segment for camera. 47 | * 48 | * @param pipeline: GstPipelineImx pipeline. 49 | */ 50 | void GstCameraImx::addCameraToPipeline(GstPipelineImx &pipeline) 51 | { 52 | std::string cmd; 53 | cmd += "v4l2src name=" + gstName + " device=" + device.string(); 54 | cmd += " num-buffers=-1 ! video/x-raw,width="; 55 | cmd += std::to_string(width) + ",height=" + std::to_string(height); 56 | cmd += ",framerate=" + std::to_string(framerate) + "/1 ! "; 57 | pipeline.addToPipeline(cmd); 58 | 59 | if ((format.length() != 0) or flip) 60 | videoscale.videoTransform(pipeline, format, -1, -1, flip, false, false); 61 | } 62 | 63 | 64 | /** 65 | * @brief Parameterized constructor. 66 | * 67 | * @param path: video path. 68 | * @param width: video width. 69 | * @param height: video height. 70 | */ 71 | GstVideoFileImx::GstVideoFileImx(const std::filesystem::path &path, 72 | const int &width, 73 | const int &height) 74 | : GstSourceImx(width, height, "") 75 | { 76 | if ((imx.isIMX93()) || (imx.isIMX95())) { 77 | log_error("video file can't be decoded with %s\n", imx.socName().c_str()); 78 | exit(-1); 79 | } else { 80 | videoPath = path; 81 | } 82 | 83 | if (videoPath.extension() == ".mkv" or 84 | videoPath.extension() == ".webm") { 85 | cmdDecoder = "matroskademux ! "; 86 | } else if (videoPath.extension() == ".mp4") { 87 | cmdDecoder = "qtdemux ! "; 88 | } else { 89 | log_error("Add a .mkv, .webm or .mp4 video format\n"); 90 | exit(-1); 91 | } 92 | 93 | cmdDecoder += "vpudec ! "; 94 | } 95 | 96 | 97 | /** 98 | * @brief Create pipeline segment for video. 99 | * 100 | * @param pipeline: GstPipelineImx pipeline. 101 | */ 102 | void GstVideoFileImx::addVideoToPipeline(GstPipelineImx &pipeline) 103 | { 104 | std::string cmd; 105 | cmd += "filesrc location=" + videoPath.string() + " ! " + cmdDecoder; 106 | pipeline.addToPipeline(cmd); 107 | 108 | if ((width > 0) && (height > 0)) 109 | videoscale.videoTransform(pipeline, "", width, height, false, true); 110 | } 111 | 112 | 113 | /** 114 | * @brief Parameterized constructor. 115 | * 116 | * @param path: slideshow path. 117 | * @param width: slideshow width. 118 | * @param height: slideshow height. 119 | */ 120 | GstSlideshowImx::GstSlideshowImx(const std::filesystem::path &path, 121 | const int &width, 122 | const int &height) 123 | : GstSourceImx(width, height, "") 124 | { 125 | slideshowPath = path; 126 | } 127 | 128 | 129 | /** 130 | * @brief Create pipeline segment for slideshow. 131 | * 132 | * @param pipeline: GstPipelineImx pipeline. 133 | */ 134 | void GstSlideshowImx::addSlideshowToPipeline(GstPipelineImx &pipeline) 135 | { 136 | std::string cmd; 137 | cmd = "multifilesrc location=" + slideshowPath.string(); 138 | cmd += " loop=true caps=image/jpeg,framerate=1/2 ! jpegdec ! "; 139 | pipeline.addToPipeline(cmd); 140 | 141 | if ((width > 0) && (height > 0)) 142 | videoscale.videoTransform(pipeline, "", width, height, false, true); 143 | } 144 | 145 | 146 | /** 147 | * @brief Parameterized constructor. 148 | * 149 | * @param gstName: name for GStreamer appsrc element. 150 | * @param isLive: whether to act as a live source. 151 | * @param emitSignal: emit need-data, enough-data and seek-data signals. 152 | * @param maxBuffers: the maximum number of buffers to queue internally. 153 | * @param leakType: whether to drop buffers once the internal queue is full. 154 | * @param formatType: the format of the segment events and seek. 155 | * @param width: appsrc width. 156 | * @param height: appsrc height. 157 | * @param format: GStreamer video format. 158 | * @param framerate: appsrc framerate. 159 | */ 160 | GstAppSrcImx::GstAppSrcImx(const std::string &gstName, 161 | const bool &isLive, 162 | const bool &emitSignal, 163 | const int &maxBuffers, 164 | const GstQueueLeaky &leakType, 165 | const int &formatType, 166 | const int &width, 167 | const int &height, 168 | const std::string &format, 169 | const int &framerate) 170 | : GstSourceImx(width, height, format), 171 | gstName(gstName), 172 | isLive(isLive), 173 | emitSignal(emitSignal), 174 | maxBuffers(maxBuffers), 175 | leakType(leakType), 176 | formatType(formatType), 177 | framerate(framerate) 178 | { 179 | } 180 | 181 | 182 | /** 183 | * @brief Create pipeline segment for appsrc element. 184 | * 185 | * @param pipeline: GstPipelineImx pipeline. 186 | */ 187 | void GstAppSrcImx::addAppSrcToPipeline(GstPipelineImx &pipeline) 188 | { 189 | std::string cmd; 190 | std::string caps; 191 | cmd = "appsrc"; 192 | 193 | if (gstName.size() != 0) 194 | cmd += " name=" + gstName; 195 | if (isLive == true) 196 | cmd += " is-live=true"; 197 | 198 | caps += "video/x-raw,width=" + std::to_string(width); 199 | caps += ",height=" + std::to_string(height); 200 | caps += ",framerate=" + std::to_string(framerate) + "/1"; 201 | 202 | if (format.size() != 0) 203 | caps += ",format=" + format; 204 | 205 | cmd += " caps=" + caps; 206 | cmd += " format=" + std::to_string(formatType); 207 | 208 | if (emitSignal == false) 209 | cmd += " emit-signals=false"; 210 | 211 | cmd += " max-buffers=" + std::to_string(maxBuffers); 212 | 213 | if (leakType != GstQueueLeaky::no) 214 | cmd += " leaky-type=" + std::to_string(static_cast(leakType)); 215 | 216 | cmd += " ! " + caps + " ! "; 217 | pipeline.addToPipeline(cmd); 218 | } -------------------------------------------------------------------------------- /common/cpp/src/gst_video_imx.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #include "gst_video_imx.hpp" 7 | 8 | /** 9 | * @brief Create pipeline segment for accelerated video formatting and csc. 10 | * 11 | * @param pipeline: GstPipelineImx pipeline. 12 | * @param format: GStreamer video format. 13 | * @param width: output video width after rescale. 14 | * @param height: output video height after rescale. 15 | * @param flip: horizontal flip, deactivated by default. 16 | * @param aspectRatio: add pixel aspect ratio of 1/1, deactivated by default. 17 | * @param useCPU: use CPU instead of acceleration (false by default). 18 | */ 19 | void GstVideoImx::videoTransform(GstPipelineImx &pipeline, 20 | const std::string &format, 21 | const int &width, 22 | const int &height, 23 | const bool &flip, 24 | const bool &aspectRatio, 25 | const bool &useCPU) 26 | { 27 | /** 28 | * imxvideoconvert_g2d and imxvideoconvert_pxp 29 | * do not support width and height lower than 16 30 | */ 31 | std::string cmd; 32 | std::string cmdFormat; 33 | int dimLimit = 16; 34 | bool validFormatG2D = true; 35 | bool validFormatPXP = true; 36 | 37 | if (format.length() != 0) { 38 | cmdFormat = ",format=" + format; 39 | /* accelerators only accept the following format*/ 40 | if ((format != "RGB16") 41 | && (format != "RGBx") 42 | && (format != "RGBA") 43 | && (format != "BGRA") 44 | && (format != "BGRx") 45 | && (format != "BGR16") 46 | && (format != "ARGB") 47 | && (format != "ABGR") 48 | && (format != "xRGB") 49 | && (format != "xBGR")) { 50 | validFormatG2D = false; 51 | } 52 | if ((format != "BGRx") 53 | && (format != "BGRA") 54 | && (format != "BGR") 55 | && (format != "RGB16") 56 | && (format != "GRAY8") 57 | && (format != "UYVY")) { 58 | validFormatPXP = false; 59 | } 60 | } 61 | 62 | if (this->imx.hasGPU2d() 63 | && (width > dimLimit || width == -1) 64 | && (height > dimLimit || height == -1) 65 | && (useCPU == false) 66 | && (validFormatG2D == true)) { 67 | cmd = "imxvideoconvert_g2d "; 68 | cmd += (flip == true) ? "rotation=4 ! " : "! "; 69 | } else if (this->imx.hasPxP() 70 | && (width > dimLimit || width == -1) 71 | && (height > dimLimit|| height == -1) 72 | && (useCPU == false) 73 | && (validFormatPXP == true)) { 74 | cmd = "imxvideoconvert_pxp "; 75 | cmd += (flip == true) ? "rotation=4 ! " : "! "; 76 | } else { 77 | /* no acceleration */ 78 | cmd = "videoscale ! videoconvert "; 79 | cmd += (flip == true) ? "! videoflip video-direction=4 ! " : "! "; 80 | } 81 | 82 | if (width > 0 && height > 0) { 83 | cmd += "video/x-raw,width=" + std::to_string(width) + ","; 84 | cmd += "height=" + std::to_string(height) + cmdFormat; 85 | cmd += (aspectRatio == true) ? ",pixel-aspect-ratio=1/1 ! " : " ! "; 86 | } else { 87 | if (format.length() != 0) 88 | cmd += "video/x-raw" + cmdFormat + " ! "; 89 | } 90 | 91 | pipeline.addToPipeline(cmd); 92 | } 93 | 94 | 95 | /** 96 | * @brief Create pipeline segment for accelerated video scaling and 97 | * conversion to RGB format. 98 | * 99 | * @param pipeline: GstPipelineImx pipeline. 100 | * @param width: output video width after rescale. 101 | * @param height: output video height after rescale. 102 | */ 103 | void GstVideoImx::videoscaleToRGB(GstPipelineImx &pipeline, 104 | const int &width, 105 | const int &height) 106 | { 107 | std::string cmd; 108 | if (this->imx.hasGPU2d()) { 109 | /** 110 | * imxvideoconvert_g2d does not support RGB sink 111 | * and use CPU to convert RGBA to RGB 112 | */ 113 | videoTransform(pipeline, "RGBA", width, height, false); 114 | cmd = "videoconvert ! video/x-raw,format=RGB ! "; 115 | pipeline.addToPipeline(cmd); 116 | } else if (this->imx.hasPxP()) { 117 | /** 118 | * imxvideoconvert_pxp does not support RGB sink 119 | * and use CPU to convert BGR to RGB 120 | */ 121 | videoTransform(pipeline, "BGR", width, height, false); 122 | cmd = "videoconvert ! video/x-raw,format=RGB ! "; 123 | pipeline.addToPipeline(cmd); 124 | } else { 125 | /* no acceleration */ 126 | videoTransform(pipeline, "RGB", width, height, false); 127 | } 128 | } 129 | 130 | 131 | /** 132 | * @brief Create pipeline segment for accelerated video cropping. 133 | * 134 | * @param pipeline: GstPipelineImx pipeline. 135 | * @param gstName: GStreamer videocrop element name. 136 | * @param width: output video width after rescale. 137 | * @param height: output video height after rescale. 138 | * @param top: top pixels to be cropped, default is 0. 139 | * @param bottom: bottom pixels to be cropped, default is 0. 140 | * @param left: left pixels to be cropped, default is 0. 141 | * @param right: right pixels to be cropped, default is 0. 142 | */ 143 | void GstVideoImx::videocrop(GstPipelineImx &pipeline, 144 | const std::string &gstName, 145 | const int &width, 146 | const int &height, 147 | const int &top, 148 | const int &bottom, 149 | const int &left, 150 | const int &right) 151 | { 152 | std::string cmd; 153 | cmd = "videocrop name=" + gstName + " "; 154 | if(top != 0) 155 | cmd += "top=" + std::to_string(top) + " "; 156 | if(bottom != 0) 157 | cmd += "bottom=" + std::to_string(bottom) + " "; 158 | if(left != 0) 159 | cmd += "left=" + std::to_string(left) + " "; 160 | if(right != 0) 161 | cmd += "right=" + std::to_string(right) + " "; 162 | cmd += "! "; 163 | if (width > 0 && height > 0) { 164 | cmd += "video/x-raw,width=" + std::to_string(width) + 165 | ",height=" + std::to_string(height) + " ! "; 166 | } 167 | pipeline.addToPipeline(cmd); 168 | } -------------------------------------------------------------------------------- /common/cpp/src/gst_video_post_process.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #include "gst_video_post_process.hpp" 7 | 8 | /** 9 | * @brief Dictionary of color in big-endian ARGB. 10 | */ 11 | std::map DictionaryColorARGB = { 12 | {"red", 0xFFFF0000}, 13 | {"green", 0xFF00FF00}, 14 | {"blue", 0xFF0000FF}, 15 | {"black", 0xFF000000}, 16 | }; 17 | 18 | 19 | /** 20 | * @brief Display GStreamer pipeline output. 21 | * 22 | * @param pipeline: GstPipelineImx pipeline. 23 | * @param sync: specifiy if we synchronized display with input buffer. 24 | */ 25 | void GstVideoPostProcess::display(GstPipelineImx &pipeline, 26 | const bool &sync) 27 | { 28 | std::string cmdSync = (sync == false) ? "sync=false " : ""; 29 | 30 | std::string cmd; 31 | if ((pipeline.isPerfAvailable().freq == true) || (pipeline.isPerfAvailable().temp == true)) { 32 | if (cairoNeeded == true) { 33 | cmd = "cairooverlay name=perf ! "; 34 | cairoNeeded = false; 35 | } 36 | cmd += "fpsdisplaysink name=img_tensor text-overlay=false video-sink=waylandsink " + cmdSync; 37 | } else { 38 | cmd = "waylandsink " + cmdSync; 39 | } 40 | 41 | pipeline.addToPipeline(cmd); 42 | } 43 | 44 | 45 | /** 46 | * @brief Add to pipeline an element to display text. 47 | * 48 | * @param pipeline: GstPipelineImx pipeline. 49 | * @param options: TextOverlayOptions structure, setup textoverlay options. 50 | */ 51 | void GstVideoPostProcess::addTextOverlay(GstPipelineImx &pipeline, 52 | const TextOverlayOptions &options) 53 | { 54 | imx::Imx imx{}; 55 | std::string colorOption; 56 | if (options.color.length() != 0) 57 | colorOption = (" color=" + std::to_string(DictionaryColorARGB[options.color])); 58 | 59 | std::string textOption; 60 | if (options.text.length() != 0) 61 | textOption = (" text=" + options.text); 62 | 63 | std::string vAlignOption; 64 | if (options.vAlignment.length() != 0) 65 | vAlignOption = (" valignment=" + options.vAlignment); 66 | 67 | std::string hAlignOption; 68 | if (options.hAlignment.length() != 0) 69 | hAlignOption = (" halignment=" + options.hAlignment); 70 | 71 | std::string cmd = "textoverlay name=" + options.gstName + " font-desc=\"" + options.fontName; 72 | cmd += ", " + std::to_string(options.fontSize) + "\"" + colorOption + textOption; 73 | if(imx.hasGPU2d()) 74 | cmd += vAlignOption + hAlignOption + " ! imxvideoconvert_g2d ! "; 75 | else if(imx.hasPxP()) 76 | cmd += vAlignOption + hAlignOption + " ! imxvideoconvert_pxp ! "; 77 | else 78 | cmd += vAlignOption + hAlignOption + " ! videoconvert ! "; 79 | 80 | pipeline.addToPipeline(cmd); 81 | } 82 | 83 | 84 | /** 85 | * @brief Add to pipeline cairooverlay for custom drawing. 86 | * 87 | * @param pipeline: GstPipelineImx pipeline. 88 | * @param gstName: name for GStreamer textoverlay element. 89 | */ 90 | void GstVideoPostProcess::addCairoOverlay(GstPipelineImx &pipeline, 91 | const std::string &gstName) 92 | { 93 | imx::Imx imx{}; 94 | if (imx.hasGPU2d()){ 95 | std::string cmd = "imxvideoconvert_g2d ! cairooverlay name=" + gstName + " ! "; 96 | pipeline.addToPipeline(cmd); 97 | } 98 | else if (imx.hasPxP()){ 99 | std::string cmd = "imxvideoconvert_pxp ! cairooverlay name=" + gstName + " ! "; 100 | pipeline.addToPipeline(cmd); 101 | } 102 | else{ 103 | std::string cmd = "videoconvert ! cairooverlay name=" + gstName + " ! "; 104 | pipeline.addToPipeline(cmd); 105 | } 106 | 107 | } 108 | 109 | 110 | /** 111 | * @brief Add to pipeline an element to save pipeline to video. 112 | * 113 | * @param pipeline: GstPipelineImx pipeline. 114 | * @param format: saved video format. Only MKV and MP4 format supported. 115 | * @param path: path to save video. 116 | */ 117 | void GstVideoPostProcess::saveToVideo(GstPipelineImx &pipeline, 118 | const std::string &format, 119 | const std::filesystem::path &path) 120 | { 121 | /** 122 | * webm not supported since no vp9 encoder in GStreamer version used 123 | */ 124 | pipeline.setSave(true); 125 | imx::Imx imx{}; 126 | if (imx.socId() == imx::IMX93) { 127 | log_error("video file can't be encoded with %s\n", imx.socName().c_str()); 128 | exit(-1); 129 | } else { 130 | std::string cmd; 131 | cmd = "v4l2h265enc ! h265parse ! "; 132 | 133 | if (format == "mkv") { 134 | cmd += "matroskamux ! filesink location="; 135 | cmd += path.string() + " "; 136 | pipeline.addToPipeline(cmd); 137 | } 138 | 139 | if (format == "mp4") { 140 | cmd = "qtmux ! filesink location="; 141 | cmd += path.string() + " "; 142 | pipeline.addToPipeline(cmd); 143 | } 144 | } 145 | } 146 | 147 | 148 | /** 149 | * @brief Add appsink element which allows the application to get access 150 | * to the raw buffer from the GStreamer pipeline. 151 | * 152 | * @param pipeline: GstPipelineImx pipeline. 153 | * @param options: AppSinkOptions structure, setup appsink options. 154 | */ 155 | void GstVideoPostProcess::addAppSink(GstPipelineImx &pipeline, 156 | const AppSinkOptions &options) 157 | { 158 | std::string cmd; 159 | cmd = "appsink"; 160 | 161 | if (options.gstName.size() != 0) 162 | cmd += " name=" + options.gstName; 163 | if (options.sync == false) 164 | cmd += " sync=false"; 165 | cmd += " max-buffers=" + std::to_string(options.maxBuffers); 166 | 167 | if (options.drop == true) 168 | cmd += " drop=true"; 169 | if (options.emitSignals == true) 170 | cmd += " emit-signals=true"; 171 | 172 | pipeline.addToPipeline(cmd + " "); 173 | } 174 | 175 | 176 | /** 177 | * @brief Link video to compositor. 178 | * 179 | * @param pipeline: GstPipelineImx pipeline. 180 | * @param inputParams: structure of parameters for an input video. 181 | */ 182 | void GstVideoCompositorImx::addToCompositor(GstPipelineImx &pipeline, 183 | const compositorInputParams &inputParams) 184 | { 185 | pipeline.addToPipeline(this->gstName + ".sink_" + std::to_string(sinkNumber) + " "); 186 | this->compositorInputs.push_back(inputParams); 187 | sinkNumber += 1; 188 | } 189 | 190 | /** 191 | * @brief Create pipeline segment for accelerated video mixing. 192 | * 193 | * @param pipeline: GstPipelineImx pipeline. 194 | * @param latency: time for a capture to reach the sink, default 0. 195 | */ 196 | void GstVideoCompositorImx::addCompositorToPipeline(GstPipelineImx &pipeline, 197 | const int &latency) 198 | { 199 | std::string cmd; 200 | std::string customParams; 201 | 202 | for (int i=0; i < this->compositorInputs.size(); i++) { 203 | compositorInputParams inputParams = compositorInputs.at(i); 204 | std::string stream = "sink_" + std::to_string(i); 205 | customParams += stream + "::zorder=" + std::to_string(inputParams.order) + " "; 206 | 207 | if (this->imx.hasPxP() && (inputParams.transparency == true)) 208 | customParams += stream + "::alpha=0.3 "; 209 | 210 | switch (inputParams.position) 211 | { 212 | case displayPosition::left: 213 | customParams += stream + "::xpos=0 "; 214 | customParams += stream + "::ypos=0 "; 215 | customParams += stream + "::width=960 "; 216 | customParams += stream + "::height=720 "; 217 | if (inputParams.keepRatio == true) 218 | customParams += stream + "::keep-ratio=true "; 219 | break; 220 | 221 | case displayPosition::right: 222 | customParams += stream + "::xpos=960 "; 223 | customParams += stream + "::ypos=0 "; 224 | customParams += stream + "::width=960 "; 225 | customParams += stream + "::height=720 "; 226 | if (inputParams.keepRatio == true) 227 | customParams += stream + "::keep-ratio=true "; 228 | break; 229 | 230 | case displayPosition::center: 231 | break; 232 | 233 | default: 234 | break; 235 | } 236 | } 237 | 238 | if(this->imx.hasGPU2d()) { 239 | cmd = "imxcompositor_g2d name=" + this->gstName + " "; 240 | cmd += customParams; 241 | } else if(this->imx.hasPxP()) { 242 | /** 243 | * imxcompositor_pxp does not support RGBA sink 244 | * and use CPU to convert RGBA to RGB 245 | */ 246 | cmd = "imxcompositor_pxp name=" + this->gstName + " "; 247 | cmd += customParams; 248 | } else { 249 | /* no acceleration */ 250 | cmd = "compositor name=" + gstName + " "; 251 | cmd += customParams; 252 | } 253 | 254 | if(latency != 0) { 255 | cmd += "latency=" + std::to_string(latency); 256 | cmd += " min-upstream-latency=" + std::to_string(latency); 257 | cmd += " "; 258 | } 259 | 260 | cmd += "! "; 261 | pipeline.addToPipeline(cmd); 262 | } -------------------------------------------------------------------------------- /common/cpp/src/model_infos.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #include "model_infos.hpp" 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | /** 14 | * @brief Dictionary of Backend identification. 15 | */ 16 | std::map inferenceHardwareBackend = { 17 | {"CPU", imx::Backend::CPU}, 18 | {"GPU", imx::Backend::GPU}, 19 | {"NPU", imx::Backend::NPU}, 20 | }; 21 | 22 | 23 | /** 24 | * @brief Parameterized constructor. 25 | * 26 | * @param path: model path. 27 | * @param backend: second argument at runtime corresponding to backend use. 28 | * @param norm: normalization to apply to input data. 29 | * @param numThreads: number of threads for XNNPACK (CPU backend). 30 | */ 31 | ModelInfos::ModelInfos(const std::filesystem::path &path, 32 | const std::string &backend, 33 | const std::string &norm, 34 | const int &numThreads) 35 | : modelPath(path), backend(backend) 36 | { 37 | setTensorFilterConfig(imx, numThreads); 38 | tensorData.tensorTransform = tensorCustomData.setTensorTransformConfig(norm); 39 | } 40 | 41 | 42 | /** 43 | * @brief Create pipeline segment for inference. 44 | * 45 | * @param pipeline: GstPipelineImx pipeline. 46 | * @param gstName: tensor_filter element name, empty by default. 47 | * @param format: tensor_filter input format, RGB by default. 48 | */ 49 | void ModelInfos::addInferenceToPipeline(GstPipelineImx &pipeline, 50 | const std::string &gstName, 51 | const std::string &format) 52 | { 53 | std::string cmd; 54 | if (format == "RGB") { 55 | videoscale.videoscaleToRGB(pipeline, modelWidth, modelHeight); 56 | } else { 57 | videoscale.videoTransform(pipeline, format, modelWidth, modelHeight, false, false, true); 58 | } 59 | cmd += "tensor_converter ! "; 60 | cmd += tensorData.tensorTransform; 61 | cmd += "tensor_filter latency=1 framework=" + framework + " model="; 62 | cmd += modelPath.string() + " "; 63 | cmd += tensorData.tensorFilterCustom; 64 | if (gstName.length() != 0) { 65 | cmd += " name=" + gstName; 66 | pipeline.addFilterName(gstName); 67 | } 68 | cmd += " ! "; 69 | pipeline.addToPipeline(cmd); 70 | } 71 | 72 | 73 | /** 74 | * @brief Setup tensor configuration, select backend use and create video 75 | * compositor segment pipeline. 76 | * 77 | * @param imx: i.MX used. 78 | * @param numThreads: number of threads for XNNPACK (CPU backend). 79 | */ 80 | void ModelInfos::setTensorFilterConfig(imx::Imx &imx, const int &numThreads) 81 | { 82 | switch (selectFromDictionary(backend, inferenceHardwareBackend)) 83 | { 84 | case imx::Backend::CPU: 85 | tensorData.tensorFilterCustom = tensorCustomData.CPU(numThreads); 86 | break; 87 | 88 | case imx::Backend::GPU: 89 | if (imx.hasVsiGPU()) { 90 | tensorData.tensorFilterCustom = tensorCustomData.vsiGPU(); 91 | } else if (imx.socId() == imx::IMX95) { 92 | tensorData.tensorFilterCustom = tensorCustomData.GPU(); 93 | } else { 94 | log_error("can't used this backend with %s\n", imx.socName().c_str()); 95 | exit(-1); 96 | } 97 | break; 98 | 99 | case imx::Backend::NPU: 100 | if (imx.isIMX8() && imx.hasNPU()) 101 | tensorData.tensorFilterCustom = tensorCustomData.vsiNPU(); 102 | 103 | if (imx.hasEthosNPU()) 104 | tensorData.tensorFilterCustom = tensorCustomData.ethosNPU(); 105 | 106 | if (imx.hasNeutronNPU()) 107 | tensorData.tensorFilterCustom = tensorCustomData.neutronNPU(); 108 | 109 | break; 110 | 111 | default: 112 | if (imx.isIMX8() && imx.hasNPU()) 113 | tensorData.tensorFilterCustom = tensorCustomData.vsiNPU(); 114 | 115 | if (imx.hasEthosNPU()) 116 | tensorData.tensorFilterCustom = tensorCustomData.ethosNPU(); 117 | 118 | if (imx.hasNeutronNPU()) 119 | tensorData.tensorFilterCustom = tensorCustomData.neutronNPU(); 120 | 121 | break; 122 | } 123 | } 124 | 125 | 126 | /** 127 | * @brief Parameterized constructor. 128 | * 129 | * @param path: TFlite model path. 130 | * @param backend: second argument at runtime corresponding to backend use. 131 | * @param norm: normalization to apply to input data. 132 | */ 133 | TFliteModelInfos::TFliteModelInfos(const std::filesystem::path &path, 134 | const std::string &backend, 135 | const std::string &norm, 136 | const int &numThreads) 137 | : ModelInfos(path, backend, norm, numThreads) 138 | { 139 | if (modelPath.extension() == ".tflite") { 140 | framework = "tensorflow-lite"; 141 | std::unique_ptr model; 142 | 143 | /* Load Model. */ 144 | model = tflite::FlatBufferModel::BuildFromFile(modelPath.string().c_str()); 145 | if (model == nullptr) { 146 | fprintf(stderr, "Failed to load model\n"); 147 | exit(-1); 148 | } 149 | 150 | /* Initiate Interpreter. */ 151 | std::unique_ptr interpreter; 152 | tflite::ops::builtin::BuiltinOpResolver resolver; 153 | tflite::InterpreterBuilder(*model.get(), resolver)(&interpreter); 154 | if (interpreter == nullptr) { 155 | log_error("Failed to initiate the interpreter\n"); 156 | exit(-1); 157 | } 158 | 159 | /* Add Ethos delegate if we use imx93 NPU. */ 160 | if (backend == "NPU" and imx.hasEthosNPU()) { 161 | const char* delegateDir = "/usr/lib/libethosu_delegate.so"; 162 | auto delegateOptions = TfLiteExternalDelegateOptionsDefault(delegateDir); 163 | auto externalDelegate = TfLiteExternalDelegateCreate(&delegateOptions); 164 | 165 | auto delegate = tflite::Interpreter::TfLiteDelegatePtr( 166 | externalDelegate, [](TfLiteDelegate* delegate) { 167 | TfLiteExternalDelegateDelete(delegate); } 168 | ); 169 | 170 | if(interpreter->ModifyGraphWithDelegate(std::move(delegate)) != kTfLiteOk) { 171 | log_error("Failed to apply delegate\n"); 172 | exit(-1); 173 | } 174 | 175 | if (interpreter->AllocateTensors() != kTfLiteOk) { 176 | log_error("Failed to allocate tensor\n"); 177 | exit(-1); 178 | } 179 | } 180 | 181 | /* Get input tensor dimensions. */ 182 | int input = interpreter->inputs()[0]; 183 | modelHeight = interpreter->tensor(input)->dims->data[1]; 184 | modelWidth = interpreter->tensor(input)->dims->data[2]; 185 | modelChannel = interpreter->tensor(input)->dims->data[3]; 186 | } else { 187 | log_error("TFlite model needed\n"); 188 | exit(-1); 189 | } 190 | } -------------------------------------------------------------------------------- /common/cpp/src/nn_decoder.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #include "nn_decoder.hpp" 7 | 8 | /** 9 | * @brief Map of models available for bounding boxes mode. 10 | */ 11 | std::map mapModeBoundingBoxes = { 12 | {ModeBoundingBoxes::yolov5, "yolov5"}, 13 | {ModeBoundingBoxes::mobilenetssd, "mobilenet-ssd"}, 14 | {ModeBoundingBoxes::mpPalmDetection, "mp-palm-detection"}, 15 | }; 16 | 17 | 18 | /** 19 | * @brief Map of models available for image segmentation mode. 20 | */ 21 | std::map mapModeImageSegment = { 22 | {ModeImageSegment::tfliteDeeplab, "tflite-deeplab"}, 23 | {ModeImageSegment::snpeDeeplab, "snpe-deeplab"}, 24 | {ModeImageSegment::snpeDepth, "snpe-depth"}, 25 | }; 26 | 27 | 28 | /** 29 | * @brief Add NNStreamer decoder for image segmentation. 30 | * 31 | * @param pipeline: GstPipelineImx pipeline. 32 | * @param options: ImageSegmentOptions structure, setup image segmetation options. 33 | */ 34 | void NNDecoder::addImageSegment(GstPipelineImx &pipeline, 35 | const ImageSegmentOptions &options) 36 | { 37 | std::string cmd; 38 | cmd = "tensor_decoder mode=image_segment option1="; 39 | cmd += mapModeImageSegment[options.modelName]; 40 | if (options.numClass != -1) { 41 | cmd += " option2=" + std::to_string(options.numClass) + "! "; 42 | } else { 43 | cmd += " ! videoconvert ! "; 44 | } 45 | pipeline.addToPipeline(cmd); 46 | } 47 | 48 | 49 | /** 50 | * @brief Add NNStreamer decoder for image labeling / classification. 51 | * 52 | * @param pipeline: GstPipelineImx pipeline. 53 | * @param labelsPath: model labels path. 54 | */ 55 | void NNDecoder::addImageLabeling(GstPipelineImx &pipeline, 56 | const std::filesystem::path &labelsPath) 57 | { 58 | std::string cmd; 59 | cmd = "tensor_decoder mode=image_labeling option1=" + labelsPath.string() + " ! "; 60 | pipeline.addToPipeline(cmd); 61 | } 62 | 63 | 64 | /** 65 | * @brief Add NNStreamer decoder for bounding boxes. 66 | * 67 | * @param pipeline: GstPipelineImx pipeline. 68 | * @param options: BoundingBoxesOptions structure, setup bouding boxes options. 69 | */ 70 | void NNDecoder::addBoundingBoxes(GstPipelineImx &pipeline, 71 | const BoundingBoxesOptions &options) 72 | { 73 | imx::Imx imx{}; 74 | std::string cmd; 75 | cmd = "tensor_decoder mode=bounding_boxes option1="; 76 | cmd += mapModeBoundingBoxes[options.modelName]; 77 | cmd += " option2=" + options.labelsPath.string(); 78 | cmd += " option3=" + options.option3; 79 | cmd += " option4=" + std::to_string(options.outDim.width) + ":"; 80 | cmd += std::to_string(options.outDim.height); 81 | cmd += " option5=" + std::to_string(options.inDim.width) + ":"; 82 | cmd += std::to_string(options.inDim.height); 83 | if (options.trackResult == true) 84 | cmd += " option6=1"; 85 | if (options.logResult == true) 86 | cmd += " option7=1"; 87 | 88 | // PxP is not supported by tensordecoder caps 89 | if(imx.hasGPU2d()) 90 | cmd += " ! imxvideoconvert_g2d ! "; 91 | else 92 | cmd += " ! videoconvert ! "; 93 | 94 | pipeline.addToPipeline(cmd); 95 | } -------------------------------------------------------------------------------- /common/cpp/src/tensor_custom_data_generator.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #include "tensor_custom_data_generator.hpp" 7 | 8 | /** 9 | * @brief Map of available normalization. 10 | */ 11 | std::map normDictionary = { 12 | {"none", Normalization::none}, 13 | {"centered", Normalization::centered}, 14 | {"reduced", Normalization::reduced}, 15 | {"centeredReduced", Normalization::centeredReduced}, 16 | {"castInt32", Normalization::castInt32}, 17 | {"castuInt8", Normalization::castuInt8}, 18 | }; 19 | 20 | 21 | /** 22 | * @brief Add tensor_filter option for CPU backend. 23 | * 24 | * @param numThreads: number of threads for XNNPACK (CPU backend). 25 | */ 26 | std::string TensorCustomGenerator::CPU(const int &numThreads) 27 | { 28 | tensorData.tensorFilterCustom = "custom=Delegate:XNNPACK,"; 29 | tensorData.tensorFilterCustom += "NumThreads:" + std::to_string(numThreads); 30 | return tensorData.tensorFilterCustom; 31 | } 32 | 33 | 34 | /** 35 | * @brief Add tensor_filter option for VSI GPU backend. 36 | */ 37 | std::string TensorCustomGenerator::vsiGPU() 38 | { 39 | tensorData.tensorFilterCustom = "custom=Delegate:External,"; 40 | tensorData.tensorFilterCustom += "ExtDelegateLib:libvx_delegate.so"; 41 | setenv("USE_GPU_INFERENCE","1",1); 42 | return tensorData.tensorFilterCustom; 43 | } 44 | 45 | 46 | /** 47 | * @brief Add tensor_filter option for VSI NPU backend. 48 | */ 49 | std::string TensorCustomGenerator::vsiNPU() 50 | { 51 | tensorData.tensorFilterCustom = "custom=Delegate:External,"; 52 | tensorData.tensorFilterCustom += "ExtDelegateLib:libvx_delegate.so"; 53 | setenv("USE_GPU_INFERENCE","0",1); 54 | return tensorData.tensorFilterCustom; 55 | } 56 | 57 | 58 | /** 59 | * @brief Add tensor_filter option for Ethos-U NPU backend. 60 | */ 61 | std::string TensorCustomGenerator::ethosNPU() 62 | { 63 | tensorData.tensorFilterCustom = "custom=Delegate:External,"; 64 | tensorData.tensorFilterCustom += "ExtDelegateLib:libethosu_delegate.so"; 65 | return tensorData.tensorFilterCustom; 66 | } 67 | 68 | 69 | /** 70 | * @brief Add tensor_filter option for Ethos-U NPU backend. 71 | */ 72 | std::string TensorCustomGenerator::neutronNPU() 73 | { 74 | tensorData.tensorFilterCustom = "custom=Delegate:External,"; 75 | tensorData.tensorFilterCustom += "ExtDelegateLib:libneutron_delegate.so"; 76 | return tensorData.tensorFilterCustom; 77 | } 78 | 79 | 80 | /** 81 | * @brief Add tensor_filter option for GPU backend on i.MX95. 82 | */ 83 | std::string TensorCustomGenerator::GPU() 84 | { 85 | tensorData.tensorFilterCustom = "custom=Delegate:GPU"; 86 | return tensorData.tensorFilterCustom; 87 | } 88 | 89 | 90 | /** 91 | * @brief Add element for normalization to pipeline. 92 | */ 93 | std::string TensorCustomGenerator::setTensorTransformConfig(const std::string &norm) 94 | { 95 | std::string tensorTransformCustom; 96 | switch (selectFromDictionary(norm, normDictionary)) 97 | { 98 | case Normalization::none: 99 | tensorTransformCustom = ""; 100 | break; 101 | 102 | case Normalization::centered: 103 | tensorTransformCustom = "tensor_transform mode=arithmetic "; 104 | tensorTransformCustom += "option=typecast:int16,add:-128 ! "; 105 | tensorTransformCustom += "tensor_transform mode=typecast "; 106 | tensorTransformCustom += "option=int8 ! "; 107 | break; 108 | 109 | case Normalization::reduced: 110 | tensorTransformCustom = "tensor_transform mode=arithmetic "; 111 | tensorTransformCustom += "option=typecast:float32,div:255 ! "; 112 | break; 113 | 114 | case Normalization::centeredReduced: 115 | tensorTransformCustom = "tensor_transform mode=arithmetic "; 116 | tensorTransformCustom += "option=typecast:float32,add:-127.5,div:127.5 ! "; 117 | break; 118 | 119 | case Normalization::castInt32: 120 | tensorTransformCustom = "tensor_transform mode=typecast "; 121 | tensorTransformCustom += "option=int32 ! "; 122 | break; 123 | 124 | case Normalization::castuInt8: 125 | tensorTransformCustom = "tensor_transform mode=typecast "; 126 | tensorTransformCustom += "option=uint8 ! "; 127 | break; 128 | 129 | default: 130 | tensorTransformCustom = ""; 131 | break; 132 | } 133 | return tensorTransformCustom; 134 | } -------------------------------------------------------------------------------- /common/python/imxpy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxp-imx/nxp-nnstreamer-examples/b8dafc13bb93b06826f9fc6b91056270cecc0fbd/common/python/imxpy/__init__.py -------------------------------------------------------------------------------- /common/python/imxpy/common_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2022-2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | import os 7 | 8 | from . import imx_dev 9 | 10 | 11 | class GstVideoImx: 12 | """Helper class for video pipeline segments handling. 13 | 14 | imx: imxdev.Imx() instance 15 | """ 16 | def __init__(self, imx): 17 | assert (isinstance(imx, imx_dev.Imx)) 18 | self.imx = imx 19 | 20 | def videoscale_to_format(self, format=None, width=None, height=None, hardware=None, flip=False): 21 | """Create pipeline segment for accelerated video formatting and csc. 22 | 23 | hardware: corresponding hardware to accelerate video composition 24 | format: GStreamer video format 25 | width: output video width after rescale 26 | height: output video height after rescale 27 | return: GStreamer pipeline segment string 28 | """ 29 | valid_dimensions = width is not None and height is not None 30 | required_format = format is not None 31 | 32 | if hardware is not None: 33 | cmd = f'imxvideoconvert_{hardware} ' 34 | if flip: 35 | cmd += 'rotation=4 ! ' 36 | else: 37 | cmd += '! ' 38 | else: 39 | # no acceleration 40 | if valid_dimensions: 41 | cmd = 'videoscale ! ' 42 | if flip: 43 | cmd += 'videoflip video-direction=4 ! ' 44 | if required_format: 45 | cmd += 'videoconvert ! ' 46 | 47 | cmd += f'video/x-raw' 48 | if valid_dimensions: 49 | cmd += f',width={width},height={height}' 50 | if required_format: 51 | cmd += f',format={format}' 52 | cmd += ' ! ' 53 | 54 | return cmd 55 | 56 | def accelerated_videoscale(self, width=None, height=None, format=None, flip=False): 57 | """Create pipeline segment for accelerated video scaling and conversion 58 | to a given GStreamer video format. 59 | 60 | width: output video width after rescale 61 | height: output video height after rescale 62 | format: GStreamer video format 63 | return: GStreamer pipeline segment string 64 | """ 65 | valid_dim_g2d = valid_dim_pxp = True 66 | valid_dimensions = width is not None and height is not None 67 | if valid_dimensions: 68 | # g2d and pxp can't resize under 64 pixels 69 | valid_dim_g2d = valid_dim_pxp = width >= 64 and height >= 64 70 | 71 | if self.imx.has_g2d() and valid_dim_g2d: 72 | # imxvideoconvert_g2d does not support RGB nor GRAY8 sink 73 | # use acceleration to RGBA 74 | if format == 'RGB' or format == 'GRAY8': 75 | cmd = self.videoscale_to_format('RGBA', width, height, 'g2d', flip) 76 | cmd += f'videoconvert ! video/x-raw,format={format} ! ' 77 | else: 78 | cmd = self.videoscale_to_format(format, width, height, 'g2d', flip) 79 | elif self.imx.has_pxp() and valid_dim_pxp: 80 | if format == 'RGB': 81 | # imxvideoconvert_pxp does not support RGB sink 82 | # use acceleration to BGR 83 | cmd = self.videoscale_to_format('BGR', width, height, 'pxp', flip) 84 | cmd += f'videoconvert ! video/x-raw,format={format} ! ' 85 | else: 86 | cmd = self.videoscale_to_format(format, width, height, 'pxp', flip) 87 | 88 | else: 89 | # no hardware acceleration 90 | cmd = self.videoscale_to_format(format, width, height, flip=flip) 91 | return cmd 92 | 93 | def videocrop_to_format(self, videocrop_name, width, height, top=None, bottom=None, 94 | left=None, right=None, format=None): 95 | """Create pipeline segment for accelerated video cropping and conversion 96 | to a given GStreamer video format. 97 | 98 | videocrop_name: GStreamer videocrop element name 99 | width: output video width after rescale 100 | height: output video height after rescale 101 | top: top pixels to be cropped - may be setup via property later 102 | bottom: bottom pixels to be cropped - may be setup via property later 103 | left: left pixels to be cropped - may be setup via property later 104 | right: right pixels to be cropped - may be setup via property later 105 | format: GStreamer video format 106 | return: GStreamer pipeline segment string 107 | """ 108 | cmd = f' videocrop name={videocrop_name} ' 109 | if top is not None: 110 | cmd += f'top={top} ' 111 | if bottom is not None: 112 | cmd += f'bottom={bottom} ' 113 | if left is not None: 114 | cmd += f'left={left} ' 115 | if right is not None: 116 | cmd += f'right={right} ' 117 | cmd += '! ' 118 | 119 | cmd += self.accelerated_videoscale(width, height, format) 120 | 121 | return cmd 122 | 123 | def video_compositor(self, latency=0): 124 | """ 125 | Create pipeline segment for accelerated video mixing 126 | """ 127 | name = "mix" 128 | nnst_stream = 'sink_0' 129 | main_stream = 'sink_1' 130 | options = { 131 | nnst_stream: {'zorder': 2}, 132 | main_stream: {'zorder': 1} 133 | } 134 | 135 | if self.imx.has_g2d(): 136 | cmd = f'imxcompositor_g2d name={name} ' 137 | elif self.imx.has_pxp(): 138 | # imxcompositor_pxp does not support RGBA sink 139 | # use acceleration to BGR 140 | cmd = f'imxcompositor_pxp name={name} ' 141 | alpha = {'alpha': 0.3} 142 | options[nnst_stream].update(alpha) 143 | else: 144 | # no acceleration 145 | cmd = f'compositor name={name} ' 146 | 147 | for sink in options: 148 | for k, v in options[sink].items(): 149 | cmd += f'{sink}::{k}={v} ' 150 | 151 | if latency != 0: 152 | cmd += f'latency={latency} min-upstream-latency={latency} ' 153 | cmd += '! ' 154 | return cmd 155 | 156 | 157 | def store_vx_graph_compilation(imx): 158 | """Store on disk .nb files that contains the result of the OpenVX graph compilation 159 | This feature is only available for iMX8 platforms to get the warmup time only once 160 | 161 | imx: imxdev.Imx() instance 162 | """ 163 | is_vsi_platform = imx.has_npu_vsi() or imx.has_gpu_vsi() 164 | if is_vsi_platform: 165 | # Set the environment variables to store graph compilation on home directory 166 | os.environ['VIV_VX_ENABLE_CACHE_GRAPH_BINARY'] = '1' 167 | HOME = os.getenv('HOME') 168 | os.environ['VIV_VX_CACHE_BINARY_GRAPH_DIR'] = HOME 169 | -------------------------------------------------------------------------------- /common/python/imxpy/imx_dev.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2022-2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | import logging 7 | import re 8 | import subprocess 9 | from enum import IntEnum, auto 10 | 11 | 12 | class SocId(IntEnum): 13 | """i.MX devices enumeration. 14 | """ 15 | def __str__(self): 16 | return str(self.name) 17 | IMX8MQ = auto() 18 | IMX8MM = auto() 19 | IMX8MN = auto() 20 | IMX8MP = auto() 21 | IMX8ULP = auto() 22 | IMX8QM = auto() 23 | IMX8QXP = auto() 24 | IMX93 = auto() 25 | IMX95 = auto() 26 | UNKNOWN = auto() 27 | 28 | 29 | class Feature(IntEnum): 30 | """i.MX Features enumeration. 31 | """ 32 | def __str__(self): 33 | return str(self.name) 34 | GPU2D = auto() 35 | GPU3D = auto() 36 | NPU = auto() 37 | 38 | 39 | # machine regex used for SoC identification 40 | mach_regex_to_soc = [('imx8mq', SocId.IMX8MQ), 41 | ('imx8mm', SocId.IMX8MM), 42 | ('imx8mn', SocId.IMX8MN), 43 | ('imx8mp', SocId.IMX8MP), 44 | ('imx8ulp', SocId.IMX8ULP), 45 | ('imx8qm', SocId.IMX8QM), 46 | ('imx8qxp', SocId.IMX8QXP), 47 | ('imx93', SocId.IMX93), 48 | ('imx95', SocId.IMX95),] 49 | 50 | # dictionary of SoC name 51 | soc_to_name = {SocId.IMX8MQ: "i.MX 8M Quad", 52 | SocId.IMX8MM: "i.MX 8M Mini", 53 | SocId.IMX8MN: "i.MX 8M Nano", 54 | SocId.IMX8MP: "i.MX 8M Plus", 55 | SocId.IMX8ULP: "i.MX 8ULP", 56 | SocId.IMX8QM: "i.MX 8QuadMax", 57 | SocId.IMX8QXP: "i.MX 8QuadXPlus", 58 | SocId.IMX95: "i.MX 95", 59 | SocId.IMX93: "i.MX 93", } 60 | 61 | # dictionary of SoC features 62 | soc_has_feature = { 63 | SocId.IMX8MQ: 64 | {Feature.GPU2D: False, Feature.GPU3D: True, Feature.NPU: False, }, 65 | SocId.IMX8MM: 66 | {Feature.GPU2D: True, Feature.GPU3D: True, Feature.NPU: False, }, 67 | SocId.IMX8MN: 68 | {Feature.GPU2D: False, Feature.GPU3D: True, Feature.NPU: False, }, 69 | SocId.IMX8MP: 70 | {Feature.GPU2D: True, Feature.GPU3D: True, Feature.NPU: True, }, 71 | SocId.IMX8ULP: 72 | {Feature.GPU2D: True, Feature.GPU3D: True, Feature.NPU: False, }, 73 | SocId.IMX8QM: 74 | {Feature.GPU2D: True, Feature.GPU3D: True, Feature.NPU: False, }, 75 | SocId.IMX8QXP: 76 | {Feature.GPU2D: True, Feature.GPU3D: True, Feature.NPU: False, }, 77 | SocId.IMX95: 78 | {Feature.GPU2D: True, Feature.GPU3D: True, Feature.NPU: True, }, 79 | SocId.IMX93: 80 | {Feature.GPU2D: True, Feature.GPU3D: False, Feature.NPU: True, }, } 81 | 82 | 83 | class Imx(): 84 | 85 | def __init__(self): 86 | """Provide helpers to query running i.MX devices properties. 87 | """ 88 | res = subprocess.run(['uname', '-a'], 89 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 90 | out = res.stdout.decode() 91 | 92 | r = re.compile("^\s*(\S+)\s+(\S+)") # noqa 93 | m = r.search(out) 94 | if m is None: 95 | raise ValueError(f"could not detect machine name in [{out}]") 96 | machine = m[2] 97 | 98 | self.soc = SocId.UNKNOWN 99 | for tuple in mach_regex_to_soc: 100 | r = re.compile(tuple[0]) 101 | m = r.search(machine) 102 | if m is not None: 103 | self.soc = tuple[1] 104 | 105 | if not (self.is_imx8() or self.is_imx93() or self.is_imx95()): 106 | raise ValueError(f"unknown imx family [{self.soc}]") 107 | 108 | if self.soc == SocId.UNKNOWN: 109 | raise ValueError(f"unknown machine name [{machine}]") 110 | 111 | def id(self): 112 | return self.soc 113 | 114 | def name(self): 115 | try: 116 | return soc_to_name[self.soc] 117 | except BaseException as e: 118 | raise ValueError(f"unknown soc name [{self.soc}]") 119 | 120 | def has_gpu2d(self): 121 | return soc_has_feature[self.soc][Feature.GPU2D] 122 | 123 | def has_gpu3d(self): 124 | return soc_has_feature[self.soc][Feature.GPU3D] 125 | 126 | def has_gpuml(self): 127 | if self.soc == SocId.IMX8MM or self.soc == SocId.IMX8ULP: 128 | return False # GPU not supported for ML acceleration 129 | else: 130 | return self.has_gpu3d() 131 | 132 | def has_npu(self): 133 | return soc_has_feature[self.soc][Feature.NPU] 134 | 135 | def has_gpu_vsi(self): 136 | return self.has_gpuml() and self.is_imx8() 137 | 138 | def has_npu_vsi(self): 139 | return self.soc == SocId.IMX8MP 140 | 141 | def has_npu_ethos(self): 142 | return self.soc == SocId.IMX93 143 | 144 | def has_npu_neutron(self): 145 | return self.soc == SocId.IMX95 146 | 147 | def has_g2d(self): 148 | return self.is_imx8() and self.soc != SocId.IMX8MQ or self.is_imx95() 149 | 150 | def has_pxp(self): 151 | return self.soc == SocId.IMX93 152 | 153 | def is_imx8(self): 154 | imx8 = [SocId.IMX8MQ, SocId.IMX8MM, SocId.IMX8MN, SocId.IMX8MP, 155 | SocId.IMX8QM, SocId.IMX8QXP, SocId.IMX8ULP] 156 | return self.soc in imx8 157 | 158 | def is_imx93(self): 159 | imx93 = [SocId.IMX93] 160 | return self.soc in imx93 161 | 162 | def is_imx95(self): 163 | imx95 = [SocId.IMX95] 164 | return self.soc in imx95 165 | 166 | 167 | if __name__ == '__main__': 168 | logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) 169 | imx = Imx() 170 | logging.info(f"id() {imx.id()}") 171 | logging.info(f"name() {imx.name()}") 172 | logging.info(f"is_imx8() {imx.is_imx8()}") 173 | logging.info(f"is_imx93() {imx.is_imx93()}") 174 | logging.info(f"is_imx95() {imx.is_imx95()}") 175 | logging.info(f"has_gpu2d() {imx.has_gpu2d()}") 176 | logging.info(f"has_gpu3d() {imx.has_gpu3d()}") 177 | logging.info(f"has_gpuml() {imx.has_gpuml()}") 178 | logging.info(f"has_npu() {imx.has_npu()}") 179 | logging.info(f"has_gpu_vsi() {imx.has_gpu_vsi()}") 180 | logging.info(f"has_npu_vsi() {imx.has_npu_vsi()}") 181 | logging.info(f"has_npu_ethos() {imx.has_npu_ethos()}") 182 | logging.info(f"has_g2d() {imx.has_g2d()}") 183 | logging.info(f"has_pxp() {imx.has_pxp()}") 184 | -------------------------------------------------------------------------------- /downloads/README.md: -------------------------------------------------------------------------------- 1 | # Download examples dependencies 2 | Examples require model files and some associated metadata (labels, ssd priors definitions...) to be downloaded so that they are made available locally to examples in top level `downloads` directory.
3 | This download procedure is to be done once from the host PC. A Jupyter notebook [download.ipynb](./download.ipynb) is available to fetch all dependencies from network and to apply necessary format-conversions.
4 | This Notebook execution requires: 5 | 1. eIQ Toolkit local install 6 | 2. Jupyter Notebook python package 7 | 8 | # eIQ Toolkit install and environment setup 9 | Last version of eIQ Toolkit installer can be downloaded from [NXP eIQ website](https://www.nxp.com/eiq). 10 | It is available for Windows and Ubuntu OS platforms.
11 | For more information, refer to eIQ Toolkit User Guide available on [the documentation section of NXP eIQ website](https://www.nxp.com/design/software/development-software/eiq-ml-development-environment/eiq-toolkit-for-end-to-end-model-development-and-deployment:EIQ-TOOLKIT#documentation), and also from top level `docs` folder within its install directory. 12 | 13 | ## Quick install procedure example on Ubuntu: 14 | Download eIQ Toolkit install package from [DOWNLOADS section of NXP eIQ website](https://www.nxp.com/design/software/development-software/eiq-ml-development-environment/eiq-toolkit-for-end-to-end-model-development-and-deployment:EIQ-TOOLKIT#downloads). 15 | Type `1.15.0` on the Downloads search bar and select the eIQ Toolkit 1.15.0 Installer for Ubuntu. 16 | No additional Extension is needed.
17 | Login to an NXP account is required to start the download, it will be possible to create one directly when the installer is selected. 18 | 19 | Then open a terminal and enter following commands: 20 | ```bash 21 | # paths below to be adapted according to package version 22 | # make package executable and execute with root privilege 23 | chmod a+x ~/Downloads/eiq-toolkit-v1.15.0.<...>.deb.bin 24 | sudo ~/Downloads/eiq-toolkit-v1.15.0.<...>.deb.bin 25 | ``` 26 | 27 | It will be asked to agree with the licence agreement. 28 | Then some extensions to be installed are proposed, 29 | none of them is necessary for the rest of the download procedure. 30 | 31 | ## eIQ Toolkit environment setup 32 | 33 | eIQ Toolkit environment configures specific python interpreter and packages to be used. It shall be done only once within same shell session: 34 | ```bash 35 | # adapt base path according to your install 36 | export EIQ_BASE="/opt/nxp/eIQ_Toolkit_v1.15.0" 37 | export EIQ_ENV="${EIQ_BASE}/bin/eiqenv.sh" 38 | source "${EIQ_ENV}" 39 | ``` 40 | 41 | ## Add supplementary packages 42 | Additional packages shall be installed on top of eIQ Portal after [setting the eIQ environment](#eiq-toolkit-environment-setup) if not already done. 43 | ```bash 44 | # Executed from eIQ environment-enabled shell 45 | # cleanup obsolete eIQ python userbase packages to avoid conflicts from past installs 46 | rm -rf "${PYTHONUSERBASE}" 47 | # install required packages in the nxp-nnstreamer-examples downloads directory 48 | python -m pip install -r /downloads/requirements.txt 49 | ``` 50 | 51 | # Jupyter Notebook setup 52 | Notebook will make use of eIQ Toolkit tools, therefore Jupyter must be started after [setting the eIQ environment](#eiq-toolkit-environment-setup) if not already done. 53 | 54 | ```bash 55 | # Executed from eIQ environment-enabled shell 56 | # go to the nxp-nnstreamer-examples downloads directory 57 | cd /downloads 58 | # Startup Jupyter Notebook web application 59 | jupyter notebook 60 | # or alternatively use 61 | python -m notebook 62 | ``` 63 | To run the notebook, from the Jupyter Notebook web page use browser to open [download.ipynb](./download.ipynb). 64 | Within this notebook, execute all sections by pressing the `▶▶` button. -------------------------------------------------------------------------------- /downloads/compile_models.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2023-2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | set -x 7 | 8 | REALPATH="$(readlink -e "$0")" 9 | BASEDIR="$(dirname "${REALPATH}")/.." 10 | MODELS_DIR="${BASEDIR}/downloads/models" 11 | REQUIRED_CAMERA=0 12 | 13 | function compile_models(){ 14 | # Compile a model if required by NPU 15 | # Argument must be destination directory ($1) 16 | 17 | # Convert quantized TFLite models to vela for i.MX 93 Ethos-U NPU 18 | if [[ "${IMX}" == "IMX93" ]] 19 | then 20 | # Convert only quantized models to vela 21 | DESTINATION_DIR="${1}" 22 | for FILE in $(find "${DESTINATION_DIR}" -name "*uint8*.tflite" -o -name "*quant*.tflite" | grep -v "vela") 23 | do 24 | vela "${FILE}" --output-dir "${DESTINATION_DIR}/output" 25 | done 26 | mv "${DESTINATION_DIR}"/output/*.tflite "${DESTINATION_DIR}" 27 | rm -r "${DESTINATION_DIR}/output" 28 | fi 29 | } 30 | 31 | source "${BASEDIR}/common/common_utils.sh" 32 | setup_env 33 | 34 | # Compile all the models once 35 | TASKS=(classification object-detection semantic-segmentation pose-estimation face-processing monocular-depth-estimation) 36 | for TASK in "${TASKS[@]}" 37 | do 38 | DEST_DIR="${MODELS_DIR}/${TASK}" 39 | compile_models "${DEST_DIR}" 40 | done 41 | -------------------------------------------------------------------------------- /downloads/requirements.txt: -------------------------------------------------------------------------------- 1 | jupyter==1.0.0 2 | tensorflow==2.16.2 -------------------------------------------------------------------------------- /tasks/classification/README.md: -------------------------------------------------------------------------------- 1 | # Image Classification 2 | 3 | ## Overview 4 | Name | Implementation | Model | ML engine | Features 5 | --- | --- | --- | --- | --- | 6 | [example_classification_mobilenet_v1_tflite.cpp](./cpp/example_classification_mobilenet_v1_tflite.cpp) | C++ | MobileNetV1 | TFLite| camera
gst-launch
7 | [example_classification_mobilenet_v1_tflite.sh](./example_classification_mobilenet_v1_tflite.sh) | Bash | MobileNetV1 | TFLite| camera
gst-launch
8 | 9 | ## MobileNetV1 image classification 10 | ### Bash 11 | | Platforms | NPU | CPU | GPU | 12 | | ------------ | --- | --- | --- | 13 | | i.MX 8M Plus | :white_check_mark: | :white_check_mark: | :white_check_mark: | 14 | | i.MX 93 | :white_check_mark: | :white_check_mark: | :x: | 15 | | i.MX 95 | :white_check_mark: | :white_check_mark: | :x: | 16 | 17 | The image classification demo in bash supports multiple backend (refers to above table), default value can be overriden by explicitly defining BACKEND variable, for instance: 18 | ```bash 19 | BACKEND=CPU ./tasks/classification/example_classification_mobilenet_v1_tflite.sh 20 | ``` 21 | 22 | ### C++ 23 | | Platforms | NPU | CPU | GPU | 24 | | ------------ | --- | --- | --- | 25 | | i.MX 8M Plus | :white_check_mark: | :white_check_mark: | :white_check_mark: | 26 | | i.MX 93 | :white_check_mark: | :white_check_mark: | :x: | 27 | | i.MX 95 | :white_check_mark: | :white_check_mark: | :white_check_mark: | 28 | 29 | C++ example script needs to be generated with [cross compilation](../). [setup_environment.sh](../tools/setup_environment.sh) script needs to be executed in [nxp-nnstreamer-examples](../) folder to define data paths: 30 | ```bash 31 | . ./tools/setup_environment.sh 32 | ``` 33 | 34 | Image classification demo can be run on different hardware (refers to above table):
35 | Inference on NPU with the following script: 36 | ```bash 37 | ./build/classification/example_classification_mobilenet_v1_tflite -p ${MOBILENETV1_QUANT} -l ${MOBILENETV1_LABELS} 38 | ``` 39 | For i.MX 93 NPU use vela converted model: 40 | ```bash 41 | ./build/classification/example_classification_mobilenet_v1_tflite -p ${MOBILENETV1_QUANT_VELA} -l ${MOBILENETV1_LABELS} 42 | ``` 43 | 44 | For i.MX 95 NPU use neutron converted model: 45 | ```bash 46 | ./build/classification/example_classification_mobilenet_v1_tflite -p ${MOBILENETV1_QUANT_NEUTRON} -l ${COCO_LABELS} -x ${MOBILENETV2_BOXES} 47 | ``` 48 | 49 | Inference on CPU with the following script: 50 | ```bash 51 | ./build/classification/example_classification_mobilenet_v1_tflite -p ${MOBILENETV1_QUANT} -l ${MOBILENETV1_LABELS} -b CPU 52 | ``` 53 | Quantized model is used for better inference performances on CPU.
54 | NOTE: inferences on i.MX8MPlus GPU have low performances, but are possible with the following script: 55 | ```bash 56 | ./build/classification/example_classification_mobilenet_v1_tflite -p ${MOBILENETV1} -l ${MOBILENETV1_LABELS} -b GPU -n centeredReduced 57 | ``` 58 | Input normalization needs to be specified, here input data needs to be centered and reduced to fit MobileNetV1 input specifications. 59 | 60 | The following execution parameters are available (Run ``` ./example_classification_mobilenet_v1_tflite -h``` to see option details): 61 | 62 | Option | Description 63 | --- | --- 64 | -b, --backend | Use the selected backend (CPU, GPU, NPU)
default: NPU 65 | -n, --normalization | Use the selected normalization (none, centered, reduced, centeredReduced, castInt32, castuInt8)
default: none 66 | -c, --camera_device | Use the selected camera device (/dev/video{number})
default: /dev/video0 for i.MX 93 and /dev/video3 for i.MX 8MP 67 | -p, --model_path | Use the selected model path 68 | -l, --labels_path | Use the selected labels path 69 | -d, --display_perf |Display performances, can specify time or freq 70 | -t, --text_color | Color of performances displayed, can choose between red, green, blue, and black
default: white 71 | -g, --graph_path | Path to store the result of the OpenVX graph compilation (only for i.MX8MPlus)
default: home directory 72 | -r, --cam_params | Use the selected camera resolution and framerate
default: 640x480, 30fps 73 | 74 | Press ```Esc or ctrl+C``` to stop the execution of the pipeline.

-------------------------------------------------------------------------------- /tasks/classification/classification_demo.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxp-imx/nxp-nnstreamer-examples/b8dafc13bb93b06826f9fc6b91056270cecc0fbd/tasks/classification/classification_demo.webp -------------------------------------------------------------------------------- /tasks/classification/classification_utils.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2022, 2024-2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | function gst_exec_classification { 7 | 8 | # accelerated video scaling before inferencing 9 | local VIDEO_SCALE=$(accelerated_video_scale_rgb_str ${MODEL_WIDTH} ${MODEL_HEIGHT}) 10 | 11 | gst-launch-1.0 \ 12 | v4l2src name=cam_src device=${CAMERA_DEVICE} num-buffers=-1 ! \ 13 | video/x-raw,width=${CAMERA_WIDTH},height=${CAMERA_HEIGHT},framerate=${CAMERA_FPS}/1 ! \ 14 | tee name=t \ 15 | t. ! queue name=thread-nn max-size-buffers=2 leaky=2 ! \ 16 | ${VIDEO_SCALE} \ 17 | tensor_converter ! \ 18 | ${TENSOR_PREPROCESS} \ 19 | ${TENSOR_FILTER} \ 20 | ${TENSOR_DECODER} \ 21 | overlay.text_sink \ 22 | t. ! queue name=thread-img max-size-buffers=2 leaky=2 ! \ 23 | textoverlay name=overlay font-desc=\"Sans, 24\" ! \ 24 | waylandsink sync=false 25 | } 26 | -------------------------------------------------------------------------------- /tasks/classification/example_classification_mobilenet_v1_tflite.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2022-2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | set -x 7 | 8 | REALPATH="$(readlink -e "$0")" 9 | BASEDIR="$(dirname "${REALPATH}")/../.." 10 | MODELS_DIR="${BASEDIR}/downloads/models/classification" 11 | 12 | source "${BASEDIR}/common/common_utils.sh" 13 | source "${BASEDIR}/tasks/classification/classification_utils.sh" 14 | 15 | setup_env 16 | 17 | # model and framework dependant variables 18 | declare -A MODEL_BACKEND_NPU 19 | MODEL_BACKEND_NPU[IMX8MP]="${MODELS_DIR}/mobilenet_v1_1.0_224_quant_uint8_float32.tflite" 20 | MODEL_BACKEND_NPU[IMX93]="${MODELS_DIR}/mobilenet_v1_1.0_224_quant_uint8_float32_vela.tflite" 21 | MODEL_BACKEND_NPU[IMX95]="${MODELS_DIR}/mobilenet_v1_1.0_224_quant_uint8_float32_neutron.tflite" 22 | 23 | declare -A MODEL_BACKEND 24 | MODEL_BACKEND[CPU]="${MODELS_DIR}/mobilenet_v1_1.0_224_quant_uint8_float32.tflite" 25 | MODEL_BACKEND[GPU]="${MODELS_DIR}/mobilenet_v1_1.0_224.tflite" 26 | MODEL_BACKEND[NPU]=${MODEL_BACKEND_NPU[${IMX}]} 27 | MODEL=${MODEL_BACKEND[${BACKEND}]} 28 | 29 | MODEL_WIDTH=224 30 | MODEL_HEIGHT=224 31 | MODEL_LABELS="${MODELS_DIR}/labels_mobilenet_quant_v1_224.txt" 32 | 33 | FRAMEWORK="tensorflow-lite" 34 | 35 | # tensor filter configuration 36 | FILTER_COMMON="tensor_filter framework=${FRAMEWORK} model=${MODEL}" 37 | 38 | declare -A FILTER_BACKEND_NPU 39 | FILTER_BACKEND_NPU[IMX8MP]=" custom=Delegate:External,ExtDelegateLib:libvx_delegate.so ! " 40 | FILTER_BACKEND_NPU[IMX93]=" custom=Delegate:External,ExtDelegateLib:libethosu_delegate.so ! " 41 | FILTER_BACKEND_NPU[IMX95]=" custom=Delegate:External,ExtDelegateLib:libneutron_delegate.so ! " 42 | 43 | declare -A FILTER_BACKEND 44 | FILTER_BACKEND[CPU]="${FILTER_COMMON}" 45 | FILTER_BACKEND[CPU]+=" custom=Delegate:XNNPACK,NumThreads:$(nproc --all) !" 46 | FILTER_BACKEND[GPU]="${FILTER_COMMON}" 47 | FILTER_BACKEND[GPU]+=" custom=Delegate:External,ExtDelegateLib:libvx_delegate.so ! " 48 | FILTER_BACKEND[NPU]="${FILTER_COMMON}" 49 | FILTER_BACKEND[NPU]+=${FILTER_BACKEND_NPU[${IMX}]} 50 | TENSOR_FILTER=${FILTER_BACKEND[${BACKEND}]} 51 | 52 | # tensor preprocessing configuration: normalize video for float input models 53 | declare -A PREPROCESS_BACKEND 54 | PREPROCESS_BACKEND[CPU]="" 55 | PREPROCESS_BACKEND[GPU]="tensor_transform mode=arithmetic option=typecast:float32,add:-127.5,div:127.5 ! " 56 | PREPROCESS_BACKEND[NPU]="" 57 | TENSOR_PREPROCESS=${PREPROCESS_BACKEND[${BACKEND}]} 58 | 59 | # tensor decoder configuration: image labeling 60 | TENSOR_DECODER="tensor_decoder mode=image_labeling option1=${MODEL_LABELS} ! " 61 | 62 | gst_exec_classification 63 | 64 | -------------------------------------------------------------------------------- /tasks/face-processing/common/deepface.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2023, 2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | import numpy as np 7 | import os 8 | 9 | 10 | class FNModel: 11 | 12 | def __init__(self, model_directory, vela=False): 13 | 14 | """Helper class for DeepFace-emotion model. 15 | 16 | model_directory: directory where tflite model is located 17 | vela: use vela version of the model 18 | """ 19 | # Face detection definitions 20 | self.MODEL_DEEPFACE_WIDTH = 48 21 | self.MODEL_DEEPFACE_HEIGHT = 48 22 | self.MODEL_DEEPFACE_NUM_CLASSES = 7 23 | self.MODEL_DEEPFACE_CLASSES = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'] 24 | 25 | # model location 26 | if vela: 27 | name = 'emotion_uint8_float32_vela.tflite' 28 | else: 29 | name = 'emotion_uint8_float32.tflite' 30 | self.tflite_model = os.path.join(model_directory, name) 31 | 32 | if not os.path.exists(self.tflite_model): 33 | raise FileExistsError(f'cannot find model [{self.tflite_model}]') 34 | 35 | def get_model_path(self): 36 | """Get full path to model file. 37 | """ 38 | return self.tflite_model 39 | 40 | def get_model_input_shape(self): 41 | """Get dimensions of model input tensor. 42 | """ 43 | return (self.MODEL_DEEPFACE_HEIGHT, self.MODEL_DEEPFACE_WIDTH, 1) 44 | 45 | def get_model_output_shape(self): 46 | """Get dimensions of model output tensors (list). 47 | """ 48 | return [(1, self.MODEL_DEEPFACE_NUM_CLASSES)] 49 | 50 | def predict_emotion(self, prediction): 51 | """Find the predicted emotion. 52 | 53 | prediction: output tensor of the emotion detection model 54 | prediction contains a confidence between 0 and 1 for each emotion 55 | returns: tuple (name, confidence [0, 1.0]) 56 | """ 57 | 58 | if not self.check_output_format(prediction): 59 | raise ValueError(f'prediction format error:\n {prediction}') 60 | # The predicted emotion is the one with the highest confidence value 61 | max_prediction = prediction.argmax() 62 | best = (self.MODEL_DEEPFACE_CLASSES[max_prediction], prediction[max_prediction]) 63 | return best 64 | 65 | def check_output_format(self, prediction): 66 | """Check consistency of prediction array with the model. 67 | 68 | prediction: output vector to be checked 69 | """ 70 | shape = prediction.shape 71 | type = prediction.dtype 72 | 73 | if len(shape) == 1 and \ 74 | shape[0] == self.MODEL_DEEPFACE_NUM_CLASSES and \ 75 | type == np.float32: 76 | return True 77 | else: 78 | return False 79 | -------------------------------------------------------------------------------- /tasks/face-processing/common/facenet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2022-2023, 2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | import numpy as np 7 | import os 8 | 9 | 10 | class FNModel: 11 | 12 | def __init__(self, model_directory, database_directory, 13 | match_threshold=1.0, vela=False): 14 | """Helper class for FaceNet model. 15 | 16 | model_directory: directory where tflite model is located 17 | database_directory: directory for face / embedding bundle records 18 | match_threshold: threshold for euclidean distance match comparison 19 | (the lower the stricter) 20 | vela: use vela version of the model 21 | """ 22 | # Face detection definitions 23 | self.MODEL_FACENET_WIDTH = 160 24 | self.MODEL_FACENET_HEIGHT = 160 25 | self.MODEL_FACENET_EMBEDDING_LEN = 512 26 | 27 | self.match_threshold = match_threshold 28 | 29 | # model location 30 | if vela: 31 | name = 'facenet512_uint8_vela.tflite' 32 | else: 33 | name = 'facenet512_uint8.tflite' 34 | self.tflite_model = os.path.join(model_directory, name) 35 | 36 | if not os.path.exists(self.tflite_model): 37 | raise FileExistsError(f'cannot find model [{self.tflite_model}]') 38 | 39 | if database_directory is not None: 40 | if not os.path.exists(database_directory): 41 | os.mkdir(database_directory) 42 | self.database_dir = database_directory 43 | 44 | def get_model_path(self): 45 | """Get full path to model file. 46 | """ 47 | return self.tflite_model 48 | 49 | def get_model_input_shape(self): 50 | """Get dimensions of model input tensor. 51 | """ 52 | return (self.MODEL_FACENET_HEIGHT, self.MODEL_FACENET_WIDTH, 3) 53 | 54 | def get_model_output_shape(self): 55 | """Get dimensions of model output tensors (list). 56 | """ 57 | return [(1, self.MODEL_FACENET_EMBEDDING_LEN)] 58 | 59 | def db_search_match(self, database, embedding): 60 | """Compare embedding with database to find a match. 61 | 62 | Criteria is based on euclidean distance of normalized vectors. 63 | 64 | database: memory database with name / vectors bundle 65 | embedding: reference embedding to be matched 66 | returns: tuple (name, confidence [0, 1.0]) 67 | """ 68 | best = (None, self.match_threshold) 69 | if not self.db_check_embedding_format(embedding): 70 | raise ValueError(f'embedding numpy format error:\n {embedding}') 71 | if database is None: 72 | return best 73 | normalized = embedding / np.linalg.norm(embedding) 74 | for _name, _embedding in database.items(): 75 | # _embedding is already L2-normalized 76 | distance = np.linalg.norm(normalized - _embedding) 77 | if (distance < best[1]): 78 | best = (_name, distance) 79 | return best 80 | 81 | def db_check_embedding_format(self, embedding): 82 | """Check consistency of embedding array with the model. 83 | 84 | embedding: vector to be checked 85 | """ 86 | shape = embedding.shape 87 | type = embedding.dtype 88 | 89 | if len(shape) == 1 and \ 90 | shape[0] == self.MODEL_FACENET_EMBEDDING_LEN and \ 91 | type == np.float32: 92 | return True 93 | else: 94 | return False 95 | 96 | def db_save_record(self, name, embedding): 97 | """Create a file to store name and embedding. 98 | 99 | name: name associated to the new record 100 | embedding: raw embedding (not normalized) associated to this name 101 | """ 102 | if not self.db_check_embedding_format(embedding): 103 | raise ValueError(f'embedding numpy format error [{embedding}]') 104 | 105 | file = os.path.join(self.database_dir, name) 106 | np.save(file, embedding) 107 | 108 | def db_add_record(self, database, name, embedding): 109 | """Add entry in memory database for an embedding associated to a name. 110 | 111 | database: memory data base for face / embedding bundles 112 | name: name associated to the new record 113 | embedding: raw embedding (not normalized) array associated to this name 114 | """ 115 | if database is None: 116 | database = {} 117 | if not self.db_check_embedding_format(embedding): 118 | raise ValueError(f'embedding numpy format error [{embedding}]') 119 | 120 | normalized = embedding / np.linalg.norm(embedding) 121 | database[name] = normalized 122 | 123 | def db_load(self): 124 | """Populate database in memory from face embeddings stored in files. 125 | 126 | File name .npy contain raw numpy array embedding for 127 | Embeddings are stored into files as raw without normalization. 128 | """ 129 | database = {} 130 | for file in os.listdir(self.database_dir): 131 | if file.endswith(".npy"): 132 | name = os.path.splitext(file)[0] 133 | npfile = os.path.join(self.database_dir, file) 134 | array = np.load(npfile) 135 | self.db_add_record(database, name, array) 136 | return database 137 | -------------------------------------------------------------------------------- /tasks/face-processing/common/facenet_create_embedding.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2022-2023, 2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | import argparse 7 | import numpy as np 8 | import os 9 | import tensorflow as tf 10 | from PIL import Image 11 | 12 | 13 | def load_image(image_path, crop_box=None, resize_wh=None): 14 | image = Image.open(image_path) 15 | image = image.convert(mode='RGB') 16 | if crop_box is not None: 17 | left = crop_box[0] 18 | upper = crop_box[1] 19 | right = crop_box[0] + crop_box[2] 20 | lower = crop_box[1] + crop_box[3] 21 | image = image.crop((left, upper, right, lower)) 22 | if resize_wh is not None: 23 | image = image.resize(resize_wh) 24 | image.show() 25 | array = np.array(image) 26 | array = np.expand_dims(array, axis=0) 27 | return array 28 | 29 | 30 | def inference(model_path, image_path, crop_box=None, resize_wh=None): 31 | # crop_box (x0,y0, w, h) tuple 32 | # resize_wh (w,y) tuple 33 | interpreter = tf.lite.Interpreter(model_path=model_path) 34 | interpreter.allocate_tensors() 35 | output = interpreter.get_output_details()[0] 36 | input = interpreter.get_input_details()[0] 37 | print(input) 38 | input_data = load_image(image_path, crop_box=crop_box, resize_wh=resize_wh) 39 | interpreter.set_tensor(input['index'], input_data) 40 | interpreter.invoke() 41 | out0 = interpreter.get_tensor(output['index']) 42 | return out0 43 | 44 | 45 | if __name__ == "__main__": 46 | pwd = os.path.dirname(os.path.abspath(__file__)) 47 | 48 | model_path = os.path.join( 49 | pwd, '../../../downloads/models/face-processing/facenet512_uint8.tflite') 50 | model_wh = (160, 160) 51 | 52 | # Default crop parameters correspond to thispersondoesnotexist.jpeg 53 | img_path_default = os.path.join(pwd, 'thispersondoesnotexist.jpeg') 54 | x0_default = 227 55 | y0_default = 317 56 | width_height_default = 610 57 | 58 | parser = argparse.ArgumentParser(description='Face Identification') 59 | parser.add_argument('--img_path', type=str, 60 | help='image path', default=img_path_default) 61 | parser.add_argument('--x0', type=int, 62 | help='cropped area x0 coordinate', default=x0_default) 63 | parser.add_argument('--y0', type=int, 64 | help='cropped area y0 coordinate', default=y0_default) 65 | parser.add_argument('--width_height', type=int, 66 | help='cropped area width/height', 67 | default=width_height_default) 68 | 69 | args = parser.parse_args() 70 | img_path = args.img_path 71 | x0 = args.x0 72 | y0 = args.y0 73 | width_height = args.width_height 74 | 75 | img_box = (x0, y0, width_height, width_height) 76 | 77 | print(f'# Compute embedding for {img_path} ') 78 | print(f'# Cropped area (x0, y0)=({x0}, {y0}) width/height={width_height}') 79 | 80 | embed = inference( 81 | model_path, 82 | img_path, 83 | crop_box=img_box, 84 | resize_wh=model_wh) 85 | embed0 = embed[0] 86 | 87 | # store resulting embedding vector as numpy file 88 | npy_file = os.path.splitext(os.path.basename(img_path))[0] + '.npy' 89 | embedding = os.path.join(pwd, npy_file) 90 | 91 | print(f'# Embedding file saved to {embedding}') 92 | np.save(embedding, embed0) 93 | -------------------------------------------------------------------------------- /tasks/face-processing/common/facenet_create_embedding_requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | pillow 3 | tensorflow 4 | 5 | -------------------------------------------------------------------------------- /tasks/face-processing/common/facenet_db/angelina_jolie.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxp-imx/nxp-nnstreamer-examples/b8dafc13bb93b06826f9fc6b91056270cecc0fbd/tasks/face-processing/common/facenet_db/angelina_jolie.npy -------------------------------------------------------------------------------- /tasks/face-processing/common/facenet_db/brad_pitt.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxp-imx/nxp-nnstreamer-examples/b8dafc13bb93b06826f9fc6b91056270cecc0fbd/tasks/face-processing/common/facenet_db/brad_pitt.npy -------------------------------------------------------------------------------- /tasks/face-processing/common/facenet_db/thispersondoesnotexist.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxp-imx/nxp-nnstreamer-examples/b8dafc13bb93b06826f9fc6b91056270cecc0fbd/tasks/face-processing/common/facenet_db/thispersondoesnotexist.npy -------------------------------------------------------------------------------- /tasks/face-processing/common/ultraface.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2022-2023, 2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | import numpy as np 7 | import os 8 | 9 | 10 | class UFModel: 11 | 12 | def __init__(self, model_directory, max_faces=16, has_post_process=True, 13 | classification_threshold=0.7, nms_iou_threshold=0.5, 14 | vela=False, neutron=False): 15 | """Helper class for UltraFace model. 16 | 17 | model_directory: directory where tflite model is located 18 | max_faces: maximum number of detected faces to be reported 19 | has_post_process: use model with embedded post-process 20 | (box decoding, nms) 21 | classification_threshold: threshold for classifier confidence 22 | nms_iou_threshold: threshold for NMS IoU dupe verdict 23 | vela: use vela version of the model 24 | """ 25 | 26 | # Face detection definitions 27 | self.MODEL_UFACE_WIDTH = 320 28 | self.MODEL_UFACE_HEIGHT = 240 29 | # No postprocessing 30 | self.MODEL_UFACE_NUMBER_BOXES = 4420 31 | self.MODEL_UFACE_NMS_IOU_THRESHOLD = nms_iou_threshold 32 | 33 | # post processing : box decoding + NMS 34 | self.MODEL_UFACE_NUMBER_NMS_BOXES = 100 35 | 36 | self.MODEL_UFACE_CLASSIFICATION_THRESHOLD = classification_threshold 37 | 38 | # Limit max number of face detections 39 | self.MODEL_UFACE_NUMBER_MAX = max_faces 40 | 41 | # model location 42 | self.has_post_process = has_post_process 43 | if has_post_process: 44 | if vela: 45 | name = 'ultraface_slim_uint8_float32_vela.tflite' 46 | elif neutron: 47 | name = 'ultraface_slim_uint8_float32_neutron.tflite' 48 | else: 49 | name = 'ultraface_slim_uint8_float32.tflite' 50 | else: 51 | name = 'version-slim_input_uint8_output_float32.tflite' 52 | self.tflite_model = os.path.join(model_directory, name) 53 | 54 | if not os.path.exists(self.tflite_model): 55 | raise FileExistsError(f'cannot find model [{self.tflite_model}]') 56 | 57 | def get_model_path(self): 58 | """Get full path to model file. 59 | """ 60 | return self.tflite_model 61 | 62 | def get_model_input_shape(self): 63 | """Get dimensions of model input tensor. 64 | """ 65 | return (self.MODEL_UFACE_HEIGHT, self.MODEL_UFACE_WIDTH, 3) 66 | 67 | def get_model_output_shape(self): 68 | """Get dimensions of model output tensors (list). 69 | """ 70 | if self.has_post_process: 71 | return [(self.MODEL_UFACE_NUMBER_NMS_BOXES, 6)] 72 | else: 73 | return [(1, self.MODEL_UFACE_NUMBER_BOXES, 6)] 74 | 75 | def iou(self, box0, box1): 76 | """Compute input boxes IoU. 77 | 78 | box0: (x1, y1, x2, y2) 79 | box1: (x1, y1, x2, y2) 80 | return: computed IoU 81 | """ 82 | assert box0[0] <= box0[2] 83 | assert box0[1] <= box0[3] 84 | assert box1[0] <= box1[2] 85 | assert box1[1] <= box1[3] 86 | 87 | x1i = max(box0[0], box1[0]) 88 | y1i = max(box0[1], box1[1]) 89 | x2i = min(box0[2], box1[2]) 90 | y2i = min(box0[3], box1[3]) 91 | # boxes individual and intersection surfaces 92 | si = max(0, x2i - x1i + 1) * max(0, y2i - y1i + 1) 93 | s0 = (box0[2] - box0[0] + 1) * (box0[3] - box0[1] + 1) 94 | s1 = (box1[2] - box1[0] + 1) * (box1[3] - box1[1] + 1) 95 | 96 | val = si / (s0 + s1 - si) 97 | assert val >= 0 and val <= 1 98 | 99 | return val 100 | 101 | def nms(self, boxes, scores, max_output_size, iou_threshold=0.5): 102 | """Execute NMS procedure on boxes list. 103 | 104 | boxes: array (N, 4) of N boxes (x1, y1, x2, y2) 105 | scores: array of N individual boxes score 106 | max_output_size: max number of boxes to be returned 107 | iou_threshold: IoU threshold for boxes duplication verdict 108 | return: array of remaining boxes after NMS filtering 109 | """ 110 | result = np.zeros((max_output_size, 4), dtype=float) 111 | count = 0 112 | 113 | sorted = np.argsort(scores, axis=0) 114 | sorted = sorted[-1::-1] 115 | _boxes = np.take(boxes, sorted, axis=0) 116 | 117 | while len(_boxes) > 0: 118 | if count >= max_output_size: 119 | return result 120 | result[count] = _boxes[0] 121 | count += 1 122 | 123 | dupe = [0] 124 | for i in range(1, len(_boxes)): 125 | if self.iou(_boxes[0], _boxes[i]) > iou_threshold: 126 | dupe += [i] 127 | _boxes = np.delete(_boxes, dupe, axis=0) 128 | 129 | return result[:count, ...] 130 | 131 | def decode_output(self, output): 132 | """Decode model output tensor. 133 | 134 | output: list of output tensors to decode 135 | output0 (no post processing): tensor (100,6) 136 | 100 boxes filtered by model built-in NMS operator 137 | output0 (no post processing): tensor (1,4480,6) 138 | 4480 raw boxes 139 | With and without post processing, box format 140 | [...,0:2] classifier output 141 | [...,2:6] decoded box (x1,y2,x2,y2) 142 | return: array (N, 4) 143 | N boxes (x1,y2,x2,y2) detected 144 | Normalized coordinates range [0, 1] 145 | """ 146 | assert len(output) == 1 147 | out0 = output[0] 148 | assert out0.dtype == np.float32 149 | assert out0.shape == self.get_model_output_shape()[0] 150 | 151 | if self.has_post_process: 152 | boxes = out0[..., 2:] 153 | scores = out0[..., 1] 154 | selected = scores > self.MODEL_UFACE_CLASSIFICATION_THRESHOLD 155 | boxes = boxes[selected] 156 | 157 | selected = boxes[:self.MODEL_UFACE_NUMBER_MAX, ...] 158 | else: 159 | boxes = out0[..., 2:] 160 | scores = out0[..., :2] 161 | boxes = boxes[0] 162 | scores = scores[0, :, 1] 163 | selected = scores > self.MODEL_UFACE_CLASSIFICATION_THRESHOLD 164 | boxes = boxes[selected] 165 | scores = scores[selected] 166 | 167 | selected = self.nms(boxes, scores, 168 | self.MODEL_UFACE_NUMBER_MAX, 169 | self.MODEL_UFACE_NMS_IOU_THRESHOLD) 170 | 171 | return selected 172 | -------------------------------------------------------------------------------- /tasks/face-processing/emotion-classification/cpp/custom_emotion_decoder.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #ifndef FACE_CUSTOM_FACE_DECODER_H_ 7 | #define FACE_CUSTOM_FACE_DECODER_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "logging.hpp" 19 | 20 | #define MODEL_UFACE_NUMBER_BOXES 100 21 | #define NUM_BOX_DATA 6 22 | #define NUMBER_OF_COORDINATES 4 23 | #define MODEL_UFACE_CLASSIFICATION_THRESHOLD 0.7f 24 | #define MODEL_UFACE_NUMBER_MAX 15 25 | 26 | 27 | typedef struct { 28 | int box[4]; 29 | std::string emotion; 30 | float confidence; 31 | } EmotionData; 32 | 33 | 34 | typedef struct { 35 | GstElement *appSrc; 36 | GstElement *videocrop; 37 | int camWidth; 38 | int camHeight; 39 | int faceCount = 0; 40 | std::vector faceBoxes; 41 | int bufferSize = NUM_BOX_DATA * MODEL_UFACE_NUMBER_BOXES; 42 | int emotionCount = 0; 43 | std::vector emotionBoxes; 44 | std::string emotionsList[7] = {"angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"}; 45 | GstBuffer *imagesBuffer = gst_buffer_new(); 46 | bool processEmotions = false; 47 | std::vector results; 48 | std::vector detections; 49 | } DecoderData; 50 | 51 | 52 | void newDataCallback(GstElement* element, 53 | GstBuffer* buffer, 54 | gpointer user_data); 55 | 56 | 57 | void secondaryNewDataCallback(GstElement* element, 58 | GstBuffer* buffer, 59 | gpointer user_data); 60 | 61 | 62 | GstFlowReturn sinkCallback(GstAppSink* appsink, gpointer user_data); 63 | 64 | 65 | void drawCallback(GstElement* overlay, 66 | cairo_t* cr, 67 | guint64 timestamp, 68 | guint64 duration, 69 | gpointer user_data); 70 | 71 | 72 | typedef struct { 73 | int size; 74 | float* bufferFP32; 75 | } BufferInfo; 76 | 77 | 78 | BufferInfo getTensorInfo(GstBuffer* buffer, int tensorIndex); 79 | 80 | 81 | void checkNumTensor(GstBuffer* buffer, int numTensor); 82 | 83 | #endif -------------------------------------------------------------------------------- /tasks/face-processing/emotion-classification/example_emotion_classification_tflite.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2023-2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | import argparse 7 | import cairo 8 | 9 | import gi 10 | import logging 11 | import math 12 | import numpy as np 13 | import os 14 | import sys 15 | 16 | gi.require_version('Gst', '1.0') 17 | gi.require_version('GstApp', '1.0') 18 | gi.require_version('GstVideo', '1.0') 19 | from gi.repository import Gst, GLib, GstApp, GstVideo # noqa 20 | 21 | python_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 22 | '../../../common/python') 23 | sys.path.append(python_path) 24 | from imxpy.imx_dev import Imx, SocId # noqa 25 | 26 | python_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 27 | '../common') 28 | sys.path.append(python_path) 29 | from facedetectpipe import FaceDetectPipe, SecondaryPipe # noqa 30 | import deepface # noqa 31 | 32 | 33 | class EmoDetectPipe(FaceDetectPipe): 34 | 35 | def __init__(self, camera_device, video_resolution=(640, 480), 36 | video_fps=30, flip=False, secondary_pipe=None, model_directory=None): 37 | """Subclass FaceDetectPipe. 38 | 39 | Derived class implements specific functions relevant to this example: 40 | - override primary pipeline cairo overlay implementation 41 | - override handling of secondary pipeline output tensors (DeepFace) 42 | - add basic UI control for video overlay 43 | 44 | camera_device: camera device node to be used as pipeline source 45 | video_resolution: camera video stream resolution 46 | video_fps: camera video stream frame rate 47 | secondary_pipe: instance of secondary pipeline for DeepFace operation 48 | model_directory: absolute path for face detection model 49 | """ 50 | super( 51 | EmoDetectPipe, 52 | self).__init__( 53 | camera_device, 54 | video_resolution, 55 | video_fps, 56 | flip, 57 | secondary_pipe, 58 | model_directory) 59 | 60 | # reference to secondary pipe DeepFace model instance 61 | self.deepface = self.secondary_pipe.model 62 | 63 | # UI 64 | self.ui = UI() 65 | 66 | def handle_secondary_output(self, buffer, boxes, index): 67 | """Override parent implementation to cater for emotion detection output. 68 | 69 | buffer: secondary pipeline inference output buffer that contains confidence value for each emotion 70 | boxes: array of boxes detected by primary pipeline 71 | index: index in boxes array used to compute buffer 72 | """ 73 | 74 | # No faces detected 75 | if buffer is None or boxes is None: 76 | self.ui.publish(None) 77 | return 78 | 79 | # extract single output tensor from GStreamer buffer 80 | dims = self.deepface.get_model_output_shape() 81 | assert buffer.n_memory() == len(dims) 82 | 83 | # tensor buffer #0 84 | mem_prediction = buffer.peek_memory(0) 85 | result, info = mem_prediction.map(Gst.MapFlags.READ) 86 | 87 | dims0 = dims[0] 88 | num = math.prod(dims0) 89 | assert info.size == num * np.dtype(np.float32).itemsize 90 | 91 | if result: 92 | # convert buffer to prediction numpy array 93 | prediction = np.frombuffer(info.data, dtype=np.float32) \ 94 | .reshape(dims0)[0] 95 | 96 | emotion, value = self.deepface.predict_emotion(prediction) 97 | 98 | if index == 0: 99 | self.ui_results = [] 100 | 101 | mem_prediction.unmap(info) 102 | 103 | box = boxes[index] 104 | entry = (box, emotion, value) 105 | self.ui_results += [entry] 106 | 107 | # Last face, publish aggregated results to UI 108 | total = len(boxes) 109 | if (index + 1 == total): 110 | self.ui.publish(self.ui_results) 111 | 112 | else: 113 | logging.error('could not map prediction GstBuffer') 114 | self.ui.publish(None, None) 115 | 116 | def display_cairo_overlay_draw_cb( 117 | self, overlay, context, timestamp, duration): 118 | """Override FaceDetectPipe implementation to implement UI specifics. 119 | 120 | Default overlay callback is changed to implement specificities of this 121 | example: 122 | - detected emotion affixed to face bounding box 123 | """ 124 | 125 | if not self.running: 126 | return 127 | 128 | results = self.ui.get_detections() 129 | total = len(results) 130 | 131 | context.set_source_rgb(0.85, 0, 1) 132 | context.move_to(14, 14) 133 | context.select_font_face( 134 | 'Arial', 135 | cairo.FONT_SLANT_NORMAL, 136 | cairo.FONT_WEIGHT_NORMAL) 137 | context.set_font_size(11.0) 138 | 139 | state = self.ui.get_state() 140 | display = self.ui.get_display_string() 141 | context.show_text(display) 142 | 143 | if total == 0: 144 | return 145 | 146 | index = 0 147 | context.set_line_width(1.0) 148 | 149 | for box, emotion, confidence in results: 150 | if index == 0: 151 | context.set_source_rgb(1, 0, 0) 152 | 153 | w = box[2] - box[0] 154 | h = box[3] - box[1] 155 | context.rectangle(box[0], box[1], w, h) 156 | context.move_to(box[0], box[1] + h + 20) 157 | 158 | context.show_text(f'{emotion} ({confidence:.3f})') 159 | 160 | context.stroke() 161 | index += 1 162 | 163 | def ui_quit(self): 164 | """Callback from UI to quit application. 165 | """ 166 | self.mainloop.quit() 167 | 168 | 169 | class UIState: 170 | """UI states definition. 171 | 172 | WARMUP: pipeline stalled pending ML accelerators warmup 173 | IDLE: standard operation 174 | """ 175 | 176 | WARMUP = 0 177 | IDLE = 1 178 | 179 | 180 | class UI: 181 | 182 | def __init__(self): 183 | """Handle basic UI rendered in video overlay. 184 | 185 | quit_cb: callback to be invoked when user quits application 186 | """ 187 | self.state = UIState.WARMUP 188 | self.detections = None 189 | 190 | def get_state(self): 191 | """Report UI state. 192 | """ 193 | return self.state 194 | 195 | def get_display_string(self): 196 | """Report UI string to be inserted by display overlay. 197 | """ 198 | if self.state == UIState.WARMUP: 199 | return 'Warm up in progress - please wait' 200 | elif self.state == UIState.IDLE: 201 | return 'Show your face in front of the camera !' 202 | else: 203 | return 'Nothing to display!' 204 | 205 | def publish(self, detections): 206 | """Publish results from pipeline into UI for use from overlay. 207 | 208 | detections: list of (box, emotion, distance) tuples 209 | box: (x1, y1, x2, y2) tuple 210 | emotion: matching emotion 211 | distance: [0-1] 212 | """ 213 | self.detections = detections 214 | 215 | # On first result, leave warmup state 216 | if self.state == UIState.WARMUP: 217 | self.state = UIState.IDLE 218 | 219 | def get_detections(self): 220 | """ Get detections (box, emotion, confidence) array coming from pipeline. 221 | """ 222 | if self.detections is None: 223 | return [] 224 | else: 225 | return self.detections.copy() 226 | 227 | 228 | if __name__ == '__main__': 229 | 230 | imx = Imx() 231 | if imx.id() == SocId.IMX8MP: 232 | default_camera = '/dev/video3' 233 | elif imx.is_imx93(): 234 | default_camera = '/dev/video0' 235 | elif imx.is_imx95(): 236 | default_camera = '/dev/video13' 237 | else: 238 | name = imx.name() 239 | raise NotImplementedError(f'Platform not supported [{name}]') 240 | 241 | parser = argparse.ArgumentParser(description='Emotion detection') 242 | parser.add_argument('--camera_device', '-c', type=str, 243 | help='camera device node', default=default_camera) 244 | parser.add_argument('--mirror', '-m', 245 | default=False, action='store_true', 246 | help='flip image to display as a mirror') 247 | args = parser.parse_args() 248 | 249 | format = '%(asctime)s.%(msecs)03d %(levelname)s:\t%(message)s' 250 | datefmt = '%Y-%m-%d %H:%M:%S' 251 | logging.basicConfig(level=logging.INFO, format=format, datefmt=datefmt) 252 | 253 | # pipelines parameters 254 | camera_device = args.camera_device 255 | flip = args.mirror 256 | vr = (640, 480) 257 | fps = 30 258 | 259 | # secondary pipeline uses DeepFace model 260 | pwd = os.path.dirname(os.path.abspath(__file__)) 261 | models_dir = os.path.join(pwd, '../../../downloads/models/face-processing') 262 | 263 | has_ethosu = imx.has_npu_ethos() 264 | model = deepface.FNModel(models_dir, vela=has_ethosu) 265 | secondary = SecondaryPipe(model, video_resolution=vr, video_fps=fps) 266 | 267 | # main pipeline for face detection 268 | pipe = EmoDetectPipe(camera_device=camera_device, video_resolution=vr, 269 | video_fps=fps, flip=flip, secondary_pipe=secondary, 270 | model_directory=models_dir) 271 | pipe.run() 272 | -------------------------------------------------------------------------------- /tasks/face-processing/face-detection/cpp/custom_face_decoder.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #include "custom_face_decoder.hpp" 7 | 8 | #include 9 | #include 10 | 11 | BufferInfo getTensorInfo(GstBuffer* buffer, int tensorIndex) 12 | { 13 | BufferInfo bufferInfo; 14 | GstMapInfo info; 15 | GstMemory* memKpts = gst_buffer_peek_memory(buffer, tensorIndex); 16 | bool result = gst_memory_map(memKpts, &info, GST_MAP_READ); 17 | if (result) { 18 | bufferInfo.bufferFP32 = reinterpret_cast(info.data); 19 | bufferInfo.size = info.size/sizeof(float); 20 | gst_memory_unmap(memKpts, &info); 21 | } else { 22 | gst_memory_unmap(memKpts, &info); 23 | log_error("Can't access buffer in memory\n"); 24 | exit(-1); 25 | } 26 | return bufferInfo; 27 | } 28 | 29 | 30 | void checkNumTensor(GstBuffer* buffer, int numTensor) 31 | { 32 | if (!GST_IS_BUFFER (buffer)) { 33 | log_error("Received invalid buffer\n"); 34 | exit(-1); 35 | } 36 | uint mem_blocks = gst_buffer_n_memory(buffer); 37 | if (mem_blocks != numTensor) { 38 | log_error("Number of tensors invalid : %d\n", mem_blocks); 39 | exit(-1); 40 | } 41 | } 42 | 43 | 44 | void newDataCallback(GstElement* element, 45 | GstBuffer* buffer, 46 | gpointer user_data) 47 | { 48 | DecoderData* boxesData = (DecoderData *) user_data; 49 | 50 | BufferInfo bufferInfo; 51 | checkNumTensor(buffer, 1); 52 | bufferInfo = getTensorInfo(buffer, 0); 53 | assert(boxesData->bufferSize == bufferInfo.size); 54 | 55 | std::vector boxes; 56 | int faceCount = 0; 57 | for (int i = 0; ((i < MODEL_UFACE_NUMBER_BOXES) 58 | && (faceCount < MODEL_UFACE_NUMBER_MAX)); i+= NUM_BOX_DATA) { 59 | // Keep only boxes with a score above the threshold 60 | if (bufferInfo.bufferFP32[i+1] > MODEL_UFACE_CLASSIFICATION_THRESHOLD) { 61 | faceCount += 1; 62 | // Store x1 63 | boxes.push_back( 64 | static_cast(bufferInfo.bufferFP32[i+2] * boxesData->camWidth) 65 | ); 66 | // Store y1 67 | boxes.push_back( 68 | static_cast(bufferInfo.bufferFP32[i+3] * boxesData->camHeight) 69 | ); 70 | // Store x2 71 | boxes.push_back( 72 | static_cast(bufferInfo.bufferFP32[i+4] * boxesData->camWidth) 73 | ); 74 | // Store y2 75 | boxes.push_back( 76 | static_cast(bufferInfo.bufferFP32[i+5] * boxesData->camHeight) 77 | ); 78 | } 79 | } 80 | 81 | // Transform rectangular to square box 82 | int w, h, cx, cy, d2; 83 | float k = 0.8; // scaling factor 84 | float minwh = 16; // minimum (imxvideoconvert constraint) 85 | float d; 86 | for (int faceIndex = 0; faceIndex < 4 * faceCount; faceIndex += 4) { 87 | w = boxes.at(2 + faceIndex) - boxes.at(0 + faceIndex) + 1; 88 | h = boxes.at(3 + faceIndex) - boxes.at(1 + faceIndex) + 1; 89 | cx = static_cast((boxes.at(0 + faceIndex) + boxes.at(2 + faceIndex))/2); 90 | cy = static_cast((boxes.at(1 + faceIndex) + boxes.at(3 + faceIndex))/2); 91 | 92 | d = std::max(w, h) * k; 93 | d = std::min(d, static_cast( 94 | std::min(boxesData->camWidth, boxesData->camHeight) 95 | )); 96 | d = std::max(d, minwh); 97 | d2 = static_cast(d/2); 98 | 99 | if ((cx + d2) >= boxesData->camWidth) 100 | cx = boxesData->camWidth - d2 - 1; 101 | if ((cx - d2) < 0) 102 | cx = d2; 103 | if ((cy + d2) >= boxesData->camHeight) 104 | cy = boxesData->camHeight - d2 - 1; 105 | if ((cy - d2) < 0) 106 | cy = d2; 107 | boxes.at(0 + faceIndex) = cx - d2; 108 | boxes.at(1 + faceIndex) = cy - d2; 109 | boxes.at(2 + faceIndex) = cx + d2; 110 | boxes.at(3 + faceIndex) = cy + d2; 111 | } 112 | 113 | boxesData->faceCount = faceCount; 114 | boxesData->selectedBoxes = boxes; 115 | } 116 | 117 | 118 | void drawCallback(GstElement* overlay, 119 | cairo_t* cr, 120 | guint64 timestamp, 121 | guint64 duration, 122 | gpointer user_data) 123 | { 124 | DecoderData* boxesData = (DecoderData *) user_data; 125 | 126 | int numFaces = boxesData->faceCount; 127 | std::vector boxes = boxesData->selectedBoxes; 128 | 129 | cairo_set_source_rgb(cr, 0.85, 0, 1); 130 | cairo_move_to(cr, boxesData->camWidth * (1 - 160.0/640), boxesData->camWidth * 18/640); 131 | cairo_select_font_face(cr, 132 | "Arial", 133 | CAIRO_FONT_SLANT_NORMAL, 134 | CAIRO_FONT_WEIGHT_NORMAL); 135 | cairo_set_font_size(cr, boxesData->camWidth * 15/640); 136 | cairo_show_text(cr, ("Faces detected: " + std::to_string(numFaces)).c_str()); 137 | 138 | cairo_set_source_rgb(cr, 1, 0, 0); 139 | cairo_set_line_width(cr, 1.0); 140 | 141 | int w, h; 142 | for (int faceIndex = 0; faceIndex < 4 * numFaces; faceIndex += 4) { 143 | w = boxes.at(2 + faceIndex) - boxes.at(0 + faceIndex); 144 | h = boxes.at(3 + faceIndex) - boxes.at(1 + faceIndex); 145 | cairo_rectangle(cr, boxes.at(0 + faceIndex), boxes.at(1 + faceIndex), w, h); 146 | } 147 | cairo_stroke(cr); 148 | } 149 | -------------------------------------------------------------------------------- /tasks/face-processing/face-detection/cpp/custom_face_decoder.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #ifndef FACE_CUSTOM_FACE_DECODER_H_ 7 | #define FACE_CUSTOM_FACE_DECODER_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "logging.hpp" 17 | 18 | #define MODEL_UFACE_NUMBER_BOXES 100 19 | #define NUM_BOX_DATA 6 20 | #define NUMBER_OF_COORDINATES 4 21 | #define MODEL_UFACE_CLASSIFICATION_THRESHOLD 0.7f 22 | #define MODEL_UFACE_NUMBER_MAX 15 23 | 24 | typedef struct { 25 | std::vector selectedBoxes; 26 | int bufferSize = NUM_BOX_DATA * MODEL_UFACE_NUMBER_BOXES; 27 | int faceCount = 0; 28 | int camWidth; 29 | int camHeight; 30 | } DecoderData; 31 | 32 | 33 | void newDataCallback(GstElement* element, 34 | GstBuffer* buffer, 35 | gpointer user_data); 36 | 37 | 38 | void drawCallback(GstElement* overlay, 39 | cairo_t* cr, 40 | guint64 timestamp, 41 | guint64 duration, 42 | gpointer user_data); 43 | 44 | 45 | typedef struct { 46 | int size; 47 | float* bufferFP32; 48 | } BufferInfo; 49 | 50 | 51 | BufferInfo getTensorInfo(GstBuffer* buffer, int tensorIndex); 52 | 53 | 54 | void checkNumTensor(GstBuffer* buffer, int numTensor); 55 | 56 | #endif -------------------------------------------------------------------------------- /tasks/face-processing/face-detection/example_face_detection_tflite.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2022-2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | import argparse 7 | import logging 8 | import os 9 | import sys 10 | 11 | python_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 12 | '../../../common/python') 13 | sys.path.append(python_path) 14 | from imxpy.imx_dev import Imx, SocId # noqa 15 | 16 | python_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 17 | '../common') 18 | sys.path.append(python_path) 19 | from facedetectpipe import FaceDetectPipe # noqa 20 | 21 | if __name__ == '__main__': 22 | 23 | imx = Imx() 24 | if imx.id() == SocId.IMX8MP: 25 | default_camera = '/dev/video3' 26 | elif imx.is_imx93(): 27 | default_camera = '/dev/video0' 28 | elif imx.is_imx95(): 29 | default_camera = '/dev/video13' 30 | else: 31 | name = imx.name() 32 | raise NotImplementedError(f'Platform not supported [{name}]') 33 | 34 | parser = argparse.ArgumentParser(description='Face Identification') 35 | parser.add_argument('--camera_device', '-c', type=str, 36 | help='camera device node', default=default_camera) 37 | parser.add_argument('--mirror', '-m', 38 | default=False, action='store_true', 39 | help='flip image to display as a mirror') 40 | args = parser.parse_args() 41 | 42 | format = '%(asctime)s.%(msecs)03d %(levelname)s:\t%(message)s' 43 | datefmt = '%Y-%m-%d %H:%M:%S' 44 | logging.basicConfig(level=logging.INFO, format=format, datefmt=datefmt) 45 | 46 | # pipeline parameters - no secondary pipeline 47 | camera_device = args.camera_device 48 | flip = args.mirror 49 | vr = (640, 480) 50 | fps = 30 51 | secondary = None 52 | 53 | pipe = FaceDetectPipe(camera_device=camera_device, video_resolution=vr, 54 | video_fps=fps, flip=flip, secondary_pipe=secondary) 55 | pipe.run() 56 | -------------------------------------------------------------------------------- /tasks/face-processing/face_demo.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxp-imx/nxp-nnstreamer-examples/b8dafc13bb93b06826f9fc6b91056270cecc0fbd/tasks/face-processing/face_demo.webp -------------------------------------------------------------------------------- /tasks/face-processing/thispersondoesnotexist.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxp-imx/nxp-nnstreamer-examples/b8dafc13bb93b06826f9fc6b91056270cecc0fbd/tasks/face-processing/thispersondoesnotexist.jpeg -------------------------------------------------------------------------------- /tasks/mixed-demos/cpp/custom_face_and_pose_decoder.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #include "custom_face_and_pose_decoder.hpp" 7 | 8 | #include 9 | #include 10 | 11 | BufferInfo getTensorInfo(GstBuffer* buffer, int tensorIndex) 12 | { 13 | BufferInfo bufferInfo; 14 | GstMapInfo info; 15 | GstMemory* memKpts = gst_buffer_peek_memory(buffer, tensorIndex); 16 | bool result = gst_memory_map(memKpts, &info, GST_MAP_READ); 17 | if (result) { 18 | bufferInfo.bufferFP32 = reinterpret_cast(info.data); 19 | bufferInfo.size = info.size/sizeof(float); 20 | gst_memory_unmap(memKpts, &info); 21 | } else { 22 | gst_memory_unmap(memKpts, &info); 23 | log_error("Can't access buffer in memory\n"); 24 | exit(-1); 25 | } 26 | return bufferInfo; 27 | } 28 | 29 | 30 | void checkNumTensor(GstBuffer* buffer, int numTensor) 31 | { 32 | if (!GST_IS_BUFFER (buffer)) { 33 | log_error("Received invalid buffer\n"); 34 | exit(-1); 35 | } 36 | uint mem_blocks = gst_buffer_n_memory(buffer); 37 | if (mem_blocks != numTensor) { 38 | log_error("Number of tensors invalid : %d\n", mem_blocks); 39 | exit(-1); 40 | } 41 | } 42 | 43 | 44 | void newDataFaceCallback(GstElement* element, 45 | GstBuffer* buffer, 46 | gpointer user_data) 47 | { 48 | FaceData* boxesData = (FaceData *) user_data; 49 | 50 | BufferInfo bufferInfo; 51 | checkNumTensor(buffer, 1); 52 | bufferInfo = getTensorInfo(buffer, 0); 53 | assert(boxesData->bufferSize == bufferInfo.size); 54 | 55 | std::vector boxes; 56 | int faceCount = 0; 57 | for (int i = 0; ((i < MODEL_UFACE_NUMBER_BOXES) 58 | && (faceCount < MODEL_UFACE_NUMBER_MAX)); i+= NUM_BOX_DATA) { 59 | // Keep only boxes with a score above the threshold 60 | if (bufferInfo.bufferFP32[i+1] > MODEL_UFACE_CLASSIFICATION_THRESHOLD) { 61 | faceCount += 1; 62 | // Store x1 63 | boxes.push_back( 64 | static_cast(bufferInfo.bufferFP32[i+2] * boxesData->inputDim) 65 | ); 66 | // Store y1 67 | boxes.push_back( 68 | static_cast(bufferInfo.bufferFP32[i+3] * boxesData->inputDim) 69 | ); 70 | // Store x2 71 | boxes.push_back( 72 | static_cast(bufferInfo.bufferFP32[i+4] * boxesData->inputDim) 73 | ); 74 | // Store y2 75 | boxes.push_back( 76 | static_cast(bufferInfo.bufferFP32[i+5] * boxesData->inputDim) 77 | ); 78 | } 79 | } 80 | 81 | // Transform rectangular to square box 82 | int w, h, cx, cy, d2; 83 | float k = 0.8; // scaling factor 84 | float minwh = 16; // minimum (imxvideoconvert constraint) 85 | float d; 86 | for (int faceIndex = 0; faceIndex < 4 * faceCount; faceIndex += 4) { 87 | w = boxes.at(2 + faceIndex) - boxes.at(0 + faceIndex) + 1; 88 | h = boxes.at(3 + faceIndex) - boxes.at(1 + faceIndex) + 1; 89 | cx = static_cast((boxes.at(0 + faceIndex) + boxes.at(2 + faceIndex))/2); 90 | cy = static_cast((boxes.at(1 + faceIndex) + boxes.at(3 + faceIndex))/2); 91 | 92 | d = std::max(w, h) * k; 93 | d = std::min(d, static_cast( 94 | std::min(boxesData->inputDim, boxesData->inputDim) 95 | )); 96 | d = std::max(d, minwh); 97 | d2 = static_cast(d/2); 98 | 99 | if ((cx + d2) >= boxesData->inputDim) 100 | cx = boxesData->inputDim - d2 - 1; 101 | if ((cx - d2) < 0) 102 | cx = d2; 103 | if ((cy + d2) >= boxesData->inputDim) 104 | cy = boxesData->inputDim - d2 - 1; 105 | if ((cy - d2) < 0) 106 | cy = d2; 107 | boxes.at(0 + faceIndex) = cx - d2; 108 | boxes.at(1 + faceIndex) = cy - d2; 109 | boxes.at(2 + faceIndex) = cx + d2; 110 | boxes.at(3 + faceIndex) = cy + d2; 111 | } 112 | 113 | boxesData->faceCount = faceCount; 114 | boxesData->selectedBoxes = boxes; 115 | } 116 | 117 | 118 | void drawFaceCallback(GstElement* overlay, 119 | cairo_t* cr, 120 | guint64 timestamp, 121 | guint64 duration, 122 | gpointer user_data) 123 | { 124 | FaceData* boxesData = (FaceData *) user_data; 125 | 126 | int numFaces = boxesData->faceCount; 127 | std::vector boxes = boxesData->selectedBoxes; 128 | 129 | cairo_set_source_rgb(cr, 0.85, 0, 1); 130 | cairo_move_to(cr, boxesData->inputDim * (1 - 160.0/640), boxesData->inputDim * 18/640); 131 | cairo_select_font_face(cr, 132 | "Arial", 133 | CAIRO_FONT_SLANT_NORMAL, 134 | CAIRO_FONT_WEIGHT_NORMAL); 135 | cairo_set_font_size(cr, boxesData->inputDim * 15/640); 136 | cairo_show_text(cr, ("Faces detected: " + std::to_string(numFaces)).c_str()); 137 | 138 | cairo_set_source_rgb(cr, 1, 0, 0); 139 | cairo_set_line_width(cr, 1.0); 140 | 141 | int w, h; 142 | for (int faceIndex = 0; faceIndex < 4 * numFaces; faceIndex += 4) { 143 | w = boxes.at(2 + faceIndex) - boxes.at(0 + faceIndex); 144 | h = boxes.at(3 + faceIndex) - boxes.at(1 + faceIndex); 145 | cairo_rectangle(cr, boxes.at(0 + faceIndex), boxes.at(1 + faceIndex), w, h); 146 | } 147 | cairo_stroke(cr); 148 | } 149 | 150 | 151 | void newDataPoseCallback(GstElement* element, 152 | GstBuffer* buffer, 153 | gpointer user_data) 154 | { 155 | PoseData* kptsData = (PoseData *) user_data; 156 | 157 | BufferInfo bufferInfo; 158 | checkNumTensor(buffer, 1); 159 | bufferInfo = getTensorInfo(buffer, 0); 160 | 161 | int row = 0; 162 | int col = 0; 163 | float score = 0; 164 | float valid = 0; 165 | for (int i = 0; i < bufferInfo.size; i++) { 166 | 167 | if ((col == X_INDEX) or (col == Y_INDEX)) { 168 | kptsData->npKpts[row][col] = bufferInfo.bufferFP32[i] * kptsData->inputDim; 169 | } else { 170 | kptsData->npKpts[row][col] = bufferInfo.bufferFP32[i]; 171 | score = kptsData->npKpts[row][SCORE_INDEX]; 172 | valid = (score >= SCORE_THRESHOLD); 173 | kptsData->npKpts[row][col] = valid; 174 | } 175 | 176 | col+=1; 177 | if (col == 3) 178 | row += 1; 179 | col = col % 3; 180 | } 181 | } 182 | 183 | 184 | void drawPoseCallback(GstElement* overlay, 185 | cairo_t* cr, 186 | guint64 timestamp, 187 | guint64 duration, 188 | gpointer user_data) 189 | { 190 | PoseData* kptsData = (PoseData *) user_data; 191 | float valid; 192 | float* npKpt; 193 | float xKpt; 194 | float yKpt; 195 | int* connections; 196 | float* npConnect; 197 | float xConnect; 198 | float yConnect; 199 | cairo_select_font_face(cr, 200 | "Arial", 201 | CAIRO_FONT_SLANT_NORMAL, 202 | CAIRO_FONT_WEIGHT_NORMAL); 203 | cairo_set_line_width(cr, 1.0); 204 | 205 | for(int i = 0; i < KPT_SIZE; i++) { 206 | npKpt = kptsData->npKpts[i]; 207 | valid = npKpt[SCORE_INDEX]; 208 | if (valid != 1.0) 209 | continue; 210 | 211 | xKpt = npKpt[X_INDEX]; 212 | yKpt = npKpt[Y_INDEX]; 213 | 214 | // Draw keypoint spot 215 | cairo_set_source_rgb(cr, 1, 0, 0); 216 | cairo_arc(cr, xKpt, yKpt, 1, 0, 2*M_PI); 217 | cairo_fill(cr); 218 | cairo_stroke(cr); 219 | 220 | // Draw keypoint label 221 | cairo_set_source_rgb(cr, 0, 1, 1); 222 | cairo_set_font_size(cr, 10.0); 223 | cairo_move_to(cr, xKpt + 5, yKpt + 5); 224 | cairo_show_text(cr, kptsData->kptLabels[i].c_str()); 225 | 226 | // Draw keypoint connections 227 | cairo_set_source_rgb(cr, 0, 1, 0); 228 | connections = kptsData->kptConnect[i]; 229 | for(int j = 0; j < 3; j++) { 230 | if (connections[j] == -1) 231 | break; 232 | npConnect = kptsData->npKpts[connections[j]]; 233 | valid = npConnect[SCORE_INDEX]; 234 | if (valid != 1.0) 235 | continue; 236 | xConnect = npConnect[X_INDEX]; 237 | yConnect = npConnect[Y_INDEX]; 238 | cairo_move_to(cr, xKpt, yKpt); 239 | cairo_line_to(cr, xConnect, yConnect); 240 | } 241 | cairo_stroke(cr); 242 | } 243 | } -------------------------------------------------------------------------------- /tasks/mixed-demos/cpp/custom_face_and_pose_decoder.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #ifndef CPP_CUSTOM_FACE_AND_POSE_DECODER_H_ 7 | #define CPP_CUSTOM_FACE_AND_POSE_DECODER_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "logging.hpp" 17 | 18 | /* Face detection constants */ 19 | #define MODEL_UFACE_NUMBER_BOXES 100 20 | #define NUM_BOX_DATA 6 21 | #define NUMBER_OF_COORDINATES 4 22 | #define MODEL_UFACE_CLASSIFICATION_THRESHOLD 0.7f 23 | #define MODEL_UFACE_NUMBER_MAX 15 24 | 25 | /* Pose detection constants */ 26 | #define KPT_SIZE 17 27 | #define Y_INDEX 0 28 | #define X_INDEX 1 29 | #define SCORE_INDEX 2 30 | #define SCORE_THRESHOLD 0.4f 31 | 32 | 33 | typedef struct { 34 | std::vector selectedBoxes; 35 | int bufferSize = NUM_BOX_DATA * MODEL_UFACE_NUMBER_BOXES; 36 | int faceCount = 0; 37 | int inputDim; 38 | } FaceData; 39 | 40 | 41 | typedef struct { 42 | float npKpts[17][3]; 43 | std::string kptLabels[17] = { 44 | "nose", "left_eye", "right_eye", "left_ear", 45 | "right_ear", "left_shoulder", "right_shoulder", 46 | "left_elbow", "right_elbow", "left_wrist", "right_wrist", 47 | "left_hip", "right_hip", "left_knee", "right_knee", "left_ankle", 48 | "right_ankle"}; 49 | int kptConnect[17][3] = { 50 | {1, 2, -1}, {0, 3, -1}, {0, 4, -1}, {1, -1, -1}, {2, -1, -1}, 51 | {6, 7, 11}, {5, 8, 12}, {5, 9, -1}, {6, 10, -1}, {7, -1, -1}, 52 | {8, -1, -1}, {5, 12, 13}, {6, 11, 14}, {11, 15, -1}, 53 | {12, 16, -1}, {13, -1, -1}, {14, -1, -1}}; 54 | int inputDim; 55 | } PoseData; 56 | 57 | 58 | void newDataPoseCallback(GstElement* element, 59 | GstBuffer* buffer, 60 | gpointer user_data); 61 | 62 | 63 | void drawPoseCallback(GstElement* overlay, 64 | cairo_t* cr, 65 | guint64 timestamp, 66 | guint64 duration, 67 | gpointer user_data); 68 | 69 | 70 | void newDataFaceCallback(GstElement* element, 71 | GstBuffer* buffer, 72 | gpointer user_data); 73 | 74 | 75 | void drawFaceCallback(GstElement* overlay, 76 | cairo_t* cr, 77 | guint64 timestamp, 78 | guint64 duration, 79 | gpointer user_data); 80 | 81 | 82 | typedef struct { 83 | int size; 84 | float* bufferFP32; 85 | } BufferInfo; 86 | 87 | 88 | BufferInfo getTensorInfo(GstBuffer* buffer, int tensorIndex); 89 | 90 | 91 | void checkNumTensor(GstBuffer* buffer, int numTensor); 92 | 93 | #endif -------------------------------------------------------------------------------- /tasks/mixed-demos/mixed_demo.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxp-imx/nxp-nnstreamer-examples/b8dafc13bb93b06826f9fc6b91056270cecc0fbd/tasks/mixed-demos/mixed_demo.webp -------------------------------------------------------------------------------- /tasks/monocular-depth-estimation/README.md: -------------------------------------------------------------------------------- 1 | # Monocular Depth Estimation 2 | 3 | ## Overview 4 | Name |Platforms | Model | ML engine | Features 5 | --- | --- | --- | --- | --- 6 | [example_depth_midas_v2_tflite.cpp](./cpp/example_depth_midas_v2_tflite.cpp) | C++ | MiDaS v2 | TFLite | camera
gst-launch
custom C++ decoding 7 | 8 | ## MiDaS v2 monocular depth estimation 9 | ### C++ 10 | | Platforms | NPU | CPU | GPU | 11 | | ------------ | --- | --- | --- | 12 | | i.MX 8M Plus | :white_check_mark: | :white_check_mark: | :white_check_mark: | 13 | | i.MX 93 | :white_check_mark: | :white_check_mark: | :x: | 14 | | i.MX 95 | :x: | :white_check_mark: | :white_check_mark: | 15 | 16 | *NOTES:* 17 | * *Distances shown on screen are relative and not absolute from the camera* 18 | * *The whitest, the closest* 19 | 20 | C++ example script needs to be generated with [cross compilation](../). [setup_environment.sh](../tools/setup_environment.sh) script needs to be executed in [nxp-nnstreamer-examples](../) folder to define data paths: 21 | ```bash 22 | . ./tools/setup_environment.sh 23 | ``` 24 | 25 | It is possible to run the monocular depth estimation demo inference on three different hardwares:
26 | Inference on NPU with the following script: 27 | ```bash 28 | ./build/monocular-depth-estimation/example_depth_midas_v2_tflite -p ${MIDASV2} 29 | ``` 30 | For i.MX 93 NPU use vela converted model: 31 | ```bash 32 | ./build/monocular-depth-estimation/example_depth_midas_v2_tflite -p ${MIDASV2_VELA} 33 | ``` 34 | Inference on CPU with the following script: 35 | ```bash 36 | ./build/monocular-depth-estimation/example_depth_midas_v2_tflite -p ${MIDASV2} -b CPU 37 | ``` 38 | NOTE: Inference on i.MX8MPlus GPU is possible but not recommended because of low performances: 39 | ```bash 40 | ./build/monocular-depth-estimation/example_depth_midas_v2_tflite -p ${MIDASV2} -b GPU 41 | ``` 42 | The following execution parameters are available (Run ``` ./example_depth_midas_v2_tflite -h``` to see option details): 43 | 44 | Option | Description 45 | --- | --- 46 | -b, --backend | Use the selected backend (CPU, GPU, NPU)
default: NPU 47 | -n, --normalization | Use the selected normalization (none, centered, reduced, centeredReduced, castInt32, castuInt8)
default: none 48 | -c, --camera_device | Use the selected camera device (/dev/video{number})
default: /dev/video0 for i.MX 93 and /dev/video3 for i.MX 8MP 49 | -p, --model_path | Use the selected model path 50 | -d, --display_perf |Display performances, can specify time or freq 51 | -t, --text_color | Color of performances displayed, can choose between red, green, blue, and black
default: white 52 | -g, --graph_path | Path to store the result of the OpenVX graph compilation (only for i.MX8MPlus)
default: home directory 53 | -r, --cam_params | Use the selected camera resolution and framerate
default: 640x480, 30fps 54 | 55 | Press ```Esc or ctrl+C``` to stop the execution of the pipeline. -------------------------------------------------------------------------------- /tasks/monocular-depth-estimation/cpp/custom_depth_decoder.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #include "custom_depth_decoder.hpp" 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | BufferInfo getTensorInfo(GstBuffer* buffer, int tensorIndex) 14 | { 15 | BufferInfo bufferInfo; 16 | GstMapInfo info; 17 | GstMemory* memKpts = gst_buffer_peek_memory(buffer, tensorIndex); 18 | bool result = gst_memory_map(memKpts, &info, GST_MAP_READ); 19 | if (result) { 20 | bufferInfo.bufferFP32 = reinterpret_cast(info.data); 21 | bufferInfo.size = info.size/sizeof(float); 22 | gst_memory_unmap(memKpts, &info); 23 | } else { 24 | gst_memory_unmap(memKpts, &info); 25 | log_error("Can't access buffer in memory\n"); 26 | exit(-1); 27 | } 28 | return bufferInfo; 29 | } 30 | 31 | 32 | void checkNumTensor(GstBuffer* buffer, int numTensor) 33 | { 34 | if (!GST_IS_BUFFER (buffer)) { 35 | log_error("Received invalid buffer\n"); 36 | exit(-1); 37 | } 38 | uint mem_blocks = gst_buffer_n_memory(buffer); 39 | if (mem_blocks != numTensor) { 40 | log_error("Number of tensors invalid : %d\n", mem_blocks); 41 | exit(-1); 42 | } 43 | } 44 | 45 | 46 | void newDataCallback(GstElement* element, 47 | GstBuffer* buffer, 48 | gpointer user_data) 49 | { 50 | DecoderData* data = (DecoderData *) user_data; 51 | 52 | BufferInfo bufferInfo; 53 | checkNumTensor(buffer, 1); 54 | bufferInfo = getTensorInfo(buffer, 0); 55 | 56 | float *min = std::min_element(bufferInfo.bufferFP32, bufferInfo.bufferFP32 + bufferInfo.size); 57 | float *max = std::max_element(bufferInfo.bufferFP32, bufferInfo.bufferFP32 + bufferInfo.size); 58 | 59 | if ((*max - *min) > MODEL_THRESHOLD) { 60 | for (int i = 0; i < bufferInfo.size; i++) { 61 | data->output[i] = (guchar)std::round(255*(bufferInfo.bufferFP32[i] - *min) / (*max - *min)); 62 | } 63 | } else { 64 | std::fill_n(&data->output[0], bufferInfo.size, 0); 65 | } 66 | 67 | data->size = bufferInfo.size; 68 | pushBuffer(data); 69 | } 70 | 71 | 72 | void pushBuffer(DecoderData* data) 73 | { 74 | static GstClockTime timestamp = 0; 75 | GstBuffer *buffer = gst_buffer_new_allocate(NULL, data->size, NULL); 76 | GstMapInfo map; 77 | gst_buffer_map(buffer, &map, GST_MAP_WRITE); 78 | memcpy((guchar *)map.data, data->output, gst_buffer_get_size(buffer)); 79 | GST_BUFFER_PTS(buffer) = timestamp; 80 | GST_BUFFER_DURATION(buffer) = gst_util_uint64_scale_int(1, GST_SECOND, 2); 81 | timestamp += GST_BUFFER_DURATION(buffer); 82 | 83 | GstFlowReturn ret; 84 | g_signal_emit_by_name(data->appSrc, "push-buffer", buffer, &ret); 85 | gst_buffer_unmap(buffer, &map); 86 | gst_buffer_unref(buffer); 87 | if (ret != GST_FLOW_OK) { 88 | log_error("Could not push buffer to appsrc\n"); 89 | exit(-1); 90 | } 91 | } -------------------------------------------------------------------------------- /tasks/monocular-depth-estimation/cpp/custom_depth_decoder.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #ifndef DEPTH_CUSTOM_DEPTH_DECODER_H_ 7 | #define DEPTH_CUSTOM_DEPTH_DECODER_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "logging.hpp" 19 | 20 | #define MODEL_THRESHOLD 1e-6 21 | #define MODEL_DIM 65536 22 | 23 | 24 | typedef struct { 25 | guchar output[MODEL_DIM]; 26 | int size = 0; 27 | GstElement *appSrc; 28 | } DecoderData; 29 | 30 | 31 | void newDataCallback(GstElement* element, 32 | GstBuffer* buffer, 33 | gpointer user_data); 34 | 35 | 36 | void pushBuffer(DecoderData* data); 37 | 38 | 39 | typedef struct { 40 | int size; 41 | float* bufferFP32; 42 | } BufferInfo; 43 | 44 | 45 | BufferInfo getTensorInfo(GstBuffer* buffer, int tensorIndex); 46 | 47 | 48 | void checkNumTensor(GstBuffer* buffer, int numTensor); 49 | 50 | #endif -------------------------------------------------------------------------------- /tasks/monocular-depth-estimation/depth_demo.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxp-imx/nxp-nnstreamer-examples/b8dafc13bb93b06826f9fc6b91056270cecc0fbd/tasks/monocular-depth-estimation/depth_demo.webp -------------------------------------------------------------------------------- /tasks/monocular-depth-estimation/export_midas-v2_to_TensorFlow.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # SPDX-License-Identifier: MIT 4 | # Copyright 2025 NXP 5 | 6 | python3.10 -m venv env 7 | 8 | source ./env/bin/activate 9 | 10 | pip install --upgrade tflite2tensorflow 11 | 12 | wget https://github.com/PINTO0309/tflite2tensorflow/raw/main/schema/schema.fbs 13 | 14 | git clone -b v2.0.8 https://github.com/google/flatbuffers.git 15 | ( 16 | cd flatbuffers && mkdir build && cd build 17 | cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release .. 18 | make -j$(nproc) 19 | ) 20 | 21 | pip install -r $1 22 | 23 | tflite2tensorflow \ 24 | --model_path midas_2_1_small_float32.tflite \ 25 | --flatc_path flatbuffers/build/flatc \ 26 | --schema_path schema.fbs \ 27 | --output_pb 28 | 29 | mv saved_model/model_float32.pb model_float32.pb 30 | 31 | #cleanup 32 | deactivate 33 | rm -rf sample_npy saved_model flatbuffers env 34 | rm schema.fbs midas_2_1_small_float32.json -------------------------------------------------------------------------------- /tasks/monocular-depth-estimation/requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py===1.4.0 2 | array-record==0.4.0 3 | astunparse==1.6.3 4 | beautifulsoup4==4.12.2 5 | cachetools==5.3.1 6 | certifi==2023.7.22 7 | charset-normalizer==3.3.0 8 | click==8.1.7 9 | contourpy==1.1.1 10 | cycler==0.12.1 11 | dm-tree==0.1.8 12 | etils==1.3.0 13 | filelock==3.12.4 14 | flatbuffers==2.0.7 15 | fonttools==4.43.1 16 | gast==0.4.0 17 | gdown==4.7.1 18 | google-auth==2.23.3 19 | google-auth-oauthlib==0.4.6 20 | google-pasta==0.2.0 21 | googleapis-common-protos==1.61.0 22 | grpcio==1.59.0 23 | h5py==3.10.0 24 | idna==3.4 25 | importlib-metadata==6.8.0 26 | importlib-resources==6.1.0 27 | keras==2.8.0rc0 28 | Keras-Preprocessing==1.1.2 29 | kiwisolver==1.4.5 30 | libclang==16.0.6 31 | Markdown==3.5 32 | MarkupSafe==2.1.3 33 | matplotlib==3.7.3 34 | numpy==1.24.4 35 | oauthlib==3.2.2 36 | opencv-python==4.8.1.78 37 | opt-einsum==3.3.0 38 | packaging==23.2 39 | pandas==2.0.3 40 | Pillow==10.1.0 41 | promise==2.3 42 | protobuf==3.20.3 43 | psutil==5.9.6 44 | pyasn1==0.5.0 45 | pyasn1-modules==0.3.0 46 | pyparsing==3.1.1 47 | PySocks==1.7.1 48 | python-dateutil==2.8.2 49 | pytz==2023.3.post1 50 | requests==2.31.0 51 | requests-oauthlib==1.3.1 52 | rsa==4.9 53 | six==1.16.0 54 | soupsieve==2.5 55 | tensorboard==2.8.0 56 | tensorboard-data-server==0.6.1 57 | tensorboard-plugin-wit==1.8.1 58 | tensorflow==2.8.0 59 | tensorflow-datasets==4.9.2 60 | tensorflow-estimator==2.7.0 61 | tensorflow-io-gcs-filesystem==0.34.0 62 | tensorflow-metadata==1.14.0 63 | termcolor==2.3.0 64 | tflite-runtime==2.13.0 65 | tflite2tensorflow==1.22.0 66 | toml==0.10.2 67 | tqdm==4.66.1 68 | typing-extensions==4.8.0 69 | tzdata==2023.3 70 | urllib3==2.0.7 71 | werkzeug==3.0.0 72 | wrapt==1.15.0 73 | zipp==3.17.0 74 | -------------------------------------------------------------------------------- /tasks/object-detection/README.md: -------------------------------------------------------------------------------- 1 | # Object Detection 2 | 3 | ## Overview 4 | Name | Implementation | Model | ML engine | Features 5 | --- | --- | --- | --- | --- | 6 | [example_detection_mobilenet_ssd_v2_tflite.cpp](./cpp/example_detection_mobilenet_ssd_v2_tflite.cpp) | C++ | SSD MobileNetV2 | TFLite | camera
gst-launch
7 | [example_detection_mobilenet_ssd_v2_tflite.sh](./example_detection_mobilenet_ssd_v2_tflite.sh) | Bash | SSD MobileNetV2 | TFLite | camera
gst-launch
8 | [example_detection_yolo_v4_tiny_tflite.sh](./example_detection_yolo_v4_tiny_tflite.sh) | Bash | YOLOv4 Tiny | TFLite | camera
gst-launch
[custom python tensor_filter](./postprocess_yolov4_tiny.py) 9 | 10 | ## YOLOv4 Tiny object detection 11 | ### Bash 12 | | Platforms | NPU | CPU | GPU | 13 | | ------------ | --- | --- | --- | 14 | | i.MX 8M Plus | :white_check_mark: | :white_check_mark: | :x: | 15 | | i.MX 93 | :white_check_mark: | :white_check_mark: | :x: | 16 | | i.MX 95 | :x: | :white_check_mark: | :x: | 17 | 18 | *NOTE: YOLOv4 Tiny output does not directly work with the YOLOv5 mode of tensor_decoder element, so a python filter is used to post-process and reshape this output as required.* 19 | 20 | The object detection demo in bash using YOLOv4 Tiny supports multiple backend (refers to above table), default value can be overriden by explicitly defining BACKEND variable, for instance: 21 | ```bash 22 | BACKEND=CPU ./tasks/object-detection/example_detection_yolo_v4_tiny_tflite.sh 23 | ``` 24 | 25 | ## SSD MobileNetV2 object detection 26 | ### Bash 27 | | Platforms | NPU | CPU | GPU | 28 | | ------------ | --- | --- | --- | 29 | | i.MX 8M Plus | :white_check_mark: | :white_check_mark: | :white_check_mark: | 30 | | i.MX 93 | :white_check_mark: | :white_check_mark: | :x: | 31 | | i.MX 95 | :white_check_mark: | :white_check_mark: | :x: | 32 | 33 | The object detection demo in bash using SSD MobiletNetV2 supports multiple backend (refers to above table), default value can be overriden by explicitly defining BACKEND variable, for instance: 34 | ```bash 35 | BACKEND=CPU ./tasks/object-detection/example_detection_mobilenet_ssd_v2_tflite.sh 36 | ``` 37 | ### C++ 38 | | Platforms | NPU | CPU | GPU | 39 | | ------------ | --- | --- | --- | 40 | | i.MX 8M Plus | :white_check_mark: | :white_check_mark: | :white_check_mark: | 41 | | i.MX 93 | :white_check_mark: | :white_check_mark: | :x: | 42 | | i.MX 95 | :white_check_mark: | :white_check_mark: | :white_check_mark: | 43 | 44 | C++ example script needs to be generated with [cross compilation](../). [setup_environment.sh](../tools/setup_environment.sh) script needs to be executed in [nxp-nnstreamer-examples](../) folder to define data paths: 45 | ```bash 46 | . ./tools/setup_environment.sh 47 | ``` 48 | 49 | It is possible to run the object detection demo inference on three different hardwares:
50 | Inference on NPU with the following script: 51 | ```bash 52 | ./build/object-detection/example_detection_mobilenet_ssd_v2_tflite -p ${MOBILENETV2_QUANT} -l ${COCO_LABELS} -x ${MOBILENETV2_BOXES} 53 | ``` 54 | For i.MX 93 NPU use vela converted model: 55 | ```bash 56 | ./build/object-detection/example_detection_mobilenet_ssd_v2_tflite -p ${MOBILENETV2_QUANT_VELA} -l ${COCO_LABELS} -x ${MOBILENETV2_BOXES} 57 | ``` 58 | 59 | For i.MX 95 NPU use neutron converted model: 60 | ```bash 61 | ./build/object-detection/example_detection_mobilenet_ssd_v2_tflite -p ${MOBILENETV2_QUANT_NEUTRON} -l ${COCO_LABELS} -x ${MOBILENETV2_BOXES} 62 | ``` 63 | 64 | Inference on CPU with the following script: 65 | ```bash 66 | ./build/object-detection/example_detection_mobilenet_ssd_v2_tflite -p ${MOBILENETV2_QUANT} -l ${COCO_LABELS} -x ${MOBILENETV2_BOXES} -b CPU 67 | ``` 68 | Quantized model is used for better inference performances on CPU.
69 | NOTE: inferences on i.MX8MPlus GPU have low performances, but are possible with the following script: 70 | ```bash 71 | ./build/object-detection/example_detection_mobilenet_ssd_v2_tflite -p ${MOBILENETV2} -l ${COCO_LABELS} -x ${MOBILENETV2_BOXES} -b GPU -n centeredReduced 72 | ``` 73 | The following execution parameters are available (Run ``` ./example_detection_mobilenet_ssd_v2_tflite -h``` to see option details): 74 | 75 | Option | Description 76 | --- | --- 77 | -b, --backend | Use the selected backend (CPU, GPU, NPU)
default: NPU 78 | -n, --normalization | Use the selected normalization (none, centered, reduced, centeredReduced, castInt32, castuInt8)
default: none 79 | -c, --camera_device | Use the selected camera device (/dev/video{number})
default: /dev/video0 for i.MX 93 and /dev/video3 for i.MX 8MP 80 | -p, --model_path | Use the selected model path 81 | -l, --labels_path | Use the selected labels path 82 | -x, --boxes_path | Use the selected boxes path 83 | -d, --display_perf |Display performances, can specify time or freq 84 | -t, --text_color | Color of performances displayed, can choose between red, green, blue, and black
default: white 85 | -g, --graph_path | Path to store the result of the OpenVX graph compilation (only for i.MX8MPlus)
default: home directory 86 | -r, --cam_params | Use the selected camera resolution and framerate
default: 640x480, 30fps 87 | 88 | Press ```Esc or ctrl+C``` to stop the execution of the pipeline. -------------------------------------------------------------------------------- /tasks/object-detection/README_postprocess_yolov4_tiny.md: -------------------------------------------------------------------------------- 1 | # Purpose of [postprocess_yolov4_tiny.py](./postprocess_yolov4_tiny.py): 2 | 3 | - This Python filter is made to postprocess and reshape the output of Yolov4-tiny, to match input format from NNStreamer tensor_decoder element using yolov5 decoder mode. 4 | - To call this Python filter, a tensor_filter element must be used in the NNStreamer pipeline flow. 5 | Path of the Python filter must be given as the model argument, and the framework argument is python3. 6 | - It operates just after the inference of Yolov4-tiny, and before the tensor_decoder element. 7 | 8 | ## Custom option available: detection threshold value 9 | 10 | Detection threshold value can be easily changed in [example_detection_yolo_v4_tiny_tflite.sh](./example_detection_yolo_v4_tiny_tflite.sh). 11 | Threshold parameter is optional, and should be defined between 0 and 1 (default = 0.25). 12 | 13 | If this threshold is close to 1, fewer objects may be detected but predictions will be more reliable. 14 | At the opposite with a threshold close to 0, the model should detect more objects but less reliably. 15 | 16 | tensor_decoder threshold value is hard-coded at 0.25 in [tensordec-boundingbox.c](https://github.com/nnstreamer/nnstreamer/blob/lts/2.4.0.b/ext/nnstreamer/tensor_decoder/tensordec-boundingbox.c#L139) in the NNStreamer repository. 17 | To set the threshold value under 0.25, hard-coded value in tensordec-boundingbox.c must also be reduced. 18 | -------------------------------------------------------------------------------- /tasks/object-detection/detection_demo.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxp-imx/nxp-nnstreamer-examples/b8dafc13bb93b06826f9fc6b91056270cecc0fbd/tasks/object-detection/detection_demo.webp -------------------------------------------------------------------------------- /tasks/object-detection/detection_utils.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2022-2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | function gst_exec_detection { 7 | 8 | # accelerated video scaling before inferencing 9 | local VIDEO_SCALE=$(accelerated_video_scale_rgb_str ${MODEL_WIDTH} ${MODEL_HEIGHT}) 10 | # accelerated video composition 11 | local VIDEO_MIXER=$(accelerated_video_mixer_str "mix" "sink_0::zorder=2 sink_1::zorder=1" "0" "0.3" "${MODEL_LATENCY}") 12 | 13 | gst-launch-1.0 \ 14 | v4l2src name=cam_src device=${CAMERA_DEVICE} num-buffers=-1 ! \ 15 | video/x-raw,width=${CAMERA_WIDTH},height=${CAMERA_HEIGHT},framerate=${CAMERA_FPS}/1 ! \ 16 | tee name=t \ 17 | t. ! queue name=thread-nn max-size-buffers=2 leaky=2 ! \ 18 | ${VIDEO_SCALE} \ 19 | tensor_converter ! \ 20 | ${TENSOR_PREPROCESS} \ 21 | ${TENSOR_FILTER} \ 22 | ${TENSOR_DECODER} \ 23 | videoconvert ! \ 24 | mix. \ 25 | t. ! queue name=thread-img max-size-buffers=2 leaky=2 ! \ 26 | videoconvert ! \ 27 | ${VIDEO_MIXER} \ 28 | waylandsink 29 | } 30 | -------------------------------------------------------------------------------- /tasks/object-detection/example_detection_mobilenet_ssd_v2_tflite.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2022-2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | set -x 7 | 8 | REALPATH="$(readlink -e "$0")" 9 | BASEDIR="$(dirname "${REALPATH}")/../.." 10 | MODELS_DIR="${BASEDIR}/downloads/models/object-detection" 11 | 12 | source "${BASEDIR}/common/common_utils.sh" 13 | source "${BASEDIR}/tasks/object-detection/detection_utils.sh" 14 | 15 | setup_env 16 | 17 | # model and framework dependant variables 18 | declare -A MODEL_BACKEND 19 | declare -A MODEL_BACKEND_NPU 20 | MODEL_BACKEND_NPU[IMX8MP]="${MODELS_DIR}/ssdlite_mobilenet_v2_coco_quant_uint8_float32_no_postprocess.tflite" 21 | MODEL_BACKEND_NPU[IMX93]="${MODELS_DIR}/ssdlite_mobilenet_v2_coco_quant_uint8_float32_no_postprocess_vela.tflite" 22 | MODEL_BACKEND_NPU[IMX95]="${MODELS_DIR}/ssdlite_mobilenet_v2_coco_quant_uint8_float32_no_postprocess_neutron.tflite" 23 | 24 | MODEL_BACKEND[CPU]="${MODELS_DIR}/ssdlite_mobilenet_v2_coco_quant_uint8_float32_no_postprocess.tflite" 25 | MODEL_BACKEND[GPU]="${MODELS_DIR}/ssdlite_mobilenet_v2_coco_no_postprocess.tflite" 26 | MODEL_BACKEND[NPU]=${MODEL_BACKEND_NPU[${IMX}]} 27 | MODEL=${MODEL_BACKEND[${BACKEND}]} 28 | 29 | declare -A MODEL_LATENCY_CPU_NS 30 | MODEL_LATENCY_CPU_NS[IMX8MP]="60000000" 31 | MODEL_LATENCY_CPU_NS[IMX93]="75000000" 32 | MODEL_LATENCY_CPU_NS[IMX95]="40000000" 33 | 34 | declare -A MODEL_LATENCY_GPU_NS 35 | MODEL_LATENCY_GPU_NS[IMX8MP]="500000000" 36 | 37 | declare -A MODEL_LATENCY_NPU_NS 38 | MODEL_LATENCY_NPU_NS[IMX8MP]="40000000" 39 | MODEL_LATENCY_NPU_NS[IMX93]="10000000" 40 | MODEL_LATENCY_NPU_NS[IMX95]="25000000" 41 | 42 | declare -A MODEL_LATENCY_NS 43 | MODEL_LATENCY_NS[CPU]=${MODEL_LATENCY_CPU_NS[${IMX}]} 44 | MODEL_LATENCY_NS[GPU]=${MODEL_LATENCY_GPU_NS[${IMX}]} 45 | MODEL_LATENCY_NS[NPU]=${MODEL_LATENCY_NPU_NS[${IMX}]} 46 | MODEL_LATENCY=${MODEL_LATENCY_NS[${BACKEND}]} 47 | 48 | MODEL_WIDTH=300 49 | MODEL_HEIGHT=300 50 | MODEL_BOXES="${MODELS_DIR}/box_priors.txt" 51 | MODEL_LABELS="${MODELS_DIR}/coco_labels_list.txt" 52 | 53 | FRAMEWORK="tensorflow-lite" 54 | 55 | # tensor filter configuration 56 | FILTER_COMMON="tensor_filter framework=${FRAMEWORK} model=${MODEL}" 57 | 58 | declare -A FILTER_BACKEND_NPU 59 | FILTER_BACKEND_NPU[IMX8MP]=" custom=Delegate:External,ExtDelegateLib:libvx_delegate.so ! " 60 | FILTER_BACKEND_NPU[IMX93]=" custom=Delegate:External,ExtDelegateLib:libethosu_delegate.so ! " 61 | FILTER_BACKEND_NPU[IMX95]=" custom=Delegate:External,ExtDelegateLib:libneutron_delegate.so ! " 62 | 63 | declare -A FILTER_BACKEND 64 | FILTER_BACKEND[CPU]="${FILTER_COMMON} custom=Delegate:XNNPACK,NumThreads:$(nproc --all) !" 65 | FILTER_BACKEND[GPU]="${FILTER_COMMON} custom=Delegate:External,ExtDelegateLib:libvx_delegate.so ! " 66 | FILTER_BACKEND[NPU]="${FILTER_COMMON} ${FILTER_BACKEND_NPU[${IMX}]}" 67 | TENSOR_FILTER=${FILTER_BACKEND[${BACKEND}]} 68 | 69 | # tensor preprocessing configuration: normalize video for float input models 70 | declare -A PREPROCESS_BACKEND 71 | PREPROCESS_BACKEND[CPU]="" 72 | PREPROCESS_BACKEND[GPU]="tensor_transform mode=arithmetic option=typecast:float32,add:-127.5,div:127.5 ! " 73 | PREPROCESS_BACKEND[NPU]="" 74 | TENSOR_PREPROCESS=${PREPROCESS_BACKEND[${BACKEND}]} 75 | 76 | # tensor decoder configuration: mobilenet ssd without post processing 77 | TENSOR_DECODER="tensor_decoder mode=bounding_boxes" 78 | TENSOR_DECODER+=" option1=mobilenet-ssd" 79 | TENSOR_DECODER+=" option2=${MODEL_LABELS}" 80 | TENSOR_DECODER+=" option3=${MODEL_BOXES}" 81 | TENSOR_DECODER+=" option4=${CAMERA_WIDTH}:${CAMERA_HEIGHT}" 82 | TENSOR_DECODER+=" option5=${MODEL_WIDTH}:${MODEL_HEIGHT} ! " 83 | 84 | gst_exec_detection 85 | 86 | -------------------------------------------------------------------------------- /tasks/object-detection/example_detection_yolo_v4_tiny_tflite.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2023-2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | set -x 7 | 8 | REALPATH="$(readlink -e "$0")" 9 | BASEDIR="$(dirname "${REALPATH}")/../.." 10 | MODELS_DIR="${BASEDIR}/downloads/models/object-detection" 11 | 12 | source "${BASEDIR}/common/common_utils.sh" 13 | source "${BASEDIR}/tasks/object-detection/detection_utils.sh" 14 | 15 | setup_env 16 | 17 | # model and framework dependant variables 18 | declare -A MODEL_BACKEND 19 | declare -A MODEL_BACKEND_NPU 20 | MODEL_BACKEND_NPU[IMX8MP]="${MODELS_DIR}/yolov4-tiny_416_quant.tflite" 21 | MODEL_BACKEND_NPU[IMX93]="${MODELS_DIR}/yolov4-tiny_416_quant_vela.tflite" 22 | 23 | MODEL_BACKEND[CPU]="${MODELS_DIR}/yolov4-tiny_416_quant.tflite" 24 | MODEL_BACKEND[NPU]=${MODEL_BACKEND_NPU[${IMX}]} 25 | MODEL=${MODEL_BACKEND[${BACKEND}]} 26 | 27 | declare -A MODEL_LATENCY_CPU_NS 28 | MODEL_LATENCY_CPU_NS[IMX8MP]="175000000" 29 | MODEL_LATENCY_CPU_NS[IMX93]="200000000" 30 | MODEL_LATENCY_CPU_NS[IMX95]="70000000" 31 | 32 | declare -A MODEL_LATENCY_NPU_NS 33 | MODEL_LATENCY_NPU_NS[IMX8MP]="60000000" 34 | MODEL_LATENCY_NPU_NS[IMX93]="60000000" 35 | 36 | declare -A MODEL_LATENCY_NS 37 | MODEL_LATENCY_NS[CPU]=${MODEL_LATENCY_CPU_NS[${IMX}]} 38 | MODEL_LATENCY_NS[NPU]=${MODEL_LATENCY_NPU_NS[${IMX}]} 39 | MODEL_LATENCY=${MODEL_LATENCY_NS[${BACKEND}]} 40 | 41 | MODEL_WIDTH=416 42 | MODEL_HEIGHT=416 43 | MODEL_LABELS="${MODELS_DIR}/coco-labels-2014_2017.txt" 44 | 45 | 46 | FRAMEWORK="tensorflow-lite" 47 | 48 | # tensor filter configuration 49 | FILTER_COMMON="tensor_filter framework=${FRAMEWORK} model=${MODEL}" 50 | 51 | declare -A FILTER_BACKEND_NPU 52 | FILTER_BACKEND_NPU[IMX8MP]=" custom=Delegate:External,ExtDelegateLib:libvx_delegate.so ! " 53 | FILTER_BACKEND_NPU[IMX93]=" custom=Delegate:External,ExtDelegateLib:libethosu_delegate.so ! " 54 | 55 | declare -A FILTER_BACKEND 56 | declare -A FILTER_BACKEND 57 | FILTER_BACKEND[CPU]="${FILTER_COMMON} custom=Delegate:XNNPACK,NumThreads:$(nproc --all) !" 58 | FILTER_BACKEND[NPU]="${FILTER_COMMON} ${FILTER_BACKEND_NPU[${IMX}]}" 59 | TENSOR_FILTER=${FILTER_BACKEND[${BACKEND}]} 60 | 61 | # python filter configuration 62 | FRAMEWORK="python3" 63 | POSTPROCESS="${BASEDIR}/tasks/object-detection/postprocess_yolov4_tiny.py" 64 | MODEL_SIZE="Height:${MODEL_HEIGHT},Width:${MODEL_WIDTH}" 65 | THRESHOLD="Threshold:0.4" # optional argument between 0 and 1 66 | if [ -z "${THRESHOLD}" ]; 67 | then 68 | TENSOR_FILTER+=" tensor_filter framework=${FRAMEWORK} model=${POSTPROCESS} custom=${MODEL_SIZE} ! "; 69 | else 70 | TENSOR_FILTER+=" tensor_filter framework=${FRAMEWORK} model=${POSTPROCESS} custom=${MODEL_SIZE},${THRESHOLD} ! "; 71 | fi 72 | 73 | # tensor preprocessing configuration: normalize video for float input models 74 | declare -A PREPROCESS_BACKEND 75 | PREPROCESS_BACKEND[CPU]="tensor_transform mode=arithmetic option=typecast:int16,add:-128 ! tensor_transform mode=typecast option=int8 !" 76 | PREPROCESS_BACKEND[NPU]="tensor_transform mode=arithmetic option=typecast:int16,add:-128 ! tensor_transform mode=typecast option=int8 !" 77 | TENSOR_PREPROCESS=${PREPROCESS_BACKEND[${BACKEND}]} 78 | 79 | 80 | # tensor decoder configuration: yolov4-tiny without post processing 81 | TENSOR_DECODER="tensor_decoder mode=bounding_boxes" 82 | TENSOR_DECODER+=" option1=yolov5" 83 | TENSOR_DECODER+=" option2=${MODEL_LABELS}" 84 | # option3 is already set to tensorflow framework by default 85 | TENSOR_DECODER+=" option4=${CAMERA_WIDTH}:${CAMERA_HEIGHT}" 86 | TENSOR_DECODER+=" option5=${MODEL_WIDTH}:${MODEL_HEIGHT} !" 87 | 88 | gst_exec_detection 89 | 90 | -------------------------------------------------------------------------------- /tasks/pose-estimation/README.md: -------------------------------------------------------------------------------- 1 | # Pose Estimation 2 | 3 | ## Overview 4 | Name | Implementation | Model | ML engine |Features 5 | --- | --- | --- | --- | --- 6 | [example_pose_movenet_tflite.py](./example_pose_movenet_tflite.py) | Python | MoveNet Lightning | TFLite | video file decoding (i.MX 8M Plus only)
camera
gst-launch
7 | [example_pose_movenet_tflite.cpp](./cpp/example_pose_movenet_tflite.cpp) | C++ | MoveNet Lightning | TFLite | video file decoding (i.MX 8M Plus only)
camera
gst-launch
8 | 9 | ## MoveNet Lightning pose estimation 10 | ### Python 11 | | Platforms | NPU | CPU | GPU | 12 | | ------------ | --- | --- | --- | 13 | | i.MX 8M Plus | :white_check_mark: | :white_check_mark: | :x: | 14 | | i.MX 93 | :x: | :white_check_mark: | :x: | 15 | | i.MX 95 | :x: | :white_check_mark: | :x: | 16 | 17 | Default backend can be overriden by explicitly defining BACKEND variable, and source can be selected as VIDEO or CAMERA, for instance: 18 | 19 | ```bash 20 | BACKEND=NPU SOURCE=CAMERA ./tasks/pose-estimation/example_pose_movenet_tflite.py 21 | ``` 22 | 23 | The following execution parameters are available (Run ``` ./example_pose_movenet_tflite.py -h``` to see option details): 24 | 25 | Option | Description 26 | --- | --- 27 | --video_file FILE | Selects another video source file with given FILE path 28 | --video_dims WIDTH HEIGHT | Provides the video source resolution 29 | --mirror | Flips the camera stream when using a front camera 30 | 31 | ### C++ 32 | | Platforms | NPU | CPU | GPU | 33 | | ------------ | --- | --- | --- | 34 | | i.MX 8M Plus | :white_check_mark: | :white_check_mark: | :white_check_mark: | 35 | | i.MX 93 | :x: | :white_check_mark: | :x: | 36 | | i.MX 95 | :x: | :white_check_mark: | :white_check_mark: | 37 | 38 | C++ example script needs to be generated with [cross compilation](../). [setup_environment.sh](../tools/setup_environment.sh) script needs to be executed in [nxp-nnstreamer-examples](../) folder to define data paths: 39 | ```bash 40 | . ./tools/setup_environment.sh 41 | ``` 42 | 43 | It is possible to run the pose estimation demo inference on three different hardwares:
44 | Inference on NPU with the following script: 45 | ```bash 46 | ./build/pose-estimation/example_pose_movenet_tflite -p ${MOVENET_QUANT} -f ${POWER_JUMP_VIDEO} 47 | ``` 48 | For i.MX 93 NPU use vela converted model: 49 | ```bash 50 | ./build/pose-estimation/example_pose_movenet_tflite -p ${MOVENET_QUANT_VELA} -f ${POWER_JUMP_VIDEO} 51 | ``` 52 | Inference on CPU with the following script: 53 | ```bash 54 | ./build/pose-estimation/example_pose_movenet_tflite -p ${MOVENET_QUANT} -f ${POWER_JUMP_VIDEO} -b CPU 55 | ``` 56 | NOTE: inferences on i.MX8MPlus GPU have low performances, but are possible with the following script: 57 | ```bash 58 | ./build/pose-estimation/example_pose_movenet_tflite -p ${MOVENET} -f ${POWER_JUMP_VIDEO} -b GPU -n castInt32 59 | ``` 60 | The following execution parameters are available (Run ``` ./example_pose_movenet_tflite -h``` to see option details): 61 | 62 | Option | Description 63 | --- | --- 64 | -b, --backend | Use the selected backend (CPU, GPU, NPU)
default: CPU 65 | -n, --normalization | Use the selected normalization (none, centered, reduced, centeredReduced, castInt32, castuInt8)
default: castuInt8 66 | -p, --model_path | Use the selected model path 67 | -f, --video_file | Use the selected video file 68 | -u, --use_camera | If we use camera or video input (true,false)
default: false (true for i.MX 93) 69 | -d, --display_perf |Display performances, can specify time or freq 70 | -t, --text_color | Color of performances displayed, can choose between red, green, blue, and black
default: white 71 | -g, --graph_path | Path to store the result of the OpenVX graph compilation (only for i.MX8MPlus)
default: home directory 72 | -r, --cam_params | Use the selected camera resolution and framerate
default: 640x480, 30fps 73 | 74 | Press ```Esc or ctrl+C``` to stop the execution of the pipeline. 75 | -------------------------------------------------------------------------------- /tasks/pose-estimation/cpp/custom_pose_decoder.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #include "custom_pose_decoder.hpp" 7 | 8 | #include 9 | 10 | BufferInfo getTensorInfo(GstBuffer* buffer, int tensorIndex) 11 | { 12 | BufferInfo bufferInfo; 13 | GstMapInfo info; 14 | GstMemory* memKpts = gst_buffer_peek_memory(buffer, tensorIndex); 15 | bool result = gst_memory_map(memKpts, &info, GST_MAP_READ); 16 | if (result) { 17 | bufferInfo.bufferFP32 = reinterpret_cast(info.data); 18 | bufferInfo.size = info.size/sizeof(float); 19 | gst_memory_unmap(memKpts, &info); 20 | } else { 21 | gst_memory_unmap(memKpts, &info); 22 | log_error("Can't access buffer in memory\n"); 23 | exit(-1); 24 | } 25 | return bufferInfo; 26 | } 27 | 28 | 29 | void checkNumTensor(GstBuffer* buffer, int numTensor) 30 | { 31 | if (!GST_IS_BUFFER (buffer)) { 32 | log_error("Received invalid buffer\n"); 33 | exit(-1); 34 | } 35 | uint mem_blocks = gst_buffer_n_memory(buffer); 36 | if (mem_blocks != numTensor) { 37 | log_error("Number of tensors invalid\n"); 38 | exit(-1); 39 | } 40 | } 41 | 42 | 43 | void newDataCallback(GstElement* element, 44 | GstBuffer* buffer, 45 | gpointer user_data) 46 | { 47 | DecoderData* kptsData = (DecoderData *) user_data; 48 | 49 | BufferInfo bufferInfo; 50 | checkNumTensor(buffer, 1); 51 | bufferInfo = getTensorInfo(buffer, 0); 52 | 53 | int row = 0; 54 | int col = 0; 55 | float score = 0; 56 | float valid = 0; 57 | for (int i = 0; i < bufferInfo.size; i++) { 58 | 59 | if ((col == kptsData->xIndex) or (col == kptsData->yIndex)) { 60 | kptsData->npKpts[row][col] = bufferInfo.bufferFP32[i] * kptsData->inputDim; 61 | } else { 62 | kptsData->npKpts[row][col] = bufferInfo.bufferFP32[i]; 63 | score = kptsData->npKpts[row][kptsData->scoreIndex]; 64 | valid = (score >= kptsData->scoreThreshold); 65 | kptsData->npKpts[row][col] = valid; 66 | } 67 | 68 | col+=1; 69 | if (col == 3) 70 | row += 1; 71 | col = col % 3; 72 | } 73 | } 74 | 75 | 76 | void drawCallback(GstElement* overlay, 77 | cairo_t* cr, 78 | guint64 timestamp, 79 | guint64 duration, 80 | gpointer user_data) 81 | { 82 | DecoderData* kptsData = (DecoderData *) user_data; 83 | float valid; 84 | float* npKpt; 85 | float xKpt; 86 | float yKpt; 87 | int* connections; 88 | float* npConnect; 89 | float xConnect; 90 | float yConnect; 91 | cairo_select_font_face(cr, 92 | "Arial", 93 | CAIRO_FONT_SLANT_NORMAL, 94 | CAIRO_FONT_WEIGHT_NORMAL); 95 | cairo_set_line_width(cr, 1.0); 96 | 97 | for(int i = 0; i < kptsData->kptSize; i++) { 98 | npKpt = kptsData->npKpts[i]; 99 | valid = npKpt[kptsData->scoreIndex]; 100 | if ((valid != 1.0)) 101 | continue; 102 | 103 | xKpt = npKpt[kptsData->xIndex]; 104 | yKpt = npKpt[kptsData->yIndex]; 105 | 106 | // Draw keypoint spot 107 | cairo_set_source_rgb(cr, 1, 0, 0); 108 | cairo_arc(cr, xKpt, yKpt, 1, 0, 2*M_PI); 109 | cairo_fill(cr); 110 | cairo_stroke(cr); 111 | 112 | // Draw keypoint label 113 | cairo_set_source_rgb(cr, 0, 1, 1); 114 | cairo_set_font_size(cr, 10.0); 115 | cairo_move_to(cr, xKpt + 5, yKpt + 5); 116 | cairo_show_text(cr, kptsData->kptLabels[i].c_str()); 117 | 118 | // Draw keypoint connections 119 | cairo_set_source_rgb(cr, 0, 1, 0); 120 | connections = kptsData->kptConnect[i]; 121 | for(int j = 0; j < 3; j++) { 122 | if (connections[j] == -1) 123 | break; 124 | npConnect = kptsData->npKpts[connections[j]]; 125 | valid = npConnect[kptsData->scoreIndex]; 126 | if (valid != 1.0) 127 | continue; 128 | xConnect = npConnect[kptsData->xIndex]; 129 | yConnect = npConnect[kptsData->yIndex]; 130 | cairo_move_to(cr, xKpt, yKpt); 131 | cairo_line_to(cr, xConnect, yConnect); 132 | } 133 | cairo_stroke(cr); 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /tasks/pose-estimation/cpp/custom_pose_decoder.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | #ifndef POSE_CUSTOM_POSE_DECODER_H_ 7 | #define POSE_CUSTOM_POSE_DECODER_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "logging.hpp" 16 | 17 | typedef struct { 18 | int kptSize = 17; 19 | int yIndex = 0; 20 | int xIndex = 1; 21 | int scoreIndex = 2; 22 | float scoreThreshold = 0.4; 23 | float npKpts[17][3]; 24 | std::string kptLabels[17] = { 25 | "nose", "left_eye", "right_eye", "left_ear", 26 | "right_ear", "left_shoulder", "right_shoulder", 27 | "left_elbow", "right_elbow", "left_wrist", "right_wrist", 28 | "left_hip", "right_hip", "left_knee", "right_knee", "left_ankle", 29 | "right_ankle"}; 30 | int kptConnect[17][3] = { 31 | {1, 2, -1}, {0, 3, -1}, {0, 4, -1}, {1, -1, -1}, {2, -1, -1}, 32 | {6, 7, 11}, {5, 8, 12}, {5, 9, -1}, {6, 10, -1}, {7, -1, -1}, 33 | {8, -1, -1}, {5, 12, 13}, {6, 11, 14}, {11, 15, -1}, 34 | {12, 16, -1}, {13, -1, -1}, {14, -1, -1}}; 35 | int inputDim; 36 | } DecoderData; 37 | 38 | 39 | void newDataCallback(GstElement* element, 40 | GstBuffer* buffer, 41 | gpointer user_data); 42 | 43 | 44 | void drawCallback(GstElement* overlay, 45 | cairo_t* cr, 46 | guint64 timestamp, 47 | guint64 duration, 48 | gpointer user_data); 49 | 50 | 51 | typedef struct { 52 | int size; 53 | float* bufferFP32; 54 | } BufferInfo; 55 | 56 | 57 | BufferInfo getTensorInfo(GstBuffer* buffer, int tensorIndex); 58 | 59 | 60 | void checkNumTensor(GstBuffer* buffer, int numTensor); 61 | 62 | #endif -------------------------------------------------------------------------------- /tasks/pose-estimation/pose_demo.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxp-imx/nxp-nnstreamer-examples/b8dafc13bb93b06826f9fc6b91056270cecc0fbd/tasks/pose-estimation/pose_demo.webp -------------------------------------------------------------------------------- /tasks/semantic-segmentation/README.md: -------------------------------------------------------------------------------- 1 | # Semantic Segmentation 2 | 3 | ## Overview 4 | Name | Implementation | Model | ML engine | Features 5 | --- | --- | --- | --- | --- 6 | [example_segmentation_deeplab_v3_tflite.cpp](./cpp/example_segmentation_deeplab_v3_tflite.cpp) | C++ | DeepLabV3 | TFLite | multifilesrc
gst-launch
7 | [example_segmentation_deeplab_v3_tflite.sh](./example_segmentation_deeplab_v3_tflite.sh) | Bash | DeepLabV3 | TFLite | multifilesrc
gst-launch
8 | 9 | ## DeepLabV3 segmentation 10 | ### Bash 11 | | Platforms | NPU | CPU | GPU | 12 | | ------------ | --- | --- | --- | 13 | | i.MX 8M Plus | :white_check_mark: | :white_check_mark: | :white_check_mark: | 14 | | i.MX 93 | :white_check_mark: | :white_check_mark: | :x: | 15 | | i.MX 95 | :x: | :white_check_mark: | :x: | 16 | 17 | The semantic segmentation demo in bash supports multiple backend (refers to above table), default value can be overriden by explicitly defining BACKEND variable, for instance: 18 | ```bash 19 | BACKEND=CPU ./tasks/semantic-segmentation/example_segmentation_deeplab_v3_tflite.sh 20 | ``` 21 | 22 | ### C++ 23 | | Platforms | NPU | CPU | GPU | 24 | | ------------ | --- | --- | --- | 25 | | i.MX 8M Plus | :white_check_mark: | :white_check_mark: | :white_check_mark: | 26 | | i.MX 93 | :white_check_mark: | :white_check_mark: | :x: | 27 | | i.MX 95 | :x: | :white_check_mark: | :white_check_mark: | 28 | 29 | C++ example script needs to be generated with [cross compilation](../). [setup_environment.sh](../tools/setup_environment.sh) script needs to be executed in [nxp-nnstreamer-examples](../) folder to define data paths: 30 | ```bash 31 | . ./tools/setup_environment.sh 32 | ``` 33 | 34 | It is possible to run the semantic segmentation demo inference on three different hardwares:
35 | Inference on NPU with the following script: 36 | ```bash 37 | ./build/semantic-segmentation/example_segmentation_deeplab_v3_tflite -p ${DEEPLABV3_QUANT} -f ${PASCAL_IMAGES} 38 | ``` 39 | For i.MX 93 NPU use vela converted model: 40 | ```bash 41 | ./build/semantic-segmentation/example_segmentation_deeplab_v3_tflite -p ${DEEPLABV3_QUANT_VELA} -f ${PASCAL_IMAGES} 42 | ``` 43 | Inference on CPU with the following script: 44 | ```bash 45 | ./build/semantic-segmentation/example_segmentation_deeplab_v3_tflite -p ${DEEPLABV3_QUANT} -f ${PASCAL_IMAGES} -b CPU 46 | ``` 47 | Quantized model is used for better inference performances on CPU.
48 | NOTE: inferences on i.MX8MPlus GPU have low performances, but are possible with the following script: 49 | ```bash 50 | ./build/semantic-segmentation/example_segmentation_deeplab_v3_tflite -p ${DEEPLABV3} -f ${PASCAL_IMAGES} -b GPU -n centeredReduced 51 | ``` 52 | The following execution parameters are available (Run ``` ./example_segmentation_deeplab_v3_tflite -h``` to see option details): 53 | 54 | Option | Description 55 | --- | --- 56 | -b, --backend | Use the selected backend (CPU, GPU, NPU)
default: NPU 57 | -n, --normalization | Use the selected normalization (none, centered, reduced, centeredReduced, castInt32, castuInt8)
default: none 58 | -p, --model_path | Use the selected model path 59 | -f, --images_file | Use the selected images file 60 | -d, --display_perf |Display performances, can specify time or freq 61 | -t, --text_color | Color of performances displayed, can choose between red, green, blue, and black
default: white 62 | -g, --graph_path | Path to store the result of the OpenVX graph compilation (only for i.MX8MPlus)
default: home directory 63 | 64 | Press ```Esc or ctrl+C``` to stop the execution of the pipeline. -------------------------------------------------------------------------------- /tasks/semantic-segmentation/cpp/example_segmentation_deeplab_v3_tflite.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024-2025 NXP 3 | * SPDX-License-Identifier: BSD-3-Clause 4 | */ 5 | 6 | /** 7 | * NNstreamer application for segmentation using tensorflow-lite. 8 | * The model used is deeplabv3_mnv2_dm05_pascal.tflite which can be retrieved from https://github.com/nxp-imx/nxp-nnstreamer-examples/blob/main/downloads/download.ipynb 9 | * 10 | * Pipeline: 11 | * multifilesrc -- jpegdec -- imxvideoconvert -- tee ----------------------------------------------------------------- 12 | * | | 13 | * | videomixer -- waylandsink 14 | * | | 15 | * --- tensor_converter -- tensor_transform -- tensor_filter -- tensor_decoder 16 | */ 17 | 18 | #include "common.hpp" 19 | 20 | #include 21 | #include 22 | 23 | #define OPTIONAL_ARGUMENT_IS_PRESENT \ 24 | ((optarg == NULL && optind < argc && argv[optind][0] != '-') \ 25 | ? (bool) (optarg = argv[optind++]) \ 26 | : (optarg != NULL)) 27 | 28 | 29 | typedef struct { 30 | std::filesystem::path modelPath; 31 | std::filesystem::path slideshowPath; 32 | std::string backend; 33 | std::string norm; 34 | bool time; 35 | bool freq; 36 | std::string textColor; 37 | char* graphPath; 38 | } ParserOptions; 39 | 40 | 41 | int cmdParser(int argc, char **argv, ParserOptions& options) 42 | { 43 | int c; 44 | int optionIndex; 45 | std::string perfDisplay; 46 | imx::Imx imx{}; 47 | static struct option longOptions[] = { 48 | {"help", no_argument, 0, 'h'}, 49 | {"backend", required_argument, 0, 'b'}, 50 | {"normalization", required_argument, 0, 'n'}, 51 | {"model_path", required_argument, 0, 'p'}, 52 | {"images_file", required_argument, 0, 'f'}, 53 | {"display_perf", optional_argument, 0, 'd'}, 54 | {"text_color", required_argument, 0, 't'}, 55 | {"graph_path", required_argument, 0, 'g'}, 56 | {0, 0, 0, 0} 57 | }; 58 | 59 | while ((c = getopt_long(argc, 60 | argv, 61 | "hb:n:p:l:f:d::t:g:", 62 | longOptions, 63 | &optionIndex)) != -1) { 64 | switch (c) 65 | { 66 | case 'h': 67 | std::cout << "Help Options:" << std::endl 68 | << std::setw(25) << std::left << " -h, --help" 69 | << std::setw(25) << std::left << "Show help options" 70 | << std::endl << std::endl 71 | << "Application Options:" << std::endl 72 | 73 | << std::setw(25) << std::left << " -b, --backend" 74 | << std::setw(25) << std::left 75 | << "Use the selected backend (CPU,GPU,NPU)" << std::endl 76 | 77 | << std::setw(25) << std::left << " -n, --normalization" 78 | << std::setw(25) << std::left 79 | << "Use the selected normalization" 80 | << " (none,centered,reduced,centeredReduced,castInt32,castuInt8)" << std::endl 81 | 82 | << std::setw(25) << std::left << " -p, --model_path" 83 | << std::setw(25) << std::left 84 | << "Use the selected model path" << std::endl 85 | 86 | << std::setw(25) << std::left << " -f, --images_file" 87 | << std::setw(25) << std::left 88 | << "Use the selected images path" << std::endl 89 | 90 | << std::setw(25) << std::left << " -d, --display_perf" 91 | << std::setw(25) << std::left 92 | << "Display performances, can specify time or freq" << std::endl 93 | 94 | << std::setw(25) << std::left << " -t, --text_color" 95 | << std::setw(25) << std::left 96 | << "Color of performances displayed," 97 | << " can choose between red, green, blue, and black (white by default)" << std::endl 98 | 99 | << std::setw(25) << std::left << " -g, --graph_path" 100 | << std::setw(25) << std::left 101 | << "Path to store the result of the OpenVX graph compilation (only for i.MX8MPlus)" << std::endl; 102 | return 1; 103 | 104 | case 'b': 105 | options.backend.assign(optarg); 106 | break; 107 | 108 | case 'n': 109 | options.norm.assign(optarg); 110 | break; 111 | 112 | case 'p': 113 | options.modelPath.assign(optarg); 114 | break; 115 | 116 | case 'f': 117 | options.slideshowPath.assign(optarg); 118 | break; 119 | 120 | case 'd': 121 | if (OPTIONAL_ARGUMENT_IS_PRESENT) 122 | perfDisplay.assign(optarg); 123 | 124 | if (perfDisplay == "freq") { 125 | options.freq = true; 126 | } else if (perfDisplay == "time") { 127 | options.time = true; 128 | } else { 129 | options.time = true; 130 | options.freq = true; 131 | } 132 | break; 133 | 134 | case 't': 135 | options.textColor.assign(optarg); 136 | break; 137 | 138 | case 'g': 139 | if (imx.socId() != imx::IMX8MP) { 140 | log_error("OpenVX graph compilation only for i.MX8MPlus\n"); 141 | return 1; 142 | } 143 | options.graphPath = optarg; 144 | break; 145 | 146 | default: 147 | break; 148 | } 149 | } 150 | return 0; 151 | } 152 | 153 | 154 | int main(int argc, char **argv) 155 | { 156 | // Initialize command line parser with default values 157 | ParserOptions options; 158 | options.backend = "NPU"; 159 | options.norm = "none"; 160 | options.time = false; 161 | options.freq = false; 162 | options.graphPath = getenv("HOME"); 163 | if (cmdParser(argc, argv, options)) 164 | return 0; 165 | 166 | imx::Imx imx{}; 167 | if (imx.isIMX95() && (options.backend == "NPU")) { 168 | log_error("Example can't run on NPU in i.MX95\n"); 169 | return 0; 170 | } 171 | 172 | // Initialize pipeline object 173 | GstPipelineImx pipeline; 174 | 175 | // Add slideshow to pipeline 176 | GstSlideshowImx slideshow(options.slideshowPath); 177 | slideshow.addSlideshowToPipeline(pipeline); 178 | 179 | // Create model object to get it's size for cropping 180 | TFliteModelInfos segmentation(options.modelPath, 181 | options.backend, 182 | options.norm); 183 | GstVideoImx gstvideoimx{}; 184 | gstvideoimx.videoTransform(pipeline, 185 | "", 186 | segmentation.getModelWidth(), 187 | segmentation.getModelHeight(), 188 | false); 189 | 190 | // Add a tee element for parallelization of tasks 191 | std::string teeName = "t"; 192 | pipeline.doInParallel(teeName); 193 | 194 | // Add a branch to tee element for inference and model post processing 195 | GstQueueOptions nnQueue = { 196 | .queueName = "thread-nn", 197 | .maxSizeBuffer = 2, 198 | .leakType = GstQueueLeaky::downstream, 199 | }; 200 | pipeline.addBranch(teeName, nnQueue); 201 | 202 | // Add model inference 203 | segmentation.addInferenceToPipeline(pipeline, "seg_filter"); 204 | 205 | // Add NNStreamer inference output decoding 206 | NNDecoder decoder; 207 | ImageSegmentOptions decOptions = { 208 | .modelName = ModeImageSegment::tfliteDeeplab, 209 | }; 210 | decoder.addImageSegment(pipeline, decOptions); 211 | 212 | // compositor class is not used because 213 | // it doesn't support 513x513 input dimension, 214 | // so videomixer is added manually. 215 | 216 | // Link decoder result to a video compositor 217 | std::string compositorName = "mix"; 218 | pipeline.addToPipeline(compositorName + ". "); 219 | 220 | // Add a branch to tee element to display result 221 | GstQueueOptions imgQueue = { 222 | .queueName = "thread-img", 223 | .maxSizeBuffer = 2, 224 | .leakType = GstQueueLeaky::downstream, 225 | }; 226 | pipeline.addBranch(teeName, imgQueue); 227 | 228 | // Add video compositing 229 | pipeline.addToPipeline("videomixer name=" 230 | + compositorName 231 | + " sink_1::alpha=0.4 " 232 | + "sink_0::alpha=1.0 background=3 ! " 233 | + "videoconvert ! "); 234 | 235 | // Display processed video 236 | GstVideoPostProcess postProcess; 237 | float scaleFactor = 15.0f/640; // Default font size is 15 pixels for a width of 640 238 | pipeline.enablePerfDisplay(options.freq, options.time, segmentation.getModelWidth() * scaleFactor, options.textColor); 239 | postProcess.display(pipeline, true); 240 | 241 | // Parse pipeline to GStreamer pipeline 242 | pipeline.parse(argc, argv, options.graphPath); 243 | 244 | // Run GStreamer pipeline 245 | pipeline.run(); 246 | 247 | return 0; 248 | } -------------------------------------------------------------------------------- /tasks/semantic-segmentation/example_segmentation_deeplab_v3_tflite.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2022-2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | set -x 7 | 8 | REALPATH="$(readlink -e "$0")" 9 | BASEDIR="$(dirname "${REALPATH}")/../.." 10 | MODELS_DIR="${BASEDIR}/downloads/models/semantic-segmentation" 11 | MEDIA_DIR="${BASEDIR}/downloads/media" 12 | IMAGES_DIR="${MEDIA_DIR}/pascal_voc_2012_images" 13 | REQUIRED_CAMERA=0 14 | 15 | source "${BASEDIR}/common/common_utils.sh" 16 | source "${BASEDIR}/tasks/semantic-segmentation/segmentation_utils.sh" 17 | 18 | setup_env 19 | 20 | case "${GPU2D_API}" in 21 | G2D) 22 | # g2d-based 23 | SCALE_FORMAT="RGBA" 24 | ;; 25 | PXP) 26 | # pxp-based 27 | SCALE_FORMAT="BGR" 28 | ;; 29 | *) 30 | # cpu-based 31 | SCALE_FORMAT="" 32 | ;; 33 | esac 34 | 35 | # model and framework dependant variables 36 | declare -A MODEL_BACKEND 37 | declare -A MODEL_BACKEND_NPU 38 | MODEL_BACKEND_NPU[IMX8MP]="${MODELS_DIR}/deeplabv3_mnv2_dm05_pascal_quant_uint8_float32.tflite" 39 | MODEL_BACKEND_NPU[IMX93]="${MODELS_DIR}/deeplabv3_mnv2_dm05_pascal_quant_uint8_float32_vela.tflite" 40 | 41 | MODEL_BACKEND[CPU]="${MODELS_DIR}/deeplabv3_mnv2_dm05_pascal_quant_uint8_float32.tflite" 42 | MODEL_BACKEND[GPU]="${MODELS_DIR}/deeplabv3_mnv2_dm05_pascal.tflite" 43 | MODEL_BACKEND[NPU]=${MODEL_BACKEND_NPU[${IMX}]} 44 | MODEL=${MODEL_BACKEND[${BACKEND}]} 45 | 46 | MODEL_WIDTH=513 47 | MODEL_HEIGHT=513 48 | 49 | FRAMEWORK="tensorflow-lite" 50 | 51 | # tensor filter configuration 52 | FILTER_COMMON="tensor_filter framework=${FRAMEWORK} model=${MODEL}" 53 | 54 | declare -A FILTER_BACKEND_NPU 55 | FILTER_BACKEND_NPU[IMX8MP]=" custom=Delegate:External,ExtDelegateLib:libvx_delegate.so ! " 56 | FILTER_BACKEND_NPU[IMX93]=" custom=Delegate:External,ExtDelegateLib:libethosu_delegate.so ! " 57 | 58 | declare -A FILTER_BACKEND 59 | FILTER_BACKEND[CPU]="${FILTER_COMMON}" 60 | FILTER_BACKEND[CPU]+=" custom=Delegate:XNNPACK,NumThreads:$(nproc --all) !" 61 | FILTER_BACKEND[GPU]="${FILTER_COMMON}" 62 | FILTER_BACKEND[GPU]+=" custom=Delegate:External,ExtDelegateLib:libvx_delegate.so ! " 63 | FILTER_BACKEND[NPU]="${FILTER_COMMON}" 64 | FILTER_BACKEND[NPU]+=${FILTER_BACKEND_NPU[${IMX}]} 65 | TENSOR_FILTER=${FILTER_BACKEND[${BACKEND}]} 66 | 67 | # tensor preprocessing configuration: normalize video for float input models 68 | declare -A PREPROCESS_BACKEND 69 | PREPROCESS_BACKEND[CPU]="" 70 | PREPROCESS_BACKEND[GPU]="tensor_transform mode=arithmetic option=typecast:float32,add:-127.5,div:127.5 ! " 71 | PREPROCESS_BACKEND[NPU]="" 72 | TENSOR_PREPROCESS=${PREPROCESS_BACKEND[${BACKEND}]} 73 | 74 | # tensor decoder configuration: tflite-deeplab (no maxargs on classes) 75 | TENSOR_DECODER="tensor_decoder mode=image_segment option1=tflite-deeplab ! " 76 | 77 | gst_exec_segmentation 78 | 79 | -------------------------------------------------------------------------------- /tasks/semantic-segmentation/segmentation_demo.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxp-imx/nxp-nnstreamer-examples/b8dafc13bb93b06826f9fc6b91056270cecc0fbd/tasks/semantic-segmentation/segmentation_demo.webp -------------------------------------------------------------------------------- /tasks/semantic-segmentation/segmentation_utils.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2022-2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | # segmentation pipeline g2d accelerated for video rescaling 7 | # video compositor elements are not working for 513x513 - nnstreamer bug 8 | # https://github.com/nnstreamer/nnstreamer/issues/3866 9 | # fall back to regular videomixer 10 | 11 | function gst_exec_segmentation { 12 | 13 | # accelerated video scaling before inferencing 14 | local VIDEO_SCALE=$(accelerated_video_scale_str ${MODEL_WIDTH} ${MODEL_HEIGHT} ${SCALE_FORMAT}) 15 | 16 | gst-launch-1.0 \ 17 | multifilesrc location="${IMAGES_DIR}/image%04d.jpg" loop=true caps=image/jpeg,framerate=1/2 ! \ 18 | jpegdec ! \ 19 | ${VIDEO_SCALE} \ 20 | tee name=t \ 21 | t. ! queue name=thread-nn max-size-buffers=2 leaky=2 ! \ 22 | videoconvert ! video/x-raw,format=RGB ! \ 23 | tensor_converter ! \ 24 | ${TENSOR_PREPROCESS} \ 25 | ${TENSOR_FILTER} \ 26 | ${TENSOR_DECODER} \ 27 | videoconvert ! mix. \ 28 | t. ! queue name=thread-img max-size-buffers=2 ! \ 29 | videomixer name=mix sink_1::alpha=0.4 sink_0::alpha=1.0 background=3 ! \ 30 | waylandsink 31 | } -------------------------------------------------------------------------------- /tools/setup_environment.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2024-2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | export REALPATH="$(readlink -f -- "$0")" 7 | BASEDIR="$(dirname "${REALPATH}")" 8 | MODELS_DIR="${BASEDIR}/downloads/models" 9 | MEDIA_DIR="${BASEDIR}/downloads/media" 10 | 11 | # Define media 12 | export POWER_JUMP_VIDEO="${MEDIA_DIR}/movies/Conditioning_Drill_1-_Power_Jump.webm.480p.vp9.webm" 13 | export PASCAL_IMAGES="${MEDIA_DIR}/pascal_voc_2012_images/image%04d.jpg" 14 | export SAVE_VIDEO_PATH="/root/video.mkv" 15 | 16 | # Define cameras path for two cameras classification demo 17 | export CAM1_PATH="/dev/video0" 18 | export CAM2_PATH="/dev/video2" 19 | 20 | # Define classification data path 21 | CLASSIFICATION_DIR="${MODELS_DIR}/classification" 22 | export MOBILENETV1_LABELS="${CLASSIFICATION_DIR}/labels_mobilenet_quant_v1_224.txt" 23 | export MOBILENETV1="${CLASSIFICATION_DIR}/mobilenet_v1_1.0_224.tflite" 24 | export MOBILENETV1_QUANT="${CLASSIFICATION_DIR}/mobilenet_v1_1.0_224_quant_uint8_float32.tflite" 25 | export MOBILENETV1_QUANT_VELA="${CLASSIFICATION_DIR}/mobilenet_v1_1.0_224_quant_uint8_float32_vela.tflite" 26 | export MOBILENETV1_QUANT_NEUTRON="${CLASSIFICATION_DIR}/mobilenet_v1_1.0_224_quant_uint8_float32_neutron.tflite" 27 | 28 | # Define depth data path 29 | DEPTH_DIR="${MODELS_DIR}/monocular-depth-estimation" 30 | export MIDASV2="${DEPTH_DIR}/midas_2_1_small_int8_quant.tflite" 31 | export MIDASV2_VELA="${DEPTH_DIR}/midas_2_1_small_int8_quant_vela.tflite" 32 | 33 | # Define detection data path 34 | DETECTION_DIR="${MODELS_DIR}/object-detection" 35 | export COCO_LABELS="${DETECTION_DIR}/coco_labels_list.txt" 36 | export MOBILENETV2_BOXES="${DETECTION_DIR}/box_priors.txt" 37 | export MOBILENETV2="${DETECTION_DIR}/ssdlite_mobilenet_v2_coco_no_postprocess.tflite" 38 | export MOBILENETV2_QUANT="${DETECTION_DIR}/ssdlite_mobilenet_v2_coco_quant_uint8_float32_no_postprocess.tflite" 39 | export MOBILENETV2_QUANT_VELA="${DETECTION_DIR}/ssdlite_mobilenet_v2_coco_quant_uint8_float32_no_postprocess_vela.tflite" 40 | export MOBILENETV2_QUANT_NEUTRON="${DETECTION_DIR}/ssdlite_mobilenet_v2_coco_quant_uint8_float32_no_postprocess_neutron.tflite" 41 | 42 | # Define face data path 43 | FACE_DIR="${MODELS_DIR}/face-processing" 44 | export ULTRAFACE_QUANT="${FACE_DIR}/ultraface_slim_uint8_float32.tflite" 45 | export ULTRAFACE_QUANT_VELA="${FACE_DIR}/ultraface_slim_uint8_float32_vela.tflite" 46 | export ULTRAFACE_QUANT_NEUTRON="${FACE_DIR}/ultraface_slim_uint8_float32_neutron.tflite" 47 | export EMOTION_QUANT="${FACE_DIR}/emotion_uint8_float32.tflite" 48 | export EMOTION_QUANT_VELA="${FACE_DIR}/emotion_uint8_float32_vela.tflite" 49 | 50 | # Define pose data path 51 | POSE_DIR="${MODELS_DIR}/pose-estimation" 52 | export MOVENET="${POSE_DIR}/movenet_single_pose_lightning.tflite" 53 | export MOVENET_QUANT="${POSE_DIR}/movenet_quant.tflite" 54 | export MOVENET_QUANT_VELA="${POSE_DIR}/movenet_quant_vela.tflite" 55 | 56 | # Define segmentation data path 57 | SEGMENTATION_DIR="${MODELS_DIR}/semantic-segmentation" 58 | export DEEPLABV3="${SEGMENTATION_DIR}/deeplabv3_mnv2_dm05_pascal.tflite" 59 | export DEEPLABV3_QUANT="${SEGMENTATION_DIR}/deeplabv3_mnv2_dm05_pascal_quant_uint8_float32.tflite" 60 | export DEEPLABV3_QUANT_VELA="${SEGMENTATION_DIR}/deeplabv3_mnv2_dm05_pascal_quant_uint8_float32_vela.tflite" -------------------------------------------------------------------------------- /tools/upload.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2022, 2025 NXP 4 | # SPDX-License-Identifier: BSD-3-Clause 5 | 6 | set -x 7 | 8 | REALPATH="$(readlink -e "$0")" 9 | BASEDIR="$(dirname "${REALPATH}")/.." 10 | BASEDIR="$(realpath ${BASEDIR})" 11 | 12 | function usage { 13 | echo "Upload models and artifact to the target for later execution." 14 | echo "syntax:" 15 | echo "upload @ [remote path]" 16 | echo "example:" 17 | echo "upload root@192.168.1.52" 18 | } 19 | 20 | REMOTE="$1" 21 | if [ -z "${REMOTE}" ]; then 22 | usage 23 | exit 1 24 | fi 25 | 26 | REMOTE_PATH=$2 27 | if [ -z "${REMOTE_PATH}" ]; then 28 | REMOTE_USER="${REMOTE%@*}" 29 | REMOTE_PATH="/home/${REMOTE_USER}" 30 | fi 31 | REMOTE_DIR="${REMOTE_PATH}/"$(basename ${BASEDIR}) 32 | 33 | # exclude unnecessary directories 34 | ssh "${REMOTE}" "rm -rf ${REMOTE_DIR} && mkdir -p ${REMOTE_DIR}" 35 | find "${BASEDIR}" -maxdepth 1 -mindepth 1 -type d \ 36 | ! -path '*/.git' \ 37 | ! -path '*/samples' \ 38 | ! -path '*/tmp' \ 39 | ! -path '*/tools' \ 40 | -exec scp -r {} ${REMOTE}:${REMOTE_DIR} \; 41 | --------------------------------------------------------------------------------