├── .gitignore ├── CMakeLists.txt ├── README.md ├── apps ├── CMakeLists.txt ├── deepstream_test1 │ ├── CMakeLists.txt │ ├── deepstream_test1.cpp │ └── deepstream_test1_config.txt ├── gstreamer │ ├── CMakeLists.txt │ └── gstreamer.cpp ├── license_plate_recognition │ ├── CMakeLists.txt │ ├── configs │ │ ├── plate_detection.txt │ │ ├── text_recognition.txt │ │ ├── tracker_config.yml │ │ ├── us_char_dict.txt │ │ ├── vehicle_detection.txt │ │ └── vehicle_tracking.txt │ └── lpr.cpp ├── save_image_deepstream │ ├── CMakeLists.txt │ ├── save_image_deepstream.cpp │ └── save_image_deepstream_config.txt ├── tiny_yolo2_deepstream │ ├── CMakeLists.txt │ ├── labels.txt │ ├── tiny_yolo2_deepstream.cpp │ └── tiny_yolo2_deepstream.txt └── using_bin │ ├── CMakeLists.txt │ ├── using_bin.cpp │ └── using_bin_config.txt ├── doc ├── bin_element.md ├── create_container.md ├── deepstream_metadata.md ├── deepstream_test1.md ├── deepstream_yolo.md ├── install_cmake.md ├── many_models.md ├── run_nvidia_deepstream_example.md ├── save_image_deepstream.md ├── sourcebin.md ├── tee_element.md ├── tensorrt_gstreamer_plugin.md └── tracking.md ├── images ├── bin-element.png ├── bin.jpg ├── element.png └── meta_data.jpg ├── include ├── deepstream_test1 │ └── deepstream_test1.hpp ├── gstreamer │ └── gstreamer_example.hpp ├── license_plate_recognition │ └── license_plate_recognition.hpp ├── save_image_deepstream │ └── save_image_deepstream.hpp ├── tiny_yolo2_deepstream │ └── tiny_yolo2_deepstream.hpp └── using_bin │ └── using_bin.hpp ├── scripts └── run.sh └── src ├── CMakeLists.txt ├── deepstream_test1 └── deepstream_test1.cpp ├── gstreamer └── gstreamer_example.cpp ├── license_plate_recognition ├── license_plate_recognition.cpp ├── nvinfer_custom_lpr_parser │ ├── Makefile │ └── nvinfer_custom_lpr_parser.cpp └── tao_converter │ ├── Readme.txt │ ├── convert.sh │ └── tao-converter ├── save_image_deepstream └── save_image_deepstream.cpp ├── tiny_yolo2_deepstream ├── custom_bbox_parser │ ├── Makefile │ └── nvdsparsebbox_tiny_yolo.cpp └── tiny_yolo2_deepstream.cpp └── using_bin └── using_bin.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | build 34 | models 35 | .cache 36 | .vscode -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10.2) 2 | project(deepstream_from_scratch VERSION 0.1.0) 3 | 4 | add_subdirectory(src) 5 | add_subdirectory(apps) 6 | 7 | set(CPACK_PROJECT_NAME ${PROJECT_NAME}) 8 | set(CPACK_PROJECT_VERSION ${PROJECT_VERSION}) 9 | include(CPack) 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deepstream_from_scratch 2 | Repo này mình viết ra với mục đích chính chia sẻ kiến thức Deepstream bằng tiếng Việt nên nhiều chỗ diễn giải hơi khó hiểu so với đọc tài liệu gốc bằng tiếng Anh. Và mình cũng chỉ tập trung vào việc giải thích Deepstream nên không chú ý đến việc code convension, nên các bạn thông cảm code hơi bẩn :)) 3 | 4 | 1. [Chạy demo một số ví dụ Nvidia deepstream](./doc/run_nvidia_deepstream_example.md) 5 | 2. [Tensorrt và Gstreamer plugin](./doc/tensorrt_gstreamer_plugin.md) 6 | 3. [Chương trình deepstream cơ bản](./doc/deepstream_test1.md) 7 | 4. [Tiny yolov2 deepstream](./doc/deepstream_yolo.md) 8 | 5. [Metadata trong deestream](./doc/deepstream_metadata.md) 9 | 6. [Lưu hình ảnh trong deepstream](./doc/save_image_deepstream.md) 10 | 7. [Sử dụng bin element trong deepstream](./doc/bin_element.md) 11 | 8. [Sử dụng tee element trong deepstream](./doc/tee_element.md) 12 | 9. [Sourcebin trong deepstream](./doc/sourcebin.md) 13 | 10. [Tracking trong deepstream](./doc/tracking.md) 14 | 11. [Hai model nối tiếp nhau trong deepstream](./doc/many_models.md) 15 | ... -------------------------------------------------------------------------------- /apps/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(deepstream_test1) 2 | add_subdirectory(gstreamer) 3 | add_subdirectory(tiny_yolo2_deepstream) 4 | add_subdirectory(save_image_deepstream) 5 | add_subdirectory(using_bin) -------------------------------------------------------------------------------- /apps/deepstream_test1/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(deepstream_test1 deepstream_test1.cpp) 2 | target_compile_features(deepstream_test1 PUBLIC cxx_std_17) 3 | target_link_libraries(deepstream_test1 PUBLIC deep_stream) -------------------------------------------------------------------------------- /apps/deepstream_test1/deepstream_test1.cpp: -------------------------------------------------------------------------------- 1 | #include "deepstream_test1/deepstream_test1.hpp" 2 | #include 3 | 4 | int main(int argc, char *argv[]) 5 | { 6 | std::cout<<"Hello_world"< 2 | #include "gstreamer/gstreamer_example.hpp" 3 | 4 | int main(int argc, char *argv[]) 5 | { 6 | std::cout<<"Hello_world"<10. Note: this value should account for the targets being tracked in shadow mode as well. Max value depends on the GPU memory capacity 10 | 11 | filterLr: 0.075 #11 #11 #0.175 #0.11 # learning rate for DCF filter in exponential moving average. Valid Range: [0.0, 1.0] 12 | gaussianSigma: 0.75 #0.75 #0.75 #0.75 # Standard deviation for Gaussian for desired response when creating DCF filter 13 | 14 | minDetectorConfidence: 0.0 # If the confidence of a detector bbox is lower than this, then it won't be considered for tracking 15 | minTrackerConfidence: 0.7 # If the confidence of an object tracker is lower than this on the fly, then it will be tracked in shadow mode. Valid Range: [0.0, 1.0] 16 | minTargetBboxSize: 5 # If the width or height of the bbox size gets smaller than this threshold, the target will be terminated 17 | 18 | featureImgSizeLevel: 2 # Size of a feature image. Valid range: {1, 2, 3, 4, 5}, from the smallest to the largest 19 | SearchRegionPaddingScale: 1 # Search region size. Determines how large the search region should be scaled from the target bbox. Valid range: {1, 2, 3}, from the smallest to the largest 20 | 21 | maxShadowTrackingAge: 30 # Max length of shadow tracking (the shadow tracking age is incremented when (1) there's detector input yet no match or (2) tracker confidence is lower than minTrackerConfidence). Once reached, the tracker will be terminated. 22 | probationAge: 3 # Once the tracker age (incremented at every frame) reaches this, the tracker is considered to be valid 23 | earlyTerminationAge: 1 # Early termination age (in terms of shadow tracking age) during the probation period 24 | 25 | # thresholds for data association 26 | minMatchingScore4Overall: 0.0 27 | minMatchingScore4Motion: 0.5 28 | minMatchingScore4Iou: 0.1 29 | minMatchingScore4VisualSimilarity: 0.2 30 | minTrackingConfidenceDuringInactive: 0.9 31 | 32 | matchingScoreWeight4VisualSimilarity: 0.8 # Weight for the visual similarity (in terms of correlation response ratio) 33 | matchingScoreWeight4Motion: 0.0 # Weight for the Size-similarity score 34 | matchingScoreWeight4Iou: 0.1 # Weight for the IOU score 35 | matchingScoreWeight4Age: 0.1 # Weight for the tracker age 36 | 37 | minDetectorBboxVisibilityTobeTracked: 0.0 38 | minVisibiilty4Tracking: 0.0 39 | 40 | bboxPaddingScaleForAssociation: 0.0 # Padding scale for bboxes when computing IOU for data association 41 | visibilityRoiFactor: 0.00 # Define the ROI of image where tracking and detection is considered to be valid. If visibilityRoiFactor = 0.05, it would shrink the ROI by 5% from the image boundary 42 | 43 | trackExponentialSmoothingLr_loc: 0.5 # Learning rate for new location 44 | trackExponentialSmoothingLr_scale: 0.3 # Learning rate for new scale 45 | trackExponentialSmoothingLr_velocity: 0.05 # Learning rate for new velocity 46 | -------------------------------------------------------------------------------- /apps/license_plate_recognition/configs/us_char_dict.txt: -------------------------------------------------------------------------------- 1 | 0 2 | 1 3 | 2 4 | 3 5 | 4 6 | 5 7 | 6 8 | 7 9 | 8 10 | 9 11 | A 12 | B 13 | C 14 | D 15 | E 16 | F 17 | G 18 | H 19 | I 20 | J 21 | K 22 | L 23 | M 24 | N 25 | P 26 | Q 27 | R 28 | S 29 | T 30 | U 31 | V 32 | W 33 | X 34 | Y 35 | Z 36 | -------------------------------------------------------------------------------- /apps/license_plate_recognition/configs/vehicle_detection.txt: -------------------------------------------------------------------------------- 1 | [property] 2 | gpu-id=0 3 | net-scale-factor=0.0039215697906911373 4 | tlt-model-key=tlt_encode 5 | tlt-encoded-model=../../../models/lpr/vehicle_det/resnet18_trafficcamnet_pruned.etlt 6 | int8-calib-file=../../../models/lpr/vehicle_det/trafficnet_int8.bin 7 | model-engine-file=../../../models/lpr/vehicle_det/resnet18_trafficcamnet_pruned.etlt_b1_gpu0_int8.engine 8 | input-dims=3;544;960;0 9 | uff-input-blob-name=input_1 10 | batch-size=1 11 | process-mode=1 12 | model-color-format=0 13 | ## 0=FP32, 1=INT8, 2=FP16 mode 14 | network-mode=1 15 | num-detected-classes=4 16 | interval=0 17 | gie-unique-id=1 18 | output-blob-names=output_bbox/BiasAdd;output_cov/Sigmoid 19 | 20 | [class-attrs-all] 21 | pre-cluster-threshold=0.2 22 | group-threshold=1 23 | ## Set eps=0.7 and minBoxes for cluster-mode=1(DBSCAN) 24 | eps=0.2 25 | #minBoxes=3 26 | -------------------------------------------------------------------------------- /apps/license_plate_recognition/configs/vehicle_tracking.txt: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a 5 | # copy of this software and associated documentation files (the "Software"), 6 | # to deal in the Software without restriction, including without limitation 7 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | # and/or sell copies of the Software, and to permit persons to whom the 9 | # Software is furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | # DEALINGS IN THE SOFTWARE. 21 | ################################################################################ 22 | 23 | # Mandatory properties for the tracker: 24 | # tracker-width: needs to be multiple of 32 for NvDCF 25 | # tracker-height: needs to be multiple of 32 for NvDCF 26 | # gpu-id 27 | # ll-lib-file: path to low-level tracker lib 28 | # ll-config-file: required for NvDCF, optional for KLT and IOU 29 | # 30 | [tracker] 31 | tracker-width=640 32 | tracker-height=384 33 | gpu-id=0 34 | ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvdcf.so 35 | ll-config-file=tracker_config.yml 36 | enable-batch-process=1 37 | -------------------------------------------------------------------------------- /apps/license_plate_recognition/lpr.cpp: -------------------------------------------------------------------------------- 1 | #include "license_plate_recognition/license_plate_recognition.hpp" 2 | #include 3 | 4 | int main(int argc, char *argv[]) 5 | { 6 | std::cout<<"Hello_world"< 3 | 4 | int main(int argc, char *argv[]) 5 | { 6 | std::cout<<"Hello_world"< 3 | 4 | int main(int argc, char *argv[]) 5 | { 6 | std::cout<<"Hello_world"< 3 | 4 | int main(int argc, char *argv[]) 5 | { 6 | std::cout<<"Hello_world"< const& outputLayersInfo, 13 | NvDsInferNetworkInfo const& networkInfo, 14 | NvDsInferParseDetectionParams const& detectionParams, 15 | std::vector& objectList); 16 | ``` 17 | - Trong đó: 18 | - outputLayersInfo: Là một vector chứa kết quả model sau khi inference 19 | - networkInfo: Thông tin về model, ví dụ: input_shape... 20 | - detectionParams: Một số thông tin config 21 | - objectList: Chứa thông tin sau khi post-process cụ thể: bbox, class_id, score... 22 | - Việc quan trọng là xử lý `outputLayersInfo` để thu được thông tin detection và đẩy vào `objectList`. Sau đó, deepstream sẽ tự động tạo các metadata ứng với thông tin trong `objectList` 23 | 24 | # Giải thích source code 25 | - Ví dụ này hoàn toàn giống với [deepstream_test1](doc/deepstream_test1.md), chỉ thay đổi duy nhất model AI, nên do đó sẽ thay đổi plugin để post-process model. Trong ví dụ trước sử dụng plugin mặc địch của deepstream viết cho model đó. 26 | # Chạy chương trình 27 | - build custom_bbox_parser 28 | ``` 29 | cd ./src/tiny_yolo2_deepstream/custom_bbox_parser 30 | make 31 | ``` 32 | - Chạy chương trình 33 | ``` 34 | rm -rf build 35 | mkdir build 36 | cd build 37 | cmake .. 38 | make 39 | cd .. 40 | ./build/apps/tiny_yolo2_deepstream/tiny_yolo2_deepstream /opt/nvidia/deepstream/deepstream-5.1/samples/streams/sample_720p.h264 41 | ``` -------------------------------------------------------------------------------- /doc/install_cmake.md: -------------------------------------------------------------------------------- 1 | ``` 2 | apt-get install software-properties-common 3 | add-apt-repository ppa:george-edison55/cmake-3.x 4 | apt-get update 5 | apt-get install cmake 6 | apt-get upgrade 7 | ``` 8 | -------------------------------------------------------------------------------- /doc/many_models.md: -------------------------------------------------------------------------------- 1 | will be updated soon -------------------------------------------------------------------------------- /doc/run_nvidia_deepstream_example.md: -------------------------------------------------------------------------------- 1 | # Cài đặt môi trường 2 | - Để thuận tiện mình sẽ hướng dẫn chạy các ví dụ Deepstream trong Docker 3 | - [Tải Deepstream image](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_docker_containers.html) 4 | Có 5 bản image khác nhau phù hợp với mục đích sử dụng, với bản deepstream:5.1-21.02-triton đầy đủ 5 | ``` 6 | $ docker pull nvcr.io/nvidia/deepstream:5.1-21.02-triton 7 | $ docker run -it --rm --gpus all --network host nvcr.io/nvidia/deepstream:5.1-21.02-triton /bin/bash 8 | ``` 9 | - Source code và các thư viện của deepstream ở ``/opt/nvidia/deepstream/deepstream-5.1`` 10 | 11 | # Chạy demo deepstream-test1 12 | - deepstream-test1: Là một ví dụ đơn giản của bài toán object detection trên Deepstream, với 4 class "Vehicle , RoadSign, TwoWheeler, 13 | Person" 14 | - Vào thư mục chứa source_code 15 | ``` 16 | $ cd /opt/nvidia/deepstream/deepstream-5.1/sources/apps/sample_apps/deepstream-test1 17 | $ deepstream-test1-app ../../../../samples/streams/sample_720p.h264 18 | ``` 19 | - Trong lần chạy đầu tiên chúng ta thấy chương trình chỉ in ra màn hình `Frame Number = 1 Number of objects = 6 Vehicle Count = 4 Person Count = 2` một lần và báo lỗi. Nguyên nhân là chương trình sử dụng `element nveglglessink` (sẽ nói rõ ở phần sau) dùng để show lên màn hình nhưng hiện tại chúng ta đang ở trong container. 20 | - Để giải quyết lỗi thực hiện như sau 21 | ``` 22 | $ vim deepstream_test1_app.c 23 | ``` 24 | - Tìm đến dòng code `sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer");` thay thế `nveglglessink` bằng `fakesink` và lưu file 25 | - Build lại chương trình 26 | ``` 27 | CUDA_VER=11.1 make install 28 | CUDA_VER=11.1 make clear 29 | ``` 30 | - Chạy lại chương trình và không còn lỗi :))) 31 | ``` 32 | $ deepstream-test1-app ../../../../samples/streams/sample_720p.h264 33 | ``` 34 | # Chạy demo với deepstream-app 35 | ``` 36 | $ deepstream-app -c ../../../../samples/configs/deepstream-app/source30_1080p_dec_infer-resnet_tiled_display_int8.txt 37 | ``` 38 | - Tương tự như ví dụ trên chương trình sẽ báo lỗi `** ERROR: : Could not open X Display` là do không có thiết bị để hiển thị (hiểu nôm na như thế) 39 | - Để giải quyết bug này thì chúng ta sửa file config của chương trình như sau 40 | ``` 41 | $ vim /opt/nvidia/deepstream/deepstream-5.1/samples/configs/deepstream-app/source30_1080p_dec_infer-resnet_tiled_display_int8.txt 42 | ``` 43 | ``` 44 | [sink0] 45 | enable=1 46 | #Type - 1=FakeSink 2=EglSink 3=File 47 | type=2 48 | sync=1 49 | source-id=0 50 | gpu-id=0 51 | nvbuf-memory-type=0 52 | ``` 53 | - Sửa enable=0 54 | - - Chạy lại chương trình và thấy các thông tin in ra màn hình là FPS của trên từng nguồn 55 | - Với config `source30_1080p_dec_infer-resnet_tiled_display_int8.txt`: chạy chương trình object detection với 30 video cùng lúc 56 | - Các bạn có thể thử với những config khác 57 | - Trong phần này mình đã hướng dẫn chạy 2 ví dụ Deepstream, để hiểu rõ các ví dụ khác đọc thêm [tại đây](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_C_Sample_Apps.html) -------------------------------------------------------------------------------- /doc/save_image_deepstream.md: -------------------------------------------------------------------------------- 1 | # Lưu hình ảnh trong deepstream 2 | - Trong phần này mình sẽ hướng dẫn lưu hình trong deepstream thông qua xử lý metadata trong hàm `probe` 3 | - Môi trường: Cần cài đặt thêm Opencv cho GPU 4 | - Chương trình deepstream dựa trên ví dụ [deepstream_test1](https://github.com/tienln4/deepstream_from_scratch/tree/main/src/deepstream_test1) 5 | - Dòng 49: Lấy thông tin từ surface 6 | - Dòng 50 - 53: Lấy các thông tin về hình ảnh trong surface 7 | - `data_ptr`: Lưu địa chỉ ảnh stream trên GPU 8 | - src_height, src_width: Kích thước ảnh (Không giữ kích thước như đầu vào, kích thước này đúng bằng thông số cài đặt cho streammuxer) 9 | - Dòng 55: Tạo một ảnh trên GPU từ các thông tin trên 10 | - Dòng 56: Tải ảnh xuống CPU và lưu 11 | - Dòng 65 - 73: Vẽ các bbox có `unique_component_id = 1` 12 | 13 | # Chạy chương trình 14 | - Tham khảo [ở đây](https://github.com/tienln4/deepstream_from_scratch/blob/main/doc/deepstream_test1.md) 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /doc/sourcebin.md: -------------------------------------------------------------------------------- 1 | will be updated soon -------------------------------------------------------------------------------- /doc/tee_element.md: -------------------------------------------------------------------------------- 1 | will be updated soon -------------------------------------------------------------------------------- /doc/tensorrt_gstreamer_plugin.md: -------------------------------------------------------------------------------- 1 | Hai thành phần quan trọng trong Deepstream là Tensorrt và Gstreamer plug in 2 | Trong phần này mình sẽ giới thiệu một số thành phần cơ bản của Gstreamer và cách dùng. Đọc thêm tài liệu của Gstreamer [tại đây](https://gstreamer.freedesktop.org/documentation/tutorials/index.html?gi-language=c) 3 | 4 | # TensorRT 5 | - TensorRT là bộ công cụ phát triển phần mềm (SDK) học sâu (deep learning) do Nvidia phát triển. Nó cho phép các ứng dụng hoạt động nhanh hơn so với training model (Pytorch, Caffe, ...) 6 | - Để convert training model sang inference model (Tensorrt) tham khảo [tại đây](https://github.com/linghu8812/tensorrt_inference) 7 | 8 | # Gstreamer plugin 9 | 10 | - GStreamer là một pipeline-based multimedia framework (Nền tảng xử lý đa phương tiện dựa trên đường ống). 11 | - GStreamer's development framework giúp bạn có thể viết bất kỳ loại ứng dụng đa phương tiện truyền trực tuyến nào với việc tối ưu xử lý dữ liệu 12 | - Một số ví dụ về sử dụng Gstreamer viết bằng C, tham khảo [tại đây](https://gstreamer.freedesktop.org/documentation/tutorials/basic/hello-world.html?gi-language=c) 13 | - Gstreamer hoạt động bằng cách ghép nối một số Element nhất định với nhau để thực hiện một ứng dụng nào đó, với mỗi Element sẽ đảm nhiệm một chức năng xử lý dữ liệu. Trong phần này mình sẽ giới thiệu một số Element quan trọng để xây dựng một pineline đọc ảnh từ máy tính đơn giản 14 | 15 | ## Element 16 | - Là đối tượng quan trọng nhất trong GStreamer. Element là khối xây dựng cơ bản cho một đường ống (pineline). 17 | - Với việc lập trình ứng dụng thì Element được xem như một hộp đen, đặt một thứ gì đó ở đầu vào và nhận một thứ khác xuất hiện ở đầu ra. Điều cần làm là hiểu rõ chức năng của các Element và ghép các Element lại với nhau để thực hiện một mục đích nào đó 18 | - Cấu tạo của một element 19 | 20 | ![alt](images/../../images/element.png) 21 | 22 | - sink (sink of element): nhận dữ liệu từ Element trước đó (Nơi để hứng dữ liệu) 23 | - src (src of element): Chứa dữ liệu của Element đó sau khi được xử lý 24 | - Ngoại trừ một số Element đặc biệt thì các Element đều có sink và src của nó 25 | - đối với ``source element`` chỉ có src mà không có sink. ngược lại, với ``sink element`` chỉ có sink mà không có src 26 | - Trong hình là một filter element có sink (bên tay trái) và src (bên tay phải) 27 | 28 | Để tạo một Element 29 | ``` 30 | GstElement * element; 31 | element = gst_element_factory_create ("filter", "fiter_name"); 32 | ``` 33 | 34 | - Các Element liên kết với nhau thông qua Pad (Pad cũng là một object của gstreamer) 35 | - Tương tự, với mỗi Element có sink_pad (bên tay trái) và src_pad (bên tay phải) 36 | 37 | Liên kết các Element với nhau 38 | - Hầu hết các element đều có thể liên kết với nhau thông qua câu lệnh (ngoại trừ một số Element như streamuxer) 39 | 40 | ``` 41 | gst_element_link_many (source, filter, sink, NULL) 42 | //or 43 | gst_element_link(source, filter) 44 | ``` 45 | ## Bins 46 | - Bin là một thùng chứa (container) các element, có thể đóng gói nhiều element vào một Bin để thực hiện một chức năng nhất định. Về bản chất Bin cũng chính là một Element nên nó có tất cả những xử lý của một element. 47 | 48 | ![alt](images/../../images/bin-element.png) 49 | 50 | Tạo Bin chứa 3 element 51 | ``` 52 | bin = gst_bin_new ("bin_example"); 53 | element_1 = gst_element_factory_make ("fakesrc", "source"); 54 | element_2 = gst_element_factory_make ("fakesink", "sink"); 55 | element_3 = gst_element_factory_make ("fakesink", "sink"); 56 | 57 | gst_bin_add_many (GST_BIN (bin_example), element_1, element_2, element_3, NULL); 58 | gst_element_link_many (element_1, element_2, element_3, NULL); 59 | ``` 60 | 61 | ## Pads 62 | - Như đã nói phần trên, Element có các pad được xem là interface với các thành phần bên ngoài. Dữ liệu từ element's source pad đến element's sink pad khác 63 | - Sử dụng Pad như thế nào sẽ được giải thích trong những ví dự sau -------------------------------------------------------------------------------- /doc/tracking.md: -------------------------------------------------------------------------------- 1 | will be updated soon -------------------------------------------------------------------------------- /images/bin-element.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tienluongngoc/deepstream_from_scratch/75159a88532405f41c17a5b22973f145c7c97afe/images/bin-element.png -------------------------------------------------------------------------------- /images/bin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tienluongngoc/deepstream_from_scratch/75159a88532405f41c17a5b22973f145c7c97afe/images/bin.jpg -------------------------------------------------------------------------------- /images/element.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tienluongngoc/deepstream_from_scratch/75159a88532405f41c17a5b22973f145c7c97afe/images/element.png -------------------------------------------------------------------------------- /images/meta_data.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tienluongngoc/deepstream_from_scratch/75159a88532405f41c17a5b22973f145c7c97afe/images/meta_data.jpg -------------------------------------------------------------------------------- /include/deepstream_test1/deepstream_test1.hpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "gstnvdsmeta.h" 6 | 7 | static GstPadProbeReturn osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data); 8 | static gboolean bus_call (GstBus * bus, GstMessage * msg, gpointer data); 9 | int deepstream_test1 (int argc, char *argv[]); -------------------------------------------------------------------------------- /include/gstreamer/gstreamer_example.hpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int gstreamer_example_1 (int argc, char *argv[]); 4 | -------------------------------------------------------------------------------- /include/license_plate_recognition/license_plate_recognition.hpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "gstnvdsmeta.h" 10 | #include "gst-nvmessage.h" 11 | #include "nvdsmeta.h" 12 | 13 | #define MAX_DISPLAY_LEN 64 14 | #define MEASURE_ENABLE 1 15 | #define PGIE_CLASS_ID_VEHICLE 0 16 | #define PGIE_CLASS_ID_PERSON 2 17 | #define SGIE_CLASS_ID_LPD 0 18 | #define MUXER_OUTPUT_WIDTH 1280 19 | #define MUXER_OUTPUT_HEIGHT 720 20 | #define MUXER_BATCH_TIMEOUT_USEC 4000000 21 | #define CONFIG_GROUP_TRACKER "tracker" 22 | #define CONFIG_GROUP_TRACKER_WIDTH "tracker-width" 23 | #define CONFIG_GROUP_TRACKER_HEIGHT "tracker-height" 24 | #define CONFIG_GROUP_TRACKER_LL_CONFIG_FILE "ll-config-file" 25 | #define CONFIG_GROUP_TRACKER_LL_LIB_FILE "ll-lib-file" 26 | #define CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS "enable-batch-process" 27 | #define CONFIG_GPU_ID "gpu-id" 28 | #define PRIMARY_DETECTOR_UID 1 29 | #define SECONDARY_DETECTOR_UID 2 30 | #define SECONDARY_CLASSIFIER_UID 3 31 | 32 | 33 | typedef struct _perf_measure{ 34 | GstClockTime pre_time; 35 | GstClockTime total_time; 36 | guint count; 37 | }perf_measure; 38 | 39 | 40 | #define CHECK_ERROR(error) \ 41 | if (error) { \ 42 | g_printerr ("Error while parsing config file: %s\n", error->message); \ 43 | goto done; \ 44 | } 45 | 46 | 47 | static gchar *get_absolute_file_path(gchar *cfg_file_path, gchar *file_path) { 48 | gchar abs_cfg_path[PATH_MAX + 1]; 49 | gchar *abs_file_path; 50 | gchar *delim; 51 | 52 | if (file_path && file_path[0] == '/') { 53 | return file_path; 54 | } 55 | 56 | if (!realpath(cfg_file_path, abs_cfg_path)) { 57 | g_free(file_path); 58 | return NULL; 59 | } 60 | 61 | if (!file_path) { 62 | abs_file_path = g_strdup(abs_cfg_path); 63 | return abs_file_path; 64 | } 65 | 66 | delim = g_strrstr(abs_cfg_path, "/"); 67 | *(delim + 1) = '\0'; 68 | 69 | abs_file_path = g_strconcat(abs_cfg_path, file_path, NULL); 70 | g_free(file_path); 71 | 72 | return abs_file_path; 73 | } 74 | 75 | 76 | static gboolean bus_call(GstBus *bus, GstMessage *msg, gpointer data) { 77 | GMainLoop *loop = (GMainLoop *)data; 78 | switch (GST_MESSAGE_TYPE(msg)) { 79 | case GST_MESSAGE_EOS: 80 | g_print("End of stream\n"); 81 | g_main_loop_quit(loop); 82 | break; 83 | case GST_MESSAGE_ERROR: { 84 | gchar *debug; 85 | GError *error; 86 | gst_message_parse_error(msg, &error, &debug); 87 | g_printerr("ERROR from element %s: %s\n", GST_OBJECT_NAME(msg->src), 88 | error->message); 89 | if (debug) g_printerr("Error details: %s\n", debug); 90 | g_free(debug); 91 | g_error_free(error); 92 | g_main_loop_quit(loop); 93 | break; 94 | } 95 | default: 96 | break; 97 | } 98 | return TRUE; 99 | } 100 | 101 | 102 | static void cb_new_pad(GstElement *element, GstPad *pad, GstElement *data) { 103 | GstCaps *new_pad_caps = NULL; 104 | GstStructure *new_pad_struct = NULL; 105 | const gchar *new_pad_type = NULL; 106 | GstPadLinkReturn ret; 107 | 108 | GstPad *sink_pad = gst_element_get_static_pad(data, "sink"); 109 | if (gst_pad_is_linked(sink_pad)) { 110 | g_print("h264parser already linked. Ignoring.\n"); 111 | goto exit; 112 | } 113 | 114 | new_pad_caps = gst_pad_get_current_caps(pad); 115 | new_pad_struct = gst_caps_get_structure(new_pad_caps, 0); 116 | new_pad_type = gst_structure_get_name(new_pad_struct); 117 | g_print("qtdemux pad %s\n", new_pad_type); 118 | 119 | if (g_str_has_prefix(new_pad_type, "video/x-h264")) { 120 | ret = gst_pad_link(pad, sink_pad); 121 | if (GST_PAD_LINK_FAILED(ret)) 122 | g_print("fail to link parser and mp4 demux.\n"); 123 | } else { 124 | g_print("%s output, not 264 stream\n", new_pad_type); 125 | } 126 | 127 | exit: 128 | gst_object_unref(sink_pad); 129 | } 130 | 131 | 132 | static gboolean set_tracker_properties(GstElement *nvtracker, char *config_file_name) { 133 | gboolean ret = FALSE; 134 | GError *error = NULL; 135 | gchar **keys = NULL; 136 | gchar **key = NULL; 137 | GKeyFile *key_file = g_key_file_new(); 138 | 139 | if (!g_key_file_load_from_file(key_file, config_file_name, G_KEY_FILE_NONE, 140 | &error)) { 141 | g_printerr("Failed to load config file: %s\n", error->message); 142 | return FALSE; 143 | } 144 | 145 | keys = g_key_file_get_keys(key_file, CONFIG_GROUP_TRACKER, NULL, &error); 146 | CHECK_ERROR(error); 147 | 148 | for (key = keys; *key; key++) { 149 | if (!g_strcmp0(*key, CONFIG_GROUP_TRACKER_WIDTH)) { 150 | gint width = g_key_file_get_integer(key_file, CONFIG_GROUP_TRACKER, 151 | CONFIG_GROUP_TRACKER_WIDTH, &error); 152 | CHECK_ERROR(error); 153 | g_object_set(G_OBJECT(nvtracker), "tracker-width", width, NULL); 154 | } else if (!g_strcmp0(*key, CONFIG_GROUP_TRACKER_HEIGHT)) { 155 | gint height = g_key_file_get_integer(key_file, CONFIG_GROUP_TRACKER, 156 | CONFIG_GROUP_TRACKER_HEIGHT, &error); 157 | CHECK_ERROR(error); 158 | g_object_set(G_OBJECT(nvtracker), "tracker-height", height, NULL); 159 | } else if (!g_strcmp0(*key, CONFIG_GPU_ID)) { 160 | guint gpu_id = g_key_file_get_integer(key_file, CONFIG_GROUP_TRACKER, 161 | CONFIG_GPU_ID, &error); 162 | CHECK_ERROR(error); 163 | g_object_set(G_OBJECT(nvtracker), "gpu_id", gpu_id, NULL); 164 | } else if (!g_strcmp0(*key, CONFIG_GROUP_TRACKER_LL_CONFIG_FILE)) { 165 | char *ll_config_file = get_absolute_file_path( 166 | config_file_name, 167 | g_key_file_get_string(key_file, CONFIG_GROUP_TRACKER, 168 | CONFIG_GROUP_TRACKER_LL_CONFIG_FILE, &error)); 169 | CHECK_ERROR(error); 170 | g_object_set(G_OBJECT(nvtracker), "ll-config-file", ll_config_file, NULL); 171 | } else if (!g_strcmp0(*key, CONFIG_GROUP_TRACKER_LL_LIB_FILE)) { 172 | char *ll_lib_file = get_absolute_file_path( 173 | config_file_name, 174 | g_key_file_get_string(key_file, CONFIG_GROUP_TRACKER, 175 | CONFIG_GROUP_TRACKER_LL_LIB_FILE, &error)); 176 | CHECK_ERROR(error); 177 | g_object_set(G_OBJECT(nvtracker), "ll-lib-file", ll_lib_file, NULL); 178 | } else if (!g_strcmp0(*key, CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS)) { 179 | gboolean enable_batch_process = g_key_file_get_integer( 180 | key_file, CONFIG_GROUP_TRACKER, 181 | CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS, &error); 182 | CHECK_ERROR(error); 183 | g_object_set(G_OBJECT(nvtracker), "enable_batch_process", 184 | enable_batch_process, NULL); 185 | } else { 186 | g_printerr("Unknown key '%s' for group [%s]", *key, CONFIG_GROUP_TRACKER); 187 | } 188 | } 189 | 190 | ret = TRUE; 191 | done: 192 | if (error) { 193 | g_error_free(error); 194 | } 195 | if (keys) { 196 | g_strfreev(keys); 197 | } 198 | if (!ret) { 199 | g_printerr("%s failed", __func__); 200 | } 201 | return ret; 202 | } 203 | 204 | 205 | static GstPadProbeReturn osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data); 206 | 207 | 208 | int lpr (int argc, char *argv[]); 209 | -------------------------------------------------------------------------------- /include/save_image_deepstream/save_image_deepstream.hpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "gstnvdsmeta.h" 6 | 7 | static GstPadProbeReturn osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data); 8 | static gboolean bus_call (GstBus * bus, GstMessage * msg, gpointer data); 9 | int save_image_deepstream (int argc, char *argv[]); -------------------------------------------------------------------------------- /include/tiny_yolo2_deepstream/tiny_yolo2_deepstream.hpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "gstnvdsmeta.h" 6 | 7 | static GstPadProbeReturn osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data); 8 | static gboolean bus_call (GstBus * bus, GstMessage * msg, gpointer data); 9 | int tiny_yolo2_deepstream (int argc, char *argv[]); -------------------------------------------------------------------------------- /include/using_bin/using_bin.hpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "gstnvdsmeta.h" 6 | 7 | static GstPadProbeReturn osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data); 8 | static gboolean bus_call (GstBus * bus, GstMessage * msg, gpointer data); 9 | int using_bin (int argc, char *argv[]); -------------------------------------------------------------------------------- /scripts/run.sh: -------------------------------------------------------------------------------- 1 | cd build 2 | make 3 | cd .. 4 | # ./build/apps/deepstream_test1/deepstream_test1 /opt/nvidia/deepstream/deepstream-5.1/samples/streams/sample_720p.h264 5 | # ./build/apps/tiny_yolo2_deepstream/tiny_yolo2_deepstream /opt/nvidia/deepstream/deepstream-5.1/samples/streams/sample_720p.h264 6 | # ./build/apps/save_image_deepstream/save_image_deepstream /opt/nvidia/deepstream/deepstream-5.1/samples/streams/sample_720p.h264 7 | ./build/apps/using_bin/using_bin /opt/nvidia/deepstream/deepstream-5.1/samples/streams/sample_720p.h264 -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(LIBRARY_NAME deep_stream) 2 | set(DEEP_STREAM_LIBS /opt/nvidia/deepstream/deepstream-5.1/lib) 3 | 4 | find_package(PkgConfig) 5 | pkg_check_modules(GSTREAMER REQUIRED gstreamer-1.0) 6 | list(APPEND INCLUDE_DIRS ${GLIB_INCLUDE_DIRS}${GSTREAMER_INCLUDE_DIRS}) 7 | list(APPEND DEPS ${GLIB_LIBRARY_DIRS}${GSTREAMER_LIBRARY_DIRS}) 8 | list(APPEND DEPS "${DEEP_STREAM_LIBS}/libnvdsgst_meta.so") 9 | list(APPEND DEPS "${DEEP_STREAM_LIBS}/libnvds_meta.so") 10 | list(APPEND DEPS "${DEEP_STREAM_LIBS}/libnvbufsurface.so") 11 | list(APPEND DEPS "${DEEP_STREAM_LIBS}/libnvbufsurftransform.so") 12 | 13 | list(APPEND INCLUDE_DIRS "${DEEP_STREAM_LIBS}/../sources/includes/") 14 | 15 | 16 | file(GLOB_RECURSE HEADER_LIST "${PROJECT_SOURCE_DIR}/include/*[.hpp, .h]") 17 | file(GLOB_RECURSE SOURCE_LIST "${PROJECT_SOURCE_DIR}/src/*.cpp") 18 | 19 | list(APPEND INCLUDE_DIRS "${PROJECT_SOURCE_DIR}/include/") 20 | 21 | find_package(CUDA REQUIRED) 22 | message(STATUS "Find CUDA include at ${CUDA_INCLUDE_DIRS}") 23 | message(STATUS "Find CUDA libraries: ${CUDA_LIBRARIES}") 24 | 25 | 26 | # OPENCV 27 | find_package(OpenCV REQUIRED core imgcodecs imgproc video videoio dnn calib3d highgui cudaimgproc) 28 | list(APPEND INCLUDE_DIRS ${OpenCV_INCLUDE_DIRS}) 29 | list(APPEND DEPS opencv_core opencv_highgui opencv_imgcodecs opencv_cudaimgproc opencv_imgproc opencv_video opencv_videoio opencv_calib3d) 30 | 31 | 32 | 33 | add_library(${LIBRARY_NAME} ${SOURCE_LIST} ${HEADER_LIST}) 34 | target_compile_features(${LIBRARY_NAME} PUBLIC cxx_std_17) 35 | target_include_directories(${LIBRARY_NAME} PUBLIC ${INCLUDE_DIRS} ${CUDA_INCLUDE_DIRS}) 36 | target_link_libraries(${LIBRARY_NAME} PUBLIC ${GSTREAMER_LIBRARIES} ${DEPS} ${CUDA_LIBRARIES}) 37 | -------------------------------------------------------------------------------- /src/deepstream_test1/deepstream_test1.cpp: -------------------------------------------------------------------------------- 1 | #include "deepstream_test1/deepstream_test1.hpp" 2 | 3 | #define MAX_DISPLAY_LEN 64 4 | #define PGIE_CLASS_ID_VEHICLE 0 5 | #define PGIE_CLASS_ID_PERSON 2 6 | #define MUXER_OUTPUT_WIDTH 1920 7 | #define MUXER_OUTPUT_HEIGHT 1080 8 | #define MUXER_BATCH_TIMEOUT_USEC 40000 9 | 10 | gint frame_number = 0; 11 | gchar pgie_classes_str[4][32] = { "Vehicle", "TwoWheeler", "Person", "Roadsign" }; 12 | 13 | static GstPadProbeReturn osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data) 14 | { 15 | GstBuffer *buf = (GstBuffer *) info->data; 16 | guint num_rects = 0; 17 | NvDsObjectMeta *obj_meta = NULL; 18 | guint vehicle_count = 0; 19 | guint person_count = 0; 20 | NvDsMetaList * l_frame = NULL; 21 | NvDsMetaList * l_obj = NULL; 22 | NvDsDisplayMeta *display_meta = NULL; 23 | 24 | NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf); 25 | 26 | for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; 27 | l_frame = l_frame->next) { 28 | NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data); 29 | int offset = 0; 30 | for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; 31 | l_obj = l_obj->next) { 32 | obj_meta = (NvDsObjectMeta *) (l_obj->data); 33 | if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) { 34 | vehicle_count++; 35 | num_rects++; 36 | } 37 | if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) { 38 | person_count++; 39 | num_rects++; 40 | } 41 | } 42 | // display_meta = nvds_acquire_display_meta_from_pool(batch_meta); 43 | // NvOSD_TextParams *txt_params = &display_meta->text_params[0]; 44 | // display_meta->num_labels = 1; 45 | // txt_params->display_text = g_malloc0 (MAX_DISPLAY_LEN); 46 | // offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, "Person = %d ", person_count); 47 | // offset = snprintf(txt_params->display_text + offset , MAX_DISPLAY_LEN, "Vehicle = %d ", vehicle_count); 48 | 49 | // /* Now set the offsets where the string should appear */ 50 | // txt_params->x_offset = 10; 51 | // txt_params->y_offset = 12; 52 | 53 | /* Font , font-color and font-size */ 54 | // txt_params->font_params.font_name = "Serif"; 55 | // txt_params->font_params.font_size = 10; 56 | // txt_params->font_params.font_color.red = 1.0; 57 | // txt_params->font_params.font_color.green = 1.0; 58 | // txt_params->font_params.font_color.blue = 1.0; 59 | // txt_params->font_params.font_color.alpha = 1.0; 60 | 61 | // /* Text background color */ 62 | // txt_params->set_bg_clr = 1; 63 | // txt_params->text_bg_clr.red = 0.0; 64 | // txt_params->text_bg_clr.green = 0.0; 65 | // txt_params->text_bg_clr.blue = 0.0; 66 | // txt_params->text_bg_clr.alpha = 1.0; 67 | 68 | // nvds_add_display_meta_to_frame(frame_meta, display_meta); 69 | } 70 | 71 | g_print ("Frame Number = %d Number of objects = %d " 72 | "Vehicle Count = %d Person Count = %d\n", 73 | frame_number, num_rects, vehicle_count, person_count); 74 | frame_number++; 75 | return GST_PAD_PROBE_OK; 76 | } 77 | 78 | static gboolean bus_call (GstBus * bus, GstMessage * msg, gpointer data) 79 | { 80 | GMainLoop *loop = (GMainLoop *) data; 81 | switch (GST_MESSAGE_TYPE (msg)) { 82 | case GST_MESSAGE_EOS: 83 | g_print ("End of stream\n"); 84 | g_main_loop_quit (loop); 85 | break; 86 | case GST_MESSAGE_ERROR:{ 87 | gchar *debug; 88 | GError *error; 89 | gst_message_parse_error (msg, &error, &debug); 90 | g_printerr ("ERROR from element %s: %s\n", 91 | GST_OBJECT_NAME (msg->src), error->message); 92 | if (debug) 93 | g_printerr ("Error details: %s\n", debug); 94 | g_free (debug); 95 | g_error_free (error); 96 | g_main_loop_quit (loop); 97 | break; 98 | } 99 | default: 100 | break; 101 | } 102 | return TRUE; 103 | } 104 | 105 | int deepstream_test1 (int argc, char *argv[]) 106 | { 107 | GMainLoop *loop = NULL; 108 | GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL, 109 | *decoder = NULL, *streammux = NULL, *sink = NULL, *pgie = NULL, *nvvidconv = NULL, 110 | *nvosd = NULL; 111 | 112 | GstElement *transform = NULL; 113 | GstBus *bus = NULL; 114 | guint bus_watch_id; 115 | GstPad *osd_sink_pad = NULL; 116 | 117 | int current_device = -1; 118 | cudaGetDevice(¤t_device); 119 | struct cudaDeviceProp prop; 120 | cudaGetDeviceProperties(&prop, current_device); 121 | /* Check input arguments */ 122 | if (argc != 2) { 123 | g_printerr ("Usage: %s \n", argv[0]); 124 | return -1; 125 | } 126 | 127 | /* Standard GStreamer initialization */ 128 | gst_init (&argc, &argv); 129 | loop = g_main_loop_new (NULL, FALSE); 130 | 131 | /* Create gstreamer elements */ 132 | /* Create Pipeline element that will form a connection of other elements */ 133 | pipeline = gst_pipeline_new ("dstest1-pipeline"); 134 | 135 | /* Source element for reading from the file */ 136 | source = gst_element_factory_make ("filesrc", "file-source"); 137 | 138 | /* Since the data format in the input file is elementary h264 stream, 139 | * we need a h264parser */ 140 | h264parser = gst_element_factory_make ("h264parse", "h264-parser"); 141 | 142 | /* Use nvdec_h264 for hardware accelerated decode on GPU */ 143 | decoder = gst_element_factory_make ("nvv4l2decoder", "nvv4l2-decoder"); 144 | 145 | /* Create nvstreammux instance to form batches from one or more sources. */ 146 | streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); 147 | 148 | if (!pipeline || !streammux) { 149 | g_printerr ("One element could not be created. Exiting.\n"); 150 | return -1; 151 | } 152 | 153 | /* Use nvinfer to run inferencing on decoder's output, 154 | * behaviour of inferencing is set through config file */ 155 | pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine"); 156 | 157 | /* Use convertor to convert from NV12 to RGBA as required by nvosd */ 158 | nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter"); 159 | 160 | /* Create OSD to draw on the converted RGBA buffer */ 161 | nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); 162 | 163 | /* Finally render the osd output */ 164 | if(prop.integrated) { 165 | transform = gst_element_factory_make ("nvegltransform", "nvegl-transform"); 166 | } 167 | sink = gst_element_factory_make ("fakesink", "nvvideo-renderer"); 168 | // sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer"); 169 | 170 | if (!source || !h264parser || !decoder || !pgie 171 | || !nvvidconv || !nvosd || !sink) { 172 | g_printerr ("One element could not be created. Exiting.\n"); 173 | return -1; 174 | } 175 | 176 | if(!transform && prop.integrated) { 177 | g_printerr ("One tegra element could not be created. Exiting.\n"); 178 | return -1; 179 | } 180 | 181 | /* we set the input filename to the source element */ 182 | g_object_set (G_OBJECT (source), "location", argv[1], NULL); 183 | 184 | g_object_set (G_OBJECT (streammux), "batch-size", 1, NULL); 185 | 186 | g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", 187 | MUXER_OUTPUT_HEIGHT, 188 | "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); 189 | 190 | /* Set all the necessary properties of the nvinfer element, 191 | * the necessary ones are : */ 192 | g_object_set (G_OBJECT (pgie), 193 | "config-file-path", "apps/deepstream_test1/deepstream_test1_config.txt", NULL); 194 | 195 | /* we add a message handler */ 196 | bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); 197 | bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); 198 | gst_object_unref (bus); 199 | 200 | /* Set up the pipeline */ 201 | /* we add all elements into the pipeline */ 202 | if(prop.integrated) { 203 | gst_bin_add_many (GST_BIN (pipeline), 204 | source, h264parser, decoder, streammux, pgie, 205 | nvvidconv, nvosd, transform, sink, NULL); 206 | } 207 | else { 208 | gst_bin_add_many (GST_BIN (pipeline), 209 | source, h264parser, decoder, streammux, pgie, 210 | nvvidconv, nvosd, sink, NULL); 211 | } 212 | 213 | GstPad *sinkpad, *srcpad; 214 | gchar pad_name_sink[16] = "sink_0"; 215 | gchar pad_name_src[16] = "src"; 216 | 217 | sinkpad = gst_element_get_request_pad (streammux, pad_name_sink); 218 | if (!sinkpad) { 219 | g_printerr ("Streammux request sink pad failed. Exiting.\n"); 220 | return -1; 221 | } 222 | 223 | srcpad = gst_element_get_static_pad (decoder, pad_name_src); 224 | if (!srcpad) { 225 | g_printerr ("Decoder request src pad failed. Exiting.\n"); 226 | return -1; 227 | } 228 | 229 | if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { 230 | g_printerr ("Failed to link decoder to stream muxer. Exiting.\n"); 231 | return -1; 232 | } 233 | 234 | gst_object_unref (sinkpad); 235 | gst_object_unref (srcpad); 236 | 237 | /* we link the elements together */ 238 | /* file-source -> h264-parser -> nvh264-decoder -> 239 | * nvinfer -> nvvidconv -> nvosd -> video-renderer */ 240 | 241 | if (!gst_element_link_many (source, h264parser, decoder, NULL)) { 242 | g_printerr ("Elements could not be linked: 1. Exiting.\n"); 243 | return -1; 244 | } 245 | 246 | if(prop.integrated) { 247 | if (!gst_element_link_many (streammux, pgie, 248 | nvvidconv, nvosd, transform, sink, NULL)) { 249 | g_printerr ("Elements could not be linked: 2. Exiting.\n"); 250 | return -1; 251 | } 252 | } 253 | else { 254 | if (!gst_element_link_many (streammux, pgie, 255 | nvvidconv, nvosd, sink, NULL)) { 256 | g_printerr ("Elements could not be linked: 2. Exiting.\n"); 257 | return -1; 258 | } 259 | } 260 | 261 | /* Lets add probe to get informed of the meta data generated, we add probe to 262 | * the sink pad of the osd element, since by that time, the buffer would have 263 | * had got all the metadata. */ 264 | osd_sink_pad = gst_element_get_static_pad (nvosd, "sink"); 265 | if (!osd_sink_pad) 266 | g_print ("Unable to get sink pad\n"); 267 | else 268 | gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, 269 | osd_sink_pad_buffer_probe, NULL, NULL); 270 | gst_object_unref (osd_sink_pad); 271 | 272 | /* Set the pipeline to "playing" state */ 273 | g_print ("Now playing"); 274 | gst_element_set_state (pipeline, GST_STATE_PLAYING); 275 | 276 | /* Wait till pipeline encounters an error or EOS */ 277 | g_print ("Running...\n"); 278 | g_main_loop_run (loop); 279 | 280 | /* Out of the main loop, clean up nicely */ 281 | g_print ("Returned, stopping playback\n"); 282 | gst_element_set_state (pipeline, GST_STATE_NULL); 283 | g_print ("Deleting pipeline\n"); 284 | gst_object_unref (GST_OBJECT (pipeline)); 285 | g_source_remove (bus_watch_id); 286 | g_main_loop_unref (loop); 287 | return 0; 288 | } 289 | -------------------------------------------------------------------------------- /src/gstreamer/gstreamer_example.cpp: -------------------------------------------------------------------------------- 1 | #include "gstreamer/gstreamer_example.hpp" 2 | 3 | int gstreamer_example_1 (int argc, char *argv[]) 4 | { 5 | GstElement *pipeline; 6 | GstBus *bus; 7 | GstMessage *msg; 8 | 9 | /* Initialize GStreamer */ 10 | gst_init (&argc, &argv); 11 | 12 | /* Build the pipeline */ 13 | pipeline = gst_parse_launch("playbin uri=https://www.freedesktop.org/software/gstreamer-sdk/data/media/sintel_trailer-480p.webm", NULL); 14 | 15 | /* Start playing */ 16 | gst_element_set_state (pipeline, GST_STATE_PLAYING); 17 | 18 | /* Wait until error or EOS */ 19 | bus = gst_element_get_bus (pipeline); 20 | msg = gst_bus_timed_pop_filtered (bus, GST_CLOCK_TIME_NONE, static_cast(GST_MESSAGE_ERROR | GST_MESSAGE_EOS)); 21 | 22 | if (msg != NULL) 23 | gst_message_unref (msg); 24 | gst_object_unref (bus); 25 | gst_element_set_state (pipeline, GST_STATE_NULL); 26 | gst_object_unref (pipeline); 27 | return 0; 28 | } 29 | -------------------------------------------------------------------------------- /src/license_plate_recognition/license_plate_recognition.cpp: -------------------------------------------------------------------------------- 1 | #include "license_plate_recognition/license_plate_recognition.hpp" 2 | 3 | #include 4 | 5 | gint frame_number = 0; 6 | gint total_plate_number = 0; 7 | gchar pgie_classes_str[4][32] = {"Vehicle", "TwoWheeler", "Person", "Roadsign"}; 8 | 9 | static GstPadProbeReturn osd_sink_pad_buffer_probe(GstPad *pad, 10 | GstPadProbeInfo *info, 11 | gpointer u_data) { 12 | GstBuffer *buf = (GstBuffer *)info->data; 13 | NvDsObjectMeta *obj_meta = NULL; 14 | guint vehicle_count = 0; 15 | guint person_count = 0; 16 | guint lp_count = 0; 17 | guint label_i = 0; 18 | NvDsMetaList *l_frame = NULL; 19 | NvDsMetaList *l_obj = NULL; 20 | NvDsMetaList *l_class = NULL; 21 | NvDsMetaList *l_label = NULL; 22 | NvDsDisplayMeta *display_meta = NULL; 23 | NvDsClassifierMeta *class_meta = NULL; 24 | NvDsLabelInfo *label_info = NULL; 25 | 26 | NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); 27 | 28 | for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; 29 | l_frame = l_frame->next) { 30 | NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); 31 | if (!frame_meta) continue; 32 | for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; 33 | l_obj = l_obj->next) { 34 | obj_meta = (NvDsObjectMeta *)(l_obj->data); 35 | if (!obj_meta) continue; 36 | 37 | if (obj_meta->unique_component_id == PRIMARY_DETECTOR_UID) { 38 | if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) vehicle_count++; 39 | if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) person_count++; 40 | } 41 | 42 | if (obj_meta->unique_component_id == SECONDARY_DETECTOR_UID) { 43 | if (obj_meta->class_id == SGIE_CLASS_ID_LPD) lp_count++; 44 | } 45 | 46 | for (l_class = obj_meta->classifier_meta_list; l_class != NULL; 47 | l_class = l_class->next) { 48 | class_meta = (NvDsClassifierMeta *)(l_class->data); 49 | if (!class_meta) continue; 50 | if (class_meta->unique_component_id == SECONDARY_CLASSIFIER_UID) { 51 | for (label_i = 0, l_label = class_meta->label_info_list; 52 | label_i < class_meta->num_labels && l_label; 53 | label_i++, l_label = l_label->next) { 54 | label_info = (NvDsLabelInfo *)(l_label->data); 55 | if (label_info) { 56 | if (label_info->label_id == 0 && 57 | label_info->result_class_id == 1) { 58 | std::cout << "Plate License: " << label_info->result_label 59 | << std::endl; 60 | } 61 | } 62 | } 63 | } 64 | } 65 | } 66 | } 67 | std::cout << "Frame Number: " << frame_number 68 | << " Vehicle Count: " << vehicle_count 69 | << " Person Count: " << person_count 70 | << " License Plate Count: " << lp_count << std::endl; 71 | frame_number++; 72 | total_plate_number += lp_count; 73 | return GST_PAD_PROBE_OK; 74 | } 75 | 76 | int lpr(int argc, char *argv[]) { 77 | GstElement *h264parser[128]; 78 | GstElement *source[128]; 79 | GstElement *decoder[128]; 80 | GstElement *mp4demux[128]; 81 | GstElement *parsequeue[128]; 82 | 83 | GstBus *bus = NULL; 84 | guint bus_watch_id; 85 | GstPad *osd_sink_pad = NULL; 86 | GstCaps *caps = NULL; 87 | GstCapsFeatures *feature = NULL; 88 | static guint src_cnt = 0; 89 | perf_measure perf_measure; 90 | 91 | gchar ele_name[64]; 92 | GstPad *sinkpad, *srcpad; 93 | gchar pad_name_sink[16] = "sink_0"; 94 | gchar pad_name_src[16] = "src"; 95 | 96 | gst_init(&argc, &argv); 97 | auto loop = g_main_loop_new(NULL, FALSE); 98 | 99 | perf_measure.pre_time = GST_CLOCK_TIME_NONE; 100 | perf_measure.total_time = GST_CLOCK_TIME_NONE; 101 | perf_measure.count = 0; 102 | 103 | auto pipeline = gst_pipeline_new("pipeline"); 104 | auto streammux = gst_element_factory_make("nvstreammux", "stream-muxer"); 105 | 106 | gst_bin_add(GST_BIN(pipeline), streammux); 107 | 108 | for (src_cnt = 0; src_cnt < (int)argc - 1; src_cnt++) { 109 | g_snprintf(ele_name, 64, "file_src_%d", src_cnt); 110 | source[src_cnt] = gst_element_factory_make("filesrc", ele_name); 111 | 112 | g_snprintf(ele_name, 64, "mp4demux_%d", src_cnt); 113 | mp4demux[src_cnt] = gst_element_factory_make("qtdemux", ele_name); 114 | 115 | g_snprintf(ele_name, 64, "h264parser_%d", src_cnt); 116 | h264parser[src_cnt] = gst_element_factory_make("h264parse", ele_name); 117 | 118 | g_snprintf(ele_name, 64, "parsequeue_%d", src_cnt); 119 | parsequeue[src_cnt] = gst_element_factory_make("queue", ele_name); 120 | 121 | g_snprintf(ele_name, 64, "decoder_%d", src_cnt); 122 | decoder[src_cnt] = gst_element_factory_make("nvv4l2decoder", ele_name); 123 | 124 | gst_bin_add_many(GST_BIN(pipeline), source[src_cnt], mp4demux[src_cnt], 125 | h264parser[src_cnt], parsequeue[src_cnt], decoder[src_cnt], NULL); 126 | 127 | g_snprintf(pad_name_sink, 64, "sink_%d", src_cnt); 128 | sinkpad = gst_element_get_request_pad(streammux, pad_name_sink); 129 | srcpad = gst_element_get_static_pad(decoder[src_cnt], pad_name_src); 130 | 131 | gst_pad_link(srcpad, sinkpad); 132 | 133 | gst_element_link_pads(source[src_cnt], "src", mp4demux[src_cnt], "sink"); 134 | g_signal_connect(mp4demux[src_cnt], "pad-added", G_CALLBACK(cb_new_pad), 135 | h264parser[src_cnt]); 136 | gst_element_link_many(h264parser[src_cnt], parsequeue[src_cnt], 137 | decoder[src_cnt], NULL); 138 | 139 | g_object_set(G_OBJECT(source[src_cnt]), "location", argv[1 + src_cnt], 140 | NULL); 141 | 142 | gst_object_unref(sinkpad); 143 | gst_object_unref(srcpad); 144 | } 145 | 146 | auto vehicle_detection = 147 | gst_element_factory_make("nvinfer", "vehicle_detection"); 148 | auto vehicle_tracking = 149 | gst_element_factory_make("nvtracker", "vehicle_tracking"); 150 | auto plate_detection = gst_element_factory_make("nvinfer", "plate_detection"); 151 | auto text_recognition = 152 | gst_element_factory_make("nvinfer", "text_recognition"); 153 | 154 | auto queue1 = gst_element_factory_make("queue", "queue1"); 155 | auto queue2 = gst_element_factory_make("queue", "queue2"); 156 | auto queue3 = gst_element_factory_make("queue", "queue3"); 157 | auto queue4 = gst_element_factory_make("queue", "queue4"); 158 | 159 | auto sink = gst_element_factory_make("fakesink", "fake-renderer"); 160 | 161 | g_object_set(G_OBJECT(streammux), "width", MUXER_OUTPUT_WIDTH, NULL); 162 | g_object_set(G_OBJECT(streammux), "height", MUXER_OUTPUT_HEIGHT, NULL); 163 | g_object_set(G_OBJECT(streammux), "batch-size", src_cnt, NULL); 164 | g_object_set(G_OBJECT(streammux), "batched-push-timeout", 165 | MUXER_BATCH_TIMEOUT_USEC, NULL); 166 | 167 | g_object_set(G_OBJECT(vehicle_detection), "config-file-path", 168 | "apps/license_plate_recognition/configs/vehicle_detection.txt", 169 | NULL); 170 | g_object_set(G_OBJECT(vehicle_detection), "unique-id", PRIMARY_DETECTOR_UID, 171 | NULL); 172 | 173 | g_object_set(G_OBJECT(plate_detection), "config-file-path", 174 | "apps/license_plate_recognition/configs/plate_detection.txt", 175 | NULL); 176 | g_object_set(G_OBJECT(plate_detection), "unique-id", SECONDARY_DETECTOR_UID, 177 | NULL); 178 | g_object_set(G_OBJECT(plate_detection), "process-mode", 2, NULL); 179 | 180 | g_object_set(G_OBJECT(text_recognition), "config-file-path", 181 | "apps/license_plate_recognition/configs/text_recognition.txt", 182 | NULL); 183 | g_object_set(G_OBJECT(text_recognition), "unique-id", 184 | SECONDARY_CLASSIFIER_UID, NULL); 185 | g_object_set(G_OBJECT(text_recognition), "process-mode", 2, NULL); 186 | 187 | char name[300]; 188 | snprintf(name, 300, 189 | "apps/license_plate_recognition/configs/vehicle_tracking.txt"); 190 | set_tracker_properties(vehicle_tracking, name); 191 | 192 | bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline)); 193 | bus_watch_id = gst_bus_add_watch(bus, bus_call, loop); 194 | gst_object_unref(bus); 195 | 196 | gst_bin_add_many(GST_BIN(pipeline), vehicle_detection, plate_detection, 197 | vehicle_tracking, queue1, queue2, queue3, queue4, 198 | text_recognition, sink, NULL); 199 | 200 | gst_element_link_many(streammux, queue1, vehicle_detection, queue2, 201 | vehicle_tracking, queue3, plate_detection, queue4, 202 | text_recognition, sink, NULL); 203 | 204 | osd_sink_pad = gst_element_get_static_pad(sink, "sink"); 205 | gst_pad_add_probe(osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, 206 | osd_sink_pad_buffer_probe, NULL, NULL); 207 | gst_object_unref(osd_sink_pad); 208 | 209 | gst_element_set_state(pipeline, GST_STATE_PLAYING); 210 | g_print("Running...\n"); 211 | g_main_loop_run(loop); 212 | gst_element_set_state(pipeline, GST_STATE_NULL); 213 | 214 | gst_object_unref(GST_OBJECT(pipeline)); 215 | g_source_remove(bus_watch_id); 216 | g_main_loop_unref(loop); 217 | return 0; 218 | } 219 | -------------------------------------------------------------------------------- /src/license_plate_recognition/nvinfer_custom_lpr_parser/Makefile: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a 5 | # copy of this software and associated documentation files (the "Software"), 6 | # to deal in the Software without restriction, including without limitation 7 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | # and/or sell copies of the Software, and to permit persons to whom the 9 | # Software is furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | # DEALINGS IN THE SOFTWARE. 21 | ################################################################################ 22 | CC:= g++ 23 | 24 | CFLAGS:= -Wall -Werror -std=c++11 -shared -fPIC -Wno-error=deprecated-declarations 25 | 26 | CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes 27 | 28 | LIBS:= -lnvinfer -lnvparsers 29 | LFLAGS:= -Wl,--start-group $(LIBS) -Wl,--end-group 30 | 31 | SRCFILES:= nvinfer_custom_lpr_parser.cpp 32 | TARGET_LIB:= libnvdsinfer_custom_impl_lpr.so 33 | 34 | all: $(TARGET_LIB) 35 | 36 | $(TARGET_LIB) : $(SRCFILES) 37 | $(CC) -o $@ $^ $(CFLAGS) $(LFLAGS) 38 | 39 | clean: 40 | rm -rf $(TARGET_LIB) 41 | -------------------------------------------------------------------------------- /src/license_plate_recognition/nvinfer_custom_lpr_parser/nvinfer_custom_lpr_parser.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include "nvdsinfer.h" 32 | #include 33 | 34 | using namespace std; 35 | using std::string; 36 | using std::vector; 37 | 38 | static bool dict_ready=false; 39 | std::vector dict_table; 40 | 41 | extern "C" 42 | { 43 | 44 | bool NvDsInferParseCustomNVPlate(std::vector const &outputLayersInfo, 45 | NvDsInferNetworkInfo const &networkInfo, float classifierThreshold, 46 | std::vector &attrList, std::string &attrString) 47 | { 48 | int *outputStrBuffer = NULL; 49 | float *outputConfBuffer = NULL; 50 | NvDsInferAttribute LPR_attr; 51 | 52 | int seq_len = 0; 53 | 54 | // Get list 55 | vector str_idxes; 56 | int prev = 100; 57 | 58 | // For confidence 59 | double bank_softmax_max[16] = {0.0}; 60 | unsigned int valid_bank_count = 0; 61 | bool do_softmax = false; 62 | ifstream fdict; 63 | 64 | setlocale(LC_CTYPE, ""); 65 | 66 | if(!dict_ready) { 67 | fdict.open("apps/license_plate_recognition/configs/us_char_dict.txt"); 68 | if(!fdict.is_open()) 69 | { 70 | cout << "open dictionary file failed." << endl; 71 | return false; 72 | } 73 | while(!fdict.eof()) { 74 | string strLineAnsi; 75 | if ( getline(fdict, strLineAnsi) ) { 76 | dict_table.push_back(strLineAnsi); 77 | } 78 | } 79 | dict_ready=true; 80 | fdict.close(); 81 | } 82 | 83 | int layer_size = outputLayersInfo.size(); 84 | 85 | LPR_attr.attributeConfidence = 1.0; 86 | 87 | seq_len = networkInfo.width/4; 88 | 89 | for( int li=0; li(outputLayersInfo[li].buffer); 94 | } 95 | else if (outputLayersInfo[li].dataType == 3) { 96 | if(!outputStrBuffer) 97 | outputStrBuffer = static_cast(outputLayersInfo[li].buffer); 98 | } 99 | } 100 | } 101 | 102 | for(int seq_id = 0; seq_id < seq_len; seq_id++) { 103 | do_softmax = false; 104 | 105 | int curr_data = outputStrBuffer[seq_id]; 106 | if (seq_id == 0) { 107 | prev = curr_data; 108 | str_idxes.push_back(curr_data); 109 | if ( curr_data != static_cast(dict_table.size()) ) do_softmax = true; 110 | } else { 111 | if (curr_data != prev) { 112 | str_idxes.push_back(curr_data); 113 | if (static_cast(curr_data) != dict_table.size()) do_softmax = true; 114 | } 115 | prev = curr_data; 116 | } 117 | 118 | // Do softmax 119 | if (do_softmax) { 120 | do_softmax = false; 121 | bank_softmax_max[valid_bank_count] = outputConfBuffer[curr_data]; 122 | valid_bank_count++; 123 | } 124 | } 125 | 126 | attrString = ""; 127 | for(unsigned int id = 0; id < str_idxes.size(); id++) { 128 | if (static_cast(str_idxes[id]) != dict_table.size()) { 129 | attrString += dict_table[str_idxes[id]]; 130 | } 131 | } 132 | 133 | //Ignore the short string, it may be wrong plate string 134 | if (valid_bank_count >= 3) { 135 | 136 | LPR_attr.attributeIndex = 0; 137 | LPR_attr.attributeValue = 1; 138 | LPR_attr.attributeLabel = strdup(attrString.c_str()); 139 | for (unsigned int count = 0; count < valid_bank_count; count++) { 140 | LPR_attr.attributeConfidence *= bank_softmax_max[count]; 141 | } 142 | attrList.push_back(LPR_attr); 143 | } 144 | 145 | return true; 146 | } 147 | 148 | }//end of extern "C" 149 | -------------------------------------------------------------------------------- /src/license_plate_recognition/tao_converter/Readme.txt: -------------------------------------------------------------------------------- 1 | Installation instructions: 2 | 3 | 4 | This document captures simple instructions to run the TAO converter for the x86 platform. 5 | 6 | 7 | 1. Copy the executable to the target device 8 | 2. Install openssl library using the following command. 9 | 1. $ sudo apt-get install libssl-dev 10 | 3. Export the following environment variables 11 | 1. $ export TRT_LIB_PATH=”/usr/lib/x86_64-linux-gnu” 12 | 2. $ export TRT_INC_PATH=”/usr/include/x86_64-linux-gnu” 13 | 4. Run the converter 14 | 15 | 16 | Note: 17 | 18 | In Int8 mode, when using calibration table generated from the TAO v1.0 GA docker, if the version of the tensorrt on the deployment device is 5.x/6.x, please make sure to update the version number in the TensorRT calibration table file. Inorder to do so, 19 | 1. Open the `calibration.bin` or `calibration.txt` file as generated by `export` using a simple text editor 20 | 2. Update the first line in calibration file from `TRT-5105-EntropyCalibration2` or `TRT-6001-EntropyCalibration2` to `TRT-7000-EntropyCalibration2`. 21 | 3. Save the edited file with the new changes. 22 | 4. Use this file with `tao-converter` to generate the engine. 23 | 24 | Sample usage: 25 | 26 | 27 | ./tao-converter -h 28 | -------------------------------------------------------------------------------- /src/license_plate_recognition/tao_converter/convert.sh: -------------------------------------------------------------------------------- 1 | /deepstream_from_scratch/src/license_plate_recognition/tao_converter/tao-converter \ 2 | -k nvidia_tlt -p image_input,1x3x48x96,4x3x48x96,16x3x48x96 \ 3 | /deepstream_from_scratch/deepstream_lpr_app/models/LP/LPR/us_lprnet_baseline18_deployable.etlt -t fp16 -e \ 4 | /deepstream_from_scratch/deepstream_lpr_app/models/LP/LPR/lpr_us_onnx_b16.engine -------------------------------------------------------------------------------- /src/license_plate_recognition/tao_converter/tao-converter: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tienluongngoc/deepstream_from_scratch/75159a88532405f41c17a5b22973f145c7c97afe/src/license_plate_recognition/tao_converter/tao-converter -------------------------------------------------------------------------------- /src/save_image_deepstream/save_image_deepstream.cpp: -------------------------------------------------------------------------------- 1 | #include "save_image_deepstream/save_image_deepstream.hpp" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "gstnvdsmeta.h" 8 | #include "nvbufsurface.h" 9 | #include "nvbufsurftransform.h" 10 | #include 11 | #include "gstnvdsinfer.h" 12 | #include "gstnvdsmeta.h" 13 | 14 | #define MAX_DISPLAY_LEN 64 15 | #define PGIE_CLASS_ID_VEHICLE 0 16 | #define PGIE_CLASS_ID_PERSON 2 17 | #define MUXER_OUTPUT_WIDTH 1920 18 | #define MUXER_OUTPUT_HEIGHT 1080 19 | #define MUXER_BATCH_TIMEOUT_USEC 40000 20 | 21 | gint frame_number = 0; 22 | gchar pgie_classes_str[4][32] = { "Vehicle", "TwoWheeler", "Person", "Roadsign" }; 23 | 24 | static GstPadProbeReturn osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data) 25 | { 26 | GstBuffer *buf = (GstBuffer *) info->data; 27 | guint num_rects = 0; 28 | NvDsObjectMeta *obj_meta = NULL; 29 | guint vehicle_count = 0; 30 | guint person_count = 0; 31 | NvDsMetaList * l_frame = NULL; 32 | NvDsMetaList * l_obj = NULL; 33 | NvDsDisplayMeta *display_meta = NULL; 34 | GstMapInfo in_map_info; 35 | cv::Mat cpu_mat; 36 | 37 | NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf); 38 | 39 | if (!gst_buffer_map(buf, &in_map_info, GST_MAP_READ)) 40 | { 41 | g_print ("failed to map gst buffer"); 42 | gst_buffer_unmap(buf, &in_map_info); 43 | } 44 | 45 | 46 | for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next) { 47 | NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data); 48 | // to get image from GPU 49 | NvBufSurface *surface = (NvBufSurface *)in_map_info.data; 50 | void *data_ptr = surface->surfaceList[frame_meta->batch_id].dataPtr; 51 | int src_height = surface->surfaceList[frame_meta->batch_id].height; 52 | int src_width = surface->surfaceList[frame_meta->batch_id].width; 53 | int data_size = surface->surfaceList[frame_meta->batch_id].dataSize; 54 | 55 | cv::cuda::GpuMat gpu_mat = cv::cuda::GpuMat(src_height, src_width, CV_8UC4, data_ptr); 56 | gpu_mat.download(cpu_mat); 57 | cv::cvtColor(cpu_mat, cpu_mat, CV_RGBA2BGRA); 58 | cv::imwrite("original_image.jpg", cpu_mat); 59 | 60 | 61 | int offset = 0; 62 | for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) { 63 | obj_meta = (NvDsObjectMeta *) (l_obj->data); 64 | 65 | if (obj_meta->unique_component_id == 1) 66 | { 67 | 68 | int x1 = obj_meta->rect_params.left; 69 | int y1 = obj_meta->rect_params.top; 70 | int x2 = x1 + obj_meta->rect_params.width; 71 | int y2 = y1 + obj_meta->rect_params.height; 72 | cv::rectangle(cpu_mat, cv::Point(x1, y1), cv::Point(x2, y2), cv::Scalar(0, 0, 255), 1, 8, 0); 73 | } 74 | 75 | 76 | if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) { 77 | vehicle_count++; 78 | num_rects++; 79 | } 80 | if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) { 81 | person_count++; 82 | num_rects++; 83 | } 84 | } 85 | 86 | cv::imwrite("draw_bbox_image.jpg", cpu_mat); 87 | if (frame_number == 10){ 88 | exit(0); 89 | } 90 | // display_meta = nvds_acquire_display_meta_from_pool(batch_meta); 91 | // NvOSD_TextParams *txt_params = &display_meta->text_params[0]; 92 | // display_meta->num_labels = 1; 93 | // txt_params->display_text = g_malloc0 (MAX_DISPLAY_LEN); 94 | // offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, "Person = %d ", person_count); 95 | // offset = snprintf(txt_params->display_text + offset , MAX_DISPLAY_LEN, "Vehicle = %d ", vehicle_count); 96 | 97 | // /* Now set the offsets where the string should appear */ 98 | // txt_params->x_offset = 10; 99 | // txt_params->y_offset = 12; 100 | 101 | /* Font , font-color and font-size */ 102 | // txt_params->font_params.font_name = "Serif"; 103 | // txt_params->font_params.font_size = 10; 104 | // txt_params->font_params.font_color.red = 1.0; 105 | // txt_params->font_params.font_color.green = 1.0; 106 | // txt_params->font_params.font_color.blue = 1.0; 107 | // txt_params->font_params.font_color.alpha = 1.0; 108 | 109 | // /* Text background color */ 110 | // txt_params->set_bg_clr = 1; 111 | // txt_params->text_bg_clr.red = 0.0; 112 | // txt_params->text_bg_clr.green = 0.0; 113 | // txt_params->text_bg_clr.blue = 0.0; 114 | // txt_params->text_bg_clr.alpha = 1.0; 115 | 116 | // nvds_add_display_meta_to_frame(frame_meta, display_meta); 117 | } 118 | 119 | g_print ("Frame Number = %d Number of objects = %d " 120 | "Vehicle Count = %d Person Count = %d\n", 121 | frame_number, num_rects, vehicle_count, person_count); 122 | frame_number++; 123 | return GST_PAD_PROBE_OK; 124 | } 125 | 126 | static gboolean bus_call (GstBus * bus, GstMessage * msg, gpointer data) 127 | { 128 | GMainLoop *loop = (GMainLoop *) data; 129 | switch (GST_MESSAGE_TYPE (msg)) { 130 | case GST_MESSAGE_EOS: 131 | g_print ("End of stream\n"); 132 | g_main_loop_quit (loop); 133 | break; 134 | case GST_MESSAGE_ERROR:{ 135 | gchar *debug; 136 | GError *error; 137 | gst_message_parse_error (msg, &error, &debug); 138 | g_printerr ("ERROR from element %s: %s\n", 139 | GST_OBJECT_NAME (msg->src), error->message); 140 | if (debug) 141 | g_printerr ("Error details: %s\n", debug); 142 | g_free (debug); 143 | g_error_free (error); 144 | g_main_loop_quit (loop); 145 | break; 146 | } 147 | default: 148 | break; 149 | } 150 | return TRUE; 151 | } 152 | 153 | int save_image_deepstream (int argc, char *argv[]) 154 | { 155 | GMainLoop *loop = NULL; 156 | GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL, 157 | *decoder = NULL, *streammux = NULL, *sink = NULL, *pgie = NULL, *nvvidconv = NULL, 158 | *nvosd = NULL; 159 | 160 | GstElement *transform = NULL; 161 | GstBus *bus = NULL; 162 | guint bus_watch_id; 163 | GstPad *osd_sink_pad = NULL; 164 | 165 | int current_device = -1; 166 | cudaGetDevice(¤t_device); 167 | struct cudaDeviceProp prop; 168 | cudaGetDeviceProperties(&prop, current_device); 169 | /* Check input arguments */ 170 | if (argc != 2) { 171 | g_printerr ("Usage: %s \n", argv[0]); 172 | return -1; 173 | } 174 | 175 | /* Standard GStreamer initialization */ 176 | gst_init (&argc, &argv); 177 | loop = g_main_loop_new (NULL, FALSE); 178 | 179 | /* Create gstreamer elements */ 180 | /* Create Pipeline element that will form a connection of other elements */ 181 | pipeline = gst_pipeline_new ("dstest1-pipeline"); 182 | 183 | /* Source element for reading from the file */ 184 | source = gst_element_factory_make ("filesrc", "file-source"); 185 | 186 | /* Since the data format in the input file is elementary h264 stream, 187 | * we need a h264parser */ 188 | h264parser = gst_element_factory_make ("h264parse", "h264-parser"); 189 | 190 | /* Use nvdec_h264 for hardware accelerated decode on GPU */ 191 | decoder = gst_element_factory_make ("nvv4l2decoder", "nvv4l2-decoder"); 192 | 193 | /* Create nvstreammux instance to form batches from one or more sources. */ 194 | streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); 195 | 196 | if (!pipeline || !streammux) { 197 | g_printerr ("One element could not be created. Exiting.\n"); 198 | return -1; 199 | } 200 | 201 | /* Use nvinfer to run inferencing on decoder's output, 202 | * behaviour of inferencing is set through config file */ 203 | pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine"); 204 | 205 | /* Use convertor to convert from NV12 to RGBA as required by nvosd */ 206 | nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter"); 207 | 208 | /* Create OSD to draw on the converted RGBA buffer */ 209 | nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); 210 | 211 | /* Finally render the osd output */ 212 | if(prop.integrated) { 213 | transform = gst_element_factory_make ("nvegltransform", "nvegl-transform"); 214 | } 215 | sink = gst_element_factory_make ("fakesink", "nvvideo-renderer"); 216 | // sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer"); 217 | 218 | if (!source || !h264parser || !decoder || !pgie 219 | || !nvvidconv || !nvosd || !sink) { 220 | g_printerr ("One element could not be created. Exiting.\n"); 221 | return -1; 222 | } 223 | 224 | if(!transform && prop.integrated) { 225 | g_printerr ("One tegra element could not be created. Exiting.\n"); 226 | return -1; 227 | } 228 | 229 | /* we set the input filename to the source element */ 230 | g_object_set (G_OBJECT (source), "location", argv[1], NULL); 231 | 232 | g_object_set (G_OBJECT (streammux), "batch-size", 1, NULL); 233 | 234 | g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", 235 | MUXER_OUTPUT_HEIGHT, 236 | "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); 237 | 238 | /* Set all the necessary properties of the nvinfer element, 239 | * the necessary ones are : */ 240 | g_object_set (G_OBJECT (pgie), 241 | "config-file-path", "apps/save_image_deepstream/save_image_deepstream_config.txt", NULL); 242 | 243 | /* we add a message handler */ 244 | bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); 245 | bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); 246 | gst_object_unref (bus); 247 | 248 | /* Set up the pipeline */ 249 | /* we add all elements into the pipeline */ 250 | if(prop.integrated) { 251 | gst_bin_add_many (GST_BIN (pipeline), 252 | source, h264parser, decoder, streammux, pgie, 253 | nvvidconv, nvosd, transform, sink, NULL); 254 | } 255 | else { 256 | gst_bin_add_many (GST_BIN (pipeline), 257 | source, h264parser, decoder, streammux, pgie, 258 | nvvidconv, nvosd, sink, NULL); 259 | } 260 | 261 | GstPad *sinkpad, *srcpad; 262 | gchar pad_name_sink[16] = "sink_0"; 263 | gchar pad_name_src[16] = "src"; 264 | 265 | sinkpad = gst_element_get_request_pad (streammux, pad_name_sink); 266 | if (!sinkpad) { 267 | g_printerr ("Streammux request sink pad failed. Exiting.\n"); 268 | return -1; 269 | } 270 | 271 | srcpad = gst_element_get_static_pad (decoder, pad_name_src); 272 | if (!srcpad) { 273 | g_printerr ("Decoder request src pad failed. Exiting.\n"); 274 | return -1; 275 | } 276 | 277 | if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { 278 | g_printerr ("Failed to link decoder to stream muxer. Exiting.\n"); 279 | return -1; 280 | } 281 | 282 | gst_object_unref (sinkpad); 283 | gst_object_unref (srcpad); 284 | 285 | /* we link the elements together */ 286 | /* file-source -> h264-parser -> nvh264-decoder -> 287 | * nvinfer -> nvvidconv -> nvosd -> video-renderer */ 288 | 289 | if (!gst_element_link_many (source, h264parser, decoder, NULL)) { 290 | g_printerr ("Elements could not be linked: 1. Exiting.\n"); 291 | return -1; 292 | } 293 | 294 | if(prop.integrated) { 295 | if (!gst_element_link_many (streammux, pgie, 296 | nvvidconv, nvosd, transform, sink, NULL)) { 297 | g_printerr ("Elements could not be linked: 2. Exiting.\n"); 298 | return -1; 299 | } 300 | } 301 | else { 302 | if (!gst_element_link_many (streammux, pgie, 303 | nvvidconv, nvosd, sink, NULL)) { 304 | g_printerr ("Elements could not be linked: 2. Exiting.\n"); 305 | return -1; 306 | } 307 | } 308 | 309 | /* Lets add probe to get informed of the meta data generated, we add probe to 310 | * the sink pad of the osd element, since by that time, the buffer would have 311 | * had got all the metadata. */ 312 | osd_sink_pad = gst_element_get_static_pad (nvosd, "sink"); 313 | if (!osd_sink_pad) 314 | g_print ("Unable to get sink pad\n"); 315 | else 316 | gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, 317 | osd_sink_pad_buffer_probe, NULL, NULL); 318 | gst_object_unref (osd_sink_pad); 319 | 320 | /* Set the pipeline to "playing" state */ 321 | g_print ("Now playing"); 322 | gst_element_set_state (pipeline, GST_STATE_PLAYING); 323 | 324 | /* Wait till pipeline encounters an error or EOS */ 325 | g_print ("Running...\n"); 326 | g_main_loop_run (loop); 327 | 328 | /* Out of the main loop, clean up nicely */ 329 | g_print ("Returned, stopping playback\n"); 330 | gst_element_set_state (pipeline, GST_STATE_NULL); 331 | g_print ("Deleting pipeline\n"); 332 | gst_object_unref (GST_OBJECT (pipeline)); 333 | g_source_remove (bus_watch_id); 334 | g_main_loop_unref (loop); 335 | return 0; 336 | } 337 | -------------------------------------------------------------------------------- /src/tiny_yolo2_deepstream/custom_bbox_parser/Makefile: -------------------------------------------------------------------------------- 1 | CUDA_VER:=11.1 2 | SRCFILES:=nvdsparsebbox_tiny_yolo.cpp 3 | TARGET_LIB:=libnvdsinfer_custom_bbox_tiny_yolo.so 4 | DEEPSTREAM_PATH:=/opt/nvidia/deepstream/deepstream-5.1 5 | 6 | ifeq ($(CUDA_VER),) 7 | $(error "CUDA_VER is not set") 8 | endif 9 | CC:= g++ 10 | 11 | CFLAGS:= -Wall -Werror -std=c++11 -shared -fPIC 12 | CFLAGS+= -I$(DEEPSTREAM_PATH)/sources/includes -I/usr/local/cuda-$(CUDA_VER)/include 13 | 14 | LIBS:= -lnvinfer -lnvparsers -L/usr/local/cuda-$(CUDA_VER)/lib64 15 | LFLAGS:= -Wl,--start-group $(LIBS) -Wl,--end-group 16 | 17 | all: $(TARGET_LIB) 18 | 19 | $(TARGET_LIB) : $(SRCFILES) 20 | $(CC) -o $@ $^ $(CFLAGS) $(LFLAGS) 21 | 22 | clean: 23 | rm -rf $(TARGET_LIB) 24 | -------------------------------------------------------------------------------- /src/tiny_yolo2_deepstream/custom_bbox_parser/nvdsparsebbox_tiny_yolo.cpp: -------------------------------------------------------------------------------- 1 | #include "nvdsinfer_custom_impl.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | /** 11 | * Function expected by DeepStream for decoding the TinyYOLOv2 output. 12 | * 13 | * C-linkage [extern "C"] was written to prevent name-mangling. This function must return true after 14 | * adding all bounding boxes to the objectList vector. 15 | * 16 | * @param [outputLayersInfo] std::vector of NvDsInferLayerInfo objects with information about the output layer. 17 | * @param [networkInfo] NvDsInferNetworkInfo object with information about the TinyYOLOv2 network. 18 | * @param [detectionParams] NvDsInferParseDetectionParams with information about some config params. 19 | * @param [objectList] std::vector of NvDsInferParseObjectInfo objects to which bounding box information must 20 | * be stored. 21 | * 22 | * @return true 23 | */ 24 | 25 | // This is just the function prototype. The definition is written at the end of the file. 26 | extern "C" bool NvDsInferParseCustomYoloV2Tiny( 27 | std::vector const& outputLayersInfo, 28 | NvDsInferNetworkInfo const& networkInfo, 29 | NvDsInferParseDetectionParams const& detectionParams, 30 | std::vector& objectList); 31 | 32 | /** 33 | * Bounds values between the range [minVal, maxVal]. 34 | * 35 | * Values that are out of bounds are set to their boundary values. 36 | * For example, consider the following: clamp(bbox.left, 0, netW). This 37 | * translates to min(netW, max(0, bbox.left)). Hence, if bbox.left was 38 | * negative, it is set to 0. If bbox.left is greater than netW, it is set 39 | * to netW. 40 | * 41 | * @param [val] Value to be bound. 42 | * @param [minVal] Lower bound. 43 | * @param [maxVal] Upper bound. 44 | * 45 | * @return A value that is bound in the range [minVal, maxVal]. 46 | */ 47 | static unsigned clamp(const uint val, const uint minVal, const uint maxVal) 48 | { 49 | assert(minVal <= maxVal); 50 | return std::min(maxVal, std::max(minVal, val)); 51 | } 52 | 53 | /** 54 | * Computes the overlap between two lines along a single axis (1D). 55 | * 56 | * Overlap is computed along either the X or Y axis as specified by the 57 | * user. If no overlap is available, 0 is returned. 58 | * 59 | * @param [x1min] Minimum coordinate of the 1st line along a single axis. 60 | * @param [x1max] Maximum coordinate of the 1st line along a single axis. 61 | * @param [x2min] Minimum coordinate of the 2nd line along a single axis. 62 | * @param [x2max] Maximum coordinate of the 2nd line along a single axis. 63 | * 64 | * @return Overlap between the two lines as a float value. 65 | */ 66 | static float overlap1D(float x1min, float x1max, float x2min, float x2max) 67 | { 68 | if (x1min > x2min) 69 | { 70 | std::swap(x1min, x2min); 71 | std::swap(x1max, x2max); 72 | } 73 | return x1max < x2min ? 0 : std::min(x1max, x2max) - x2min; 74 | } 75 | 76 | /** 77 | * Computes the Intersection over Union (IoU) of two rectangular bounding boxes. 78 | * 79 | * IoU is computed as follows. First, the 1D overlap of the two boxes along the 80 | * X and Y axis is computed. They are multiplied to get the 2D overlap of the two 81 | * boxes. The areas of both boxes are also computed. The 2D overlap gives us the 82 | * intersection. The sum of the areas of both boxes minus the intersection gives us 83 | * the union. IoU is just the ratio of the intersection to the union. A 0 is returned 84 | * if there is union is 0. 85 | * 86 | * @param [bbox1] NvDsInferParseObjectInfo object containing the 1st bounding box info. 87 | * @param [bbox2] NvDsInferParseObjectInfo object containing the 2nd bounding box info. 88 | * 89 | * @return IoU between the two bounding boxes as a float value. 90 | */ 91 | static float computeIoU(const NvDsInferParseObjectInfo& bbox1, const NvDsInferParseObjectInfo& bbox2) 92 | { 93 | float overlapX 94 | = overlap1D(bbox1.left, bbox1.left + bbox1.width, bbox2.left, bbox2.left + bbox2.width); 95 | float overlapY 96 | = overlap1D(bbox1.top, bbox1.top + bbox1.height, bbox2.top, bbox2.top + bbox2.height); 97 | float area1 = (bbox1.width) * (bbox1.height); 98 | float area2 = (bbox2.width) * (bbox2.height); 99 | float overlap2D = overlapX * overlapY; 100 | float u = area1 + area2 - overlap2D; 101 | return u == 0 ? 0 : overlap2D / u; 102 | } 103 | 104 | /** 105 | * Returns a boolean value indicating whether a bounding box confidence value is greater. 106 | * 107 | * Given two NvDsInferParseObjectInfo bounding box object, we compare the confidence values 108 | * and return true if the 1st bounding box confidence value is greater than 2nd bounding box 109 | * confidence value. This function is passed as an argument for the sorting function. 110 | * 111 | * @param [bbox1] NvDsInferParseObjectInfo object containing the 1st bounding box info. 112 | * @param [bbox2] NvDsInferParseObjectInfo object containing the 2nd bounding box info. 113 | * 114 | * @return Boolean value indicating whether confidence of bbox1 is greater than confidence of bbox2. 115 | */ 116 | static bool compareBBoxConfidence(const NvDsInferParseObjectInfo& bbox1, const NvDsInferParseObjectInfo& bbox2) 117 | { 118 | return bbox1.detectionConfidence > bbox2.detectionConfidence; 119 | } 120 | 121 | /** 122 | * Creates the NvDsInferParseObjectInfo Bounding Box object given attributes. 123 | * 124 | * @param [bx] Bounding Box center X-Coordinate. 125 | * @param [by] Bounding Box center Y-Coordinate. 126 | * @param [bw] Bounding Box width. 127 | * @param [bh] Bounding Box height. 128 | * @param [stride] Ratio of the image width to the grid size. 129 | * @param [netW] Width of the image. 130 | * @param [netH] Height of the image. 131 | * 132 | * @return NvDsInferParseObjectInfo Bounding Box object. 133 | */ 134 | static NvDsInferParseObjectInfo createBBox(const float& bx, const float& by, const float& bw, 135 | const float& bh, const int& stride, const uint& netW, 136 | const uint& netH) 137 | { 138 | NvDsInferParseObjectInfo bbox; 139 | // Restore coordinates to network input resolution 140 | float x = bx * stride; 141 | float y = by * stride; 142 | 143 | bbox.left = x - bw / 2; 144 | bbox.width = bw; 145 | 146 | bbox.top = y - bh / 2; 147 | bbox.height = bh; 148 | 149 | // Bounds bbox values between [minVal, maxVal] 150 | bbox.left = clamp(bbox.left, 0, netW); 151 | bbox.width = clamp(bbox.width, 0, netW); 152 | bbox.top = clamp(bbox.top, 0, netH); 153 | bbox.height = clamp(bbox.height, 0, netH); 154 | 155 | return bbox; 156 | } 157 | 158 | /** 159 | * Adds an NvDsInferParseObjectInfo Bounding Box object to a vector. 160 | * 161 | * This function is used to accumulate all the bounding boxes present in a single 162 | * frame of the video into a vector called bboxInfo. 163 | * 164 | * @param [bx] Bounding Box center X-Coordinate. 165 | * @param [by] Bounding Box center Y-Coordinate. 166 | * @param [bw] Bounding Box width. 167 | * @param [bh] Bounding Box height. 168 | * @param [stride] Ratio of the image width to the grid size. 169 | * @param [netW] Width of the image. 170 | * @param [netH] Height of the image. 171 | * @param [maxIndex] Class ID of the detected bounding box. 172 | * @param [maxProb] Confidence of the detected bounding box 173 | * @param [bboxInfo] std::vector of bounding boxes. 174 | */ 175 | static void addBBoxProposal(const float bx, const float by, const float bw, const float bh, 176 | const uint stride, const uint& netW, const uint& netH, const int maxIndex, 177 | const float maxProb, std::vector& bboxInfo) 178 | { 179 | NvDsInferParseObjectInfo bbox = createBBox(bx, by, bw, bh, stride, netW, netH); 180 | if (((bbox.left + bbox.width) > netW) || ((bbox.top + bbox.height) > netH)) return; 181 | 182 | bbox.detectionConfidence = maxProb; 183 | bbox.classId = maxIndex; 184 | bboxInfo.push_back(bbox); 185 | } 186 | 187 | /** 188 | * Performs Non Maximum Suppression (NMS) over all bounding boxes of a single class. 189 | * 190 | * The bounding boxes are first sorted by their confidence values in decending order. Then, 191 | * the bounding boxes are iterated over to remove multiple detections of the same object 192 | * and only retain the bounding box of the highest confidence of that object. Two bounding 193 | * boxes are considered to belong to the same object if they have a large overlap. 194 | * 195 | * @param [inputBBoxInfo] std::vector of bounding boxes belonging to a single class. 196 | * @param [nmsThresh] Overlap threshold for NMS. 197 | * 198 | * @return std::vector of bounding boxes of a single class after NMS is applied. 199 | */ 200 | static std::vector 201 | nonMaximumSuppression(std::vector inputBBoxInfo, const float nmsThresh) 202 | { 203 | std::stable_sort(inputBBoxInfo.begin(), inputBBoxInfo.end(), compareBBoxConfidence); 204 | std::vector outputBBoxInfo; 205 | 206 | for (auto bbox1 : inputBBoxInfo) 207 | { 208 | bool keep = true; 209 | for (auto bbox2 : outputBBoxInfo) 210 | { 211 | if (keep) 212 | { 213 | float overlap = computeIoU(bbox1, bbox2); 214 | keep = overlap <= nmsThresh; 215 | } 216 | else 217 | break; 218 | } 219 | if (keep) outputBBoxInfo.push_back(bbox1); 220 | } 221 | return outputBBoxInfo; 222 | } 223 | 224 | /** 225 | * Performs Non Maximum Suppression (NMS) over all classes. 226 | * 227 | * Interatively performs NMS over all classes using the single class NMS function. 228 | * 229 | * @param [bboxInfo] [description] 230 | * @param [numClasses] Number of classes. 231 | * @param [nmsThresh] Overlap threshold for NMS. 232 | * 233 | * @return std::vector of bounding boxes of all classes after NMS is applied. 234 | */ 235 | 236 | static std::vector 237 | nmsAllClasses(std::vector& bboxInfo, const uint numClasses, const float nmsThresh) 238 | { 239 | std::vector resultBBoxes; 240 | 241 | // std::vector of std::vector (of size numClasses) to hold classwise bounding boxes. 242 | std::vector> splitBoxes(numClasses); 243 | 244 | // Bounding box with attribute "classID" is pushed into the index "classID" of "splitBoxes". This 245 | // way, splitBoxes will have bounding boxes belonging to the same class at each index. 246 | for (auto &bbox : bboxInfo) 247 | { 248 | splitBoxes.at(bbox.classId).push_back(bbox); 249 | } 250 | 251 | // Applying NMS for bounding boxes belonging to the same class and collecting the resultant 252 | // bounding boxes in resultBBoxes. 253 | for (auto &bboxesPerClass : splitBoxes) 254 | { 255 | bboxesPerClass = nonMaximumSuppression(bboxesPerClass, nmsThresh); 256 | resultBBoxes.insert(resultBBoxes.end(), bboxesPerClass.begin(), bboxesPerClass.end()); 257 | } 258 | return resultBBoxes; 259 | } 260 | 261 | /** 262 | * Deocodes the output of TinyYOLOv2 and outputs a vector of NvDsInferParseObjectInfo bounding boxes. 263 | * 264 | * Loops over the output of TinyYOLOv2 and calculates the bounding box locations. Ignores bounding 265 | * boxes with detection probability less than the probability threshold. 266 | * 267 | * @param [detections] Output of TinyYOLOv2 as a flattened object. 268 | * @param [netW] Width of the image. 269 | * @param [netH] Height of the image. 270 | * @param [anchors] XY locations of the Anchor Boxes multiplied by the stride value provided as a vector. 271 | * The anchor box vector is of the form [x_1, y_1, x_2, y_2, ..., x_5, y_5]. 272 | * @param [numBBoxes] Number of Anchors. This is equal to the number of predicted bounding boxes per grid cell. 273 | * @param [gridSize] Size of the grid. 274 | * @param [stride] Ratio of the image width to the grid size. 275 | * @param [probThresh] Bounding box probability threshold. 276 | * @param [numOutputClasses] Overlap threshold for NMS. 277 | * 278 | * @return std::vector of bounding boxes of all classes obtained by decoding the TinyYOLOv2 output. 279 | */ 280 | 281 | static std::vector 282 | decodeYoloV2Tensor( 283 | const float* detections, const uint& netW, const uint& netH, 284 | const std::vector &anchors, const uint numBBoxes, 285 | const uint gridSize, const uint stride, const float probThresh, 286 | const uint numOutputClasses) 287 | { 288 | std::vector bboxInfo; 289 | const int b_offset = gridSize * gridSize; 290 | 291 | for (uint y = 0; y < gridSize; ++y) 292 | { 293 | for (uint x = 0; x < gridSize; ++x) 294 | { 295 | const int xy_offset = (y * gridSize + x); 296 | 297 | for (uint b = 0; b < numBBoxes; ++b) 298 | { 299 | const float pw = anchors[b * 2]; 300 | const float ph = anchors[b * 2 + 1]; 301 | 302 | const int start_idx = xy_offset + b_offset * b * (5 + numOutputClasses); 303 | 304 | const float sigmoid_tx 305 | = 1 / (1 + exp (-detections[start_idx + 0 * b_offset])); 306 | const float sigmoid_ty 307 | = 1 / (1 + exp (-detections[start_idx + 1 * b_offset])); 308 | const float bx 309 | = x + sigmoid_tx; 310 | const float by 311 | = y + sigmoid_ty; 312 | const float bw 313 | = pw * exp (detections[start_idx + 2 * b_offset]); 314 | const float bh 315 | = ph * exp (detections[start_idx + 3 * b_offset]); 316 | const float objectness 317 | = 1 / (1 + exp(-detections[start_idx + 4 * b_offset])); 318 | 319 | int maxIndex = -1; 320 | float maxProb = 0.0f; 321 | float max_class_val = 0.0f; 322 | 323 | // Finding the maximum value and well as the index with maximum value 324 | // prior to applying softmax. Since softmax is monotonically increasing, 325 | // maxIndex can be calculated here itself. 326 | for (uint i = 0; i < numOutputClasses; ++i) 327 | { 328 | float class_val = detections[start_idx + (5 + i) * b_offset]; 329 | 330 | if (class_val > max_class_val) 331 | { 332 | max_class_val = class_val; 333 | maxIndex = i; 334 | } 335 | } 336 | 337 | float sum_exp = 0.0f; 338 | // Calculating the denominator of the softmax function. Note that, we are 339 | // performing softmax(x - max(x)) where x is the list of class outputs. 340 | // Note that softmax(x + a) gives the same result as softmax(x) where, a is 341 | // a constant value. By replacing a with -max(x) softmax becomes more 342 | // stable since exp does not have to deal with large numbers. 343 | for (uint i = 0; i < numOutputClasses; ++i) 344 | { 345 | float class_val = detections[start_idx + (5 + i) * b_offset]; 346 | float class_exp = exp(class_val - max_class_val); 347 | sum_exp = sum_exp + class_exp; 348 | } 349 | 350 | // The largest softmax probability among all x values will be softmax(max(x)) 351 | // since softmax is monotonically increasing. Since we are actually calculating 352 | // softmax(x_i - max(x)), when x_i = max(x), we get softmax(max(x) - max(x)), 353 | // which is just 1 / sum_exp. 354 | float max_softmax_prob = 1 / sum_exp; 355 | maxProb = objectness * max_softmax_prob; 356 | 357 | if (maxProb > probThresh) 358 | { 359 | addBBoxProposal(bx, by, bw, bh, stride, netW, netH, maxIndex, maxProb, bboxInfo); 360 | } 361 | } 362 | } 363 | } 364 | return bboxInfo; 365 | } 366 | 367 | 368 | /** 369 | * Function expected by DeepStream for decoding the TinyYOLOv2 output. 370 | * 371 | * C-linkage [extern "C"] was written to prevent name-mangling. This function must return true after 372 | * adding all bounding boxes to the objectList vector. 373 | * 374 | * @param [outputLayersInfo] std::vector of NvDsInferLayerInfo objects with information about the output layer. 375 | * @param [networkInfo] NvDsInferNetworkInfo object with information about the TinyYOLOv2 network. 376 | * @param [detectionParams] NvDsInferParseDetectionParams with information about some config params. 377 | * @param [objectList] std::vector of NvDsInferParseObjectInfo objects to which bounding box information must 378 | * be stored. 379 | * 380 | * @return true 381 | */ 382 | 383 | /* C-linkage to prevent name-mangling */ 384 | extern "C" bool NvDsInferParseCustomYoloV2Tiny( 385 | std::vector const& outputLayersInfo, 386 | NvDsInferNetworkInfo const& networkInfo, 387 | NvDsInferParseDetectionParams const& detectionParams, 388 | std::vector& objectList) 389 | { 390 | 391 | // Initializing some parameters. 392 | 393 | /** 394 | * In our case, we know stride and gridSize beforehand. If this is 395 | * not the case, they can be calculated using the following formulae: 396 | * 397 | * const uint gridSize = layer.dims.d[1]; 398 | * const uint stride = networkInfo.width / gridSize; 399 | */ 400 | 401 | static const uint kSTRIDE = 32; 402 | static const uint kGRID_SIZE = 13; 403 | static const uint kNUM_ANCHORS = 5; 404 | static const float kNMS_THRESH = 0.2f; 405 | static const float kPROB_THRESH = 0.6f; 406 | static const uint kNUM_CLASSES_YOLO = 20; 407 | 408 | /** 409 | * The vector kANCHORS is actually the anchor box coordinates 410 | * multiplied by the stride variable. Since we know the stride 411 | * value before hand, we store the multiplied values as it saves 412 | * some computation. [For our case, stride = 32] 413 | */ 414 | 415 | static const std::vector kANCHORS = { 416 | 34.56, 38.08, 109.44, 141.12, 417 | 212.16, 364.16, 301.44, 163.52, 418 | 531.84, 336.64 }; 419 | 420 | // Some assertions and error checking. 421 | if (outputLayersInfo.empty()) { 422 | std::cerr << "Could not find output layer in bbox parsing" << std::endl;; 423 | return false; 424 | } 425 | 426 | if (kNUM_CLASSES_YOLO != detectionParams.numClassesConfigured) 427 | { 428 | std::cerr << "WARNING: Num classes mismatch. Configured:" 429 | << detectionParams.numClassesConfigured 430 | << ", detected by network: " << kNUM_CLASSES_YOLO << std::endl; 431 | } 432 | 433 | // Obtaining the output layer. 434 | const NvDsInferLayerInfo &layer = outputLayersInfo[0]; 435 | assert (layer.inferDims.numDims == 3); 436 | 437 | // Decoding the output tensor of TinyYOLOv2 to the NvDsInferParseObjectInfo format. 438 | std::vector objects = 439 | decodeYoloV2Tensor( 440 | (const float*)(layer.buffer), networkInfo.width, networkInfo.height, 441 | kANCHORS, kNUM_ANCHORS, kGRID_SIZE, kSTRIDE, kPROB_THRESH, 442 | kNUM_CLASSES_YOLO 443 | ); 444 | 445 | // Applying Non Maximum Suppression to remove multiple detections of the same object. 446 | objectList.clear(); 447 | objectList = nmsAllClasses(objects, kNUM_CLASSES_YOLO, kNMS_THRESH); 448 | 449 | return true; 450 | } 451 | 452 | /* Check that the custom function has been defined correctly */ 453 | CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomYoloV2Tiny); 454 | -------------------------------------------------------------------------------- /src/tiny_yolo2_deepstream/tiny_yolo2_deepstream.cpp: -------------------------------------------------------------------------------- 1 | #include "tiny_yolo2_deepstream/tiny_yolo2_deepstream.hpp" 2 | 3 | #define MAX_DISPLAY_LEN 64 4 | #define PGIE_CLASS_ID_VEHICLE 6 5 | #define PGIE_CLASS_ID_PERSON 14 6 | #define MUXER_OUTPUT_WIDTH 1920 7 | #define MUXER_OUTPUT_HEIGHT 1080 8 | #define MUXER_BATCH_TIMEOUT_USEC 40000 9 | 10 | gint frame_number = 0; 11 | gchar pgie_classes_str[4][32] = { "Vehicle", "TwoWheeler", "Person", "Roadsign" }; 12 | 13 | static GstPadProbeReturn osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data) 14 | { 15 | GstBuffer *buf = (GstBuffer *) info->data; 16 | guint num_rects = 0; 17 | NvDsObjectMeta *obj_meta = NULL; 18 | guint vehicle_count = 0; 19 | guint person_count = 0; 20 | NvDsMetaList * l_frame = NULL; 21 | NvDsMetaList * l_obj = NULL; 22 | NvDsDisplayMeta *display_meta = NULL; 23 | 24 | NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf); 25 | 26 | for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; 27 | l_frame = l_frame->next) { 28 | NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data); 29 | int offset = 0; 30 | for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; 31 | l_obj = l_obj->next) { 32 | obj_meta = (NvDsObjectMeta *) (l_obj->data); 33 | if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) { 34 | vehicle_count++; 35 | num_rects++; 36 | } 37 | if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) { 38 | person_count++; 39 | num_rects++; 40 | } 41 | } 42 | // display_meta = nvds_acquire_display_meta_from_pool(batch_meta); 43 | // NvOSD_TextParams *txt_params = &display_meta->text_params[0]; 44 | // display_meta->num_labels = 1; 45 | // txt_params->display_text = g_malloc0 (MAX_DISPLAY_LEN); 46 | // offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, "Person = %d ", person_count); 47 | // offset = snprintf(txt_params->display_text + offset , MAX_DISPLAY_LEN, "Vehicle = %d ", vehicle_count); 48 | 49 | // /* Now set the offsets where the string should appear */ 50 | // txt_params->x_offset = 10; 51 | // txt_params->y_offset = 12; 52 | 53 | /* Font , font-color and font-size */ 54 | // txt_params->font_params.font_name = "Serif"; 55 | // txt_params->font_params.font_size = 10; 56 | // txt_params->font_params.font_color.red = 1.0; 57 | // txt_params->font_params.font_color.green = 1.0; 58 | // txt_params->font_params.font_color.blue = 1.0; 59 | // txt_params->font_params.font_color.alpha = 1.0; 60 | 61 | // /* Text background color */ 62 | // txt_params->set_bg_clr = 1; 63 | // txt_params->text_bg_clr.red = 0.0; 64 | // txt_params->text_bg_clr.green = 0.0; 65 | // txt_params->text_bg_clr.blue = 0.0; 66 | // txt_params->text_bg_clr.alpha = 1.0; 67 | 68 | // nvds_add_display_meta_to_frame(frame_meta, display_meta); 69 | } 70 | 71 | g_print ("Frame Number = %d Number of objects = %d " 72 | "Vehicle Count = %d Person Count = %d\n", 73 | frame_number, num_rects, vehicle_count, person_count); 74 | frame_number++; 75 | return GST_PAD_PROBE_OK; 76 | } 77 | 78 | static gboolean bus_call (GstBus * bus, GstMessage * msg, gpointer data) 79 | { 80 | GMainLoop *loop = (GMainLoop *) data; 81 | switch (GST_MESSAGE_TYPE (msg)) { 82 | case GST_MESSAGE_EOS: 83 | g_print ("End of stream\n"); 84 | g_main_loop_quit (loop); 85 | break; 86 | case GST_MESSAGE_ERROR:{ 87 | gchar *debug; 88 | GError *error; 89 | gst_message_parse_error (msg, &error, &debug); 90 | g_printerr ("ERROR from element %s: %s\n", 91 | GST_OBJECT_NAME (msg->src), error->message); 92 | if (debug) 93 | g_printerr ("Error details: %s\n", debug); 94 | g_free (debug); 95 | g_error_free (error); 96 | g_main_loop_quit (loop); 97 | break; 98 | } 99 | default: 100 | break; 101 | } 102 | return TRUE; 103 | } 104 | 105 | int tiny_yolo2_deepstream (int argc, char *argv[]) 106 | { 107 | GMainLoop *loop = NULL; 108 | GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL, 109 | *decoder = NULL, *streammux = NULL, *sink = NULL, *pgie = NULL, *nvvidconv = NULL, 110 | *nvosd = NULL; 111 | 112 | GstElement *transform = NULL; 113 | GstBus *bus = NULL; 114 | guint bus_watch_id; 115 | GstPad *osd_sink_pad = NULL; 116 | 117 | int current_device = -1; 118 | cudaGetDevice(¤t_device); 119 | struct cudaDeviceProp prop; 120 | cudaGetDeviceProperties(&prop, current_device); 121 | /* Check input arguments */ 122 | if (argc != 2) { 123 | g_printerr ("Usage: %s \n", argv[0]); 124 | return -1; 125 | } 126 | 127 | /* Standard GStreamer initialization */ 128 | gst_init (&argc, &argv); 129 | loop = g_main_loop_new (NULL, FALSE); 130 | 131 | /* Create gstreamer elements */ 132 | /* Create Pipeline element that will form a connection of other elements */ 133 | pipeline = gst_pipeline_new ("dstest1-pipeline"); 134 | 135 | /* Source element for reading from the file */ 136 | source = gst_element_factory_make ("filesrc", "file-source"); 137 | 138 | /* Since the data format in the input file is elementary h264 stream, 139 | * we need a h264parser */ 140 | h264parser = gst_element_factory_make ("h264parse", "h264-parser"); 141 | 142 | /* Use nvdec_h264 for hardware accelerated decode on GPU */ 143 | decoder = gst_element_factory_make ("nvv4l2decoder", "nvv4l2-decoder"); 144 | 145 | /* Create nvstreammux instance to form batches from one or more sources. */ 146 | streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); 147 | 148 | if (!pipeline || !streammux) { 149 | g_printerr ("One element could not be created. Exiting.\n"); 150 | return -1; 151 | } 152 | 153 | /* Use nvinfer to run inferencing on decoder's output, 154 | * behaviour of inferencing is set through config file */ 155 | pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine"); 156 | 157 | /* Use convertor to convert from NV12 to RGBA as required by nvosd */ 158 | nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter"); 159 | 160 | /* Create OSD to draw on the converted RGBA buffer */ 161 | nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); 162 | 163 | /* Finally render the osd output */ 164 | if(prop.integrated) { 165 | transform = gst_element_factory_make ("nvegltransform", "nvegl-transform"); 166 | } 167 | sink = gst_element_factory_make ("fakesink", "nvvideo-renderer"); 168 | // sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer"); 169 | 170 | if (!source || !h264parser || !decoder || !pgie 171 | || !nvvidconv || !nvosd || !sink) { 172 | g_printerr ("One element could not be created. Exiting.\n"); 173 | return -1; 174 | } 175 | 176 | if(!transform && prop.integrated) { 177 | g_printerr ("One tegra element could not be created. Exiting.\n"); 178 | return -1; 179 | } 180 | 181 | /* we set the input filename to the source element */ 182 | g_object_set (G_OBJECT (source), "location", argv[1], NULL); 183 | 184 | g_object_set (G_OBJECT (streammux), "batch-size", 1, NULL); 185 | 186 | g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", 187 | MUXER_OUTPUT_HEIGHT, 188 | "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); 189 | 190 | /* Set all the necessary properties of the nvinfer element, 191 | * the necessary ones are : */ 192 | g_object_set (G_OBJECT (pgie), 193 | "config-file-path", "apps/tiny_yolo2_deepstream/tiny_yolo2_deepstream.txt", NULL); 194 | 195 | /* we add a message handler */ 196 | bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); 197 | bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); 198 | gst_object_unref (bus); 199 | 200 | /* Set up the pipeline */ 201 | /* we add all elements into the pipeline */ 202 | if(prop.integrated) { 203 | gst_bin_add_many (GST_BIN (pipeline), 204 | source, h264parser, decoder, streammux, pgie, 205 | nvvidconv, nvosd, transform, sink, NULL); 206 | } 207 | else { 208 | gst_bin_add_many (GST_BIN (pipeline), 209 | source, h264parser, decoder, streammux, pgie, 210 | nvvidconv, nvosd, sink, NULL); 211 | } 212 | 213 | GstPad *sinkpad, *srcpad; 214 | gchar pad_name_sink[16] = "sink_0"; 215 | gchar pad_name_src[16] = "src"; 216 | 217 | sinkpad = gst_element_get_request_pad (streammux, pad_name_sink); 218 | if (!sinkpad) { 219 | g_printerr ("Streammux request sink pad failed. Exiting.\n"); 220 | return -1; 221 | } 222 | 223 | srcpad = gst_element_get_static_pad (decoder, pad_name_src); 224 | if (!srcpad) { 225 | g_printerr ("Decoder request src pad failed. Exiting.\n"); 226 | return -1; 227 | } 228 | 229 | if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { 230 | g_printerr ("Failed to link decoder to stream muxer. Exiting.\n"); 231 | return -1; 232 | } 233 | 234 | gst_object_unref (sinkpad); 235 | gst_object_unref (srcpad); 236 | 237 | /* we link the elements together */ 238 | /* file-source -> h264-parser -> nvh264-decoder -> 239 | * nvinfer -> nvvidconv -> nvosd -> video-renderer */ 240 | 241 | if (!gst_element_link_many (source, h264parser, decoder, NULL)) { 242 | g_printerr ("Elements could not be linked: 1. Exiting.\n"); 243 | return -1; 244 | } 245 | 246 | if(prop.integrated) { 247 | if (!gst_element_link_many (streammux, pgie, 248 | nvvidconv, nvosd, transform, sink, NULL)) { 249 | g_printerr ("Elements could not be linked: 2. Exiting.\n"); 250 | return -1; 251 | } 252 | } 253 | else { 254 | if (!gst_element_link_many (streammux, pgie, 255 | nvvidconv, nvosd, sink, NULL)) { 256 | g_printerr ("Elements could not be linked: 2. Exiting.\n"); 257 | return -1; 258 | } 259 | } 260 | 261 | /* Lets add probe to get informed of the meta data generated, we add probe to 262 | * the sink pad of the osd element, since by that time, the buffer would have 263 | * had got all the metadata. */ 264 | osd_sink_pad = gst_element_get_static_pad (nvosd, "sink"); 265 | if (!osd_sink_pad) 266 | g_print ("Unable to get sink pad\n"); 267 | else 268 | gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, 269 | osd_sink_pad_buffer_probe, NULL, NULL); 270 | gst_object_unref (osd_sink_pad); 271 | 272 | /* Set the pipeline to "playing" state */ 273 | g_print ("Now playing"); 274 | gst_element_set_state (pipeline, GST_STATE_PLAYING); 275 | 276 | /* Wait till pipeline encounters an error or EOS */ 277 | g_print ("Running...\n"); 278 | g_main_loop_run (loop); 279 | 280 | /* Out of the main loop, clean up nicely */ 281 | g_print ("Returned, stopping playback\n"); 282 | gst_element_set_state (pipeline, GST_STATE_NULL); 283 | g_print ("Deleting pipeline\n"); 284 | gst_object_unref (GST_OBJECT (pipeline)); 285 | g_source_remove (bus_watch_id); 286 | g_main_loop_unref (loop); 287 | return 0; 288 | } 289 | -------------------------------------------------------------------------------- /src/using_bin/using_bin.cpp: -------------------------------------------------------------------------------- 1 | #include "save_image_deepstream/save_image_deepstream.hpp" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "gstnvdsmeta.h" 8 | #include "nvbufsurface.h" 9 | #include "nvbufsurftransform.h" 10 | #include 11 | #include "gstnvdsinfer.h" 12 | #include "gstnvdsmeta.h" 13 | 14 | #define MAX_DISPLAY_LEN 64 15 | #define PGIE_CLASS_ID_VEHICLE 0 16 | #define PGIE_CLASS_ID_PERSON 2 17 | #define MUXER_OUTPUT_WIDTH 1920 18 | #define MUXER_OUTPUT_HEIGHT 1080 19 | #define MUXER_BATCH_TIMEOUT_USEC 40000 20 | 21 | gint frame_number = 0; 22 | gchar pgie_classes_str[4][32] = { "Vehicle", "TwoWheeler", "Person", "Roadsign" }; 23 | 24 | static GstPadProbeReturn osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data) 25 | { 26 | GstBuffer *buf = (GstBuffer *) info->data; 27 | guint num_rects = 0; 28 | NvDsObjectMeta *obj_meta = NULL; 29 | guint vehicle_count = 0; 30 | guint person_count = 0; 31 | NvDsMetaList * l_frame = NULL; 32 | NvDsMetaList * l_obj = NULL; 33 | NvDsDisplayMeta *display_meta = NULL; 34 | GstMapInfo in_map_info; 35 | cv::Mat cpu_mat; 36 | 37 | NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf); 38 | 39 | if (!gst_buffer_map(buf, &in_map_info, GST_MAP_READ)) 40 | { 41 | g_print ("failed to map gst buffer"); 42 | gst_buffer_unmap(buf, &in_map_info); 43 | } 44 | 45 | 46 | for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next) { 47 | NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data); 48 | // to get image from GPU 49 | NvBufSurface *surface = (NvBufSurface *)in_map_info.data; 50 | void *data_ptr = surface->surfaceList[frame_meta->batch_id].dataPtr; 51 | int src_height = surface->surfaceList[frame_meta->batch_id].height; 52 | int src_width = surface->surfaceList[frame_meta->batch_id].width; 53 | int data_size = surface->surfaceList[frame_meta->batch_id].dataSize; 54 | 55 | cv::cuda::GpuMat gpu_mat = cv::cuda::GpuMat(src_height, src_width, CV_8UC4, data_ptr); 56 | gpu_mat.download(cpu_mat); 57 | cv::cvtColor(cpu_mat, cpu_mat, CV_RGBA2BGRA); 58 | cv::imwrite("original_image.jpg", cpu_mat); 59 | 60 | 61 | int offset = 0; 62 | for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) { 63 | obj_meta = (NvDsObjectMeta *) (l_obj->data); 64 | 65 | if (obj_meta->unique_component_id == 1) 66 | { 67 | 68 | int x1 = obj_meta->rect_params.left; 69 | int y1 = obj_meta->rect_params.top; 70 | int x2 = x1 + obj_meta->rect_params.width; 71 | int y2 = y1 + obj_meta->rect_params.height; 72 | cv::rectangle(cpu_mat, cv::Point(x1, y1), cv::Point(x2, y2), cv::Scalar(0, 0, 255), 1, 8, 0); 73 | } 74 | 75 | 76 | if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) { 77 | vehicle_count++; 78 | num_rects++; 79 | } 80 | if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) { 81 | person_count++; 82 | num_rects++; 83 | } 84 | } 85 | 86 | cv::imwrite("draw_bbox_image.jpg", cpu_mat); 87 | if (frame_number == 10){ 88 | exit(0); 89 | } 90 | // display_meta = nvds_acquire_display_meta_from_pool(batch_meta); 91 | // NvOSD_TextParams *txt_params = &display_meta->text_params[0]; 92 | // display_meta->num_labels = 1; 93 | // txt_params->display_text = g_malloc0 (MAX_DISPLAY_LEN); 94 | // offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, "Person = %d ", person_count); 95 | // offset = snprintf(txt_params->display_text + offset , MAX_DISPLAY_LEN, "Vehicle = %d ", vehicle_count); 96 | 97 | // /* Now set the offsets where the string should appear */ 98 | // txt_params->x_offset = 10; 99 | // txt_params->y_offset = 12; 100 | 101 | /* Font , font-color and font-size */ 102 | // txt_params->font_params.font_name = "Serif"; 103 | // txt_params->font_params.font_size = 10; 104 | // txt_params->font_params.font_color.red = 1.0; 105 | // txt_params->font_params.font_color.green = 1.0; 106 | // txt_params->font_params.font_color.blue = 1.0; 107 | // txt_params->font_params.font_color.alpha = 1.0; 108 | 109 | // /* Text background color */ 110 | // txt_params->set_bg_clr = 1; 111 | // txt_params->text_bg_clr.red = 0.0; 112 | // txt_params->text_bg_clr.green = 0.0; 113 | // txt_params->text_bg_clr.blue = 0.0; 114 | // txt_params->text_bg_clr.alpha = 1.0; 115 | 116 | // nvds_add_display_meta_to_frame(frame_meta, display_meta); 117 | } 118 | 119 | g_print ("Frame Number = %d Number of objects = %d " 120 | "Vehicle Count = %d Person Count = %d\n", 121 | frame_number, num_rects, vehicle_count, person_count); 122 | frame_number++; 123 | return GST_PAD_PROBE_OK; 124 | } 125 | 126 | static gboolean bus_call (GstBus * bus, GstMessage * msg, gpointer data) 127 | { 128 | GMainLoop *loop = (GMainLoop *) data; 129 | switch (GST_MESSAGE_TYPE (msg)) { 130 | case GST_MESSAGE_EOS: 131 | g_print ("End of stream\n"); 132 | g_main_loop_quit (loop); 133 | break; 134 | case GST_MESSAGE_ERROR:{ 135 | gchar *debug; 136 | GError *error; 137 | gst_message_parse_error (msg, &error, &debug); 138 | g_printerr ("ERROR from element %s: %s\n", 139 | GST_OBJECT_NAME (msg->src), error->message); 140 | if (debug) 141 | g_printerr ("Error details: %s\n", debug); 142 | g_free (debug); 143 | g_error_free (error); 144 | g_main_loop_quit (loop); 145 | break; 146 | } 147 | default: 148 | break; 149 | } 150 | return TRUE; 151 | } 152 | 153 | int using_bin (int argc, char *argv[]) 154 | { 155 | GMainLoop *loop = NULL; 156 | GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL, 157 | *decoder = NULL, *streammux = NULL, *sink = NULL, *pgie = NULL, *nvvidconv = NULL, 158 | *nvosd = NULL, *bin = NULL; 159 | 160 | GstElement *transform = NULL; 161 | GstBus *bus = NULL; 162 | guint bus_watch_id; 163 | GstPad *osd_sink_pad = NULL; 164 | GstPad *source_pad = NULL, *decoder_pad = NULL; 165 | 166 | int current_device = -1; 167 | cudaGetDevice(¤t_device); 168 | struct cudaDeviceProp prop; 169 | cudaGetDeviceProperties(&prop, current_device); 170 | /* Check input arguments */ 171 | if (argc != 2) { 172 | g_printerr ("Usage: %s \n", argv[0]); 173 | return -1; 174 | } 175 | 176 | /* Standard GStreamer initialization */ 177 | gst_init (&argc, &argv); 178 | loop = g_main_loop_new (NULL, FALSE); 179 | 180 | /* Create gstreamer elements */ 181 | /* Create Pipeline element that will form a connection of other elements */ 182 | pipeline = gst_pipeline_new ("dstest1-pipeline"); 183 | bin = gst_bin_new("test_src_bin"); 184 | 185 | source = gst_element_factory_make ("filesrc", "file-source"); 186 | h264parser = gst_element_factory_make ("h264parse", "h264-parser"); 187 | decoder = gst_element_factory_make ("nvv4l2decoder", "nvv4l2-decoder"); 188 | g_object_set (G_OBJECT (source), "location", argv[1], NULL); 189 | 190 | gst_bin_add_many(GST_BIN(bin), source, h264parser, decoder, NULL); 191 | if (!gst_element_link_many (source, h264parser, decoder, NULL)) { 192 | g_printerr ("Elements could not be linked: 1. Exiting.\n"); 193 | return -1; 194 | } 195 | 196 | decoder_pad = gst_element_get_static_pad(decoder, "src"); 197 | // gst_element_add_pad(bin, gst_ghost_pad_new("sink", source_pad)); 198 | gst_element_add_pad(bin, gst_ghost_pad_new("src", decoder_pad)); 199 | 200 | 201 | // gst_bin_add_many(GST_BIN(pipeline), bin, NULL); 202 | 203 | /* Create nvstreammux instance to form batches from one or more sources. */ 204 | streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); 205 | 206 | if (!pipeline || !streammux) { 207 | g_printerr ("One element could not be created. Exiting.\n"); 208 | return -1; 209 | } 210 | 211 | /* Use nvinfer to run inferencing on decoder's output, 212 | * behaviour of inferencing is set through config file */ 213 | pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine"); 214 | 215 | /* Use convertor to convert from NV12 to RGBA as required by nvosd */ 216 | nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter"); 217 | 218 | /* Create OSD to draw on the converted RGBA buffer */ 219 | nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); 220 | 221 | /* Finally render the osd output */ 222 | if(prop.integrated) { 223 | transform = gst_element_factory_make ("nvegltransform", "nvegl-transform"); 224 | } 225 | sink = gst_element_factory_make ("fakesink", "nvvideo-renderer"); 226 | // sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer"); 227 | 228 | if (!source || !h264parser || !decoder || !pgie 229 | || !nvvidconv || !nvosd || !sink) { 230 | g_printerr ("One element could not be created. Exiting.\n"); 231 | return -1; 232 | } 233 | 234 | if(!transform && prop.integrated) { 235 | g_printerr ("One tegra element could not be created. Exiting.\n"); 236 | return -1; 237 | } 238 | 239 | /* we set the input filename to the source element */ 240 | 241 | g_object_set (G_OBJECT (streammux), "batch-size", 1, NULL); 242 | 243 | g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", 244 | MUXER_OUTPUT_HEIGHT, 245 | "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); 246 | 247 | /* Set all the necessary properties of the nvinfer element, 248 | * the necessary ones are : */ 249 | g_object_set (G_OBJECT (pgie), 250 | "config-file-path", "apps/using_bin/using_bin_config.txt", NULL); 251 | 252 | /* we add a message handler */ 253 | bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); 254 | bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); 255 | gst_object_unref (bus); 256 | 257 | /* Set up the pipeline */ 258 | /* we add all elements into the pipeline */ 259 | if(prop.integrated) { 260 | gst_bin_add_many (GST_BIN (pipeline), 261 | bin, streammux, pgie, 262 | nvvidconv, nvosd, transform, sink, NULL); 263 | } 264 | else { 265 | gst_bin_add_many (GST_BIN (pipeline), 266 | bin, streammux, pgie, 267 | nvvidconv, nvosd, sink, NULL); 268 | } 269 | 270 | GstPad *sinkpad, *srcpad; 271 | gchar pad_name_sink[16] = "sink_0"; 272 | gchar pad_name_src[16] = "src"; 273 | 274 | sinkpad = gst_element_get_request_pad (streammux, pad_name_sink); 275 | if (!sinkpad) { 276 | g_printerr ("Streammux request sink pad failed. Exiting.\n"); 277 | return -1; 278 | } 279 | 280 | srcpad = gst_element_get_static_pad (bin, pad_name_src); 281 | if (!srcpad) { 282 | g_printerr ("Decoder request src pad failed. Exiting.\n"); 283 | return -1; 284 | } 285 | 286 | if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { 287 | g_printerr ("Failed to link decoder to stream muxer. Exiting.\n"); 288 | return -1; 289 | } 290 | 291 | gst_object_unref (sinkpad); 292 | gst_object_unref (srcpad); 293 | 294 | /* we link the elements together */ 295 | /* file-source -> h264-parser -> nvh264-decoder -> 296 | * nvinfer -> nvvidconv -> nvosd -> video-renderer */ 297 | 298 | // if (!gst_element_link_many (source, h264parser, decoder, NULL)) { 299 | // g_printerr ("Elements could not be linked: 1. Exiting.\n"); 300 | // return -1; 301 | // } 302 | 303 | if(prop.integrated) { 304 | if (!gst_element_link_many (streammux, pgie, 305 | nvvidconv, nvosd, transform, sink, NULL)) { 306 | g_printerr ("Elements could not be linked: 2. Exiting.\n"); 307 | return -1; 308 | } 309 | } 310 | else { 311 | if (!gst_element_link_many (streammux, pgie, 312 | nvvidconv, nvosd, sink, NULL)) { 313 | g_printerr ("Elements could not be linked: 2. Exiting.\n"); 314 | return -1; 315 | } 316 | } 317 | 318 | /* Lets add probe to get informed of the meta data generated, we add probe to 319 | * the sink pad of the osd element, since by that time, the buffer would have 320 | * had got all the metadata. */ 321 | osd_sink_pad = gst_element_get_static_pad (nvosd, "sink"); 322 | if (!osd_sink_pad) 323 | g_print ("Unable to get sink pad\n"); 324 | else 325 | gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, 326 | osd_sink_pad_buffer_probe, NULL, NULL); 327 | gst_object_unref (osd_sink_pad); 328 | 329 | /* Set the pipeline to "playing" state */ 330 | g_print ("Now playing"); 331 | gst_element_set_state (pipeline, GST_STATE_PLAYING); 332 | 333 | /* Wait till pipeline encounters an error or EOS */ 334 | g_print ("Running...\n"); 335 | g_main_loop_run (loop); 336 | 337 | /* Out of the main loop, clean up nicely */ 338 | g_print ("Returned, stopping playback\n"); 339 | gst_element_set_state (pipeline, GST_STATE_NULL); 340 | g_print ("Deleting pipeline\n"); 341 | gst_object_unref (GST_OBJECT (pipeline)); 342 | g_source_remove (bus_watch_id); 343 | g_main_loop_unref (loop); 344 | return 0; 345 | } 346 | --------------------------------------------------------------------------------