├── jetmulticam ├── bins │ ├── __init__.py │ ├── encoders.py │ └── cameras.py ├── pipelines │ ├── __init__.py │ ├── basepipeline.py │ ├── multicam.py │ └── multicamDNN.py ├── utils │ ├── __init__.py │ ├── v4l.py │ └── gst.py ├── models │ ├── __init__.py │ ├── dashcamnet │ │ ├── __init__.py │ │ ├── dashcamnet_gpu.txt │ │ ├── dashcamnet_dla0.txt │ │ ├── dashcamnet_dla1.txt │ │ └── dashcamnet.py │ └── peoplenet │ │ ├── __init__.py │ │ ├── peoplenet_gpu.txt │ │ ├── peoplenet_dla0.txt │ │ ├── peoplenet_dla1.txt │ │ └── peoplenet.py └── __init__.py ├── docs ├── ready_pipelines │ ├── 04_image_save.md │ ├── README.md │ ├── 05_streaming.md │ ├── 03_encode.md │ ├── 01_capture.md │ └── 02_inference.md ├── CLA.pdf └── simple_python_pipelines │ ├── 00_basic_pipeline.py │ ├── 01_encode_pipeline.py │ ├── 02_encode_pipeline_bin.py │ ├── 04_tap_into_appsink.py │ └── 03_encode_pipeline_bin_v4l2_bin.py ├── pyproject.toml ├── scripts ├── env_vars.sh └── install_dependencies.sh ├── examples ├── example-no-ai.py ├── example.py └── example-person-following.py ├── LICENSE.md ├── setup.py ├── .gitignore └── README.md /jetmulticam/bins/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jetmulticam/pipelines/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jetmulticam/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/ready_pipelines/04_image_save.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/CLA.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-AI-IOT/jetson-multicamera-pipelines/HEAD/docs/CLA.pdf -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools>=42", 4 | "wheel" 5 | ] 6 | build-backend = "setuptools.build_meta" 7 | 8 | -------------------------------------------------------------------------------- /jetmulticam/models/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | MODELS_PATH = os.path.dirname(os.path.realpath(__file__)) 4 | 5 | from .dashcamnet import DashCamNet 6 | from .peoplenet import PeopleNet 7 | -------------------------------------------------------------------------------- /scripts/env_vars.sh: -------------------------------------------------------------------------------- 1 | export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libgomp.so.1; 2 | export OPENBLAS_CORETYPE=ARMV8; # Workaround of numpy/OpenBLAS bug on aarch64 https://github.com/numpy/numpy/issues/18131 3 | export DISPLAY=:0; # To display over SSH 4 | -------------------------------------------------------------------------------- /jetmulticam/models/dashcamnet/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from .dashcamnet import DashCamNet 4 | 5 | FILEPATH = os.path.abspath(__file__) 6 | DIRPATH = os.path.dirname(FILEPATH) 7 | 8 | if not os.path.isfile( 9 | DIRPATH + "./dashcamnet_pruned_v1.0/resnet18_dashcamnet_pruned.etlt" 10 | ): 11 | DashCamNet._download() 12 | -------------------------------------------------------------------------------- /examples/example-no-ai.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from jetmulticam import CameraPipeline 4 | 5 | if __name__ == "__main__": 6 | 7 | p = CameraPipeline([0, 1]) 8 | 9 | print(p.running()) 10 | 11 | for _ in range(100): 12 | arr = p.read(0) 13 | if arr is not None: 14 | print(arr.shape) 15 | time.sleep(0.1) 16 | -------------------------------------------------------------------------------- /jetmulticam/models/peoplenet/__init__.py: -------------------------------------------------------------------------------- 1 | # Here we do lazy loading of the model weights 2 | 3 | import os 4 | 5 | from .peoplenet import PeopleNet 6 | 7 | FILEPATH = os.path.abspath(__file__) 8 | DIRPATH = os.path.dirname(FILEPATH) 9 | 10 | if not os.path.isfile( 11 | DIRPATH + "/tlt_peoplenet_pruned_v2.0/resnet18_peoplenet_int8_dla.txt" 12 | ): 13 | PeopleNet._download() 14 | -------------------------------------------------------------------------------- /jetmulticam/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | 3 | # For logging 4 | import logging 5 | import sys 6 | 7 | # Expose top-level API for the package 8 | from .pipelines.multicam import CameraPipeline 9 | from .pipelines.multicamDNN import CameraPipelineDNN 10 | 11 | # Setup logging 12 | log = logging.getLogger("jetmulticam") 13 | log.setLevel(logging.WARN) 14 | 15 | # By default stream to stdout 16 | handler = logging.StreamHandler(sys.stdout) 17 | handler.setLevel(logging.WARN) 18 | formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") 19 | handler.setFormatter(formatter) 20 | log.addHandler(handler) 21 | -------------------------------------------------------------------------------- /jetmulticam/models/peoplenet/peoplenet_gpu.txt: -------------------------------------------------------------------------------- 1 | 2 | [property] 3 | gpu-id=0 4 | net-scale-factor=0.0039215697906911373 5 | tlt-model-key=tlt_encode 6 | tlt-encoded-model=./tlt_peoplenet_pruned_v2.0/resnet18_peoplenet_pruned.etlt 7 | labelfile-path=./tlt_peoplenet_pruned_v2.0/labels.txt 8 | model-engine-file=./tlt_peoplenet_pruned_v2.0/resnet18_peoplenet_pruned.etlt_b3_gpu0_int8.engine 9 | infer-dims=3;544;960 10 | uff-input-blob-name=input_1 11 | batch-size=3 12 | process-mode=1 13 | model-color-format=0 14 | ## 0=FP32, 1=INT8, 2=FP16 mode 15 | network-mode=1 16 | num-detected-classes=3 17 | cluster-mode=1 18 | interval=0 19 | gie-unique-id=1 20 | output-blob-names=output_bbox/BiasAdd;output_cov/Sigmoid 21 | 22 | [class-attrs-all] 23 | pre-cluster-threshold=0.5 24 | ## Set eps=0.7 and minBoxes for cluster-mode=1(DBSCAN) 25 | eps=0.7 26 | minBoxes=1 27 | -------------------------------------------------------------------------------- /jetmulticam/models/dashcamnet/dashcamnet_gpu.txt: -------------------------------------------------------------------------------- 1 | 2 | [property] 3 | gpu-id=0 4 | net-scale-factor=0.0039215697906911373 5 | tlt-model-key=tlt_encode 6 | tlt-encoded-model=./dashcamnet_pruned_v1.0/resnet18_dashcamnet_pruned.etlt 7 | labelfile-path=./dashcamnet_pruned_v1.0/labels.txt 8 | int8-calib-file=./dashcamnet_pruned_v1.0/dashcamnet_int8.txt 9 | model-engine-file=./dashcamnet_pruned_v1.0/resnet18_dashcamnet_pruned.etlt_b3_gpu0_int8.engine 10 | infer-dims=3;544;960 11 | uff-input-blob-name=input_1 12 | process-mode=1 13 | model-color-format=0 14 | network-mode=1 15 | num-detected-classes=3 16 | cluster-mode=1 17 | interval=0 18 | gie-unique-id=1 19 | output-blob-names=output_bbox/BiasAdd;output_cov/Sigmoid 20 | 21 | [class-attrs-all] 22 | pre-cluster-threshold=0.3 23 | ## Set eps=0.7 and minBoxes for cluster-mode=1(DBSCAN) 24 | eps=0.7 25 | minBoxes=1 26 | -------------------------------------------------------------------------------- /jetmulticam/utils/v4l.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | 3 | import glob 4 | 5 | 6 | def find_dev_by_name(query: str): 7 | v4l_name_files = glob.glob("/sys/class/video4linux/video*/name") 8 | v4l_name_files = sorted(v4l_name_files) 9 | 10 | dev2name = {} 11 | for nf in v4l_name_files: 12 | with open(nf) as f: 13 | dev = nf.split("/")[-2] 14 | name = f.read() 15 | name = name.strip("\n") # remove newline from the end of device name 16 | dev2name["/dev/" + dev] = name 17 | 18 | name2dev = {v: k for k, v in dev2name.items()} 19 | 20 | matching = [] 21 | for (d, n) in dev2name.items(): 22 | if query in n: 23 | matching.append(d) 24 | 25 | return matching 26 | 27 | 28 | print(find_dev_by_name("imx185")) 29 | print(find_dev_by_name("ar0234")) 30 | -------------------------------------------------------------------------------- /jetmulticam/models/dashcamnet/dashcamnet_dla0.txt: -------------------------------------------------------------------------------- 1 | 2 | [property] 3 | enable-dla=1 4 | use-dla-core=0 5 | net-scale-factor=0.0039215697906911373 6 | tlt-model-key=tlt_encode 7 | tlt-encoded-model=./dashcamnet_pruned_v1.0/resnet18_dashcamnet_pruned.etlt 8 | labelfile-path=./dashcamnet_pruned_v1.0/labels.txt 9 | int8-calib-file=./dashcamnet_pruned_v1.0/dashcamnet_int8.txt 10 | model-engine-file=./dashcamnet_pruned_v1.0/resnet18_dashcamnet_pruned.etlt_b3_dla0_int8.engine 11 | infer-dims=3;544;960 12 | uff-input-blob-name=input_1 13 | process-mode=1 14 | model-color-format=0 15 | network-mode=1 16 | num-detected-classes=3 17 | cluster-mode=1 18 | interval=0 19 | gie-unique-id=1 20 | output-blob-names=output_bbox/BiasAdd;output_cov/Sigmoid 21 | 22 | [class-attrs-all] 23 | pre-cluster-threshold=0.4 24 | ## Set eps=0.7 and minBoxes for cluster-mode=1(DBSCAN) 25 | eps=0.7 26 | minBoxes=1 27 | -------------------------------------------------------------------------------- /jetmulticam/models/dashcamnet/dashcamnet_dla1.txt: -------------------------------------------------------------------------------- 1 | 2 | [property] 3 | enable-dla=1 4 | use-dla-core=1 5 | net-scale-factor=0.0039215697906911373 6 | tlt-model-key=tlt_encode 7 | tlt-encoded-model=./dashcamnet_pruned_v1.0/resnet18_dashcamnet_pruned.etlt 8 | labelfile-path=./dashcamnet_pruned_v1.0/labels.txt 9 | int8-calib-file=./dashcamnet_pruned_v1.0/dashcamnet_int8.txt 10 | model-engine-file=./dashcamnet_pruned_v1.0/resnet18_dashcamnet_pruned.etlt_b3_dla1_int8.engine 11 | infer-dims=3;544;960 12 | uff-input-blob-name=input_1 13 | process-mode=1 14 | model-color-format=0 15 | network-mode=1 16 | num-detected-classes=3 17 | cluster-mode=1 18 | interval=0 19 | gie-unique-id=1 20 | output-blob-names=output_bbox/BiasAdd;output_cov/Sigmoid 21 | 22 | [class-attrs-all] 23 | pre-cluster-threshold=0.4 24 | ## Set eps=0.7 and minBoxes for cluster-mode=1(DBSCAN) 25 | eps=0.7 26 | minBoxes=1 27 | -------------------------------------------------------------------------------- /docs/simple_python_pipelines/00_basic_pipeline.py: -------------------------------------------------------------------------------- 1 | import gi 2 | import sys 3 | 4 | gi.require_version("Gst", "1.0") 5 | from gi.repository import GObject, Gst 6 | 7 | sys.path.append("../..") 8 | from jetmulticam.gstutils import _make_element_safe, _sanitize 9 | 10 | # Standard GStreamer initialization 11 | GObject.threads_init() 12 | Gst.init(None) 13 | 14 | pipeline = _sanitize(Gst.Pipeline()) 15 | cam = _make_element_safe("nvarguscamerasrc") 16 | display = _make_element_safe("nvoverlaysink") 17 | fakesink = _make_element_safe("fakesink") 18 | 19 | sink = display 20 | # sink = fakesink 21 | 22 | pipeline.add(cam) 23 | pipeline.add(sink) 24 | cam.link(sink) 25 | 26 | # create an event loop and feed gstreamer bus mesages to it 27 | loop = GObject.MainLoop() 28 | pipeline.set_state(Gst.State.PLAYING) 29 | 30 | try: 31 | loop.run() 32 | except: 33 | pass 34 | finally: 35 | pipeline.set_state(Gst.State.NULL) 36 | -------------------------------------------------------------------------------- /docs/ready_pipelines/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Pipeline examples 3 | 4 | All examples should be ran from the root of the repository. 5 | This ensures that all the paths for model configs stay valid. 6 | 7 | For example, to run the first example in `02_infer.md` do: 8 | 9 | ```shell 10 | git clone ssh://git@gitlab-master.nvidia.com:12051/tlewicki/multicamera-robot.git 11 | cd multicamera-robot 12 | gst-launch-1.0 nvarguscamerasrc bufapi-version=1 ! 'video/x-raw(memory:NVMM),width=1920,height=1080,framerate=30/1' ! mx.sink_0 nvstreammux width=1920 height=1080 batch-size=1 name=mx ! nvinfer config-file-path=jetmulticam/models/peoplenet/peoplenet_dla0.txt ! nvvideoconvert ! nvdsosd ! nvoverlaysink sync=0 13 | ``` 14 | 15 | - [01 Camera Capture & Display](01_capture.md) 16 | - [02 Deepstream Inference](02_infer.md) 17 | - [03 Video Encoding](03_video_encoding.md) 18 | - [04 Image Saving](04_image_save.md) 19 | - [05 Streaming Pipelines](05_streaming.md) 20 | 21 | -------------------------------------------------------------------------------- /scripts/install_dependencies.sh: -------------------------------------------------------------------------------- 1 | build_gst_python() 2 | { 3 | # Builds and installs gst-python (python bindings for gstreamer) 4 | sudo apt-get install python3-dev python-gi-dev libgstreamer1.0-dev -y; 5 | export GST_LIBS="-lgstreamer-1.0 -lgobject-2.0 -lglib-2.0"; 6 | export GST_CFLAGS="-pthread -I/usr/include/gstreamer-1.0 -I/usr/include/glib-2.0 -I/usr/lib/x86_64-linux-gnu/glib-2.0/include"; 7 | git clone https://github.com/GStreamer/gst-python.git /tmp/gst-python; 8 | cd /tmp/gst-python; 9 | git checkout 1a8f48a; 10 | ./autogen.sh PYTHON=python3; 11 | ./configure PYTHON=python3; 12 | make -j$(nproc); 13 | sudo make install; # TODO: is sudo necessary? 14 | } 15 | 16 | install_ds() 17 | { 18 | sudo apt-get install deepstream-5.1 -y; 19 | } 20 | 21 | install_pyds() 22 | { 23 | cd /opt/nvidia/deepstream/deepstream/lib; 24 | sudo python3 setup.py install; 25 | } 26 | 27 | build_gst_python 28 | install_ds 29 | install_pyds 30 | 31 | 32 | -------------------------------------------------------------------------------- /jetmulticam/models/peoplenet/peoplenet_dla0.txt: -------------------------------------------------------------------------------- 1 | 2 | [property] 3 | enable-dla=1 4 | use-dla-core=0 5 | net-scale-factor=0.0039215697906911373 6 | tlt-model-key=tlt_encode 7 | tlt-encoded-model=./tlt_peoplenet_pruned_v2.0/resnet18_peoplenet_pruned.etlt 8 | labelfile-path=./tlt_peoplenet_pruned_v2.0/labels.txt 9 | int8-calib-file=./tlt_peoplenet_pruned_v2.0/resnet18_peoplenet_int8_dla.txt 10 | model-engine-file=./tlt_peoplenet_pruned_v2.0/resnet18_peoplenet_pruned.etlt_b3_dla0_int8.engine 11 | ## model-engine-file=./tlt_peoplenet_pruned_v2.0/resnet18_peoplenet_pruned.etlt_b3_dla0_fp16.engine 12 | infer-dims=3;544;960 13 | uff-input-blob-name=input_1 14 | process-mode=1 15 | model-color-format=0 16 | ## 0=FP32, 1=INT8, 2=FP16 mode 17 | network-mode=1 18 | num-detected-classes=3 19 | cluster-mode=1 20 | interval=0 21 | gie-unique-id=1 22 | output-blob-names=output_bbox/BiasAdd;output_cov/Sigmoid 23 | 24 | [class-attrs-all] 25 | pre-cluster-threshold=0.4 26 | ## Set eps=0.7 and minBoxes for cluster-mode=1(DBSCAN) 27 | eps=0.7 28 | minBoxes=1 29 | -------------------------------------------------------------------------------- /jetmulticam/models/peoplenet/peoplenet_dla1.txt: -------------------------------------------------------------------------------- 1 | 2 | [property] 3 | enable-dla=1 4 | use-dla-core=1 5 | net-scale-factor=0.0039215697906911373 6 | tlt-model-key=tlt_encode 7 | tlt-encoded-model=./tlt_peoplenet_pruned_v2.0/resnet18_peoplenet_pruned.etlt 8 | labelfile-path=./tlt_peoplenet_pruned_v2.0/labels.txt 9 | int8-calib-file=./tlt_peoplenet_pruned_v2.0/resnet18_peoplenet_int8_dla.txt 10 | model-engine-file=./tlt_peoplenet_pruned_v2.0/resnet18_peoplenet_pruned.etlt_b3_dla1_int8.engine 11 | ## model-engine-file=./tlt_peoplenet_pruned_v2.0/resnet18_peoplenet_pruned.etlt_b3_dla1_fp16.engine 12 | infer-dims=3;544;960 13 | uff-input-blob-name=input_1 14 | process-mode=1 15 | model-color-format=0 16 | ## 0=FP32, 1=INT8, 2=FP16 mode 17 | network-mode=1 18 | num-detected-classes=3 19 | cluster-mode=1 20 | interval=0 21 | gie-unique-id=1 22 | output-blob-names=output_bbox/BiasAdd;output_cov/Sigmoid 23 | 24 | [class-attrs-all] 25 | pre-cluster-threshold=0.4 26 | ## Set eps=0.7 and minBoxes for cluster-mode=1(DBSCAN) 27 | eps=0.7 28 | minBoxes=1 29 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r", encoding="utf-8") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="jetmulticam", 8 | version="0.1.0", 9 | author="Tomasz Lewicki", 10 | author_email="tlewicki@nvidia.com", 11 | description="JetMutliCam: Easy-to-use realtime CV/AI pipelines for Nvidia Jetson Platform.", 12 | long_description=long_description, 13 | long_description_content_type="text/markdown", 14 | url="https://github.com/tomek-l/nv-jetmulticam", 15 | project_urls={ 16 | "Bug Tracker": "https://github.com/tomek-l/nv-jetmulticam/issues", 17 | }, 18 | classifiers=[ 19 | "Programming Language :: Python :: 3", 20 | "License :: OSI Approved :: MIT License", 21 | ], 22 | packages=setuptools.find_packages(where="."), 23 | package_dir={"jetmulticam": "jetmulticam"}, 24 | include_package_data=True, 25 | package_data={ 26 | "": ["*.txt"], # For nvinfer config files 27 | }, 28 | python_requires=">=3.6", 29 | install_requires=[ 30 | 'numpy>=1.18' # TODO: test with >1.0-1.19 31 | ] 32 | ) 33 | -------------------------------------------------------------------------------- /jetmulticam/models/dashcamnet/dashcamnet.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | 3 | import os 4 | 5 | FILEPATH = os.path.abspath(__file__) 6 | DIRPATH = os.path.dirname(FILEPATH) 7 | 8 | 9 | class DashCamNet: 10 | DLA0 = DIRPATH + "/dashcamnet_dla0.txt" 11 | DLA1 = DIRPATH + "/dashcamnet_dla1.txt" 12 | GPU = DIRPATH + "/dashcamnet_gpu.txt" 13 | 14 | @staticmethod 15 | def _download(): 16 | # Lazy loading of dashcamnet model 17 | MODEL_URL = "https://api.ngc.nvidia.com/v2/models/nvidia/tao/dashcamnet/versions/pruned_v1.0/zip" 18 | 19 | import urllib.request 20 | from zipfile import PyZipFile 21 | 22 | print( 23 | "Downloading pretrained weights for DashCamNet model. This may take a while..." 24 | ) 25 | urllib.request.urlretrieve( 26 | MODEL_URL, filename="/tmp/dashcamnet_pruned_v1.0.zip" 27 | ) 28 | # TODO: progressbar here would be nice. Keras has something like that: 29 | # https://github.com/keras-team/keras/blob/5550cb0c96c508211b1f0af4aa5af6caff7385a2/keras/utils/data_utils.py#L276 30 | 31 | extract_to = DIRPATH + "/dashcamnet_pruned_v1.0" 32 | pzf = PyZipFile("/tmp/dashcamnet_pruned_v1.0.zip") 33 | pzf.extractall(path=extract_to) 34 | print(f"Dowloaded pre-trained DashCamNet model to: {extract_to}") 35 | -------------------------------------------------------------------------------- /jetmulticam/models/peoplenet/peoplenet.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | 3 | import os 4 | 5 | FILEPATH = os.path.abspath(__file__) 6 | DIRPATH = os.path.dirname(FILEPATH) 7 | 8 | 9 | class PeopleNet: 10 | DLA0 = DIRPATH + "/peoplenet_dla0.txt" 11 | DLA1 = DIRPATH + "/peoplenet_dla1.txt" 12 | GPU = DIRPATH + "/peoplenet_gpu.txt" 13 | 14 | @staticmethod 15 | def _download(): 16 | # Lazy loading of peoplenet model 17 | MODEL_URL = "https://api.ngc.nvidia.com/v2/models/nvidia/tlt_peoplenet/versions/pruned_v2.0/zip" 18 | 19 | import urllib.request 20 | from zipfile import PyZipFile 21 | 22 | print( 23 | "Downloading pretrained weights for PeopleNet model. This may take a while..." 24 | ) 25 | urllib.request.urlretrieve( 26 | MODEL_URL, filename="/tmp/tlt_peoplenet_pruned_v2.0.zip" 27 | ) 28 | # TODO: progressbar here would be nice. Keras has something like that: 29 | # https://github.com/keras-team/keras/blob/5550cb0c96c508211b1f0af4aa5af6caff7385a2/keras/utils/data_utils.py#L276 30 | 31 | extract_to = DIRPATH + "/tlt_peoplenet_pruned_v2.0" 32 | pzf = PyZipFile("/tmp/tlt_peoplenet_pruned_v2.0.zip") 33 | pzf.extractall(path=extract_to) 34 | print(f"Dowloaded pre-trained PeopleNet model to: {extract_to}") 35 | -------------------------------------------------------------------------------- /docs/simple_python_pipelines/01_encode_pipeline.py: -------------------------------------------------------------------------------- 1 | import gi 2 | 3 | gi.require_version("Gst", "1.0") 4 | from gi.repository import GObject, Gst 5 | 6 | from gstutils import _make_element_safe, _sanitize 7 | 8 | # Standard GStreamer initialization 9 | GObject.threads_init() 10 | Gst.init(None) 11 | 12 | pipeline = _sanitize(Gst.Pipeline()) 13 | 14 | cam = _make_element_safe("nvarguscamerasrc") 15 | cam.set_property("sensor-id", 0) 16 | cam.set_property("bufapi-version", 1) 17 | cam = _make_element_safe("videotestsrc") 18 | 19 | conv = _make_element_safe("nvvideoconvert") 20 | 21 | enc = _make_element_safe("nvv4l2h264enc") 22 | enc.set_property("bitrate", 10000000) 23 | 24 | parser = _make_element_safe("h264parse") 25 | mux = _make_element_safe("matroskamux") 26 | 27 | filesink = _make_element_safe("filesink") 28 | filesink.set_property("sync", 1) 29 | filesink.set_property("location", "test.mkv") 30 | 31 | 32 | elements = [cam, conv, enc, parser, mux, filesink] 33 | 34 | for el in elements: 35 | pipeline.add(el) 36 | 37 | cam.link(conv) 38 | conv.link(enc) 39 | enc.link(parser) 40 | parser.link(mux) 41 | mux.link(filesink) 42 | 43 | # create an event loop and feed gstreamer bus mesages to it 44 | loop = GObject.MainLoop() 45 | pipeline.set_state(Gst.State.PLAYING) 46 | 47 | try: 48 | loop.run() 49 | except Exception as e: 50 | print(e) 51 | finally: 52 | pipeline.set_state(Gst.State.NULL) 53 | -------------------------------------------------------------------------------- /examples/example.py: -------------------------------------------------------------------------------- 1 | import time 2 | from jetmulticam import CameraPipelineDNN 3 | from jetmulticam.models import PeopleNet, DashCamNet 4 | 5 | if __name__ == "__main__": 6 | 7 | pipeline = CameraPipelineDNN( 8 | cameras=[2, 5, 8], 9 | models=[ 10 | PeopleNet.DLA1, 11 | DashCamNet.DLA0, 12 | # PeopleNet.GPU 13 | ], 14 | save_video=True, 15 | save_video_folder="/home/nx/logs/videos", 16 | display=True, 17 | ) 18 | 19 | while pipeline.running(): 20 | 21 | # We can access the captured images here. 22 | # For example `pipeline.images` is a list of numpy arrays for each camera 23 | # In my case (RGB 1080p image), `arr` will be np.ndarray with shape: (1080, 1920, 3) 24 | arr = pipeline.images[0] 25 | print(type(arr)) 26 | 27 | # Detections in each image are available here as a list of dicts: 28 | dets = pipeline.detections[0] 29 | print(dets) 30 | 31 | # Assuming there's one detection in `image[0]`, `dets` can look like so: 32 | # [{ 33 | # 'class': 'person', 34 | # 'position': (361.31, 195.60, 891.96, 186.05), # bbox (left, width, top, height) 35 | # 'confidence': 0.92 36 | # }] 37 | 38 | # Main thread is not tied in any computation. 39 | # We can perform any operations on our images. 40 | avg = pipeline.images[0].mean() 41 | print(avg) 42 | time.sleep(1 / 30) 43 | -------------------------------------------------------------------------------- /docs/ready_pipelines/05_streaming.md: -------------------------------------------------------------------------------- 1 | # Streaming 2 | 3 | ### Stream videotestsrc to UDP:5000 4 | 5 | source: 6 | ```shell 7 | gst-launch-1.0 videotestsrc ! nvvideoconvert ! nvv4l2h264enc insert-sps-pps=true bitrate=16000000 ! rtph264pay ! udpsink port=5000 host=127.0.0.1 8 | ``` 9 | 10 | receiver: 11 | ```shell 12 | gst-launch-1.0 -v udpsrc port=5000 ! "application/x-rtp, media=(string)video, clock-rate=(int)90000, encoding-name=(string)H264, payload=(int)96" ! rtph264depay ! h264parse ! decodebin ! videoconvert ! autovideosink sync=false 13 | ``` 14 | 15 | ### The same, but with h265 16 | 17 | source: 18 | ```shell 19 | gst-launch-1.0 videotestsrc ! nvvideoconvert ! nvv4l2h265enc insert-sps-pps=true bitrate=16000000 ! rtph265pay ! udpsink port=5000 host=127.0.0.1 20 | ``` 21 | 22 | receiver: 23 | ```shell 24 | gst-launch-1.0 -v udpsrc port=5000 ! "application/x-rtp, media=(string)video, clock-rate=(int)90000, encoding-name=(string)H265, payload=(int)96" ! rtph265depay ! h265parse ! decodebin ! videoconvert ! autovideosink sync=false 25 | ``` 26 | 27 | 28 | ### H265 streaming between Jetson and remote host 29 | 30 | source: 31 | (change `10.0.0.167` to your PC's IP address) 32 | ```shell 33 | gst-launch-1.0 videotestsrc ! nvvideoconvert ! nvv4l2h264enc insert-sps-pps=true bitrate=16000000 ! rtph264pay ! udpsink port=5000 host=10.0.0.167 34 | ``` 35 | 36 | host: 37 | ```shell 38 | gst-launch-1.0 -v udpsrc port=5000 ! "application/x-rtp, media=(string)video, clock-rate=(int)90000, encoding-name=(string)H265, payload=(int)96" ! rtph265depay ! h265parse ! decodebin ! videoconvert ! autovideosink sync=false 39 | ``` 40 | -------------------------------------------------------------------------------- /jetmulticam/pipelines/basepipeline.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | 3 | import logging 4 | import time 5 | 6 | # Gstreamer imports 7 | import gi 8 | 9 | gi.require_version("Gst", "1.0") 10 | from gi.repository import GObject, Gst 11 | 12 | from ..utils.gst import _err_if_none, _make_element_safe, _sanitize, bus_call 13 | 14 | 15 | class BasePipeline: 16 | def __init__(self, **kwargs): 17 | 18 | # Gstreamer init 19 | GObject.threads_init() 20 | Gst.init(None) 21 | 22 | # create an event loop and feed gstreamer bus mesages to it 23 | self._mainloop = GObject.MainLoop() 24 | 25 | self._p = self._create_pipeline(**kwargs) 26 | self._log = logging.getLogger("jetmulticam") 27 | 28 | self._bus = self._p.get_bus() 29 | self._bus.add_signal_watch() 30 | self._bus.connect("message", bus_call, self._mainloop) 31 | 32 | # self.start() 33 | self._p.set_state(Gst.State.PLAYING) 34 | self.wait_ready() 35 | self._start_ts = time.perf_counter() 36 | 37 | def __del__(self): 38 | self.stop() 39 | 40 | def stop(self): 41 | self._p.send_event(Gst.Event.new_eos()) 42 | self._p.set_state(Gst.State.PAUSED) 43 | # Sometimes nvargus-deamon will segfault when transitioning nvarguscamerasrc from PAUSED->NULL 44 | # https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/704#note_947201 45 | # https://forums.developer.nvidia.com/t/nvargus-daemon-freeze-hang-on-pipeline-stop-on-r32-1/80849/58 46 | self._p.set_state(Gst.State.NULL) 47 | 48 | def running(self): 49 | _, state, _ = self._p.get_state(1) 50 | return True if state == Gst.State.PLAYING else False 51 | 52 | def wait_ready(self): 53 | while not self.running(): 54 | time.sleep(0.1) 55 | -------------------------------------------------------------------------------- /docs/simple_python_pipelines/02_encode_pipeline_bin.py: -------------------------------------------------------------------------------- 1 | import gi 2 | 3 | gi.require_version("Gst", "1.0") 4 | from gi.repository import GObject, Gst 5 | from gstutils import _make_element_safe, _sanitize 6 | 7 | 8 | def make_nvenc_bin() -> Gst.Bin: 9 | h264sink = Gst.Bin() 10 | 11 | # Create video converter 12 | conv = _make_element_safe("nvvideoconvert") 13 | 14 | # H264 encoder 15 | enc = _make_element_safe("nvv4l2h264enc") 16 | enc.set_property("bitrate", 10000000) 17 | 18 | # parser, mux 19 | parser = _make_element_safe("h264parse") 20 | mux = _make_element_safe("matroskamux") 21 | 22 | # filesink 23 | filesink = _make_element_safe("filesink") 24 | filesink.set_property("sync", 0) 25 | filesink.set_property("location", "test.mkv") 26 | 27 | # Add elements to bin before linking 28 | for el in [conv, enc, parser, mux, filesink]: 29 | h264sink.add(el) 30 | 31 | # Link bin elements 32 | conv.link(enc) 33 | enc.link(parser) 34 | parser.link(mux) 35 | mux.link(filesink) 36 | 37 | enter_pad = _sanitize(conv.get_static_pad("sink")) 38 | gp = Gst.GhostPad.new(name="sink", target=enter_pad) 39 | h264sink.add_pad(gp) 40 | 41 | return h264sink 42 | 43 | 44 | if __name__ == "__main__": 45 | # Standard GStreamer initialization 46 | GObject.threads_init() 47 | Gst.init(None) 48 | 49 | pipeline = _sanitize(Gst.Pipeline()) 50 | 51 | cam = _make_element_safe("nvarguscamerasrc") 52 | cam.set_property("sensor-id", 0) 53 | cam.set_property("bufapi-version", 1) 54 | 55 | # cam = _make_element_safe("videotestsrc") 56 | h264sink = make_nvenc_bin() 57 | 58 | for el in [cam, h264sink]: 59 | pipeline.add(el) 60 | 61 | cam.link(h264sink) 62 | loop = GObject.MainLoop() 63 | pipeline.set_state(Gst.State.PLAYING) 64 | 65 | try: 66 | loop.run() 67 | except Exception as e: 68 | print(e) 69 | finally: 70 | pipeline.set_state(Gst.State.NULL) 71 | -------------------------------------------------------------------------------- /jetmulticam/bins/encoders.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | 3 | import gi 4 | 5 | gi.require_version("Gst", "1.0") 6 | from gi.repository import Gst 7 | 8 | from ..utils.gst import _make_element_safe, _sanitize 9 | 10 | 11 | def make_nvenc_bin(filepath, bitrate=int(20e6)) -> Gst.Bin: 12 | h264sink = Gst.Bin() 13 | 14 | # Create video converter 15 | conv = _make_element_safe("nvvideoconvert") 16 | 17 | # H264 encoder 18 | enc = _make_element_safe("nvv4l2h264enc") 19 | enc.set_property("bitrate", bitrate) 20 | enc.set_property("bufapi-version", 1) 21 | enc.set_property("maxperf-enable", True) 22 | 23 | # parser, mux 24 | parser = _make_element_safe("h264parse") 25 | mux = _make_element_safe("matroskamux") 26 | 27 | # filesink 28 | filesink = _make_element_safe("filesink") 29 | filesink.set_property("sync", 0) 30 | filesink.set_property("location", filepath) 31 | 32 | # Add elements to bin before linking 33 | for el in [conv, enc, parser, mux, filesink]: 34 | h264sink.add(el) 35 | 36 | # Link bin elements 37 | conv.link(enc) 38 | enc.link(parser) 39 | parser.link(mux) 40 | mux.link(filesink) 41 | 42 | enter_pad = _sanitize(conv.get_static_pad("sink")) 43 | gp = Gst.GhostPad.new(name="sink", target=enter_pad) 44 | h264sink.add_pad(gp) 45 | 46 | return h264sink 47 | 48 | 49 | def make_nvenc_bin_no_ds(filepath, bitrate=int(20e6)) -> Gst.Bin: 50 | h264sink = Gst.Bin() 51 | 52 | # Create video converter 53 | conv = _make_element_safe("nvvidconv") 54 | 55 | # H264 encoder 56 | enc = _make_element_safe("nvv4l2h264enc") 57 | enc.set_property("bitrate", bitrate) 58 | enc.set_property("bufapi-version", 0) 59 | 60 | # parser, mux 61 | parser = _make_element_safe("h264parse") 62 | mux = _make_element_safe("matroskamux") 63 | 64 | # filesink 65 | filesink = _make_element_safe("filesink") 66 | filesink.set_property("sync", 0) 67 | filesink.set_property("location", filepath) 68 | 69 | # Add elements to bin before linking 70 | for el in [conv, enc, parser, mux, filesink]: 71 | h264sink.add(el) 72 | 73 | # Link bin elements 74 | conv.link(enc) 75 | enc.link(parser) 76 | parser.link(mux) 77 | mux.link(filesink) 78 | 79 | enter_pad = _sanitize(conv.get_static_pad("sink")) 80 | gp = Gst.GhostPad.new(name="sink", target=enter_pad) 81 | h264sink.add_pad(gp) 82 | 83 | return h264sink 84 | 85 | -------------------------------------------------------------------------------- /docs/simple_python_pipelines/04_tap_into_appsink.py: -------------------------------------------------------------------------------- 1 | import gi 2 | 3 | gi.require_version("Gst", "1.0") 4 | from gi.repository import GObject, Gst 5 | from jetmulticam.gstutils import _make_element_safe, _sanitize 6 | 7 | 8 | def make_nvenc_bin() -> Gst.Bin: 9 | h264sink = Gst.Bin() 10 | 11 | # Create video converter 12 | conv = _make_element_safe("nvvideoconvert") 13 | 14 | # H264 encoder 15 | enc = _make_element_safe("nvv4l2h264enc") 16 | enc.set_property("bitrate", 10000000) 17 | 18 | # parser, mux 19 | parser = _make_element_safe("h264parse") 20 | mux = _make_element_safe("matroskamux") 21 | 22 | # filesink 23 | filesink = _make_element_safe("filesink") 24 | filesink.set_property("sync", 0) 25 | filesink.set_property("location", "test.mkv") 26 | 27 | # Add elements to bin before linking 28 | for el in [conv, enc, parser, mux, filesink]: 29 | h264sink.add(el) 30 | 31 | # Link bin elements 32 | conv.link(enc) 33 | enc.link(parser) 34 | parser.link(mux) 35 | mux.link(filesink) 36 | 37 | enter_pad = _sanitize(conv.get_static_pad("sink")) 38 | gp = Gst.GhostPad.new(name="sink", target=enter_pad) 39 | h264sink.add_pad(gp) 40 | 41 | return h264sink 42 | 43 | 44 | if __name__ == "__main__": 45 | # Standard GStreamer initialization 46 | GObject.threads_init() 47 | Gst.init(None) 48 | 49 | pipeline = _sanitize(Gst.Pipeline()) 50 | 51 | cam = _make_element_safe("nvarguscamerasrc") 52 | cam.set_property("sensor-id", 0) 53 | cam.set_property("bufapi-version", 1) 54 | 55 | tee = _make_element_safe("tee") 56 | 57 | # cam = _make_element_safe("videotestsrc") 58 | h264sink = make_nvenc_bin() 59 | appsink = _make_element_safe("appsink") 60 | 61 | for el in [cam, tee, h264sink, appsink]: 62 | pipeline.add(el) 63 | 64 | sinks = [h264sink, appsink] 65 | for idx, sink in enumerate(sinks): 66 | # Use queues for each sink. This ensures the sinks can execute in separate threads 67 | queue = _make_element_safe("queue") 68 | pipeline.add(queue) 69 | # tee.src_%d -> queue 70 | srcpad_or_none = tee.get_request_pad(f"src_{idx}") 71 | sinkpad_or_none = queue.get_static_pad("sink") 72 | srcpad = _sanitize(srcpad_or_none) 73 | sinkpad = _sanitize(sinkpad_or_none) 74 | srcpad.link(sinkpad) 75 | # queue -> sink 76 | queue.link(sink) 77 | 78 | sample = self.appsink.emit("pull-sample") # blocks until sample avaialable 79 | buf = sample.get_buffer() 80 | (result, mapinfo) = buf.map(Gst.MapFlags.READ) 81 | 82 | cam.link(tee) 83 | loop = GObject.MainLoop() 84 | pipeline.set_state(Gst.State.PLAYING) 85 | 86 | try: 87 | loop.run() 88 | except Exception as e: 89 | print(e) 90 | finally: 91 | pipeline.set_state(Gst.State.NULL) 92 | -------------------------------------------------------------------------------- /jetmulticam/utils/gst.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | 3 | import sys 4 | 5 | import gi 6 | 7 | gi.require_version("Gst", "1.0") 8 | from gi.repository import GObject, Gst 9 | 10 | 11 | def _err_if_none(element): 12 | if element is None: 13 | raise Exception("Element is none!") 14 | else: 15 | return True 16 | 17 | 18 | def _sanitize(element) -> Gst.Element: 19 | """ 20 | Passthrough function which sure element is not `None` 21 | Returns `Gst.Element` or raises Error 22 | """ 23 | if element is None: 24 | raise Exception("Element is none!") 25 | else: 26 | return element 27 | 28 | 29 | def _make_element_safe(el_type: str, el_name=None) -> Gst.Element: 30 | """ 31 | Creates a gstremer element using el_type factory. 32 | Returns Gst.Element or throws an error if we fail. 33 | This is to avoid `None` elements in our pipeline 34 | """ 35 | 36 | # name=None parameter asks Gstreamer to uniquely name the elements for us 37 | el = Gst.ElementFactory.make(el_type, name=el_name) 38 | 39 | if el is not None: 40 | return el 41 | else: 42 | print(f"Pipeline element is None!") 43 | # TODO: narrow down the error 44 | # TODO: use Gst.ElementFactory.find to generate a more informative error message 45 | raise NameError(f"Could not create element {el_type}") 46 | 47 | 48 | def bus_call(bus, message, loop): 49 | # Taken from: 50 | # https://github.com/NVIDIA-AI-IOT/deepstream_python_apps/blob/6aabcaf85a9e8f11e9a4c39ab1cd46554de7c578/apps/common/bus_call.py 51 | 52 | t = message.type 53 | if t == Gst.MessageType.EOS: 54 | sys.stdout.write("End-of-stream\n") 55 | loop.quit() 56 | elif t == Gst.MessageType.WARNING: 57 | err, debug = message.parse_warning() 58 | sys.stderr.write("Warning: %s: %s\n" % (err, debug)) 59 | elif t == Gst.MessageType.ERROR: 60 | err, debug = message.parse_error() 61 | sys.stderr.write("Error: %s: %s\n" % (err, debug)) 62 | loop.quit() 63 | return True 64 | 65 | 66 | class GListIterator: 67 | """ 68 | Implements python iterator protocol for pyds.GList type 69 | This is not perfect, but concise than the altenative of manually iterating the list using l_obj.next 70 | https://github.com/NVIDIA-AI-IOT/deepstream_python_apps/blob/6aabcaf85a9e8f11e9a4c39ab1cd46554de7c578/apps/deepstream-imagedata-multistream/deepstream_imagedata-multistream.py#L112 71 | """ 72 | 73 | def __init__(self, pyds_glist): 74 | self._head = pyds_glist 75 | self._curr = pyds_glist 76 | 77 | def __iter__(self): 78 | self._curr = self._head 79 | return self 80 | 81 | def __next__(self): 82 | # Advance the _curr node of linked list 83 | self._curr = self._curr.next 84 | 85 | if self._curr is None: 86 | # TODO: can we take care of the type cast here? 87 | raise StopIteration("Glist object reached termination node") 88 | else: 89 | return self._curr 90 | -------------------------------------------------------------------------------- /jetmulticam/bins/cameras.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | 3 | import gi 4 | 5 | gi.require_version("Gst", "1.0") 6 | from gi.repository import Gst 7 | 8 | from ..utils.gst import _make_element_safe, _sanitize 9 | 10 | 11 | def make_argus_camera_configured(sensor_id, bufapi_version=1) -> Gst.Element: 12 | """ 13 | Make pre-configured camera source, so we have consistent setting across sensors 14 | Switch off defaults which are not helpful for machine vision like edge-enhancement 15 | """ 16 | cam = _make_element_safe("nvarguscamerasrc") 17 | cam.set_property("sensor-id", sensor_id) 18 | cam.set_property("bufapi-version", bufapi_version) 19 | cam.set_property("wbmode", 1) # 1=auto, 0=off, 20 | cam.set_property("aeantibanding", 3) # 3=60Hz, 2=50Hz, 1=auto, 0=off 21 | cam.set_property("tnr-mode", 0) 22 | cam.set_property("ee-mode", 0) 23 | cam.set_property("silent", True) 24 | 25 | return cam 26 | 27 | 28 | def make_argus_cam_bin(sensor_id) -> Gst.Bin: 29 | bin = Gst.Bin() 30 | 31 | # Create v4l2 camera 32 | src = make_argus_camera_configured(sensor_id) 33 | conv = _make_element_safe("nvvideoconvert") 34 | conv_cf = _make_element_safe("capsfilter") 35 | conv_cf.set_property( 36 | "caps", Gst.Caps.from_string("video/x-raw(memory:NVMM),format=(string)RGBA") 37 | ) 38 | 39 | # Add elements to bin before linking 40 | for el in [src, conv, conv_cf]: 41 | bin.add(el) 42 | 43 | # Link bin elements 44 | src.link(conv) 45 | conv.link(conv_cf) 46 | 47 | # We exit via nvvidconv source pad 48 | exit_pad = _sanitize(conv_cf.get_static_pad("src")) 49 | gp = _sanitize(Gst.GhostPad.new(name="src", target=exit_pad)) 50 | bin.add_pad(gp) 51 | 52 | return bin 53 | 54 | 55 | def make_v4l2_cam_bin(dev: str) -> Gst.Bin: 56 | # dev: v4l2 node e.g. "/dev/video3" 57 | bin = Gst.Bin() 58 | 59 | # Create v4l2 camera 60 | src = _make_element_safe("v4l2src") 61 | src.set_property("device", dev) 62 | src.set_property("framerate", 30) 63 | 64 | vidconv = _make_element_safe("videoconvert") 65 | vidconv_cf = _make_element_safe("capsfilter") 66 | # Ensure we output something nvvideoconvert has caps for 67 | vidconv_cf.set_property( 68 | "caps", 69 | Gst.Caps.from_string( 70 | "video/x-raw, format=(string)RGBA, framerate=(fraction)30/1" 71 | ), 72 | ) 73 | 74 | nvvidconv = _make_element_safe("nvvideoconvert") 75 | nvvidconv_cf = _make_element_safe("capsfilter") 76 | nvvidconv_cf.set_property("caps", Gst.Caps.from_string("video/x-raw(memory:NVMM)")) 77 | 78 | # Add elements to bin before linking 79 | for el in [src, vidconv, vidconv_cf, nvvidconv, nvvidconv_cf]: 80 | bin.add(el) 81 | 82 | # Link bin elements 83 | src.link(vidconv) 84 | vidconv.link(vidconv_cf) 85 | vidconv_cf.link(nvvidconv) 86 | nvvidconv.link(nvvidconv_cf) 87 | 88 | # We exit via nvvidconv source pad 89 | exit_pad = _sanitize(nvvidconv_cf.get_static_pad("src")) 90 | gp = Gst.GhostPad.new(name="src", target=exit_pad) 91 | bin.add_pad(gp) 92 | 93 | return bin 94 | -------------------------------------------------------------------------------- /docs/simple_python_pipelines/03_encode_pipeline_bin_v4l2_bin.py: -------------------------------------------------------------------------------- 1 | import gi 2 | import sys 3 | 4 | sys.path.append("/home/nx/nv-multi-camera-robot/") 5 | 6 | gi.require_version("Gst", "1.0") 7 | from gi.repository import GObject, Gst 8 | from jetmulticam.gstutils import _make_element_safe, _sanitize 9 | 10 | 11 | def make_nvenc_bin() -> Gst.Bin: 12 | h264sink = Gst.Bin() 13 | 14 | # Create video converter 15 | conv = _make_element_safe("nvvideoconvert") 16 | 17 | # H264 encoder 18 | enc = _make_element_safe("nvv4l2h264enc") 19 | enc.set_property("bitrate", 10000000) 20 | 21 | # parser, mux 22 | parser = _make_element_safe("h264parse") 23 | mux = _make_element_safe("matroskamux") 24 | 25 | # filesink 26 | filesink = _make_element_safe("filesink") 27 | filesink.set_property("sync", 0) 28 | filesink.set_property("location", "test.mkv") 29 | 30 | # Add elements to bin before linking 31 | for el in [conv, enc, parser, mux, filesink]: 32 | h264sink.add(el) 33 | 34 | # Link bin elements 35 | conv.link(enc) 36 | enc.link(parser) 37 | parser.link(mux) 38 | mux.link(filesink) 39 | 40 | enter_pad = _sanitize(conv.get_static_pad("sink")) 41 | gp = Gst.GhostPad.new(name="sink", target=enter_pad) 42 | h264sink.add_pad(gp) 43 | 44 | return h264sink 45 | 46 | 47 | def make_v4l2_cam_bin(dev="/dev/video3") -> Gst.Bin: 48 | bin = Gst.Bin() 49 | 50 | # Create v4l2 camera 51 | src = _make_element_safe("v4l2src") 52 | src.set_property("device", dev) 53 | 54 | vidconv = _make_element_safe("videoconvert") 55 | vidconv_cf = _make_element_safe("capsfilter") 56 | # Ensure we output something nvvideoconvert has caps for 57 | vidconv_cf.set_property( 58 | "caps", Gst.Caps.from_string("video/x-raw, format=(string)RGBA") 59 | ) 60 | 61 | nvvidconv = _make_element_safe("nvvideoconvert") 62 | nvvidconv_cf = _make_element_safe("capsfilter") 63 | nvvidconv_cf.set_property("caps", Gst.Caps.from_string("video/x-raw(memory:NVMM)")) 64 | 65 | # Add elements to bin before linking 66 | for el in [src, vidconv, vidconv_cf, nvvidconv, nvvidconv_cf]: 67 | bin.add(el) 68 | 69 | # Link bin elements 70 | src.link(vidconv) 71 | vidconv.link(vidconv_cf) 72 | vidconv_cf.link(nvvidconv) 73 | nvvidconv.link(nvvidconv_cf) 74 | 75 | # We exit via nvvidconv source pad 76 | exit_pad = _sanitize(nvvidconv_cf.get_static_pad("src")) 77 | gp = Gst.GhostPad.new(name="src", target=exit_pad) 78 | bin.add_pad(gp) 79 | 80 | return bin 81 | 82 | 83 | if __name__ == "__main__": 84 | # Standard GStreamer initialization 85 | GObject.threads_init() 86 | Gst.init(None) 87 | 88 | pipeline = _sanitize(Gst.Pipeline()) 89 | 90 | cam = make_v4l2_cam_bin() 91 | # cam = _make_element_safe("videotestsrc") 92 | h264sink = make_nvenc_bin() 93 | 94 | for el in [cam, h264sink]: 95 | pipeline.add(el) 96 | 97 | cam.link(h264sink) 98 | loop = GObject.MainLoop() 99 | pipeline.set_state(Gst.State.PLAYING) 100 | 101 | try: 102 | loop.run() 103 | except Exception as e: 104 | print(e) 105 | finally: 106 | pipeline.set_state(Gst.State.NULL) 107 | -------------------------------------------------------------------------------- /docs/ready_pipelines/03_encode.md: -------------------------------------------------------------------------------- 1 | # Encoding 2 | 3 | ### Record video from one camera 4 | ```shell 5 | gst-launch-1.0 \ 6 | nvarguscamerasrc bufapi-version=1 sensor-id=0 ! \ 7 | nvvideoconvert ! \ 8 | nvv4l2h264enc bitrate=10000000 ! \ 9 | h264parse ! qtmux ! filesink sync=true location=test.mp4 -e; 10 | ``` 11 | 12 | ### Inference + encode + save to mp4 13 | ```shell 14 | gst-launch-1.0 nvarguscamerasrc bufapi-version=1 ! 'video/x-raw(memory:NVMM),width=1920,height=1080,framerate=30/1' ! mx.sink_0 nvstreammux width=1920 height=1080 batch-size=1 name=mx ! \ 15 | nvinfer config-file-path=jetmulticam/models/peoplenet/peoplenet_dla0.txt ! nvvideoconvert ! nvdsosd ! \ 16 | nvvideoconvert ! nvv4l2h264enc bitrate=10000000 ! 'video/x-h264, stream-format=(string)byte-stream' ! \ 17 | h264parse ! qtmux ! filesink sync=true location=output.mp4 -e; 18 | ``` 19 | 20 | ### Same, but with matroskamux 21 | If the pipeline is killed abruptly (without end-of-stream signal) qtmux will output a corrput file. 22 | Matroskamux does not have this issue (notice no `-e` switch) 23 | 24 | ```shell 25 | gst-launch-1.0 nvarguscamerasrc bufapi-version=1 ! 'video/x-raw(memory:NVMM),width=1920,height=1080,framerate=30/1' ! mx.sink_0 nvstreammux width=1920 height=1080 batch-size=1 name=mx ! \ 26 | nvinfer config-file-path=jetmulticam/models/peoplenet/peoplenet_dla0.txt ! nvvideoconvert ! nvdsosd ! \ 27 | nvvideoconvert ! nvv4l2h264enc bitrate=10000000 ! 'video/x-h264, stream-format=(string)byte-stream' ! \ 28 | h264parse ! matroskamux ! filesink sync=true location=output.mkv; 29 | ``` 30 | 31 | ### Inferenece -> tiler -> h264 -> mp4 32 | 33 | ``` 34 | gst-launch-1.0 \ 35 | nvarguscamerasrc bufapi-version=1 sensor-id=1 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_0 \ 36 | nvarguscamerasrc bufapi-version=1 sensor-id=0 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_1 \ 37 | nvarguscamerasrc bufapi-version=1 sensor-id=2 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_2 \ 38 | nvstreammux name=m width=1920 height=1080 batch-size=3 ! \ 39 | nvinfer config-file-path=jetmulticam/models/peoplenet/peoplenet_dla0.txt ! \ 40 | nvmultistreamtiler rows=2 columns=2 width=1920 height=1080 ! \ 41 | nvdsosd ! nvvideoconvert ! nvv4l2h264enc bitrate=10000000 ! 'video/x-h264, stream-format=(string)byte-stream' ! \ 42 | h264parse ! qtmux ! filesink sync=true location=test_apartment_video.mp4 -e; 43 | ``` 44 | 45 | 46 | ### Infer and save to mp4 47 | ```bash 48 | gst-launch-1.0 \ 49 | nvarguscamerasrc bufapi-version=1 sensor-id=0 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_0 \ 50 | nvarguscamerasrc bufapi-version=1 sensor-id=1 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_1 \ 51 | nvarguscamerasrc bufapi-version=1 sensor-id=2 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_2 \ 52 | nvstreammux name=m width=640 height=360 batch-size=3 ! \ 53 | nvinfer config-file-path=jetmulticam/models/peoplenet/peoplenet_dla0.txt ! \ 54 | nvmultistreamtiler rows=1 columns=3 width=2880 height=360 ! \ 55 | nvdsosd ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! \ 56 | nvvideoconvert ! nvv4l2h264enc bitrate=10000000 ! 'video/x-h264, stream-format=(string)byte-stream' ! \ 57 | h264parse ! qtmux ! filesink sync=true location=pano.mp4 -e; 58 | ``` 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | build 34 | *mp4 35 | 10-deepstream-python/models/* 36 | # Byte-compiled / optimized / DLL files 37 | __pycache__/ 38 | *.py[cod] 39 | *$py.class 40 | 41 | # C extensions 42 | *.so 43 | 44 | # Distribution / packaging 45 | .Python 46 | build/ 47 | develop-eggs/ 48 | dist/ 49 | downloads/ 50 | eggs/ 51 | .eggs/ 52 | lib/ 53 | lib64/ 54 | parts/ 55 | sdist/ 56 | var/ 57 | wheels/ 58 | share/python-wheels/ 59 | *.egg-info/ 60 | .installed.cfg 61 | *.egg 62 | MANIFEST 63 | 64 | # PyInstaller 65 | # Usually these files are written by a python script from a template 66 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 67 | *.manifest 68 | *.spec 69 | 70 | # Installer logs 71 | pip-log.txt 72 | pip-delete-this-directory.txt 73 | 74 | # Unit test / coverage reports 75 | htmlcov/ 76 | .tox/ 77 | .nox/ 78 | .coverage 79 | .coverage.* 80 | .cache 81 | nosetests.xml 82 | coverage.xml 83 | *.cover 84 | *.py,cover 85 | .hypothesis/ 86 | .pytest_cache/ 87 | cover/ 88 | 89 | # Translations 90 | *.mo 91 | *.pot 92 | 93 | # Django stuff: 94 | *.log 95 | local_settings.py 96 | db.sqlite3 97 | db.sqlite3-journal 98 | 99 | # Flask stuff: 100 | instance/ 101 | .webassets-cache 102 | 103 | # Scrapy stuff: 104 | .scrapy 105 | 106 | # Sphinx documentation 107 | docs/_build/ 108 | 109 | # PyBuilder 110 | .pybuilder/ 111 | target/ 112 | 113 | # Jupyter Notebook 114 | .ipynb_checkpoints 115 | 116 | # IPython 117 | profile_default/ 118 | ipython_config.py 119 | 120 | # pyenv 121 | # For a library or package, you might want to ignore these files since the code is 122 | # intended to run in multiple environments; otherwise, check them in: 123 | # .python-version 124 | 125 | # pipenv 126 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 127 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 128 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 129 | # install all needed dependencies. 130 | #Pipfile.lock 131 | 132 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 133 | __pypackages__/ 134 | 135 | # Celery stuff 136 | celerybeat-schedule 137 | celerybeat.pid 138 | 139 | # SageMath parsed files 140 | *.sage.py 141 | 142 | # Environments 143 | .env 144 | .venv 145 | env/ 146 | venv/ 147 | ENV/ 148 | env.bak/ 149 | venv.bak/ 150 | 151 | # Spyder project settings 152 | .spyderproject 153 | .spyproject 154 | 155 | # Rope project settings 156 | .ropeproject 157 | 158 | # mkdocs documentation 159 | /site 160 | 161 | # mypy 162 | .mypy_cache/ 163 | .dmypy.json 164 | dmypy.json 165 | 166 | # Pyre type checker 167 | .pyre/ 168 | 169 | # pytype static type analyzer 170 | .pytype/ 171 | 172 | # Cython debug symbols 173 | cython_debug/ 174 | 175 | 176 | # Project-specific files 177 | 178 | *.pt # pytorch weights 179 | remap_rectify_params.yml # Stereo rectify parameters 180 | 181 | # video files 182 | *.h264 183 | *.mkv 184 | 185 | # Deepstream + TRT 186 | *.etlt 187 | *.engine 188 | 189 | # Downloaded NGC files 190 | */models/dashcamnet/dashcamnet_pruned_v1.0/ 191 | */models/peoplenet/tlt_peoplenet_pruned_v2.0/ -------------------------------------------------------------------------------- /docs/ready_pipelines/01_capture.md: -------------------------------------------------------------------------------- 1 | # Camera Capture 2 | 3 | 4 | ## Display single camera 5 | ```shell 6 | gst-launch-1.0 nvarguscamerasrc ! 'video/x-raw(memory:NVMM), width=(int)1920, height=(int)1080, format=(string)NV12, framerate=(fraction)30/1' ! nvoverlaysink 7 | ``` 8 | 9 | ## Display single camera (windowed) 10 | ```shell 11 | gst-launch-1.0 nvarguscamerasrc ! 'video/x-raw(memory:NVMM), width=(int)1920, height=(int)1080, format=(string)NV12, framerate=(fraction)30/1' ! nvegltransform ! nveglglessink 12 | ``` 13 | 14 | ## Display single USB camera (xvimagesink) 15 | ```shell 16 | gst-launch-1.0 v4l2src device=/dev/video3 ! videoconvert ! "video/x-raw, format=(string)RGBA" ! videoconvert ! xvimagesink sync=false 17 | ``` 18 | 19 | ## Display single USB camera (nvvideoconvert + nvoverlaysink) 20 | Explicitly set memory:NVMM caps 21 | ```shell 22 | gst-launch-1.0 v4l2src device=/dev/video3 ! videoconvert ! "video/x-raw, format=(string)RGBA" ! videoconvert ! nvvideoconvert ! "video/x-raw(memory:NVMM)" ! nvoverlaysink sync=false 23 | ``` 24 | 25 | ## Display test pattern with nvmultistreamtiler 26 | ```shell 27 | gst-launch-1.0 videotestsrc pattern=1 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_0 videotestsrc ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_1 nvstreammux name=m width=1920 height=1080 batch-size=2 ! nvmultistreamtiler rows=2 columns=2 width=1920 height=1080 ! nvdsosd ! nvegltransform ! nveglglessink sync=0 28 | ``` 29 | 30 | ## Display test pattern + 1 video 31 | ```shell 32 | gst-launch-1.0 \ 33 | nvarguscamerasrc bufapi-version=1 sensor-id=0 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_0 \ 34 | videotestsrc ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_1 \ 35 | nvstreammux name=m width=1920 height=1080 batch-size=2 ! nvmultistreamtiler rows=2 columns=2 width=1920 height=1080 ! nvdsosd ! nvegltransform ! nveglglessink sync=0 36 | ``` 37 | 38 | ## Display 3 cameras in tile stream 39 | ```shell 40 | gst-launch-1.0 \ 41 | nvarguscamerasrc bufapi-version=1 sensor-id=0 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_0 \ 42 | nvarguscamerasrc bufapi-version=1 sensor-id=1 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_1 \ 43 | nvarguscamerasrc bufapi-version=1 sensor-id=2 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_2 \ 44 | nvstreammux name=m width=1920 height=1080 batch-size=3 ! nvmultistreamtiler rows=2 columns=2 width=1920 height=1080 ! nvdsosd ! nvegltransform ! nveglglessink sync=0 45 | ``` 46 | 47 | 48 | ## Display 2 USB cameras in tile stream 49 | ```shell 50 | gst-launch-1.0 \ 51 | v4l2src device=/dev/video0 ! videoconvert ! "video/x-raw, format=(string)RGBA" ! videoconvert ! nvvideoconvert ! "video/x-raw(memory:NVMM)" ! m.sink_0 \ 52 | v4l2src device=/dev/video1 ! videoconvert ! "video/x-raw, format=(string)RGBA" ! videoconvert ! nvvideoconvert ! "video/x-raw(memory:NVMM)" ! m.sink_1 \ 53 | nvstreammux name=m width=1920 height=1080 batch-size=2 ! nvmultistreamtiler rows=2 columns=2 width=1920 height=1080 ! nvdsosd ! nvegltransform ! nveglglessink sync=0 54 | ``` 55 | 56 | ## Display 3 USB cameras in tile stream 57 | ```shell 58 | gst-launch-1.0 \ 59 | v4l2src device=/dev/video0 ! videoconvert ! "video/x-raw, format=(string)RGBA" ! videoconvert ! nvvideoconvert ! "video/x-raw(memory:NVMM)" ! m.sink_0 \ 60 | v4l2src device=/dev/video1 ! videoconvert ! "video/x-raw, format=(string)RGBA" ! videoconvert ! nvvideoconvert ! "video/x-raw(memory:NVMM)" ! m.sink_1 \ 61 | v4l2src device=/dev/video2 ! videoconvert ! "video/x-raw, format=(string)RGBA" ! videoconvert ! nvvideoconvert ! "video/x-raw(memory:NVMM)" ! m.sink_2 \ 62 | nvstreammux name=m width=1920 height=1080 batch-size=3 ! nvmultistreamtiler rows=2 columns=2 width=1920 height=1080 ! nvdsosd ! nvegltransform ! nveglglessink sync=0 63 | ``` -------------------------------------------------------------------------------- /docs/ready_pipelines/02_inference.md: -------------------------------------------------------------------------------- 1 | # Inference Pipelines 2 | 3 | ### Simple pipeline with 1 nvarguscamera camera and 1 nvinfer 4 | ```shell 5 | gst-launch-1.0 nvarguscamerasrc bufapi-version=1 ! 'video/x-raw(memory:NVMM),width=1920,height=1080,framerate=30/1' ! mx.sink_0 nvstreammux width=1920 height=1080 batch-size=1 name=mx ! nvinfer config-file-path=/home/nx/nv-multi-camera-robot/jetmulticam/models/peoplenet/peoplenet_dla0.txt ! nvvideoconvert ! nvdsosd ! nvoverlaysink sync=0 6 | ``` 7 | 8 | ### Simple pipeline with 1 USB camera and 1 nvinfer 9 | ```shell 10 | gst-launch-1.0 v4l2src device=/dev/video3 ! videoconvert ! "video/x-raw, format=(string)RGBA" ! nvvideoconvert ! "video/x-raw(memory:NVMM)" ! mx.sink_0 nvstreammux width=640 height=480 batch-size=1 name=mx ! nvinfer config-file-path=/home/nx/nv-multi-camera-robot/jetmulticam/models/peoplenet/peoplenet_dla0.txt ! nvvideoconvert ! nvdsosd ! nvoverlaysink sync=0 11 | ``` 12 | 13 | ### Two infers back-to-back on one camera 14 | 15 | Two different DNNs running on DLA0 + DLA1 16 | ```shell 17 | gst-launch-1.0 nvarguscamerasrc bufapi-version=1 ! 'video/x-raw(memory:NVMM),width=1920,height=1080,framerate=30/1' ! mx.sink_0 nvstreammux width=1920 height=1080 batch-size=1 name=mx ! \ 18 | nvinfer config-file-path=jetmulticam/models/peoplenet/peoplenet_dla1.txt ! \ 19 | nvinfer config-file-path=jetmulticam/models/dashcamnet/dashcamnet_dla0.txt ! \ 20 | nvvideoconvert ! nvdsosd ! nvoverlaysink sync=0 21 | ``` 22 | 23 | ### Run an object detector on a batch from 3 cameras, display everything in a tile grid 24 | ```shell 25 | gst-launch-1.0 \ 26 | nvarguscamerasrc bufapi-version=1 sensor-id=1 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_0 \ 27 | nvarguscamerasrc bufapi-version=1 sensor-id=0 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_1 \ 28 | nvarguscamerasrc bufapi-version=1 sensor-id=2 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_2 \ 29 | nvstreammux name=m width=1920 height=1080 batch-size=3 ! \ 30 | nvinfer config-file-path=jetmulticam/models/peoplenet/peoplenet_gpu.txt ! \ 31 | nvmultistreamtiler rows=2 columns=2 width=1920 height=1080 ! \ 32 | nvdsosd ! nvoverlaysink sync=0 33 | ``` 34 | 35 | 36 | ### 3 cameras with 2 DNN inferences in chained configuration 37 | 38 | ```bash 39 | gst-launch-1.0 \ 40 | nvarguscamerasrc bufapi-version=1 sensor-id=1 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_0 \ 41 | nvarguscamerasrc bufapi-version=1 sensor-id=0 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_1 \ 42 | nvarguscamerasrc bufapi-version=1 sensor-id=2 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! m.sink_2 \ 43 | nvstreammux name=m width=1920 height=1080 batch-size=3 ! \ 44 | nvinfer config-file-path=jetmulticam/models/peoplenet/peoplenet_gpu.txt ! \ 45 | nvinfer config-file-path=jetmulticam/models/dashcamnet/dashcamnet_dla0.txt ! \ 46 | nvmultistreamtiler rows=2 columns=2 width=1920 height=1080 ! \ 47 | nvdsosd ! nvoverlaysink sync=0 48 | ``` 49 | 50 | ### 3 cameras with 2 DNN inferences in side-by-side configuration 51 | ```shell 52 | gst-launch-1.0 \ 53 | nvarguscamerasrc bufapi-version=1 sensor-id=1 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! tee name=t0 ! queue ! muxA.sink_0 \ 54 | nvarguscamerasrc bufapi-version=1 sensor-id=0 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! tee name=t1 ! queue ! muxA.sink_1 \ 55 | nvarguscamerasrc bufapi-version=1 sensor-id=2 ! nvvideoconvert ! "video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1" ! tee name=t2 ! queue ! muxA.sink_2 \ 56 | nvstreammux name=muxA width=1920 height=1080 batch-size=3 ! nvinfer config-file-path=jetmulticam/models/peoplenet/peoplenet_dla0.txt ! nvmultistreamtiler rows=2 columns=2 width=1920 height=1080 ! nvdsosd ! nvegltransform ! nveglglessink sync=0 \ 57 | t0. ! queue ! muxB.sink_0 \ 58 | t1. ! queue ! muxB.sink_1 \ 59 | t2. ! queue ! muxB.sink_2 \ 60 | nvstreammux name=muxB width=1920 height=1080 batch-size=3 ! nvinfer config-file-path=jetmulticam/models/peoplenet/peoplenet_dla1.txt ! nvmultistreamtiler rows=2 columns=2 width=1920 height=1080 ! fakesink 61 | ``` -------------------------------------------------------------------------------- /jetmulticam/pipelines/multicam.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | 3 | import logging # TODO: remove 4 | import os 5 | import time 6 | from threading import Thread 7 | 8 | # Gstreamer imports 9 | import gi 10 | import numpy as np 11 | 12 | gi.require_version("Gst", "1.0") 13 | from gi.repository import Gst 14 | 15 | from .basepipeline import BasePipeline 16 | from ..utils.gst import _make_element_safe, _sanitize 17 | from ..bins.cameras import make_argus_camera_configured 18 | from ..bins.encoders import make_nvenc_bin_no_ds 19 | 20 | 21 | def make_conv_bin(caps="video/x-raw, format=(string)RGBA") -> Gst.Bin: 22 | bin = Gst.Bin() 23 | 24 | nvvidconv = _make_element_safe("nvvidconv") 25 | conv_cf = _make_element_safe("capsfilter") 26 | conv_cf.set_property( 27 | "caps", 28 | Gst.Caps.from_string( 29 | caps 30 | ), # NOTE: make parametric? i.e. height=1080, width=1920 31 | ) 32 | 33 | nvvidconv.link(conv_cf) 34 | 35 | # We enter via conv sink pad 36 | enter_pad = _sanitize(nvvidconv.get_static_pad("sink")) 37 | gp_enter = _sanitize(Gst.GhostPad.new(name="sink", target=enter_pad)) 38 | bin.add_pad(gp_enter) 39 | 40 | # We exit via conv_cf source pad 41 | exit_pad = _sanitize(conv_cf.get_static_pad("src")) 42 | gp = _sanitize(Gst.GhostPad.new(name="src", target=exit_pad)) 43 | bin.add_pad(gp) 44 | 45 | return bin 46 | 47 | 48 | def make_appsink_configured() -> Gst.Element: 49 | appsink = _make_element_safe("appsink") 50 | appsink.set_property("max-buffers", 1) 51 | appsink.set_property("drop", True) 52 | return appsink 53 | 54 | 55 | class CameraPipeline(BasePipeline): 56 | def __init__(self, cameras, logdir="/home/nx/logs/videos", **kwargs): 57 | """ 58 | cameras: list of sensor ids for argus cameras 59 | """ 60 | self._cams = cameras 61 | self._logdir = logdir 62 | os.makedirs(self._logdir, exist_ok=True) 63 | super().__init__(**kwargs) 64 | 65 | def _create_pipeline(self): 66 | 67 | pipeline = _sanitize(Gst.Pipeline()) 68 | cameras = [ 69 | make_argus_camera_configured(c, bufapi_version=0) for c in self._cams 70 | ] 71 | 72 | convs = [_make_element_safe("nvvidconv") for _ in self._cams] 73 | conv_cfs = [_make_element_safe("capsfilter") for _ in self._cams] 74 | 75 | for cf in conv_cfs: 76 | cf.set_property( 77 | "caps", 78 | Gst.Caps.from_string( 79 | "video/x-raw, format=(string)RGBA" 80 | ), # NOTE: This could be parametric. I.e. height=1080, width=1920 81 | ) 82 | 83 | tees = [_make_element_safe("tee") for _ in self._cams] 84 | ts = time.strftime("%Y-%m-%dT%H-%M-%S%z") 85 | h264sinks = [ 86 | make_nvenc_bin_no_ds(f"{self._logdir}/jetmulticam-{ts}-cam{c}.mkv") 87 | for c in self._cams 88 | ] 89 | self._appsinks = appsinks = [make_appsink_configured() for _ in self._cams] 90 | 91 | for el in [*cameras, *convs, *conv_cfs, *tees, *h264sinks, *appsinks]: 92 | pipeline.add(el) 93 | 94 | for cam, conv in zip(cameras, convs): 95 | print(cam, conv) 96 | cam.link(conv) 97 | 98 | for conv, cf in zip(convs, conv_cfs): 99 | conv.link(cf) 100 | 101 | for cf, tee in zip(conv_cfs, tees): 102 | cf.link(tee) 103 | 104 | for (tee, enc, app) in zip(tees, h264sinks, appsinks): 105 | 106 | for idx, sink in enumerate([enc, app]): 107 | # Use queues for each sink. This ensures the sinks can execute in separate threads 108 | queue = _make_element_safe("queue") 109 | pipeline.add(queue) 110 | # tee.src_%d -> queue 111 | srcpad_or_none = tee.get_request_pad(f"src_{idx}") 112 | sinkpad_or_none = queue.get_static_pad("sink") 113 | srcpad = _sanitize(srcpad_or_none) 114 | sinkpad = _sanitize(sinkpad_or_none) 115 | srcpad.link(sinkpad) 116 | # queue -> sink 117 | queue.link(sink) 118 | 119 | return pipeline 120 | 121 | def read(self, cam_idx): 122 | """ 123 | Returns np.array or None 124 | """ 125 | sample = self._appsinks[cam_idx].emit("pull-sample") 126 | if sample is None: 127 | return None 128 | buf = sample.get_buffer() 129 | buf2 = buf.extract_dup(0, buf.get_size()) 130 | # Get W, H, C: 131 | caps_format = sample.get_caps().get_structure(0) # Gst.Structure 132 | w, h = caps_format.get_value("width"), caps_format.get_value("height") 133 | c = 4 # Earlier we converted to RGBA 134 | # To check format: print(caps_format.get_value("format")) 135 | arr = np.ndarray(shape=(h, w, c), buffer=buf2, dtype=np.uint8) 136 | 137 | return arr 138 | -------------------------------------------------------------------------------- /examples/example-person-following.py: -------------------------------------------------------------------------------- 1 | from threading import Thread 2 | import time 3 | import collections 4 | from concurrent.futures import ThreadPoolExecutor 5 | 6 | from jetmulticam import CameraPipelineDNN 7 | from jetmulticam.models import PeopleNet, DashCamNet 8 | 9 | # robot related packages 10 | from controller import ManualController 11 | from vehicle import Vehicle 12 | 13 | 14 | def find_closest_human(dets_l, dets_c, dets_r): 15 | 16 | htf = None 17 | htf_area = -1 18 | htf_im = None 19 | 20 | for dets, im in zip((dets_l, dets_c, dets_r), ["l", "c", "r"]): 21 | humans = [det for det in dets if det["class"] == "person"] 22 | for human in humans: 23 | (l, w, t, h) = human["position"] 24 | if w * h > htf_area: 25 | htf = human # htf -> human to follow 26 | htf_area = w * h 27 | htf_im = im 28 | 29 | return (htf, htf_im) 30 | 31 | 32 | def dets2steer(dets): 33 | W = 1920 # Image width 34 | human, which_im = find_closest_human(*dets) 35 | steer = 0 36 | if which_im == "c": 37 | (l, w, _, _) = human["position"] 38 | center = l + w / 2 39 | steer = (center - W / 2) / W * 2 40 | elif which_im == "l": 41 | steer = -1 42 | elif which_im == "r": 43 | steer = 1 44 | return steer 45 | 46 | 47 | class Filter: 48 | def __init__(self, window=10): 49 | self._q = collections.deque(maxlen=window) 50 | 51 | def __call__(self, sample): 52 | self._q.append(sample) 53 | value = np.array(self._q).mean() 54 | return value 55 | 56 | 57 | def main_follow_person(): 58 | controller = ManualController() 59 | vehicle = Vehicle() 60 | filter = Filter(20) 61 | 62 | pipeline = CameraPipelineDNN( 63 | cameras=[5, 2, 8], 64 | models=[ 65 | PeopleNet.DLA1, 66 | DashCamNet.DLA0, 67 | ], 68 | save_video=True, 69 | save_video_folder="/home/nx/video-output", 70 | display=True, # just for visual debug 71 | ) 72 | 73 | try: 74 | while pipeline.running(): 75 | 76 | if controller.right_pressed(): 77 | # If button pressed, follow person 78 | velocity = 0.3 79 | steering = dets2steer(pipeline.detections) 80 | steering = filter(steering) 81 | print(steering) 82 | vehicle.set_throttle(velocity) 83 | vehicle.set_steering(steering) 84 | else: 85 | # Fall back to manual steering 86 | velocity = controller.throttle() 87 | steering = controller.steering() 88 | vehicle.set_throttle(velocity) 89 | vehicle.set_steering(steering) 90 | 91 | # # Debug messages 92 | dbg_str = "" 93 | dbg_str += f"Center: {pipeline.fps()[0]:0.2f} FPS Front: {pipeline.fps()[1]:0.2f} FPS Right: {pipeline.fps()[2]:0.2f}FPS | " 94 | dbg_str += ( 95 | f"Throttle: {round(velocity, 2)} Steering: {round(controller.s, 2)} | " 96 | ) 97 | dbg_str += f"Train: {controller.train_mode()} Auto: {controller.autonomous_mode()} N. dets: {len(pipeline.detections[0])} | " 98 | # dbg_str += f"Dets: {pipeline.detections}" 99 | print(dbg_str) 100 | 101 | time.sleep(1 / 30) # Cap the main thread at 30 FPS 102 | 103 | except Exception as e: 104 | print(e) 105 | finally: 106 | pipeline.stop() 107 | controller.stop() 108 | vehicle.stop() 109 | 110 | 111 | def main_manual(): 112 | controller = ManualController() 113 | vehicle = Vehicle() 114 | 115 | pipeline = CameraPipelineDNN( 116 | cameras=[5, 2, 8], 117 | models=[ 118 | # PeopleNet.DLA1, 119 | # DashCamNet.DLA0, 120 | DashCamNet.GPU 121 | ], 122 | save_video=True, 123 | save_video_folder="/home/nx/jetvision-output", 124 | display=True, # just for visual debug 125 | ) 126 | 127 | try: 128 | while pipeline.running(): 129 | 130 | velocity, human_steering = controller.t, controller.s 131 | 132 | vehicle.set_throttle(velocity) 133 | vehicle.set_steering(human_steering) 134 | 135 | # # Debug messages 136 | dbg_str = "" 137 | dbg_str += f"Center: {pipeline.fps()[0]:0.2f} FPS Front: {pipeline.fps()[1]:0.2f} FPS Right: {pipeline.fps()[2]:0.2f}FPS | " 138 | dbg_str += f"Throttle: {round(controller.t, 2)} Steering: {round(controller.s, 2)} | " 139 | dbg_str += f"Train: {controller.train_mode()} Auto: {controller.autonomous_mode()} N. dets: {len(pipeline.detections[0])} | " 140 | print(dbg_str) 141 | 142 | time.sleep(1 / 30) # Cap the main thread at 30 FPS 143 | 144 | except Exception as e: 145 | print(e) 146 | finally: 147 | pipeline.stop() 148 | controller.stop() 149 | vehicle.stop() 150 | 151 | 152 | if __name__ == "__main__": 153 | main_follow_person() 154 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Jetson Multicamera Pipelines 2 | 3 | Easy-to-use realtime CV/AI pipelines for Nvidia Jetson Platform. This project: 4 | - Builds a typical multi-camera pipeline, i.e. `N×(capture)->preprocess->batch->DNN-> <> ->encode->file I/O + display`. Uses `gstreamer` and `deepstream` under-the-hood. 5 | - Gives programatic acces to configure the pipeline in python via `jetmulticam` package. 6 | - Utilizes Nvidia HW accleration for minimal CPU usage. For example, you can perform object detection in real-time on 6 camera streams using as little as `16.5%` CPU. See benchmarks below for details. 7 | 8 | ## Demos 9 | 10 | You can easily build your custom logic in python by accessing image data (via `np.array`), as well object detection results. 11 | See examples of person following below: 12 | 13 | #### DashCamNet _(DLA0)_ + PeopleNet _(DLA1)_ on 3 camera streams. 14 | We have 3 intependent cameras with ~270° field of view. 15 | Red Boxes correspond to DashCamNet detections, green ones to PeopleNet. 16 | The PeopleNet detections are used to perform person following logic. 17 | 18 | 19 | https://github.com/NVIDIA-AI-IOT/jetson-multicamera-pipelines/assets/26127866/9fe0b080-4964-49ac-b086-3cca9db01cef 20 | 21 | 22 | #### PeopleNet (GPU) on 3 cameras streams. 23 | Robot is operated in manual mode. 24 | 25 | 26 | https://github.com/NVIDIA-AI-IOT/jetson-multicamera-pipelines/assets/26127866/e56b4acb-ce93-4f7c-b7af-eb04297b087e 27 | 28 | 29 | #### DashCamNet (GPU) on 3 camera streams. 30 | Robot is operated in manual mode. 31 | 32 | 33 | https://github.com/NVIDIA-AI-IOT/jetson-multicamera-pipelines/assets/26127866/8dda2023-8d2b-41ac-8c29-890b1cfc14e9 34 | 35 | 36 | All demos are performed in real-time onboard Nvidia Jetson Xavier NX. 37 | 38 | ## Quickstart 39 | 40 | Install: 41 | ```shell 42 | git clone https://github.com/NVIDIA-AI-IOT/jetson-multicamera-pipelines.git 43 | cd jetson-multicamera-pipelines 44 | bash scripts/install_dependencies.sh 45 | pip3 install Cython 46 | pip3 install . 47 | ``` 48 | Run example with your cameras: 49 | ```shell 50 | source scripts/env_vars.sh 51 | cd examples 52 | python3 example.py 53 | ``` 54 | 55 | ## Usage example 56 | 57 | ```python 58 | import time 59 | from jetmulticam import CameraPipelineDNN 60 | from jetmulticam.models import PeopleNet, DashCamNet 61 | 62 | if __name__ == "__main__": 63 | 64 | pipeline = CameraPipelineDNN( 65 | cameras=[2, 5, 8], 66 | models=[ 67 | PeopleNet.DLA1, 68 | DashCamNet.DLA0, 69 | # PeopleNet.GPU 70 | ], 71 | save_video=True, 72 | save_video_folder="/home/nx/logs/videos", 73 | display=True, 74 | ) 75 | 76 | while pipeline.running(): 77 | arr = pipeline.images[0] # np.array with shape (1080, 1920, 3), i.e. (1080p RGB image) 78 | dets = pipeline.detections[0] # Detections from the DNNs 79 | time.sleep(1/30) 80 | ``` 81 | 82 | ## Benchmarks 83 | 84 | | # | Scenario | # cams | CPU util.
(jetmulticam) | CPU util.
(nvargus-deamon) | CPU
total | GPU % | EMC util % | Power draw | Inference Hardware | 85 | | --- | -------------------------------------- | ------ | -------------------------- | ------------------------------- | ------------ | ----- | ---------- | ---------- | -------------------------------------------------------------- | 86 | | 1. | 1xGMSL -> 2xDNNs + disp + encode | 1 | 5.3% | 4% | 9.3% | <3% | 57% | 8.5W | DLA0: PeopleNet DLA1: DashCamNet | 87 | | 2. | 2xGMSL -> 2xDNNs + disp + encode | 2 | 7.2% | 7.7% | 14.9% | <3% | 62% | 9.4W | DLA0: PeopleNet DLA1: DashCamNet | 88 | | 3. | 3xGMSL -> 2xDNNs + disp + encode | 3 | 9.2% | 11.3% | 20.5% | <3% | 68% | 10.1W | DLA0: PeopleNet DLA1: DashCamNet | 89 | | 4. | Same as _#3_ with CPU @ 1.9GHz | 3 | 7.5% | 9.0% | 16.5% | <3% | 68% | 10.4w | DLA0: PeopleNet DLA1: DashCamNet | 90 | | 5. | 3xGMSL+2xV4L -> 2xDNNs + disp + encode | 5 | 9.5% | 11.3% | 20.8% | <3% | 45% | 9.1W | DLA0: PeopleNet _(interval=1)_ DLA1: DashCamNet _(interval=1)_ | 91 | | 6. | 3xGMSL+2xV4L -> 2xDNNs + disp + encode | 5 | 8.3% | 11.3% | 19.6% | <3% | 25% | 7.5W | DLA0: PeopleNet _(interval=6)_ DLA1: DashCamNet _(interval=6)_ | 92 | | 7. | 3xGMSL -> DNN + disp + encode | 5 | 10.3% | 12.8% | 23.1% | 99% | 25% | 15W | GPU: PeopleNet | 93 | 94 | 95 | Notes: 96 | - All figures are in `15W 6 core mode`. To reproduce do: `sudo nvpmodel -m 2; sudo jetson_clocks;` 97 | - Test platform: [Jetson Xavier NX](https://developer.nvidia.com/embedded/jetson-xavier-nx-devkit) and [XNX Box](https://www.leopardimaging.com/product/nvidia-jetson-cameras/nvidia-nx-mipi-camera-kits/li-xnx-box-gmsl2/) running [JetPack](https://developer.nvidia.com/embedded/jetpack) v4.5.1 98 | - The residual GPU usage in DLA-accelerated nets is caused by Sigmoid activations being computed with CUDA backend. Remaining layers are computed on DLA. 99 | - CPU usage will vary depending on factors such as camera resolution, framerate, available video formats and driver implementation. 100 | 101 | ## More 102 | 103 | ### Supported models / acceleratorss 104 | ```python 105 | pipeline = CameraPipelineDNN( 106 | cam_ids = [0, 1, 2] 107 | models=[ 108 | models.PeopleNet.DLA0, 109 | models.PeopleNet.DLA1, 110 | models.PeopleNet.GPU, 111 | models.DashCamNet.DLA0, 112 | models.DashCamNet.DLA1, 113 | models.DashCamNet.GPU 114 | ] 115 | # ... 116 | ) 117 | ``` 118 | -------------------------------------------------------------------------------- /jetmulticam/pipelines/multicamDNN.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | 3 | import logging 4 | import time 5 | 6 | # Gstreamer imports 7 | import gi 8 | import numpy as np 9 | import pyds 10 | 11 | gi.require_version("Gst", "1.0") 12 | from gi.repository import Gst 13 | 14 | from .basepipeline import BasePipeline 15 | from ..utils.gst import _err_if_none, _make_element_safe, _sanitize, bus_call 16 | from ..bins.cameras import make_argus_cam_bin, make_v4l2_cam_bin 17 | from ..bins.encoders import make_nvenc_bin 18 | 19 | 20 | class CameraPipelineDNN(BasePipeline): 21 | def __init__(self, cameras, models, model_intervals=None, *args, **kwargs): 22 | """ 23 | models parameter can be: 24 | - `dict`: mapping of models->sensor-ids to infer on 25 | - `list`: list of models to use on frames from all cameras 26 | """ 27 | 28 | self._m = models 29 | self._c = cameras 30 | 31 | # model interval is 0 by default 32 | if model_intervals is None: 33 | model_intervals = [0 for _ in models] 34 | 35 | self._model_intervals = model_intervals 36 | 37 | if len(self._m) != len(self._model_intervals): 38 | raise ValueError("len(model_intervals) must be equal to len(models)!") 39 | 40 | # Runtime parameters 41 | N_CAMS = len(cameras) 42 | self.images = [np.empty((1080, 1920, 3)) for _ in range(0, N_CAMS)] 43 | self.detections = [[] for _ in range(0, N_CAMS)] # dets for each camera 44 | self.frame_n = [-1 for _ in range(0, N_CAMS)] 45 | self.det_n = [-1 for _ in range(0, N_CAMS)] 46 | 47 | super().__init__(**kwargs) 48 | 49 | def _create_pipeline(self, **kwargs) -> Gst.Pipeline: 50 | # gst pipeline object 51 | if type(self._m) is list: 52 | p = self._create_pipeline_fully_connected(self._c, self._m, **kwargs) 53 | elif type(self._m) is dict: 54 | p = self._create_pipeline_sparsely_connected(self._c, self._m, **kwargs) 55 | 56 | return p 57 | 58 | def _create_pipeline_fully_connected( 59 | self, 60 | cameras, 61 | model_list, 62 | save_video=True, 63 | save_video_folder="/home/nx/logs/videos", 64 | display=True, 65 | streaming=False, 66 | ): 67 | """ 68 | Images from all sources will go through all DNNs 69 | """ 70 | 71 | pipeline = Gst.Pipeline() 72 | _err_if_none(pipeline) 73 | 74 | n_cams = len(cameras) 75 | sources = self._make_sources(cameras) 76 | 77 | # Create muxer 78 | mux = _make_element_safe("nvstreammux") 79 | mux.set_property("live-source", True) 80 | mux.set_property("width", 1920) 81 | mux.set_property("height", 1080) 82 | mux.set_property("batch-size", n_cams) 83 | mux.set_property("batched-push-timeout", 4000000) 84 | 85 | # Create nvinfers 86 | nvinfers = [_make_element_safe("nvinfer") for _ in model_list] 87 | for m_path, nvinf, interval in zip(model_list, nvinfers, self._model_intervals): 88 | nvinf.set_property("config-file-path", m_path) 89 | nvinf.set_property("batch-size", n_cams) 90 | nvinf.set_property("interval", interval) # to infer every n batches 91 | 92 | # nvvideoconvert -> nvdsosd -> nvegltransform -> sink 93 | nvvidconv = _make_element_safe("nvvideoconvert") 94 | nvosd = _make_element_safe("nvdsosd") 95 | 96 | tiler = _make_element_safe("nvmultistreamtiler") 97 | 98 | n_cols = min(n_cams, 3) # max 3 cameras in a row. More looks overcrowded. 99 | tiler.set_property("rows", n_cams // n_cols) 100 | tiler.set_property("columns", n_cols) 101 | # Encoder crashes when we attempt encoding 5760 x 1080, so we set it lower 102 | # TODO: Is that a bug, or hw limitation? 103 | 104 | tiler.set_property("width", 1920) 105 | tiler.set_property("height", 360) 106 | 107 | # Render with EGL GLE sink 108 | # transform = _make_element_safe("nvegltransform") 109 | # renderer = _make_element_safe("nveglglessink") 110 | 111 | tee = _make_element_safe("tee") 112 | 113 | sinks = [] 114 | if save_video: 115 | ts = time.strftime("%Y-%m-%dT%H-%M-%S%z") 116 | encodebin = make_nvenc_bin( 117 | filepath=save_video_folder + f"/jetmulticam{ts}.mkv" 118 | ) 119 | sinks.append(encodebin) 120 | if display: 121 | overlay = _make_element_safe("nvoverlaysink") 122 | overlay.set_property("sync", 0) # crucial for performance of the pipeline 123 | sinks.append(overlay) 124 | if streaming: 125 | # TODO: 126 | raise NotImplementedError 127 | 128 | if len(sinks) == 0: 129 | # If no other sinks are added we terminate with fakesink 130 | fakesink = _make_element_safe("fakesink") 131 | sinks.append(fakesink) 132 | 133 | # Add all elements to the pipeline 134 | elements = [*sources, mux, *nvinfers, nvvidconv, tiler, nvosd, tee, *sinks] 135 | for el in elements: 136 | pipeline.add(el) 137 | 138 | for (idx, source) in enumerate(sources): 139 | srcpad_or_none = source.get_static_pad(f"src") 140 | sinkpad_or_none = mux.get_request_pad(f"sink_{idx}") 141 | srcpad = _sanitize(srcpad_or_none) 142 | sinkpad = _sanitize(sinkpad_or_none) 143 | srcpad.link(sinkpad) 144 | 145 | # Chain multiple nvinfers back-to-back 146 | mux.link(nvinfers[0]) 147 | for i in range(1, len(nvinfers)): 148 | nvinfers[i - 1].link(nvinfers[i]) 149 | nvinfers[-1].link(nvvidconv) 150 | 151 | nvvidconv.link(tiler) 152 | tiler.link(nvosd) 153 | nvosd.link(tee) 154 | 155 | # Link tees to sinks 156 | for idx, sink in enumerate(sinks): 157 | # Use queues for each sink. This ensures the sinks can execute in separate threads 158 | queue = _make_element_safe("queue") 159 | pipeline.add(queue) 160 | # tee.src_%d -> queue 161 | srcpad_or_none = tee.get_request_pad(f"src_{idx}") 162 | sinkpad_or_none = queue.get_static_pad("sink") 163 | srcpad = _sanitize(srcpad_or_none) 164 | sinkpad = _sanitize(sinkpad_or_none) 165 | srcpad.link(sinkpad) 166 | # queue -> sink 167 | queue.link(sink) 168 | 169 | # Alternative renderer 170 | # tiler.link(transform) 171 | # transform.link(renderer) 172 | 173 | # Register callback on the last nvinfer sinkpad. 174 | # This way we get access to object detection results 175 | nvinfer_sinkpad = _sanitize(nvinfers[-1].get_static_pad("sink")) 176 | nvinfer_sinkpad.add_probe(Gst.PadProbeType.BUFFER, self._parse_dets_callback, 0) 177 | 178 | for idx, source in enumerate(sources): 179 | sourcepad = _sanitize(source.get_static_pad("src")) 180 | cb_args = {"image_idx": idx} 181 | sourcepad.add_probe( 182 | Gst.PadProbeType.BUFFER, self._get_np_img_callback, cb_args 183 | ) 184 | 185 | return pipeline 186 | 187 | @staticmethod 188 | def _make_sources(cameras: list) -> list: 189 | # Create pre-configured sources with appropriate type: argus or v4l 190 | sources = [] 191 | for c in cameras: 192 | # int -> bin with arguscamerasrc (e.g. 0) 193 | # str -> bin with nv4l2src (e.g. '/dev/video0) 194 | if type(c) is int: 195 | source = make_argus_cam_bin(c) 196 | elif type(c) is str: 197 | source = make_v4l2_cam_bin(c) 198 | else: 199 | raise TypeError( 200 | f"Error parsing 'cameras' argument. Valid cameras must be either:\n\ 201 | 1) 'str' type for v4l2 (e.g. '/dev/video0')\n\ 202 | 2) 'int' type for argus (0)\n\ 203 | Got '{type(c)}'" 204 | ) 205 | 206 | sources.append(source) 207 | 208 | return sources 209 | 210 | def _get_np_img_callback(self, pad, info, u_data): 211 | """ 212 | Callback responsible for extracting the numpy array from image 213 | """ 214 | cb_start = time.perf_counter() 215 | 216 | gst_buffer = info.get_buffer() 217 | if not gst_buffer: 218 | logging.warn( 219 | "_np._img_callback was unable to get GstBuffer. Dropping frame." 220 | ) 221 | return Gst.PadProbeReturn.DROP 222 | 223 | buff_ptr = hash(gst_buffer) # Memory address of gst_buffer 224 | idx = u_data["image_idx"] 225 | img = pyds.get_nvds_buf_surface(buff_ptr, 0) 226 | self.images[idx] = img[:, :, :3] # RGBA->RGB 227 | self.frame_n[idx] += 1 228 | 229 | self._log.info( 230 | f"Image ingest callback for image {idx} took {1000 * (time.perf_counter() - cb_start):0.2f} ms" 231 | ) 232 | return Gst.PadProbeReturn.OK 233 | 234 | def _parse_dets_callback(self, pad, info, u_data): 235 | cb_start = time.perf_counter() 236 | 237 | gst_buffer = info.get_buffer() 238 | if not gst_buffer: 239 | logging.error("Detection callback unable to get GstBuffer ") 240 | return Gst.PadProbeReturn.DROP 241 | 242 | buff_ptr = hash(gst_buffer) # Memory address of gst_buffer 243 | batch_meta = pyds.gst_buffer_get_nvds_batch_meta(buff_ptr) 244 | 245 | # Iterate frames in a batch 246 | # TODO: for loop 247 | l_frame = batch_meta.frame_meta_list 248 | while l_frame is not None: 249 | 250 | detections = [] 251 | frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data) 252 | cam_id = frame_meta.source_id # there's also frame_meta.batch_id 253 | 254 | # Iterate objects in a frame 255 | l_obj = frame_meta.obj_meta_list 256 | while l_obj is not None: 257 | obj_meta = pyds.NvDsObjectMeta.cast(l_obj.data) 258 | 259 | # This is all in image frame, e.g.: 1092.7200927734375 93.68058776855469 248.01895141601562 106.38716125488281 260 | l, w = obj_meta.rect_params.left, obj_meta.rect_params.width 261 | t, h = obj_meta.rect_params.top, obj_meta.rect_params.height 262 | 263 | position = (l, w, t, h) 264 | cls_id = obj_meta.class_id 265 | conf = obj_meta.confidence 266 | label = obj_meta.obj_label 267 | 268 | detections.append( 269 | {"class": label, "position": position, "confidence": conf} 270 | ) 271 | 272 | obj_meta.rect_params.border_color.set(0.0, 1.0, 0.0, 0.0) 273 | 274 | l_obj = l_obj.next 275 | 276 | self.detections[cam_id] = detections 277 | self.det_n[cam_id] += 1 278 | 279 | l_frame = l_frame.next 280 | 281 | self._log.info( 282 | f"Detection parsing callback took {1000 * (time.perf_counter() - cb_start):0.2f} ms" 283 | ) 284 | return Gst.PadProbeReturn.OK 285 | 286 | def elapsed_time(self): 287 | delta = time.perf_counter() - self._start_ts 288 | return delta 289 | 290 | def fps(self): 291 | t = self.elapsed_time() 292 | fps_list = [cnt / t for cnt in self.frame_n] 293 | return fps_list 294 | --------------------------------------------------------------------------------