├── .gitignore ├── CHANGES.md ├── MANIFEST.in ├── README.md ├── build-3rd-party.sh ├── build-gst-python.sh ├── docs └── api_reference.md ├── examples ├── gst_capture.py ├── gst_capture_and_display.py ├── gst_capture_and_split_dvr.py ├── gst_display.py ├── gst_launch.py ├── gst_launch_many.py ├── gst_launch_many_mt.py ├── pipeline_with_factory.py ├── pipeline_with_parse_launch.py ├── run_appsink.py ├── run_appsrc.py ├── run_rtsp.py └── run_wrt_rank.py ├── gstreamer ├── 3rd_party │ ├── build.sh │ └── gstreamer │ │ ├── CMakeLists.txt │ │ ├── build.sh │ │ ├── gst_objects_info_meta.c │ │ └── gst_objects_info_meta.h ├── __init__.py ├── gst_hacks.py ├── gst_objects_info_meta.py ├── gst_tools.py ├── logging.py └── utils.py ├── requirements.txt ├── setup.py └── tests ├── conftest.py └── test_utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ### v0.0.2 (16.04.2020) 4 | - fixed "iterator not iterable" error (by edumotya) 5 | - support for python3.5 6 | - added blocking to appsrc to reduce memory consumption by queue 7 | 8 | ### v0.0.1 (18.01.2020) 9 | - Released stable version 10 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.so 2 | include gstreamer/3rd_party/gstreamer/build/*.so 3 | 4 | recursive-include *.so -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## gstreamer-python 2 | ### Purpose 3 | - abstraction over [PyGOBJECT API](https://lazka.github.io/pgi-docs/) for Gstreamer 4 | - work with Gstreamer metadata 5 | - common tools for Gstreamer pipeline management 6 | - easy [gst-python](https://github.com/GStreamer/gst-python) installation 7 | 8 | ### Install 9 | #### Install OS packages 10 | - [How to install Gstreamer on Ubuntu](http://lifestyletransfer.com/how-to-install-gstreamer-on-ubuntu/) 11 | - [How to install Gstreamer Python Bindings](http://lifestyletransfer.com/how-to-install-gstreamer-python-bindings/) 12 | 13 | #### in-place 14 | ```bash 15 | python3 -m venv venv 16 | 17 | source venv/bin/activate 18 | pip install --upgrade wheel pip setuptools 19 | pip install --upgrade --requirement requirements.txt 20 | 21 | ./build-3rd-party.sh 22 | ./build-gst-python.sh 23 | ``` 24 | 25 | #### pip-package 26 | ```bash 27 | 28 | pip install git+https://github.com/jackersson/gstreamer-python.git@{tag_name}#egg=gstreamer-python 29 | 30 | ### to skip ./build-gst-python.sh 31 | pip install . -v --install-option "build_py" --install-option "--skip-gst-python" 32 | 33 | ### to set specific gstreamer version 34 | export GST_VERSION=1.14.5 35 | ``` 36 | ### Test 37 | ```bash 38 | PYTHONPATH=. pytest tests/ -s --verbose 39 | ``` 40 | 41 | ### Tools 42 | 43 | #### Setup 44 | - By default Gstreamer tools use **libgstreamer-1.0.so.0** 45 | ```bash 46 | export LIB_GSTREAMER_PATH=libgstreamer-1.0.so.0 47 | ``` 48 | Export **LIB_GSTREAMER_PATH** with custom path to **libgstreamer.so** 49 | 50 | ##### Setup Log Level 51 | ```bash 52 | export GST_PYTHON_LOG_LEVEL=0, 1, 2, 3, 4, 5 53 | ``` 54 | 55 | #### [Make Gst.Buffer writable](http://lifestyletransfer.com/how-to-make-gstreamer-buffer-writable-in-python/) 56 | from gstreamer import map_gst_buffer 57 | with map_gst_buffer(pbuffer, Gst.MapFlags.READ | Gst.MapFlags.WRITE) as mapped: 58 | // do_something with mapped 59 | 60 | #### Make Gst.Memory writable 61 | from gstreamer import map_gst_memory 62 | with map_gst_memory(memory, Gst.MapFlags.READ | Gst.MapFlags.WRITE) as mapped: 63 | // do_something with mapped 64 | 65 | #### Get Gst.Buffer shape (width,height) from Gst.Caps 66 | from gstreamer import get_buffer_size 67 | ret, (width, height) = get_buffer_size(Gst.Caps) 68 | 69 | #### Convert Gst.Buffer to np.ndarray 70 | from gstreamer import gst_buffer_to_ndarray, gst_buffer_with_pad_to_ndarray 71 | 72 | array = gst_buffer_to_ndarray(Gst.Buffer, width, height, channels) 73 | # or 74 | array = gst_buffer_with_pad_to_ndarray(Gst.Buffer, Gst.Pad, channels) 75 | 76 | ### GstPipeline 77 | - With **GstPipeline** run any **gst-launch** pipeline in Python 78 | ```bash 79 | from gstreamer import GstPipeline 80 | 81 | command = "videotestsrc num-buffers=100 ! fakesink sync=false" 82 | with GstPipeline(command) as pipeline: 83 | ... 84 | ``` 85 | 86 | 87 | #### GstVideoSource based on AppSink 88 | - With **GstVideoSource** run any **gst-launch** pipeline and receive buffers in Python 89 | ```bash 90 | from gstreamer import GstVideoSource 91 | 92 | width, height, num_buffers = 1920, 1080, 100 93 | caps_filter = 'capsfilter caps=video/x-raw,format=RGB,width={},height={}'.format(width, height) 94 | command = 'videotestsrc num-buffers={} ! {} ! appsink emit-signals=True sync=false'.format( 95 | num_buffers, caps_filter) 96 | with GstVideoSource(command) as pipeline: 97 | buffers = [] 98 | while len(buffers) < num_buffers: 99 | buffer = pipeline.pop() 100 | if buffer: 101 | buffers.append(buffer) 102 | print('Got: {} buffers'.format(len(buffers))) 103 | ``` 104 | 105 | #### GstVideoSink based on AppSrc 106 | - With **GstVideoSink** push buffers in Python to any **gst-launch** pipeline 107 | ```bash 108 | from gstreamer import GstVideoSink 109 | 110 | width, height = 1920, 1080 111 | command = "appsrc emit-signals=True is-live=True ! videoconvert ! fakesink sync=false" 112 | with GstVideoSink(command, width=width, height=height) as pipeline: 113 | for _ in range(10): 114 | pipeline.push(buffer=np.random.randint(low=0, high=255, size=(height, width, 3), dtype=np.uint8)) 115 | ``` 116 | 117 | ### Metadata 118 | 119 | #### [Object Info MedataData](https://github.com/jackersson/gstreamer-python/blob/master/gstreamer/gst_objects_info_meta.py) 120 | 121 | x 122 | y 123 | width 124 | height 125 | confidence 126 | class_name 127 | track_id 128 | 129 | 130 | ### Examples 131 | #### Run Gstreamer pipeline in Python using Gst.ElementFactory 132 | ```bash 133 | python examples/pipeline_with_factory.py 134 | ``` 135 | 136 | #### Run Gstreamer pipeline in Python using Gst.parse_launch 137 | ```bash 138 | python examples/pipeline_with_parse_launch.py -p "videotestsrc num-buffers=100 pattern=1 ! autovideosink" 139 | ``` 140 | 141 | #### Capture frames (np.ndarray) from any Gstreamer pipeline 142 | ```bash 143 | PYTHONPATH=. python examples/run_appsink.py -p "videotestsrc num-buffers=100 ! capsfilter caps=video/x-raw,format=RGB,width=640,height=480 ! appsink emit-signals=True" 144 | ``` 145 | 146 | #### Push images (np.ndarray) to any Gstreamer pipeline 147 | ```bash 148 | PYTHONPATH=. python examples/run_appsrc.py -p "appsrc emit-signals=True is-live=True caps=video/x-raw,format=RGB,width=640,height=480 ! queue ! videoconvert ! autovideosink" -n 1000 149 | ``` 150 | 151 | 152 | 153 | -------------------------------------------------------------------------------- /build-3rd-party.sh: -------------------------------------------------------------------------------- 1 | cd gstreamer/3rd_party 2 | ./build.sh -------------------------------------------------------------------------------- /build-gst-python.sh: -------------------------------------------------------------------------------- 1 | 2 | # After PyGObject (https://lazka.github.io/pgi-docs/) installed 3 | # Run current script to override Gstreamer related Scripts 4 | 5 | LIBPYTHONPATH="" 6 | PYTHON=${PYTHON:-/usr/bin/python3} 7 | GST_VERSION=${GST_VERSION:-$(gst-launch-1.0 --version | grep version | tr -s ' ' '\n' | tail -1)} 8 | 9 | # Ensure pygst to be installed in current environment 10 | LIBPYTHON=$($PYTHON -c 'from distutils import sysconfig; print(sysconfig.get_config_var("LDLIBRARY"))') 11 | LIBPYTHONPATH=$(dirname $(ldconfig -p | grep -w $LIBPYTHON | head -1 | tr ' ' '\n' | grep /)) 12 | 13 | GST_PREFIX=${GST_PREFIX:-$(dirname $(dirname $(which python)))} 14 | 15 | echo "Python Executable: $PYTHON" 16 | echo "Python Library Path: $LIBPYTHONPATH" 17 | echo "Current Python Path $GST_PREFIX" 18 | echo "Gstreamer Version: $GST_VERSION" 19 | 20 | TEMP_DIR="temp" 21 | mkdir $TEMP_DIR 22 | cd $TEMP_DIR 23 | 24 | # Build gst-python 25 | git clone https://github.com/GStreamer/gst-python.git 26 | cd gst-python 27 | 28 | export PYTHON=$PYTHON 29 | git checkout $GST_VERSION 30 | 31 | ./autogen.sh --disable-gtk-doc --noconfigure 32 | ./configure --with-libpython-dir=$LIBPYTHONPATH --prefix $GST_PREFIX 33 | make 34 | make install 35 | 36 | cd ../.. 37 | 38 | # Clear folder 39 | rm -rf $TEMP_DIR 40 | 41 | -------------------------------------------------------------------------------- /docs/api_reference.md: -------------------------------------------------------------------------------- 1 | ## GstContext 2 | ### Purpose 3 | - Hides [GObject.MainLoop](https://lazka.github.io/pgi-docs/GLib-2.0/structs/MainLoop.html) routine in single class 4 | 5 | ### Example 6 | ```python 7 | import time 8 | from gstreamer import GstContext 9 | 10 | with GstContext(): 11 | ... 12 | 13 | # run pipeline 1 14 | ... 15 | # run pipeline N 16 | 17 | while any(pipeline): 18 | time.sleep(1) 19 | ``` 20 | 21 | ## GstPipeline 22 | ### Purpose 23 | - Hides [Gst.Pipeline](https://lazka.github.io/pgi-docs/Gst-1.0/classes/Pipeline.html) creation and message handling in single class 24 | 25 | ### Example 26 | ```python 27 | import time 28 | from gstreamer import GstContext, GstPipeline 29 | 30 | with GstContext(): 31 | with GstPipeline("videotestsrc num-buffers=100 ! autovideosink") as p: 32 | while not p.is_done: 33 | time.sleep(1) 34 | ``` 35 | -------------------------------------------------------------------------------- /examples/gst_capture.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import time 3 | import threading 4 | 5 | from gstreamer import GstVideoSource, GstVideo, Gst, GLib, GstContext 6 | 7 | WIDTH, HEIGHT, CHANNELS = 640, 480, 3 8 | NUM_BUFFERS = 50 9 | VIDEO_FORMAT = GstVideo.VideoFormat.RGB 10 | 11 | video_format_str = GstVideo.VideoFormat.to_string(VIDEO_FORMAT) 12 | caps_filter = "capsfilter caps=video/x-raw,format={video_format_str},width={WIDTH},height={HEIGHT}".format(**locals()) 13 | command = "videotestsrc num-buffers={NUM_BUFFERS} ! {caps_filter} ! appsink emit-signals=True sync=false".format(**locals()) 14 | 15 | last_buffer = None 16 | with GstContext(), GstVideoSource(command) as pipeline: 17 | 18 | while pipeline.is_active or pipeline.queue_size > 0: 19 | buffer = pipeline.pop() 20 | if buffer: 21 | print("{}: shape {}".format(Gst.TIME_ARGS(buffer.pts), buffer.data.shape)) 22 | last_buffer = buffer 23 | 24 | print("Read {} buffers".format(last_buffer.offset)) 25 | -------------------------------------------------------------------------------- /examples/gst_capture_and_display.py: -------------------------------------------------------------------------------- 1 | import time 2 | import numpy as np 3 | import threading 4 | 5 | from gstreamer import GstVideoSource, GstVideoSink, GstVideo, Gst, GLib, GstContext 6 | 7 | WIDTH, HEIGHT, CHANNELS = 640, 480, 3 8 | NUM_BUFFERS = 1000 9 | VIDEO_FORMAT = GstVideo.VideoFormat.RGB 10 | 11 | video_format_str = GstVideo.VideoFormat.to_string(VIDEO_FORMAT) 12 | caps_filter = "capsfilter caps=video/x-raw,format={video_format_str},width={WIDTH},height={HEIGHT}".format(**locals()) 13 | capture_cmd = "videotestsrc num-buffers={NUM_BUFFERS} ! {caps_filter} ! appsink emit-signals=True sync=false".format(**locals()) 14 | 15 | display_cmd = "appsrc emit-signals=True is-live=True ! videoconvert ! gtksink sync=false" 16 | 17 | 18 | with GstContext(), GstVideoSource(capture_cmd) as capture, \ 19 | GstVideoSink(display_cmd, width=WIDTH, height=HEIGHT, video_frmt=VIDEO_FORMAT) as display: 20 | 21 | # wait pipeline to initialize 22 | max_num_tries, num_tries = 5, 0 23 | while not display.is_active and num_tries <= max_num_tries: 24 | time.sleep(.1) 25 | num_tries += 1 26 | 27 | while not capture.is_done or capture.queue_size > 0: 28 | buffer = capture.pop() 29 | if buffer: 30 | display.push(buffer.data, pts=buffer.pts, 31 | dts=buffer.dts, offset=buffer.offset) 32 | # print("{}: shape {}".format(Gst.TIME_ARGS(buffer.pts), buffer.data.shape)) 33 | -------------------------------------------------------------------------------- /examples/gst_capture_and_split_dvr.py: -------------------------------------------------------------------------------- 1 | import time 2 | import numpy as np 3 | import threading 4 | 5 | from gstreamer import GstVideoSource, GstVideoSink, GstVideo, Gst, GLib, GstContext 6 | 7 | WIDTH, HEIGHT, CHANNELS = 640, 480, 3 8 | NUM_BUFFERS = 1000 9 | VIDEO_FORMAT = GstVideo.VideoFormat.RGB 10 | 11 | video_format_str = GstVideo.VideoFormat.to_string(VIDEO_FORMAT) 12 | 13 | # capturing pipeline 14 | caps_filter = "capsfilter caps=video/x-raw,format={video_format_str},width={WIDTH},height={HEIGHT}".format( 15 | **locals()) 16 | capture_cmd = "videotestsrc num-buffers={NUM_BUFFERS} ! {caps_filter} ! appsink emit-signals=True sync=false".format( 17 | **locals()) 18 | 19 | # video record pipeline 20 | dvr_cmd = "appsrc emit-signals=True is-live=True ! videoconvert ! x264enc tune=zerolatency ! mp4mux ! filesink location={}" 21 | 22 | NUM_VIDEO_FILES = 2 23 | NUM_FRAMES_PER_VIDEO_FILE = NUM_BUFFERS // NUM_VIDEO_FILES 24 | with GstContext(), GstVideoSource(capture_cmd) as capture: 25 | 26 | idx_video_file, num_read = 0, -1 27 | video_writer = None 28 | try: 29 | while not capture.is_done or capture.queue_size > 0: 30 | buffer = capture.pop() # GstBuffer 31 | 32 | # restart video_writer is necessary 33 | if num_read == -1 or num_read > NUM_FRAMES_PER_VIDEO_FILE: 34 | num_read = 0 35 | 36 | # shutdown previous video writer 37 | if video_writer: 38 | video_writer.shutdown() 39 | 40 | # initialize new one 41 | video_writer = GstVideoSink(dvr_cmd.format(f"video_{idx_video_file}.mp4"), 42 | width=WIDTH, height=HEIGHT, video_frmt=VIDEO_FORMAT) 43 | video_writer.startup() 44 | 45 | idx_video_file += 1 46 | 47 | if buffer: 48 | num_read += 1 49 | video_writer.push(buffer.data) # np.ndarray 50 | 51 | except Exception as e: 52 | print("Exception :", e) 53 | finally: 54 | if video_writer: 55 | video_writer.shutdown() 56 | -------------------------------------------------------------------------------- /examples/gst_display.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from gstreamer import GstVideoSink, GstVideo, GstContext 4 | 5 | WIDTH, HEIGHT, CHANNELS = 640, 480, 3 6 | NUM_BUFFERS = 1000 7 | VIDEO_FORMAT = GstVideo.VideoFormat.RGB 8 | command = "appsrc emit-signals=True is-live=True ! videoconvert ! gtksink sync=false" 9 | 10 | with GstContext(), GstVideoSink(command, width=WIDTH, height=HEIGHT, video_frmt=VIDEO_FORMAT) as pipeline: 11 | 12 | for _ in range(NUM_BUFFERS): 13 | buffer = np.random.randint(low=0, high=255, size=( 14 | HEIGHT, WIDTH, CHANNELS), dtype=np.uint8) 15 | pipeline.push(buffer) 16 | 17 | while pipeline.is_done: 18 | pass 19 | 20 | print("Displayed {} buffers".format(pipeline.total_buffers_count)) 21 | -------------------------------------------------------------------------------- /examples/gst_launch.py: -------------------------------------------------------------------------------- 1 | import time 2 | import argparse 3 | 4 | from gstreamer import GstPipeline, GstContext 5 | 6 | DEFAULT_PIPELINE = "videotestsrc num-buffers=100 ! fakesink sync=false" 7 | 8 | ap = argparse.ArgumentParser() 9 | ap.add_argument("-p", "--pipeline", required=True, 10 | default=DEFAULT_PIPELINE, help="Gstreamer pipeline without gst-launch") 11 | 12 | args = vars(ap.parse_args()) 13 | 14 | if __name__ == '__main__': 15 | with GstContext(), GstPipeline(args['pipeline']) as pipeline: 16 | while not pipeline.is_done: 17 | time.sleep(.1) 18 | -------------------------------------------------------------------------------- /examples/gst_launch_many.py: -------------------------------------------------------------------------------- 1 | import time 2 | from random import randint 3 | 4 | from gstreamer import GstPipeline, GstContext 5 | 6 | 7 | if __name__ == '__main__': 8 | with GstContext(): 9 | pipelines = [GstPipeline( 10 | "videotestsrc num-buffers={} ! gtksink".format(randint(50, 300))) for _ in range(5)] 11 | 12 | for p in pipelines: 13 | p.startup() 14 | 15 | while any(p.is_active for p in pipelines): 16 | time.sleep(.5) 17 | 18 | for p in pipelines: 19 | p.shutdown() 20 | -------------------------------------------------------------------------------- /examples/gst_launch_many_mt.py: -------------------------------------------------------------------------------- 1 | """ 2 | Launching multiple pipelines in different threads. Graceful shutdown 3 | """ 4 | 5 | import time 6 | import threading 7 | 8 | from gstreamer import GstPipeline, GstContext 9 | 10 | DEFAULT_PIPELINE = "videotestsrc num-buffers=100 ! fakesink" 11 | 12 | 13 | def launch(stop_event: threading.Event): 14 | with GstContext(), GstPipeline(DEFAULT_PIPELINE) as pipeline: 15 | while not pipeline.is_done and not stop_event.is_set(): 16 | time.sleep(.1) 17 | 18 | 19 | def launch_context(event: threading.Event): 20 | with GstContext(): 21 | while not event.is_set(): 22 | time.sleep(1) 23 | 24 | 25 | def launch_pipeline(event: threading.Event): 26 | with GstPipeline(DEFAULT_PIPELINE) as pipeline: 27 | while not pipeline.is_done and not event.is_set(): 28 | time.sleep(.1) 29 | 30 | 31 | if __name__ == '__main__': 32 | 33 | num_pipeline = 3 34 | num_threads = num_pipeline + 1 # thread for context 35 | events = [threading.Event() for _ in range(num_threads)] 36 | for e in events: 37 | e.clear() 38 | 39 | context = threading.Thread(target=launch_context, args=(events[0],)) 40 | pipelines = [threading.Thread(target=launch_pipeline, args=(e,)) 41 | for e in events[1:]] 42 | 43 | threads = [context] + pipelines 44 | for t in threads: 45 | t.start() 46 | 47 | try: 48 | # check if any thread (except context) is alive 49 | while any([t.isAlive() for t in threads[1:]]): 50 | time.sleep(.1) 51 | except KeyboardInterrupt as e: 52 | print("Pressed Ctrl-C") 53 | finally: 54 | # reverse, so the context will be stopped the last one 55 | for e, t in zip(reversed(events), reversed(threads)): 56 | e.set() 57 | try: 58 | t.join(timeout=1) 59 | except Exception as e: 60 | pass 61 | -------------------------------------------------------------------------------- /examples/pipeline_with_factory.py: -------------------------------------------------------------------------------- 1 | import traceback 2 | import sys 3 | 4 | import gi 5 | gi.require_version('Gst', '1.0') 6 | from gi.repository import Gst, GObject # noqa:F401,F402 7 | 8 | 9 | # Initializes Gstreamer, it's variables, paths 10 | Gst.init(sys.argv) 11 | 12 | 13 | def on_message(bus: Gst.Bus, message: Gst.Message, loop: GObject.MainLoop): 14 | mtype = message.type 15 | """ 16 | Gstreamer Message Types and how to parse 17 | https://lazka.github.io/pgi-docs/Gst-1.0/flags.html#Gst.MessageType 18 | """ 19 | if mtype == Gst.MessageType.EOS: 20 | print("End of stream") 21 | loop.quit() 22 | 23 | elif mtype == Gst.MessageType.ERROR: 24 | err, debug = message.parse_error() 25 | print(err, debug) 26 | loop.quit() 27 | elif mtype == Gst.MessageType.WARNING: 28 | err, debug = message.parse_warning() 29 | print(err, debug) 30 | 31 | return True 32 | 33 | 34 | # Gst.Pipeline https://lazka.github.io/pgi-docs/Gst-1.0/classes/Pipeline.html 35 | pipeline = Gst.Pipeline() 36 | 37 | # Creates element by name 38 | # https://lazka.github.io/pgi-docs/Gst-1.0/classes/ElementFactory.html#Gst.ElementFactory.make 39 | src_name = "my_video_test_src" 40 | src = Gst.ElementFactory.make("videotestsrc", "my_video_test_src") 41 | src.set_property("num-buffers", 50) 42 | src.set_property("pattern", "ball") 43 | 44 | sink = Gst.ElementFactory.make("gtksink") 45 | 46 | pipeline.add(src, sink) 47 | 48 | src.link(sink) 49 | 50 | # https://lazka.github.io/pgi-docs/Gst-1.0/classes/Bus.html 51 | bus = pipeline.get_bus() 52 | 53 | # allow bus to emit messages to main thread 54 | bus.add_signal_watch() 55 | 56 | # Start pipeline 57 | pipeline.set_state(Gst.State.PLAYING) 58 | 59 | # Init GObject loop to handle Gstreamer Bus Events 60 | loop = GObject.MainLoop() 61 | 62 | # Add handler to specific signal 63 | # https://lazka.github.io/pgi-docs/GObject-2.0/classes/Object.html#GObject.Object.connect 64 | bus.connect("message", on_message, loop) 65 | 66 | try: 67 | loop.run() 68 | except Exception: 69 | traceback.print_exc() 70 | loop.quit() 71 | 72 | # Stop Pipeline 73 | pipeline.set_state(Gst.State.NULL) 74 | del pipeline 75 | -------------------------------------------------------------------------------- /examples/pipeline_with_parse_launch.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import traceback 3 | import argparse 4 | 5 | import gi 6 | gi.require_version('Gst', '1.0') 7 | from gi.repository import Gst, GObject # noqa:F401,F402 8 | 9 | 10 | # Initializes Gstreamer, it's variables, paths 11 | Gst.init(sys.argv) 12 | 13 | DEFAULT_PIPELINE = "videotestsrc num-buffers=100 ! autovideosink" 14 | 15 | ap = argparse.ArgumentParser() 16 | ap.add_argument("-p", "--pipeline", required=False, 17 | default=DEFAULT_PIPELINE, help="Gstreamer pipeline without gst-launch") 18 | 19 | args = vars(ap.parse_args()) 20 | 21 | 22 | def on_message(bus: Gst.Bus, message: Gst.Message, loop: GObject.MainLoop): 23 | mtype = message.type 24 | """ 25 | Gstreamer Message Types and how to parse 26 | https://lazka.github.io/pgi-docs/Gst-1.0/flags.html#Gst.MessageType 27 | """ 28 | if mtype == Gst.MessageType.EOS: 29 | print("End of stream") 30 | loop.quit() 31 | 32 | elif mtype == Gst.MessageType.ERROR: 33 | err, debug = message.parse_error() 34 | print(err, debug) 35 | loop.quit() 36 | 37 | elif mtype == Gst.MessageType.WARNING: 38 | err, debug = message.parse_warning() 39 | print(err, debug) 40 | 41 | return True 42 | 43 | 44 | command = args["pipeline"] 45 | 46 | # Gst.Pipeline https://lazka.github.io/pgi-docs/Gst-1.0/classes/Pipeline.html 47 | # https://lazka.github.io/pgi-docs/Gst-1.0/functions.html#Gst.parse_launch 48 | pipeline = Gst.parse_launch(command) 49 | 50 | # https://lazka.github.io/pgi-docs/Gst-1.0/classes/Bus.html 51 | bus = pipeline.get_bus() 52 | 53 | # allow bus to emit messages to main thread 54 | bus.add_signal_watch() 55 | 56 | # Start pipeline 57 | pipeline.set_state(Gst.State.PLAYING) 58 | 59 | # Init GObject loop to handle Gstreamer Bus Events 60 | loop = GObject.MainLoop() 61 | 62 | # Add handler to specific signal 63 | # https://lazka.github.io/pgi-docs/GObject-2.0/classes/Object.html#GObject.Object.connect 64 | bus.connect("message", on_message, loop) 65 | 66 | try: 67 | loop.run() 68 | except Exception: 69 | traceback.print_exc() 70 | loop.quit() 71 | 72 | # Stop Pipeline 73 | pipeline.set_state(Gst.State.NULL) 74 | -------------------------------------------------------------------------------- /examples/run_appsink.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import traceback 3 | import argparse 4 | import typing as typ 5 | import time 6 | import attr 7 | 8 | import numpy as np 9 | 10 | from gstreamer import GstContext, GstPipeline, GstApp, Gst, GstVideo 11 | import gstreamer.utils as utils 12 | 13 | # Converts list of plugins to gst-launch string 14 | # ['plugin_1', 'plugin_2', 'plugin_3'] => plugin_1 ! plugin_2 ! plugin_3 15 | DEFAULT_PIPELINE = utils.to_gst_string([ 16 | "videotestsrc num-buffers=100", 17 | "capsfilter caps=video/x-raw,format=GRAY16_LE,width=640,height=480", 18 | "queue", 19 | "appsink emit-signals=True" 20 | ]) 21 | 22 | ap = argparse.ArgumentParser() 23 | ap.add_argument("-p", "--pipeline", required=False, 24 | default=DEFAULT_PIPELINE, help="Gstreamer pipeline without gst-launch") 25 | 26 | args = vars(ap.parse_args()) 27 | 28 | command = args["pipeline"] 29 | 30 | 31 | def extract_buffer(sample: Gst.Sample) -> np.ndarray: 32 | """Extracts Gst.Buffer from Gst.Sample and converts to np.ndarray""" 33 | 34 | buffer = sample.get_buffer() # Gst.Buffer 35 | 36 | print("timestamp: ", Gst.TIME_ARGS(buffer.pts), "offset: ", buffer.offset) 37 | 38 | caps_format = sample.get_caps().get_structure(0) # Gst.Structure 39 | 40 | # GstVideo.VideoFormat 41 | video_format = GstVideo.VideoFormat.from_string( 42 | caps_format.get_value('format')) 43 | 44 | w, h = caps_format.get_value('width'), caps_format.get_value('height') 45 | c = utils.get_num_channels(video_format) 46 | 47 | buffer_size = buffer.get_size() 48 | format_info = GstVideo.VideoFormat.get_info(video_format) # GstVideo.VideoFormatInfo 49 | array = np.ndarray(shape=buffer_size // (format_info.bits // utils.BITS_PER_BYTE), 50 | buffer=buffer.extract_dup(0, buffer_size), 51 | dtype=utils.get_np_dtype(video_format)) 52 | if c > 0: 53 | array = array.reshape(h, w, c).squeeze() 54 | return np.squeeze(array) # remove single dimension if exists 55 | 56 | 57 | def on_buffer(sink: GstApp.AppSink, data: typ.Any) -> Gst.FlowReturn: 58 | """Callback on 'new-sample' signal""" 59 | # Emit 'pull-sample' signal 60 | # https://lazka.github.io/pgi-docs/GstApp-1.0/classes/AppSink.html#GstApp.AppSink.signals.pull_sample 61 | 62 | sample = sink.emit("pull-sample") # Gst.Sample 63 | 64 | if isinstance(sample, Gst.Sample): 65 | array = extract_buffer(sample) 66 | print( 67 | "Received {type} with shape {shape} of type {dtype}".format(type=type(array), 68 | shape=array.shape, 69 | dtype=array.dtype)) 70 | return Gst.FlowReturn.OK 71 | 72 | return Gst.FlowReturn.ERROR 73 | 74 | 75 | with GstContext(): # create GstContext (hides MainLoop) 76 | # create GstPipeline (hides Gst.parse_launch) 77 | with GstPipeline(command) as pipeline: 78 | appsink = pipeline.get_by_cls(GstApp.AppSink)[0] # get AppSink 79 | # subscribe to signal 80 | appsink.connect("new-sample", on_buffer, None) 81 | while not pipeline.is_done: 82 | time.sleep(.1) 83 | -------------------------------------------------------------------------------- /examples/run_appsrc.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import traceback 3 | import argparse 4 | import typing as typ 5 | import random 6 | import time 7 | from fractions import Fraction 8 | 9 | import numpy as np 10 | 11 | from gstreamer import GstContext, GstPipeline, GstApp, Gst, GstVideo, GLib, GstVideoSink 12 | import gstreamer.utils as utils 13 | 14 | VIDEO_FORMAT = "RGB" 15 | WIDTH, HEIGHT = 640, 480 16 | FPS = Fraction(30) 17 | GST_VIDEO_FORMAT = GstVideo.VideoFormat.from_string(VIDEO_FORMAT) 18 | 19 | 20 | def fraction_to_str(fraction: Fraction) -> str: 21 | """Converts fraction to str""" 22 | return '{}/{}'.format(fraction.numerator, fraction.denominator) 23 | 24 | 25 | def parse_caps(pipeline: str) -> dict: 26 | """Parses appsrc's caps from pipeline string into a dict 27 | 28 | :param pipeline: "appsrc caps=video/x-raw,format=RGB,width=640,height=480 ! videoconvert ! autovideosink" 29 | 30 | Result Example: 31 | { 32 | "width": "640", 33 | "height": "480" 34 | "format": "RGB", 35 | "fps": "30/1", 36 | ... 37 | } 38 | """ 39 | 40 | try: 41 | # typ.List[typ.Tuple[str, str]] 42 | caps = [prop for prop in pipeline.split( 43 | "!")[0].split(" ") if "caps" in prop][0] 44 | return dict([p.split('=') for p in caps.split(',') if "=" in p]) 45 | except IndexError as err: 46 | return None 47 | 48 | 49 | FPS_STR = fraction_to_str(FPS) 50 | DEFAULT_CAPS = "video/x-raw,format={VIDEO_FORMAT},width={WIDTH},height={HEIGHT},framerate={FPS_STR}".format(**locals()) 51 | 52 | # Converts list of plugins to gst-launch string 53 | # ['plugin_1', 'plugin_2', 'plugin_3'] => plugin_1 ! plugin_2 ! plugin_3 54 | DEFAULT_PIPELINE = utils.to_gst_string([ 55 | "appsrc emit-signals=True is-live=True caps={DEFAULT_CAPS}".format(**locals()), 56 | "queue", 57 | "videoconvert", 58 | "autovideosink" 59 | ]) 60 | 61 | 62 | ap = argparse.ArgumentParser() 63 | ap.add_argument("-p", "--pipeline", required=False, 64 | default=DEFAULT_PIPELINE, help="Gstreamer pipeline without gst-launch") 65 | 66 | ap.add_argument("-n", "--num_buffers", required=False, 67 | default=100, help="Num buffers to pass") 68 | 69 | args = vars(ap.parse_args()) 70 | 71 | command = args["pipeline"] 72 | 73 | args_caps = parse_caps(command) 74 | NUM_BUFFERS = int(args['num_buffers']) 75 | 76 | WIDTH = int(args_caps.get("width", WIDTH)) 77 | HEIGHT = int(args_caps.get("height", HEIGHT)) 78 | FPS = Fraction(args_caps.get("framerate", FPS)) 79 | 80 | GST_VIDEO_FORMAT = GstVideo.VideoFormat.from_string( 81 | args_caps.get("format", VIDEO_FORMAT)) 82 | CHANNELS = utils.get_num_channels(GST_VIDEO_FORMAT) 83 | DTYPE = utils.get_np_dtype(GST_VIDEO_FORMAT) 84 | 85 | FPS_STR = fraction_to_str(FPS) 86 | CAPS = "video/x-raw,format={VIDEO_FORMAT},width={WIDTH},height={HEIGHT},framerate={FPS_STR}".format(**locals()) 87 | 88 | with GstContext(): # create GstContext (hides MainLoop) 89 | 90 | pipeline = GstPipeline(command) 91 | 92 | def on_pipeline_init(self): 93 | """Setup AppSrc element""" 94 | appsrc = self.get_by_cls(GstApp.AppSrc)[0] # get AppSrc 95 | 96 | # instructs appsrc that we will be dealing with timed buffer 97 | appsrc.set_property("format", Gst.Format.TIME) 98 | 99 | # instructs appsrc to block pushing buffers until ones in queue are preprocessed 100 | # allows to avoid huge queue internal queue size in appsrc 101 | appsrc.set_property("block", True) 102 | 103 | # set input format (caps) 104 | appsrc.set_caps(Gst.Caps.from_string(CAPS)) 105 | 106 | # override on_pipeline_init to set specific properties before launching pipeline 107 | pipeline._on_pipeline_init = on_pipeline_init.__get__(pipeline) 108 | 109 | try: 110 | pipeline.startup() 111 | appsrc = pipeline.get_by_cls(GstApp.AppSrc)[0] # GstApp.AppSrc 112 | 113 | pts = 0 # buffers presentation timestamp 114 | duration = 10**9 / (FPS.numerator / FPS.denominator) # frame duration 115 | for _ in range(NUM_BUFFERS): 116 | 117 | # create random np.ndarray 118 | array = np.random.randint(low=0, high=255, 119 | size=(HEIGHT, WIDTH, CHANNELS), dtype=DTYPE) 120 | 121 | # convert np.ndarray to Gst.Buffer 122 | gst_buffer = utils.ndarray_to_gst_buffer(array) 123 | 124 | # set pts and duration to be able to record video, calculate fps 125 | pts += duration # Increase pts by duration 126 | gst_buffer.pts = pts 127 | gst_buffer.duration = duration 128 | 129 | # emit event with Gst.Buffer 130 | appsrc.emit("push-buffer", gst_buffer) 131 | 132 | # emit event 133 | appsrc.emit("end-of-stream") 134 | 135 | while not pipeline.is_done: 136 | time.sleep(.1) 137 | except Exception as e: 138 | print("Error: ", e) 139 | finally: 140 | pipeline.shutdown() -------------------------------------------------------------------------------- /examples/run_rtsp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 vi:ts=4:noexpandtab 3 | # Simple RTSP server. Run as-is or with a command-line to replace the default pipeline 4 | 5 | import time 6 | import sys 7 | import abc 8 | import numpy as np 9 | import typing as typ 10 | from fractions import Fraction 11 | import functools 12 | 13 | import gi 14 | 15 | gi.require_version('Gst', '1.0') 16 | gi.require_version('GstRtspServer', '1.0') 17 | gi.require_version('GstApp', '1.0') 18 | gi.require_version('GstVideo', '1.0') 19 | 20 | from gi.repository import Gst, GLib, GstRtspServer, GObject, GstApp, GstVideo # noqa:F401,F402 21 | 22 | import gstreamer as gst # noqa:F401,F402 23 | import gstreamer.utils as utils # noqa:F401,F402 24 | 25 | # Examples 26 | # https://github.com/tamaggo/gstreamer-examples 27 | # https://github.com/GStreamer/gst-rtsp-server/tree/master/examples 28 | # https://stackoverflow.com/questions/47396372/write-opencv-frames-into-gstreamer-rtsp-server-pipeline 29 | 30 | 31 | VIDEO_FORMAT = "RGB" 32 | # VIDEO_FORMAT = "I420" 33 | 34 | WIDTH, HEIGHT = 320, 240 35 | FPS = Fraction(30) 36 | GST_VIDEO_FORMAT = GstVideo.VideoFormat.from_string(VIDEO_FORMAT) 37 | 38 | 39 | class GstBufferGenerator(metaclass=abc.ABCMeta): 40 | 41 | @abc.abstractmethod 42 | def get(self) -> Gst.Buffer: 43 | pass 44 | 45 | @property 46 | def caps(self) -> Gst.Caps: 47 | pass 48 | 49 | def __enter__(self): 50 | self.startup() 51 | return self 52 | 53 | def __exit__(self, exc_type, exc_val, exc_tb): 54 | self.shutdown() 55 | 56 | def startup(self): 57 | pass 58 | 59 | def shutdown(self): 60 | pass 61 | 62 | 63 | class FakeGstBufferGenerator(GstBufferGenerator): 64 | 65 | def __init__(self, *, width: int, height: int, 66 | fps: typ.Union[Fraction, int] = Fraction('30/1'), 67 | video_type: gst.gst_tools.VideoType = gst.gst_tools.VideoType.VIDEO_RAW, 68 | video_frmt: GstVideo.VideoFormat = GstVideo.VideoFormat.RGB): 69 | 70 | self._width = width 71 | self._height = height 72 | 73 | self._fps = Fraction(fps) 74 | 75 | self._pts = 0 76 | self._dts = GLib.MAXUINT64 77 | 78 | self._duration = Gst.SECOND / (self._fps.numerator / self._fps.denominator) 79 | self._video_frmt = video_frmt 80 | self._video_type = video_type 81 | 82 | # Gst.Caps 83 | self._caps = gst.gst_tools.gst_video_format_plugin( 84 | width=width, height=height, fps=self._fps, 85 | video_type=video_type, video_frmt=video_frmt 86 | ) 87 | 88 | @property 89 | def caps(self) -> Gst.Caps: 90 | return Gst.Caps.from_string(self._caps) 91 | 92 | def get(self) -> Gst.Buffer: 93 | 94 | np_dtype = gst.utils.get_np_dtype(self._video_frmt) 95 | channels = gst.utils.get_num_channels(self._video_frmt) 96 | 97 | array = np.random.randint(low=0, high=255, 98 | size=(self._height, self._width, channels), dtype=np_dtype) 99 | 100 | self._pts += self._duration 101 | 102 | gst_buffer = gst.utils.ndarray_to_gst_buffer(array) # Gst.Buffer 103 | 104 | gst_buffer.pts = self._pts 105 | gst_buffer.dts = self._dts 106 | gst_buffer.duration = self._duration 107 | gst_buffer.offset = self._pts // self._duration 108 | 109 | return gst_buffer 110 | 111 | 112 | class GstBufferGeneratorFromPipeline(GstBufferGenerator): 113 | 114 | def __init__(self, gst_launch: str, loop: bool = False): 115 | self._loop = loop 116 | self._gst_launch = gst_launch 117 | self._num_loops = 0 118 | 119 | self._pipeline = None # gst.GstVideoSource 120 | 121 | def startup(self): 122 | self._pipeline = gst.GstVideoSource(self._gst_launch, max_buffers_size=8) 123 | self._pipeline.startup() 124 | 125 | self._num_loops += 1 126 | print(f"Starting {self._num_loops} loop") 127 | 128 | def shutdown(self): 129 | if isinstance(self._pipeline, gst.GstVideoSource): 130 | self._pipeline.shutdown() 131 | 132 | @property 133 | def caps(self) -> Gst.Caps: 134 | appsink = self._pipeline.get_by_cls(GstApp.AppSink)[0] 135 | return appsink.sinkpad.get_current_caps() 136 | 137 | def get(self) -> Gst.Buffer: 138 | 139 | buffer = self._pipeline.pop() 140 | if not buffer: 141 | if self._pipeline.is_done and self._loop: 142 | self.shutdown() 143 | self.startup() 144 | return None 145 | 146 | gst_buffer = gst.utils.ndarray_to_gst_buffer(buffer.data) # Gst.Buffer 147 | 148 | gst_buffer.pts = buffer.pts 149 | gst_buffer.dts = buffer.dts 150 | gst_buffer.duration = buffer.duration 151 | gst_buffer.offset = buffer.offset 152 | 153 | return gst_buffer 154 | 155 | @classmethod 156 | def clone(cls) -> 'GstBufferGeneratorFromPipeline': 157 | return cls(self._gst_launch) 158 | 159 | 160 | def get_child_by_cls(element: Gst.Element, cls: GObject.GType) -> typ.List[Gst.Element]: 161 | """ Get Gst.Element[] from pipeline by GType """ 162 | return [e for e in element.iterate_elements() if isinstance(e, cls)] 163 | 164 | 165 | # https://lazka.github.io/pgi-docs/GstRtspServer-1.0/classes/RTSPMediaFactory.html#gstrtspserver-rtspmediafactory 166 | class RTSPMediaFactoryCustom(GstRtspServer.RTSPMediaFactory): 167 | 168 | def __init__(self, source: typ.Callable[..., GstBufferGenerator]): 169 | super().__init__() 170 | 171 | self._source = source 172 | self._sources = {} 173 | 174 | # def do_create_element(self, url) -> Gst.Element: 175 | # # https://lazka.github.io/pgi-docs/GstRtspServer-1.0/classes/RTSPMediaFactory.html#GstRtspServer.RTSPMediaFactory.do_create_element 176 | 177 | # src = "appsrc emit-signals=True is-live=True" 178 | # encoder = "x264enc tune=zerolatency" # pass=quant 179 | # color_convert = "videoconvert n-threads=0 ! video/x-raw,format=I420" 180 | # # rtp = "rtph264pay config-interval=1 name=pay0 pt=96" 181 | # rtp = "rtpvrawpay name=pay0 pt=96" 182 | # pipeline = "{src} ! {color_convert} ! {encoder} ! queue max-size-buffers=8 ! {rtp}".format(**locals()) 183 | # pipeline = "{src} ! queue max-size-buffers=8 ! {rtp}".format(**locals()) 184 | 185 | # print(f"gst-launch-1.0 {pipeline}") 186 | # return Gst.parse_launch(pipeline) 187 | 188 | def on_need_data(self, src: GstApp.AppSrc, length: int): 189 | """ Callback on "need-data" signal 190 | 191 | Signal: 192 | https://lazka.github.io/pgi-docs/GstApp-1.0/classes/AppSrc.html#GstApp.AppSrc.signals.need_data 193 | :param length: amount of bytes needed 194 | """ 195 | 196 | buffer = None 197 | while not buffer: # looping pipeline 198 | buffer = self._sources[src.name].get() # Gst.Buffer 199 | time.sleep(.1) 200 | 201 | retval = src.emit('push-buffer', buffer) 202 | # print(f'Pushed buffer, frame {buffer.offset}, duration {Gst.TIME_ARGS(buffer.pts)}') 203 | if retval != Gst.FlowReturn.OK: 204 | print(retval) 205 | 206 | def do_configure(self, rtsp_media: GstRtspServer.RTSPMedia): 207 | # https://lazka.github.io/pgi-docs/GstRtspServer-1.0/classes/RTSPMedia.html#GstRtspServer.RTSPMedia 208 | 209 | appsrcs = get_child_by_cls(rtsp_media.get_element(), GstApp.AppSrc) 210 | if not appsrcs: 211 | return 212 | 213 | appsrc = appsrcs[0] 214 | 215 | self._sources[appsrc.name] = self._source() 216 | self._sources[appsrc.name].startup() 217 | time.sleep(.5) # wait to start 218 | 219 | # this instructs appsrc that we will be dealing with timed buffer 220 | appsrc.set_property("format", Gst.Format.TIME) 221 | 222 | # instructs appsrc to block pushing buffers until ones in queue are preprocessed 223 | # allows to avoid huge queue internal queue size in appsrc 224 | appsrc.set_property("block", True) 225 | 226 | appsrc.set_property("caps", self._sources[appsrc.name].caps) 227 | 228 | appsrc.connect('need-data', self.on_need_data) 229 | 230 | def __del__(self): 231 | for source in self._sources.values(): 232 | source.shutdown() 233 | 234 | 235 | class GstServer(): 236 | def __init__(self, shared: bool = False): 237 | # https://lazka.github.io/pgi-docs/GstRtspServer-1.0/classes/RTSPServer.html 238 | self.server = GstRtspServer.RTSPServer() 239 | 240 | # https://lazka.github.io/pgi-docs/GstRtspServer-1.0/classes/RTSPMediaFactory.html#GstRtspServer.RTSPMediaFactory.set_shared 241 | # f.set_shared(True) 242 | 243 | # https://lazka.github.io/pgi-docs/GstRtspServer-1.0/classes/RTSPServer.html#GstRtspServer.RTSPServer.get_mount_points 244 | # https://lazka.github.io/pgi-docs/GstRtspServer-1.0/classes/RTSPMountPoints.html#GstRtspServer.RTSPMountPoints 245 | m = self.server.get_mount_points() 246 | 247 | # pipeline 248 | # Launch test buffers streaming 249 | pipeline = utils.to_gst_string([ 250 | "videotestsrc num-buffers=1000", 251 | "video/x-raw,format={fmt},width={width},height={height}".format(fmt=VIDEO_FORMAT, 252 | width=WIDTH, 253 | height=HEIGHT), 254 | "appsink emit-signals=True" 255 | ]) 256 | 257 | # Launch file streaming 258 | # pipeline = [ 259 | # "filesrc location=video.mp4", 260 | # "decodebin" 261 | # "videoconvert n-threads=0", 262 | # "video/x-raw,format=RGB", 263 | # "appsink emit-signals=True" 264 | # ] 265 | 266 | # Buffers streaming from pipeline (Gst) 267 | generator = functools.partial( 268 | GstBufferGeneratorFromPipeline, gst_launch=pipeline, loop=True 269 | ) 270 | 271 | # Fake buffers streaming from generator (numpy) 272 | # generator = functools.partial(FakeGstBufferGenerator, width=WIDTH, height=HEIGHT, 273 | # fps=FPS, video_frmt=GST_VIDEO_FORMAT) 274 | 275 | # https://lazka.github.io/pgi-docs/GstRtspServer-1.0/classes/RTSPMountPoints.html#GstRtspServer.RTSPMountPoints.add_factory 276 | mount_point = "/stream.rtp" 277 | factory = RTSPMediaFactoryCustom(generator) 278 | 279 | # Launch Raw Stream 280 | pipeline = [ 281 | "appsrc emit-signals=True is-live=True", 282 | "queue max-size-buffers=8", 283 | "rtpvrawpay name=pay0 pt=96" 284 | ] 285 | 286 | # Launch H264 Stream 287 | # pipeline = [ 288 | # "appsrc emit-signals=True is-live=True", 289 | # "queue max-size-buffers=8", 290 | # "videoconvert n-threads=0 ! video/x-raw,format=I420", 291 | # "264enc tune=zerolatency", # pass quant 292 | # "queue max-size-buffers=8", 293 | # "rtph264pay config-interval=1 name=pay0 pt=96" 294 | # ] 295 | 296 | factory.set_launch(utils.to_gst_string(pipeline)) 297 | factory.set_shared(shared) 298 | m.add_factory(mount_point, factory) # adding streams 299 | 300 | port = self.server.get_property("service") 301 | print(f"rtsp://localhost:{port}/{mount_point}") 302 | 303 | # https://lazka.github.io/pgi-docs/GstRtspServer-1.0/classes/RTSPServer.html#GstRtspServer.RTSPServer.attach 304 | self.server.attach(None) 305 | 306 | 307 | if __name__ == '__main__': 308 | with gst.GstContext(): 309 | s = GstServer(shared=True) 310 | 311 | while True: 312 | time.sleep(.1) 313 | -------------------------------------------------------------------------------- /examples/run_wrt_rank.py: -------------------------------------------------------------------------------- 1 | """ 2 | Set decoder priority for video processing pipelines 3 | 4 | Examples: 5 | python examples/run_wrt_rank.py -d "avdec_h264" 6 | 7 | python examples/run_wrt_rank.py -p "rtspsrc location=rtsp://... ! decodebin ! nvvideoconvert ! gtksink" -d "nvv4l2decoder" 8 | """ 9 | 10 | import time 11 | import argparse 12 | 13 | from gstreamer import Gst, GstContext, GstPipeline, GObject 14 | import gstreamer.utils as utils 15 | 16 | TARGET_PLUGIN_NAME = "avdec_h264" 17 | FREE_RTSP_SOURCE = [ 18 | "rtsp://freja.hiof.no:1935/rtplive/definst/hessdalen03.stream", 19 | "rtsp://wowzaec2demo.streamlock.net/vod/mp4:BigBuckBunny_115k.mov" 20 | ] 21 | DEFAULT_PIPELINE = "rtspsrc location={} ! decodebin ! videoconvert ! gtksink".format( 22 | FREE_RTSP_SOURCE[0]) 23 | 24 | ap = argparse.ArgumentParser() 25 | ap.add_argument("-p", "--pipeline", required=False, 26 | default=DEFAULT_PIPELINE, help="Gstreamer pipeline without gst-launch") 27 | 28 | ap.add_argument("-d", "--decoder", required=False, 29 | default=TARGET_PLUGIN_NAME, help="Specify plugin name (decoder) to use") 30 | 31 | args = vars(ap.parse_args()) 32 | 33 | pipeline = args['pipeline'] 34 | target_element_name = args['decoder'] 35 | 36 | # Filter target elements with Gst.Constans (Gst.ELEMENT_FACTORY_TYPE_*) 37 | # https://lazka.github.io/pgi-docs/#Gst-1.0/constants.html#constants 38 | filt = Gst.ELEMENT_FACTORY_TYPE_DECODER # only decoders 39 | filt |= Gst.ELEMENT_FACTORY_TYPE_MEDIA_VIDEO # only for video 40 | 41 | # https://lazka.github.io/pgi-docs/Gst-1.0/classes/ElementFactory.html#Gst.ElementFactory.list_get_elements 42 | # list all elements by filter 43 | factories = Gst.ElementFactory.list_get_elements( 44 | filt, Gst.Rank.MARGINAL) # Gst.ElementFactory[] 45 | 46 | # get target 47 | target_element = Gst.ElementFactory.find( 48 | target_element_name) # Gst.ElementFactory 49 | 50 | # factory object extends Gst.PluginFeature 51 | # with get_name(), get_rank() 52 | # https://lazka.github.io/pgi-docs/Gst-1.0/classes/PluginFeature.html#gst-pluginfeature 53 | # sort by plugin rank in descending order 54 | factories = sorted(factories, key=lambda f: f.get_rank(), reverse=True) 55 | 56 | # Gst.Rank 57 | # https://lazka.github.io/pgi-docs/index.html#Gst-1.0/enums.html#Gst.Rank 58 | # get max rank element 59 | max_rank_element = factories[0] 60 | if max_rank_element.get_name() != target_element_name: 61 | print("--- Before ---") 62 | print("Max rank plugin:", max_rank_element.get_name(), 63 | "(", max_rank_element.get_rank(), ")") 64 | print("Rank of target plugin:", target_element.get_name(), 65 | "(", target_element.get_rank(), ")") 66 | 67 | print("--- After ---") 68 | 69 | # Increase target's element rank 70 | target_element.set_rank(max_rank_element.get_rank() + 1) 71 | print("Rank of target plugin:", target_element.get_name(), 72 | "(", target_element.get_rank(), ")") 73 | 74 | pipeline_str = pipeline 75 | 76 | with GstContext(), GstPipeline(pipeline_str) as p: 77 | try: 78 | while not p.is_done: 79 | time.sleep(1) 80 | except Exception: 81 | pass 82 | finally: 83 | # print all elements and notify of target plugin presence 84 | elements = [el.get_factory().get_name() 85 | for el in p.pipeline.iterate_recurse()] 86 | print("All elements: ", elements) 87 | print("Target element ({}) is {}".format(target_element_name, 88 | 'present' if target_element_name in set(elements) else "missing")) 89 | -------------------------------------------------------------------------------- /gstreamer/3rd_party/build.sh: -------------------------------------------------------------------------------- 1 | ROOT=$PWD 2 | 3 | echo "PWD: $PWD" 4 | 5 | # Gstreamer 6 | cd gstreamer 7 | ./build.sh -------------------------------------------------------------------------------- /gstreamer/3rd_party/gstreamer/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | 3 | project (annotations_meta) 4 | set(CMAKE_BUILD_TYPE Release) 5 | 6 | include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) 7 | add_library(gst_objects_info_meta SHARED gst_objects_info_meta.c) 8 | 9 | # Gstreamer linkage 10 | include(${CMAKE_ROOT}/Modules/FindPkgConfig.cmake) 11 | 12 | # Set CMAKE_C_FLAGS variable with info from pkg-util 13 | execute_process(COMMAND pkg-config --cflags gstreamer-1.0 14 | OUTPUT_VARIABLE CMAKE_C_FLAGS) 15 | string(REPLACE "\n" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) 16 | message("CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}") 17 | 18 | # Set CMAKE_EXE_LINKER_FLAGS variable with info from pkg-util 19 | 20 | execute_process(COMMAND pkg-config --libs gstreamer-1.0 21 | OUTPUT_VARIABLE CMAKE_EXE_LINKER_FLAGS) 22 | string(REPLACE "\n" "" CMAKE_LINKER_FLAGS ${CMAKE_EXE_LINKER_FLAGS}) 23 | message("CMAKE_EXE_LINKER_FLAGS: ${CMAKE_EXE_LINKER_FLAGS}") 24 | 25 | set_target_properties(gst_objects_info_meta 26 | PROPERTIES COMPILE_FLAGS ${CMAKE_C_FLAGS} 27 | LINKER_FLAGS ${CMAKE_EXE_LINKER_FLAGS}) -------------------------------------------------------------------------------- /gstreamer/3rd_party/gstreamer/build.sh: -------------------------------------------------------------------------------- 1 | 2 | rm -rf build 3 | mkdir build 4 | cd build 5 | cmake .. 6 | make -------------------------------------------------------------------------------- /gstreamer/3rd_party/gstreamer/gst_objects_info_meta.c: -------------------------------------------------------------------------------- 1 | #include "gst_objects_info_meta.h" 2 | 3 | #include "stdio.h" 4 | #include "stdlib.h" 5 | #include "string.h" 6 | 7 | static gboolean gst_objects_info_meta_init(GstMeta *meta, gpointer params, GstBuffer *buffer); 8 | static gboolean gst_objects_info_meta_transform(GstBuffer *transbuf, GstMeta *meta, GstBuffer *buffer, 9 | GQuark type, gpointer data); 10 | 11 | GstObjectsInfoArray* empty(){ 12 | static GstObjectsInfoMeta meta; 13 | meta.objects.size = 0; 14 | return &meta.objects; 15 | } 16 | 17 | // Register metadata type and returns Gtype 18 | // https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gstreamer/html/gstreamer-GstMeta.html#gst-meta-api-type-register 19 | GType gst_objects_info_meta_api_get_type(void) 20 | { 21 | static const gchar *tags[] = {NULL}; 22 | static volatile GType type; 23 | if (g_once_init_enter (&type)) { 24 | GType _type = gst_meta_api_type_register("GstObjectsInfoMetaAPI", tags); 25 | g_once_init_leave(&type, _type); 26 | } 27 | return type; 28 | } 29 | 30 | // GstMetaInfo provides info for specific metadata implementation 31 | // https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gstreamer/html/gstreamer-GstMeta.html#GstMetaInfo 32 | const GstMetaInfo *gst_objects_info_meta_get_info(void) 33 | { 34 | static const GstMetaInfo *gst_objects_info_meta_info = NULL; 35 | 36 | if (g_once_init_enter (&gst_objects_info_meta_info)) { 37 | // Explanation of fields 38 | // https://gstreamer.freedesktop.org/documentation/design/meta.html#gstmeta1 39 | const GstMetaInfo *meta = gst_meta_register (GST_OBJECTS_INFO_META_API_TYPE, 40 | "GstObjectsInfoMeta", 41 | sizeof (GstObjectsInfoMeta), 42 | gst_objects_info_meta_init, 43 | (GstMetaFreeFunction) NULL, 44 | gst_objects_info_meta_transform); 45 | g_once_init_leave (&gst_objects_info_meta_info, meta); 46 | } 47 | return gst_objects_info_meta_info; 48 | } 49 | 50 | // Meta init function 51 | // 4-th field in GstMetaInfo 52 | static gboolean gst_objects_info_meta_init(GstMeta *meta, gpointer params, GstBuffer *buffer) 53 | { 54 | GstObjectsInfoMeta *gst_objects_info_meta = (GstObjectsInfoMeta *)meta; 55 | gst_objects_info_meta->objects.size = 0; 56 | return TRUE; 57 | } 58 | 59 | // Meta transform function 60 | // 5-th field in GstMetaInfo 61 | // https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gstreamer/html/gstreamer-GstMeta.html#GstMetaTransformFunction 62 | static gboolean gst_objects_info_meta_transform(GstBuffer *transbuf, GstMeta *meta, GstBuffer *buffer, 63 | GQuark type, gpointer data) 64 | { 65 | GstObjectsInfoMeta *gst_objects_info_meta = (GstObjectsInfoMeta *)meta; 66 | gst_buffer_add_objects_info_meta(transbuf, &(gst_objects_info_meta->objects)); 67 | return TRUE; 68 | } 69 | 70 | 71 | // Only for Python : return GstObjectsInfoArray instead of GsObjectsInfoMeta 72 | // // To avoid GstMeta (C) map to Gst.Meta (Python) 73 | GstObjectsInfoArray* gst_buffer_get_objects_info_meta(GstBuffer* b) 74 | { 75 | GstObjectsInfoMeta* meta = (GstObjectsInfoMeta*)gst_buffer_get_meta((b), GST_OBJECTS_INFO_META_API_TYPE); 76 | 77 | if (meta == NULL) 78 | return empty(); 79 | else 80 | return &meta->objects; 81 | } 82 | 83 | 84 | GstObjectsInfoMeta * gst_buffer_add_objects_info_meta(GstBuffer *buffer, GstObjectsInfoArray* objects) 85 | { 86 | GstObjectsInfoMeta *gst_objects_info_meta = NULL; 87 | 88 | g_return_val_if_fail(GST_IS_BUFFER(buffer), NULL); 89 | g_return_val_if_fail(gst_buffer_is_writable(buffer), NULL); 90 | 91 | gst_objects_info_meta = (GstObjectsInfoMeta *) gst_buffer_add_meta (buffer, GST_OBJECTS_INFO_META_INFO, NULL); 92 | 93 | guint32 size = objects->size; 94 | if (size > 0){ 95 | gst_objects_info_meta->objects.size = size; 96 | guint32 total_size = sizeof(GstObjectInfo)*size; 97 | gst_objects_info_meta->objects.items = malloc(total_size); 98 | memcpy ( gst_objects_info_meta->objects.items, objects->items, total_size ); 99 | } 100 | return gst_objects_info_meta; 101 | } 102 | 103 | 104 | // Removes metadata (GstBufferInfo) from buffer 105 | gboolean gst_buffer_remove_objects_info_meta(GstBuffer *buffer) 106 | { 107 | g_return_val_if_fail(GST_IS_BUFFER(buffer), NULL); 108 | 109 | GstObjectsInfoMeta* meta = (GstObjectsInfoMeta*)gst_buffer_get_meta((buffer), GST_OBJECTS_INFO_META_API_TYPE); 110 | 111 | if (meta == NULL) 112 | return TRUE; 113 | 114 | if ( !gst_buffer_is_writable(buffer)) 115 | return FALSE; 116 | 117 | // https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gstreamer/html/GstBuffer.html#gst-buffer-remove-meta 118 | return gst_buffer_remove_meta(buffer, &meta->meta); 119 | } 120 | -------------------------------------------------------------------------------- /gstreamer/3rd_party/gstreamer/gst_objects_info_meta.h: -------------------------------------------------------------------------------- 1 | /* 2 | Meta implementation example 3 | https://github.com/EricssonResearch/openwebrtc-gst-plugins/tree/master/gst-libs/gst/sctp 4 | */ 5 | 6 | 7 | #ifndef __GST_OBJECTS_INFO_META_H__ 8 | #define __GST_OBJECTS_INFO_META_H__ 9 | 10 | #include 11 | 12 | 13 | G_BEGIN_DECLS 14 | 15 | typedef enum { 16 | GST_OBJECTS_INFO_META_PARTIAL_RELIABILITY_NONE, 17 | GST_OBJECTS_INFO_META_PARTIAL_RELIABILITY_TTL, 18 | GST_OBJECTS_INFO_META_PARTIAL_RELIABILITY_BUF, 19 | GST_OBJECTS_INFO_META_PARTIAL_RELIABILITY_RTX 20 | 21 | } GstObjectsInfoMetaPartiallyReliability; 22 | 23 | 24 | // Api Type 25 | // 1-st field of GstMetaInfo 26 | #define GST_OBJECTS_INFO_META_API_TYPE (gst_objects_info_meta_api_get_type()) 27 | #define GST_OBJECTS_INFO_META_INFO (gst_objects_info_meta_get_info()) 28 | 29 | typedef struct _GstObjectsInfoMeta GstObjectsInfoMeta; 30 | typedef struct _GstObjectInfo GstObjectInfo ; 31 | typedef struct _GstObjectsInfoArray GstObjectsInfoArray; 32 | 33 | struct _GstObjectInfo { 34 | 35 | // bounding box 36 | guint32 x, y, width, height; 37 | gfloat confidence; 38 | gchar* class_name; 39 | 40 | guint32 track_id; 41 | }; 42 | 43 | struct _GstObjectsInfoArray { 44 | GstObjectInfo* items; 45 | guint32 size; 46 | }; 47 | 48 | 49 | struct _GstObjectsInfoMeta { 50 | 51 | // Required as it is base structure for metadata 52 | // https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gstreamer/html/gstreamer-GstMeta.html 53 | GstMeta meta; 54 | 55 | // Custom fields 56 | GstObjectsInfoArray objects; 57 | }; 58 | 59 | GType gst_objects_info_meta_api_get_type(void); 60 | 61 | GST_EXPORT 62 | const GstMetaInfo * gst_objects_info_meta_get_info(void); 63 | 64 | GST_EXPORT 65 | GstObjectsInfoMeta * gst_buffer_add_objects_info_meta(GstBuffer *buffer, GstObjectsInfoArray*); 66 | 67 | GST_EXPORT 68 | GstObjectsInfoArray* gst_buffer_get_objects_info_meta(GstBuffer* buffer); 69 | 70 | GST_EXPORT 71 | gboolean gst_buffer_remove_objects_info_meta(GstBuffer *buffer); 72 | 73 | G_END_DECLS 74 | 75 | #endif /* __GST_SCTP_SEND_META_H__ */ -------------------------------------------------------------------------------- /gstreamer/__init__.py: -------------------------------------------------------------------------------- 1 | from .logging import setup_logging, get_log_level 2 | 3 | import gi 4 | gi.require_version('Gst', '1.0') 5 | gi.require_version('GstBase', '1.0') 6 | gi.require_version('GstApp', '1.0') 7 | gi.require_version('GstVideo', '1.0') 8 | from gi.repository import Gst, GLib, GObject, GstApp, GstVideo, GstBase # noqa:F401,F402 9 | 10 | from .gst_hacks import map_gst_buffer, map_gst_memory # noqa:F401,F402 11 | 12 | from .utils import gst_buffer_to_ndarray, gst_buffer_with_pad_to_ndarray, ndarray_to_gst_buffer # noqa:F401,F402 13 | 14 | from .gst_tools import GstVideoSink, GstVideoSource, GstPipeline, GstContext # noqa:F401,F402 15 | 16 | setup_logging(verbose=get_log_level()) 17 | -------------------------------------------------------------------------------- /gstreamer/gst_hacks.py: -------------------------------------------------------------------------------- 1 | """ 2 | Source: https://github.com/stb-tester/stb-tester/blob/master/_stbt/gst_hacks.py 3 | """ 4 | import os 5 | from ctypes import * 6 | from typing import Tuple 7 | from contextlib import contextmanager 8 | 9 | import gi 10 | gi.require_version('Gst', '1.0') 11 | from gi.repository import Gst # noqa:F401,F402 12 | 13 | _GST_PADDING = 4 # From gstconfig.h 14 | 15 | 16 | class _GstMapInfo(Structure): 17 | _fields_ = [("memory", c_void_p), # GstMemory *memory 18 | ("flags", c_int), # GstMapFlags flags 19 | ("data", POINTER(c_byte)), # guint8 *data 20 | ("size", c_size_t), # gsize size 21 | ("maxsize", c_size_t), # gsize maxsize 22 | ("user_data", c_void_p * 4), # gpointer user_data[4] 23 | ("_gst_reserved", c_void_p * _GST_PADDING)] 24 | 25 | 26 | _GST_MAP_INFO_POINTER = POINTER(_GstMapInfo) 27 | 28 | _libgst = CDLL(os.getenv("LIB_GSTREAMER_PATH", "libgstreamer-1.0.so.0")) 29 | _libgst.gst_buffer_map.argtypes = [c_void_p, _GST_MAP_INFO_POINTER, c_int] 30 | _libgst.gst_buffer_map.restype = c_int 31 | 32 | _libgst.gst_buffer_unmap.argtypes = [c_void_p, _GST_MAP_INFO_POINTER] 33 | _libgst.gst_buffer_unmap.restype = None 34 | 35 | _libgst.gst_mini_object_is_writable.argtypes = [c_void_p] 36 | _libgst.gst_mini_object_is_writable.restype = c_int 37 | 38 | _libgst.gst_memory_map.argtypes = [c_void_p, _GST_MAP_INFO_POINTER, c_int] 39 | _libgst.gst_memory_map.restype = c_int 40 | 41 | _libgst.gst_memory_unmap.argtypes = [c_void_p, _GST_MAP_INFO_POINTER] 42 | _libgst.gst_memory_unmap.restype = None 43 | 44 | 45 | @contextmanager 46 | def map_gst_buffer(pbuffer: Gst.Buffer, flags: Gst.MapFlags) -> _GST_MAP_INFO_POINTER: 47 | """ Map Gst.Buffer with READ/WRITE flags 48 | 49 | Example: 50 | with map_gst_buffer(pbuffer, Gst.MapFlags.READ | Gst.MapFlags.WRITE) as mapped: 51 | // do_something with mapped 52 | """ 53 | 54 | if pbuffer is None: 55 | raise TypeError("Cannot pass NULL to _map_gst_buffer") 56 | 57 | ptr = hash(pbuffer) 58 | if flags & Gst.MapFlags.WRITE and _libgst.gst_mini_object_is_writable(ptr) == 0: 59 | raise ValueError( 60 | "Writable array requested but buffer is not writeable") 61 | 62 | mapping = _GstMapInfo() 63 | success = _libgst.gst_buffer_map(ptr, mapping, flags) 64 | if not success: 65 | raise RuntimeError("Couldn't map buffer") 66 | try: 67 | yield cast( 68 | mapping.data, POINTER(c_byte * mapping.size)).contents 69 | finally: 70 | _libgst.gst_buffer_unmap(ptr, mapping) 71 | 72 | 73 | @contextmanager 74 | def map_gst_memory(memory: Gst.Memory, flags: Gst.MapFlags) -> _GST_MAP_INFO_POINTER: 75 | """Map Gst.Memory with READ/WRITE flags 76 | 77 | Example: 78 | with map_gst_memory(memory, Gst.MapFlags.READ | Gst.MapFlags.WRITE) as mapped: 79 | // do_something with mapped 80 | """ 81 | 82 | if memory is None: 83 | raise TypeError("Cannot pass NULL to _map_gst_buffer") 84 | 85 | ptr = hash(memory) 86 | if flags & Gst.MapFlags.WRITE and _libgst.gst_mini_object_is_writable(ptr) == 0: 87 | raise ValueError( 88 | "Writable array requested but buffer is not writeable") 89 | 90 | mapping = _GstMapInfo() 91 | success = _libgst.gst_memory_map(ptr, mapping, flags) 92 | if not success: 93 | raise RuntimeError("Couldn't map buffer") 94 | try: 95 | yield cast( 96 | mapping.data, POINTER(c_byte * mapping.size)).contents 97 | finally: 98 | _libgst.gst_memory_unmap(ptr, mapping) 99 | -------------------------------------------------------------------------------- /gstreamer/gst_objects_info_meta.py: -------------------------------------------------------------------------------- 1 | import os 2 | from ctypes import * 3 | from typing import List 4 | 5 | import gi 6 | gi.require_version('Gst', '1.0') 7 | from gi.repository import Gst # noqa:F401,F402 8 | 9 | 10 | class GstObjectInfo(Structure): 11 | _fields_ = [("x", c_int), 12 | ("y", c_int), 13 | ("width", c_int), 14 | ("height", c_int), 15 | ("confidence", c_float), 16 | ("class_name", c_char_p), 17 | ("track_id", c_int)] 18 | 19 | 20 | class GstObjectInfoArray(Structure): 21 | _fields_ = [("items", POINTER(GstObjectInfo)), 22 | ("size", c_int)] 23 | 24 | 25 | GstObjectInfoArrayPtr = POINTER(GstObjectInfoArray) 26 | 27 | cwd = os.path.dirname(os.path.abspath(__file__)) 28 | libc = CDLL(os.path.join(cwd, "3rd_party/gstreamer/build/libgst_objects_info_meta.so")) 29 | 30 | libc.gst_buffer_add_objects_info_meta.argtypes = [c_void_p, GstObjectInfoArrayPtr] 31 | libc.gst_buffer_add_objects_info_meta.restype = c_void_p 32 | 33 | libc.gst_buffer_get_objects_info_meta.argtypes = [c_void_p] 34 | libc.gst_buffer_get_objects_info_meta.restype = GstObjectInfoArrayPtr 35 | 36 | libc.gst_buffer_remove_objects_info_meta.argtypes = [c_void_p] 37 | libc.gst_buffer_remove_objects_info_meta.restype = c_bool 38 | 39 | 40 | def to_gst_objects_info(objects: List[dict]) -> GstObjectInfoArray: 41 | """Converts List of objects to GstObjectInfoMeta """ 42 | gst_objects_info = GstObjectInfoArray() 43 | gst_objects_info.size = len(objects) 44 | gst_objects_info.items = (GstObjectInfo * gst_objects_info.size)() 45 | 46 | for i, obj in enumerate(objects): 47 | x, y, width, height = obj['bounding_box'] 48 | gst_objects_info.items[i] = (x, y, width, height, 49 | obj["confidence"], 50 | obj["class_name"].encode("utf-8"), 51 | obj.get("track_id", 0)) 52 | 53 | return gst_objects_info 54 | 55 | 56 | def to_list(gst_object_info: GstObjectInfoArray) -> List[dict]: 57 | """ Converts GstObjectInfoMeta to List of objects""" 58 | objects = [] 59 | for i in range(gst_object_info.size): 60 | obj = gst_object_info.items[i] 61 | class_name = "" 62 | try: 63 | class_name = obj.class_name.decode("utf-8") 64 | except Exception: 65 | pass 66 | objects.append({"bounding_box": [obj.x, obj.y, obj.width, obj.height], 67 | "confidence": obj.confidence, 68 | "class_name": class_name, 69 | "track_id": obj.track_id}) 70 | return objects 71 | 72 | 73 | def gst_meta_write(buffer: Gst.Buffer, objects: List[dict]): 74 | """ Writes List of objects to Gst.Buffer""" 75 | gst_objects_info = to_gst_objects_info(objects) 76 | _ = libc.gst_buffer_add_objects_info_meta(hash(buffer), gst_objects_info) 77 | 78 | 79 | def gst_meta_get(buffer: Gst.Buffer) -> List[dict]: 80 | """ Gets List of objects from Gst.Buffer""" 81 | res = libc.gst_buffer_get_objects_info_meta(hash(buffer)) 82 | return to_list(res.contents) 83 | 84 | 85 | def gst_meta_remove(buffer: Gst.Buffer): 86 | """ Removes all objects from Gst.Buffer """ 87 | libc.gst_buffer_remove_objects_info_meta(hash(buffer)) 88 | -------------------------------------------------------------------------------- /gstreamer/gst_tools.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Usage Example: 4 | >>> width, height, num_buffers = 1920, 1080, 100 5 | >>> caps_filter = 'capsfilter caps=video/x-raw,format=RGB,width={},height={}'.format(width, height) 6 | >>> source_cmd = 'videotestsrc num-buffers={} ! {} ! appsink emit-signals=True sync=false'.format( 7 | ... num_buffers, caps_filter) 8 | >>> display_cmd = "appsrc emit-signals=True is-live=True ! videoconvert ! gtksink sync=false" 9 | >>> 10 | >>> with GstVideoSource(source_cmd) as pipeline, GstVideoSink(display_cmd, width=width, height=height) as display: 11 | ... current_num_buffers = 0 12 | ... while current_num_buffers < num_buffers: 13 | ... buffer = pipeline.pop() 14 | ... if buffer: 15 | ... display.push(buffer.data) 16 | ... current_num_buffers += 1 17 | >>> 18 | """ 19 | 20 | import sys 21 | import os 22 | import time 23 | import queue 24 | import logging 25 | import threading 26 | import typing as typ 27 | from enum import Enum 28 | from functools import partial 29 | from fractions import Fraction 30 | 31 | import attr 32 | import numpy as np 33 | 34 | import gi 35 | 36 | gi.require_version("Gst", "1.0") 37 | gi.require_version("GstApp", "1.0") 38 | gi.require_version("GstVideo", "1.0") 39 | from gi.repository import Gst, GLib, GObject, GstApp, GstVideo # noqa:F401,F402 40 | 41 | from .utils import * # noqa:F401,F402 42 | 43 | Gst.init(sys.argv if hasattr(sys, "argv") else None) 44 | 45 | 46 | class NamedEnum(Enum): 47 | def __repr__(self): 48 | return str(self) 49 | 50 | @classmethod 51 | def names(cls) -> typ.List[str]: 52 | return list(cls.__members__.keys()) 53 | 54 | 55 | class VideoType(NamedEnum): 56 | """ 57 | https://gstreamer.freedesktop.org/documentation/plugin-development/advanced/media-types.html?gi-language=c 58 | """ 59 | 60 | VIDEO_RAW = "video/x-raw" 61 | VIDEO_GL_RAW = "video/x-raw(memory:GLMemory)" 62 | VIDEO_NVVM_RAW = "video/x-raw(memory:NVMM)" 63 | 64 | 65 | class GstContext: 66 | def __init__(self): 67 | # SIGINT handle issue: 68 | # https://github.com/beetbox/audioread/issues/63#issuecomment-390394735 69 | self._main_loop = GLib.MainLoop.new(None, False) 70 | 71 | self._main_loop_thread = threading.Thread(target=self._main_loop_run) 72 | 73 | self._log = logging.getLogger("pygst.{}".format(self.__class__.__name__)) 74 | 75 | def __str__(self) -> str: 76 | return self.__class__.__name__ 77 | 78 | def __repr__(self) -> str: 79 | return "<{}>".format(self) 80 | 81 | def __enter__(self): 82 | self.startup() 83 | return self 84 | 85 | def __exit__(self, exc_type, exc_val, exc_tb): 86 | self.shutdown() 87 | 88 | @property 89 | def log(self) -> logging.Logger: 90 | return self._log 91 | 92 | def startup(self): 93 | if self._main_loop_thread.is_alive(): 94 | return 95 | 96 | self._main_loop_thread.start() 97 | 98 | def _main_loop_run(self): 99 | try: 100 | self._main_loop.run() 101 | except Exception: 102 | pass 103 | 104 | def shutdown(self, timeout: int = 2): 105 | self.log.debug("%s Quitting main loop ...", self) 106 | 107 | if self._main_loop.is_running(): 108 | self._main_loop.quit() 109 | 110 | self.log.debug("%s Joining main loop thread...", self) 111 | try: 112 | if self._main_loop_thread.is_alive(): 113 | self._main_loop_thread.join(timeout=timeout) 114 | except Exception as err: 115 | self.log.error("%s.main_loop_thread : %s", self, err) 116 | pass 117 | 118 | 119 | class GstPipeline: 120 | """Base class to initialize any Gstreamer Pipeline from string""" 121 | 122 | def __init__(self, command: str): 123 | """ 124 | :param command: gst-launch string 125 | """ 126 | self._command = command 127 | self._pipeline = None # Gst.Pipeline 128 | self._bus = None # Gst.Bus 129 | 130 | self._log = logging.getLogger("pygst.{}".format(self.__class__.__name__)) 131 | self._log.info("%s \n gst-launch-1.0 %s", self, command) 132 | 133 | self._end_stream_event = threading.Event() 134 | 135 | @property 136 | def log(self) -> logging.Logger: 137 | return self._log 138 | 139 | def __str__(self) -> str: 140 | return self.__class__.__name__ 141 | 142 | def __repr__(self) -> str: 143 | return "<{}>".format(self) 144 | 145 | def __enter__(self): 146 | self.startup() 147 | return self 148 | 149 | def __exit__(self, exc_type, exc_val, exc_tb): 150 | self.shutdown() 151 | 152 | def get_by_cls(self, cls: GObject.GType) -> typ.List[Gst.Element]: 153 | """ Get Gst.Element[] from pipeline by GType """ 154 | elements = self._pipeline.iterate_elements() 155 | if isinstance(elements, Gst.Iterator): 156 | # Patch "TypeError: ‘Iterator’ object is not iterable." 157 | # For versions we have to get a python iterable object from Gst iterator 158 | _elements = [] 159 | while True: 160 | ret, el = elements.next() 161 | if ret == Gst.IteratorResult(1): # GST_ITERATOR_OK 162 | _elements.append(el) 163 | else: 164 | break 165 | elements = _elements 166 | 167 | return [e for e in elements if isinstance(e, cls)] 168 | 169 | def get_by_name(self, name: str) -> Gst.Element: 170 | """Get Gst.Element from pipeline by name 171 | :param name: plugins name (name={} in gst-launch string) 172 | """ 173 | return self._pipeline.get_by_name(name) 174 | 175 | def startup(self): 176 | """ Starts pipeline """ 177 | if self._pipeline: 178 | raise RuntimeError("Can't initiate %s. Already started") 179 | 180 | self._pipeline = Gst.parse_launch(self._command) 181 | 182 | # Initialize Bus 183 | self._bus = self._pipeline.get_bus() 184 | self._bus.add_signal_watch() 185 | self.bus.connect("message::error", self.on_error) 186 | self.bus.connect("message::eos", self.on_eos) 187 | self.bus.connect("message::warning", self.on_warning) 188 | 189 | # Initalize Pipeline 190 | self._on_pipeline_init() 191 | self._pipeline.set_state(Gst.State.READY) 192 | 193 | self.log.info("Starting %s", self) 194 | 195 | self._end_stream_event.clear() 196 | 197 | self.log.debug( 198 | "%s Setting pipeline state to %s ... ", 199 | self, 200 | gst_state_to_str(Gst.State.PLAYING), 201 | ) 202 | self._pipeline.set_state(Gst.State.PLAYING) 203 | self.log.debug( 204 | "%s Pipeline state set to %s ", self, gst_state_to_str(Gst.State.PLAYING) 205 | ) 206 | 207 | def _on_pipeline_init(self) -> None: 208 | """Sets additional properties for plugins in Pipeline""" 209 | pass 210 | 211 | @property 212 | def bus(self) -> Gst.Bus: 213 | return self._bus 214 | 215 | @property 216 | def pipeline(self) -> Gst.Pipeline: 217 | return self._pipeline 218 | 219 | def _shutdown_pipeline(self, timeout: int = 1, eos: bool = False) -> None: 220 | """ Stops pipeline 221 | :param eos: if True -> send EOS event 222 | - EOS event necessary for FILESINK finishes properly 223 | - Use when pipeline crushes 224 | """ 225 | 226 | if self._end_stream_event.is_set(): 227 | return 228 | 229 | self._end_stream_event.set() 230 | 231 | if not self.pipeline: 232 | return 233 | 234 | self.log.debug("%s Stopping pipeline ...", self) 235 | 236 | # https://lazka.github.io/pgi-docs/Gst-1.0/classes/Element.html#Gst.Element.get_state 237 | if self._pipeline.get_state(timeout=1)[1] == Gst.State.PLAYING: 238 | self.log.debug("%s Sending EOS event ...", self) 239 | try: 240 | thread = threading.Thread( 241 | target=self._pipeline.send_event, args=(Gst.Event.new_eos(),) 242 | ) 243 | thread.start() 244 | thread.join(timeout=timeout) 245 | except Exception: 246 | pass 247 | 248 | self.log.debug("%s Reseting pipeline state ....", self) 249 | try: 250 | self._pipeline.set_state(Gst.State.NULL) 251 | self._pipeline = None 252 | except Exception: 253 | pass 254 | 255 | self.log.debug("%s Gst.Pipeline successfully destroyed", self) 256 | 257 | def shutdown(self, timeout: int = 1, eos: bool = False) -> None: 258 | """Shutdown pipeline 259 | :param timeout: time to wait when pipeline fully stops 260 | :param eos: if True -> send EOS event 261 | - EOS event necessary for FILESINK finishes properly 262 | - Use when pipeline crushes 263 | """ 264 | self.log.info("%s Shutdown requested ...", self) 265 | 266 | self._shutdown_pipeline(timeout=timeout, eos=eos) 267 | 268 | self.log.info("%s successfully destroyed", self) 269 | 270 | @property 271 | def is_active(self) -> bool: 272 | return self.pipeline is not None and not self.is_done 273 | 274 | @property 275 | def is_done(self) -> bool: 276 | return self._end_stream_event.is_set() 277 | 278 | def on_error(self, bus: Gst.Bus, message: Gst.Message): 279 | err, debug = message.parse_error() 280 | self.log.error("Gstreamer.%s: Error %s: %s. ", self, err, debug) 281 | self._shutdown_pipeline() 282 | 283 | def on_eos(self, bus: Gst.Bus, message: Gst.Message): 284 | self.log.debug("Gstreamer.%s: Received stream EOS event", self) 285 | self._shutdown_pipeline() 286 | 287 | def on_warning(self, bus: Gst.Bus, message: Gst.Message): 288 | warn, debug = message.parse_warning() 289 | self.log.warning("Gstreamer.%s: %s. %s", self, warn, debug) 290 | 291 | 292 | def gst_video_format_plugin( 293 | *, 294 | width: int = None, 295 | height: int = None, 296 | fps: Fraction = None, 297 | video_type: VideoType = VideoType.VIDEO_RAW, 298 | video_frmt: GstVideo.VideoFormat = GstVideo.VideoFormat.RGB 299 | ) -> typ.Optional[str]: 300 | """ 301 | https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gstreamer-plugins/html/gstreamer-plugins-capsfilter.html 302 | Returns capsfilter 303 | video/x-raw,width=widht,height=height 304 | video/x-raw,framerate=fps/1 305 | video/x-raw,format=RGB 306 | video/x-raw,format=RGB,width=widht,height=height,framerate=1/fps 307 | :param width: image width 308 | :param height: image height 309 | :param fps: video fps 310 | :param video_type: gst specific (raw, h264, ..) 311 | https://gstreamer.freedesktop.org/documentation/design/mediatype-video-raw.html 312 | :param video_frmt: gst specific (RGB, BGR, RGBA) 313 | https://gstreamer.freedesktop.org/documentation/design/mediatype-video-raw.html 314 | https://lazka.github.io/pgi-docs/index.html#GstVideo-1.0/enums.html#GstVideo.VideoFormat 315 | """ 316 | 317 | plugin = str(video_type.value) 318 | n = len(plugin) 319 | if video_frmt: 320 | plugin += ",format={}".format(GstVideo.VideoFormat.to_string(video_frmt)) 321 | if width and width > 0: 322 | plugin += ",width={}".format(width) 323 | if height and height > 0: 324 | plugin += ",height={}".format(height) 325 | if fps and fps > 0: 326 | plugin += ",framerate={}".format(fraction_to_str(fps)) 327 | 328 | if n == len(plugin): 329 | return None 330 | 331 | return plugin 332 | 333 | 334 | class GstVideoSink(GstPipeline): 335 | """Gstreamer Video Sink Base Class 336 | 337 | Usage Example: 338 | >>> width, height = 1920, 1080 339 | ... command = "appsrc emit-signals=True is-live=True ! videoconvert ! fakesink sync=false" 340 | ... with GstVideoSink(command, width=width, height=height) as pipeline: 341 | ... for _ in range(10): 342 | ... pipeline.push(buffer=np.random.randint(low=0, high=255, size=(height, width, 3), dtype=np.uint8)) 343 | >>> 344 | """ 345 | 346 | def __init__( 347 | self, 348 | command: str, 349 | *, 350 | width: int, 351 | height: int, 352 | fps: typ.Union[Fraction, int] = Fraction("30/1"), 353 | video_type: VideoType = VideoType.VIDEO_RAW, 354 | video_frmt: GstVideo.VideoFormat = GstVideo.VideoFormat.RGB 355 | ): 356 | 357 | super(GstVideoSink, self).__init__(command) 358 | 359 | self._fps = Fraction(fps) 360 | self._width = width 361 | self._height = height 362 | self._video_type = video_type # VideoType 363 | self._video_frmt = video_frmt # GstVideo.VideoFormat 364 | 365 | self._pts = 0 366 | self._dts = GLib.MAXUINT64 367 | self._duration = 10 ** 9 / (fps.numerator / fps.denominator) 368 | 369 | self._src = None # GstApp.AppSrc 370 | 371 | @property 372 | def video_frmt(self): 373 | return self._video_frmt 374 | 375 | def _on_pipeline_init(self): 376 | """Sets additional properties for plugins in Pipeline""" 377 | # find src element 378 | appsrcs = self.get_by_cls(GstApp.AppSrc) 379 | self._src = appsrcs[0] if len(appsrcs) == 1 else None 380 | if not self._src: 381 | raise ValueError("%s not found", GstApp.AppSrc) 382 | 383 | if self._src: 384 | # this instructs appsrc that we will be dealing with timed buffer 385 | self._src.set_property("format", Gst.Format.TIME) 386 | 387 | # instructs appsrc to block pushing buffers until ones in queue are preprocessed 388 | # allows to avoid huge queue internal queue size in appsrc 389 | self._src.set_property("block", True) 390 | 391 | # set src caps 392 | caps = gst_video_format_plugin( 393 | width=self._width, 394 | height=self._height, 395 | fps=self._fps, 396 | video_type=self._video_type, 397 | video_frmt=self._video_frmt, 398 | ) 399 | 400 | self.log.debug("%s Caps: %s", self, caps) 401 | if caps is not None: 402 | self._src.set_property("caps", Gst.Caps.from_string(caps)) 403 | 404 | @staticmethod 405 | def to_gst_buffer( 406 | buffer: typ.Union[Gst.Buffer, np.ndarray], 407 | *, 408 | pts: typ.Optional[int] = None, 409 | dts: typ.Optional[int] = None, 410 | offset: typ.Optional[int] = None, 411 | duration: typ.Optional[int] = None 412 | ) -> Gst.Buffer: 413 | """Convert buffer to Gst.Buffer. Updates required fields 414 | Parameters explained: 415 | https://lazka.github.io/pgi-docs/Gst-1.0/classes/Buffer.html#gst-buffer 416 | """ 417 | gst_buffer = buffer 418 | if isinstance(gst_buffer, np.ndarray): 419 | gst_buffer = Gst.Buffer.new_wrapped(bytes(buffer)) 420 | 421 | if not isinstance(gst_buffer, Gst.Buffer): 422 | raise ValueError( 423 | "Invalid buffer format {} != {}".format(type(gst_buffer), Gst.Buffer) 424 | ) 425 | 426 | gst_buffer.pts = pts or GLib.MAXUINT64 427 | gst_buffer.dts = dts or GLib.MAXUINT64 428 | gst_buffer.offset = offset or GLib.MAXUINT64 429 | gst_buffer.duration = duration or GLib.MAXUINT64 430 | return gst_buffer 431 | 432 | def push( 433 | self, 434 | buffer: typ.Union[Gst.Buffer, np.ndarray], 435 | *, 436 | pts: typ.Optional[int] = None, 437 | dts: typ.Optional[int] = None, 438 | offset: typ.Optional[int] = None 439 | ) -> None: 440 | 441 | # FIXME: maybe put in queue first 442 | if not self.is_active: 443 | self.log.warning("Warning %s: Can't push buffer. Pipeline not active") 444 | return 445 | 446 | if not self._src: 447 | raise RuntimeError("Src {} is not initialized".format(Gst.AppSrc)) 448 | 449 | self._pts += self._duration 450 | offset_ = int(self._pts / self._duration) 451 | 452 | gst_buffer = self.to_gst_buffer( 453 | buffer, 454 | pts=pts or self._pts, 455 | dts=dts or self._dts, 456 | offset=offset or offset_, 457 | duration=self._duration, 458 | ) 459 | 460 | # Emit 'push-buffer' signal 461 | # https://lazka.github.io/pgi-docs/GstApp-1.0/classes/AppSrc.html#GstApp.AppSrc.signals.push_buffer 462 | self._src.emit("push-buffer", gst_buffer) 463 | 464 | @property 465 | def total_buffers_count(self) -> int: 466 | """Total pushed buffers count """ 467 | return int(self._pts / self._duration) 468 | 469 | def shutdown(self, timeout: int = 1, eos: bool = False): 470 | 471 | if self.is_active: 472 | if isinstance(self._src, GstApp.AppSrc): 473 | # Emit 'end-of-stream' signal 474 | # https://lazka.github.io/pgi-docs/GstApp-1.0/classes/AppSrc.html#GstApp.AppSrc.signals.end_of_stream 475 | self._src.emit("end-of-stream") 476 | 477 | super().shutdown(timeout=timeout, eos=eos) 478 | 479 | 480 | class LeakyQueue(queue.Queue): 481 | """Queue that contains only the last actual items and drops the oldest one.""" 482 | 483 | def __init__( 484 | self, 485 | maxsize: int = 100, 486 | on_drop: typ.Optional[typ.Callable[["LeakyQueue", "object"], None]] = None, 487 | ): 488 | super().__init__(maxsize=maxsize) 489 | self._dropped = 0 490 | self._on_drop = on_drop or (lambda queue, item: None) 491 | 492 | def put(self, item, block=True, timeout=None): 493 | if self.full(): 494 | dropped_item = self.get_nowait() 495 | self._dropped += 1 496 | self._on_drop(self, dropped_item) 497 | super().put(item, block=block, timeout=timeout) 498 | 499 | @property 500 | def dropped(self): 501 | return self._dropped 502 | 503 | 504 | # Struct copies fields from Gst.Buffer 505 | # https://lazka.github.io/pgi-docs/Gst-1.0/classes/Buffer.html 506 | @attr.s(slots=True, frozen=True) 507 | class GstBuffer: 508 | data = attr.ib() # type: np.ndarray 509 | pts = attr.ib(default=GLib.MAXUINT64) # type: int 510 | dts = attr.ib(default=GLib.MAXUINT64) # type: int 511 | offset = attr.ib(default=GLib.MAXUINT64) # type: int 512 | duration = attr.ib(default=GLib.MAXUINT64) # type: int 513 | 514 | 515 | class GstVideoSource(GstPipeline): 516 | """Gstreamer Video Source Base Class 517 | 518 | Usage Example: 519 | >>> width, height, num_buffers = 1920, 1080, 100 520 | >>> caps_filter = 'capsfilter caps=video/x-raw,format=RGB,width={},height={}'.format(width, height) 521 | >>> command = 'videotestsrc num-buffers={} ! {} ! appsink emit-signals=True sync=false'.format( 522 | ... num_buffers, caps_filter) 523 | >>> with GstVideoSource(command) as pipeline: 524 | ... buffers = [] 525 | ... while len(buffers) < num_buffers: 526 | ... buffer = pipeline.pop() 527 | ... if buffer: 528 | ... buffers.append(buffer) 529 | ... print('Got: {} buffers'.format(len(buffers))) 530 | >>> 531 | """ 532 | 533 | def __init__(self, command: str, leaky: bool = False, max_buffers_size: int = 100): 534 | """ 535 | :param command: gst-launch-1.0 command (last element: appsink) 536 | """ 537 | super(GstVideoSource, self).__init__(command) 538 | 539 | self._sink = None # GstApp.AppSink 540 | self._counter = 0 # counts number of received buffers 541 | 542 | queue_cls = partial(LeakyQueue, on_drop=self._on_drop) if leaky else queue.Queue 543 | self._queue = queue_cls(maxsize=max_buffers_size) # Queue of GstBuffer 544 | 545 | @property 546 | def total_buffers_count(self) -> int: 547 | """Total read buffers count """ 548 | return self._counter 549 | 550 | @staticmethod 551 | def _clean_queue(q: queue.Queue): 552 | while not q.empty(): 553 | try: 554 | q.get_nowait() 555 | except queue.Empty: 556 | break 557 | 558 | def _on_drop(self, queue: LeakyQueue, buffer: GstBuffer) -> None: 559 | self.log.warning( 560 | "Buffer #%d for %s is dropped (totally dropped %d buffers)", 561 | int(buffer.pts / buffer.duration), 562 | self, 563 | queue.dropped, 564 | ) 565 | 566 | def _on_pipeline_init(self): 567 | """Sets additional properties for plugins in Pipeline""" 568 | 569 | appsinks = self.get_by_cls(GstApp.AppSink) 570 | self._sink = appsinks[0] if len(appsinks) == 1 else None 571 | if not self._sink: 572 | # TODO: force pipeline to have appsink 573 | raise AttributeError("%s not found", GstApp.AppSink) 574 | 575 | # Listen to 'new-sample' event 576 | # https://lazka.github.io/pgi-docs/GstApp-1.0/classes/AppSink.html#GstApp.AppSink.signals.new_sample 577 | if self._sink: 578 | self._sink.connect("new-sample", self._on_buffer, None) 579 | 580 | def _extract_buffer(self, sample: Gst.Sample) -> typ.Optional[GstBuffer]: 581 | """Converts Gst.Sample to GstBuffer 582 | 583 | Gst.Sample: 584 | https://lazka.github.io/pgi-docs/Gst-1.0/classes/Sample.html 585 | """ 586 | buffer = sample.get_buffer() 587 | caps = sample.get_caps() 588 | 589 | cnt = buffer.n_memory() 590 | if cnt <= 0: 591 | self.log.warning("%s No data in Gst.Buffer", self) 592 | return None 593 | 594 | memory = buffer.get_memory(0) 595 | if not memory: 596 | self.log.warning("%s No Gst.Memory in Gst.Buffer", self) 597 | return None 598 | 599 | array = gst_buffer_with_caps_to_ndarray(buffer, caps, do_copy=True) 600 | 601 | return GstBuffer( 602 | data=array, 603 | pts=buffer.pts, 604 | dts=buffer.dts, 605 | duration=buffer.duration, 606 | offset=buffer.offset, 607 | ) 608 | 609 | def _on_buffer(self, sink: GstApp.AppSink, data: typ.Any) -> Gst.FlowReturn: 610 | """Callback on 'new-sample' signal""" 611 | # Emit 'pull-sample' signal 612 | # https://lazka.github.io/pgi-docs/GstApp-1.0/classes/AppSink.html#GstApp.AppSink.signals.pull_sample 613 | 614 | sample = sink.emit("pull-sample") 615 | if isinstance(sample, Gst.Sample): 616 | self._queue.put(self._extract_buffer(sample)) 617 | self._counter += 1 618 | 619 | return Gst.FlowReturn.OK 620 | 621 | self.log.error( 622 | "Error : Not expected buffer type: %s != %s. %s", 623 | type(sample), 624 | Gst.Sample, 625 | self, 626 | ) 627 | return Gst.FlowReturn.ERROR 628 | 629 | def pop(self, timeout: float = 0.1) -> typ.Optional[GstBuffer]: 630 | """ Pops GstBuffer """ 631 | if not self._sink: 632 | raise RuntimeError("Sink {} is not initialized".format(Gst.AppSink)) 633 | 634 | buffer = None 635 | while (self.is_active or not self._queue.empty()) and not buffer: 636 | try: 637 | buffer = self._queue.get(timeout=timeout) 638 | except queue.Empty: 639 | pass 640 | 641 | return buffer 642 | 643 | @property 644 | def queue_size(self) -> int: 645 | """Returns queue size of GstBuffer""" 646 | return self._queue.qsize() 647 | 648 | def shutdown(self, timeout: int = 1, eos: bool = False): 649 | super().shutdown(timeout=timeout, eos=eos) 650 | 651 | self._clean_queue(self._queue) 652 | -------------------------------------------------------------------------------- /gstreamer/logging.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | 4 | LOG_BASE_NAME = 'pygst' 5 | LOG_FORMAT = '%(levelname)-6.6s | %(name)-20s | %(asctime)s.%(msecs)03d | %(threadName)s | %(message)s' 6 | LOG_DATE_FORMAT = '%d.%m %H:%M:%S' 7 | 8 | 9 | def get_log_level(): 10 | return int(os.getenv("GST_PYTHON_LOG_LEVEL", logging.DEBUG / 10)) * 10 11 | 12 | 13 | def setup_logging(verbose: int = logging.DEBUG): 14 | """Configure console logging. Info and below go to stdout, others go to stderr. """ 15 | 16 | root_logger = logging.getLogger('') 17 | root_logger.setLevel(logging.DEBUG if verbose > 0 else logging.INFO) 18 | 19 | log_handler = logging.StreamHandler() 20 | log_handler.setFormatter(logging.Formatter( 21 | fmt=LOG_FORMAT, datefmt=LOG_DATE_FORMAT)) 22 | 23 | local_logger = logging.getLogger(LOG_BASE_NAME) 24 | local_logger.setLevel(verbose) 25 | 26 | root_logger.addHandler(log_handler) 27 | -------------------------------------------------------------------------------- /gstreamer/utils.py: -------------------------------------------------------------------------------- 1 | import math 2 | import typing as typ 3 | from fractions import Fraction 4 | 5 | import numpy as np 6 | 7 | import gi 8 | gi.require_version('Gst', '1.0') 9 | gi.require_version('GstVideo', '1.0') 10 | from gi.repository import Gst, GstVideo # noqa:F401,F402 11 | 12 | from .gst_hacks import map_gst_buffer # noqa:F401,F402 13 | 14 | 15 | BITS_PER_BYTE = 8 16 | 17 | _ALL_VIDEO_FORMATS = [GstVideo.VideoFormat.from_string( 18 | f.strip()) for f in GstVideo.VIDEO_FORMATS_ALL.strip('{ }').split(',')] 19 | 20 | 21 | def has_flag(value: GstVideo.VideoFormatFlags, 22 | flag: GstVideo.VideoFormatFlags) -> bool: 23 | 24 | # in VideoFormatFlags each new value is 1 << 2**{0...8} 25 | return bool(value & (1 << max(1, math.ceil(math.log2(int(flag)))))) 26 | 27 | 28 | def _get_num_channels(fmt: GstVideo.VideoFormat) -> int: 29 | """ 30 | -1: means complex format (YUV, ...) 31 | """ 32 | frmt_info = GstVideo.VideoFormat.get_info(fmt) 33 | 34 | # temporal fix 35 | if fmt == GstVideo.VideoFormat.BGRX: 36 | return 4 37 | 38 | if has_flag(frmt_info.flags, GstVideo.VideoFormatFlags.ALPHA): 39 | return 4 40 | 41 | if has_flag(frmt_info.flags, GstVideo.VideoFormatFlags.RGB): 42 | return 3 43 | 44 | if has_flag(frmt_info.flags, GstVideo.VideoFormatFlags.GRAY): 45 | return 1 46 | 47 | return -1 48 | 49 | 50 | _ALL_VIDEO_FORMAT_CHANNELS = {fmt: _get_num_channels(fmt) for fmt in _ALL_VIDEO_FORMATS} 51 | 52 | 53 | def get_num_channels(fmt: GstVideo.VideoFormat): 54 | return _ALL_VIDEO_FORMAT_CHANNELS[fmt] 55 | 56 | 57 | _DTYPES = { 58 | 16: np.int16, 59 | } 60 | 61 | 62 | def get_np_dtype(fmt: GstVideo.VideoFormat) -> np.number: 63 | format_info = GstVideo.VideoFormat.get_info(fmt) 64 | return _DTYPES.get(format_info.bits, np.uint8) 65 | 66 | 67 | def fraction_to_str(fraction: Fraction) -> str: 68 | """Converts fraction to str""" 69 | return '{}/{}'.format(fraction.numerator, fraction.denominator) 70 | 71 | 72 | def gst_state_to_str(state: Gst.State) -> str: 73 | """Converts Gst.State to str representation 74 | 75 | Explained: https://lazka.github.io/pgi-docs/Gst-1.0/classes/Element.html#Gst.Element.state_get_name 76 | """ 77 | return Gst.Element.state_get_name(state) 78 | 79 | 80 | def gst_video_format_from_string(frmt: str) -> GstVideo.VideoFormat: 81 | return GstVideo.VideoFormat.from_string(frmt) 82 | 83 | 84 | def gst_buffer_to_ndarray(buffer: Gst.Buffer, *, width: int, height: int, channels: int, 85 | dtype: np.dtype, bpp: int = 1, do_copy: bool = False) -> np.ndarray: 86 | """Converts Gst.Buffer with known format (w, h, c, dtype) to np.ndarray""" 87 | 88 | result = None 89 | if do_copy: 90 | result = np.ndarray(buffer.get_size() // (bpp // BITS_PER_BYTE), 91 | buffer=buffer.extract_dup(0, buffer.get_size()), dtype=dtype) 92 | else: 93 | with map_gst_buffer(buffer, Gst.MapFlags.READ) as mapped: 94 | result = np.ndarray(buffer.get_size() // (bpp // BITS_PER_BYTE), 95 | buffer=mapped, dtype=dtype) 96 | if channels > 0: 97 | result = result.reshape(height, width, channels).squeeze() 98 | return result 99 | 100 | 101 | def gst_buffer_with_pad_to_ndarray(buffer: Gst.Buffer, pad: Gst.Pad, do_copy: bool = False) -> np.ndarray: 102 | """Converts Gst.Buffer with Gst.Pad (stores Gst.Caps) to np.ndarray """ 103 | return gst_buffer_with_caps_to_ndarray(buffer, pad.get_current_caps(), do_copy=do_copy) 104 | 105 | 106 | def gst_buffer_with_caps_to_ndarray(buffer: Gst.Buffer, caps: Gst.Caps, do_copy: bool = False) -> np.ndarray: 107 | """ Converts Gst.Buffer with Gst.Caps (stores buffer info) to np.ndarray """ 108 | 109 | structure = caps.get_structure(0) # Gst.Structure 110 | 111 | width, height = structure.get_value("width"), structure.get_value("height") 112 | 113 | # GstVideo.VideoFormat 114 | video_format = gst_video_format_from_string(structure.get_value('format')) 115 | 116 | channels = get_num_channels(video_format) 117 | 118 | dtype = get_np_dtype(video_format) # np.dtype 119 | 120 | format_info = GstVideo.VideoFormat.get_info(video_format) # GstVideo.VideoFormatInfo 121 | 122 | return gst_buffer_to_ndarray(buffer, width=width, height=height, channels=channels, 123 | dtype=dtype, bpp=format_info.bits, do_copy=do_copy) 124 | 125 | 126 | def get_buffer_size_from_gst_caps(caps: Gst.Caps) -> typ.Tuple[int, int]: 127 | """Returns buffers width, height from Gst.Caps """ 128 | structure = caps.get_structure(0) # Gst.Structure 129 | return structure.get_value("width"), structure.get_value("height") 130 | 131 | 132 | def ndarray_to_gst_buffer(array: np.ndarray) -> Gst.Buffer: 133 | """Converts numpy array to Gst.Buffer""" 134 | return Gst.Buffer.new_wrapped(array.tobytes()) 135 | 136 | 137 | def flatten_list(in_list: typ.List) -> typ.List: 138 | """Flattens list""" 139 | result = [] 140 | for item in in_list: 141 | if isinstance(item, list): 142 | result.extend(flatten_list(item)) 143 | else: 144 | result.append(item) 145 | return result 146 | 147 | 148 | def to_gst_string(plugins: typ.List[str]) -> str: 149 | """ Generates string representation from list of plugins """ 150 | 151 | if len(plugins) < 2: 152 | return "" 153 | 154 | plugins_ = flatten_list(plugins) 155 | 156 | # between plugins (except tee) 157 | return plugins_[0] + "".join(["{} {}".format('' if pl[-1] == '.' else ' !', pl) for pl in plugins_[1:]]) 158 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy>=1.14.3 2 | pycairo>=1.18.2 3 | PyGObject 4 | pytest>=5.3.2 5 | pytest-benchmark>=3.2.2 6 | attrs>=19.3.0 7 | autopep8>=1.4.4 8 | flake8>=3.7.9 9 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | setup for gstreamer-python package 6 | """ 7 | import os 8 | from pathlib import Path 9 | 10 | from setuptools import setup 11 | from setuptools.command.build_py import build_py as _build_py 12 | 13 | 14 | def read(file): 15 | return Path(file).read_text('utf-8').strip() 16 | 17 | 18 | class build_py(_build_py): 19 | 20 | user_options = _build_py.user_options + [ 21 | ('skip-gst-python', None, "Skip gst-python build"), 22 | ] 23 | 24 | boolean_options = _build_py.boolean_options + ['skip-gst-python'] 25 | 26 | def initialize_options(self): 27 | _build_py.initialize_options(self) 28 | self.skip_gst_python = None 29 | 30 | def finalize_options(self): 31 | _build_py.finalize_options(self) 32 | 33 | def run(self): 34 | import subprocess 35 | 36 | def _run_bash_file(bash_file: str): 37 | if os.path.isfile(bash_file): 38 | print("Running ... ", bash_file) 39 | _ = subprocess.run(bash_file, shell=True, 40 | executable="/bin/bash") 41 | else: 42 | print("Not found ", bash_file) 43 | 44 | cwd = os.path.dirname(os.path.abspath(__file__)) 45 | if not bool(self.skip_gst_python): 46 | _run_bash_file(os.path.join(cwd, 'build-gst-python.sh')) 47 | _run_bash_file(os.path.join(cwd, 'build-3rd-party.sh')) 48 | 49 | _build_py.run(self) 50 | 51 | 52 | install_requires = [ 53 | r for r in read('requirements.txt').split('\n') if r] 54 | 55 | setup( 56 | name='gstreamer-python', 57 | use_scm_version=True, 58 | setup_requires=['setuptools_scm'], 59 | description="PyGst Utils package", 60 | long_description='\n\n'.join((read('README.md'))), 61 | author="LifeStyleTransfer", 62 | author_email='taras@lifestyletransfer.com', 63 | url='https://github.com/jackersson/pygst-utils', 64 | packages=[ 65 | 'gstreamer', 66 | ], 67 | include_package_data=True, 68 | install_requires=install_requires, 69 | license="Apache Software License 2.0", 70 | zip_safe=True, 71 | keywords='gstreamer-python', 72 | classifiers=[ 73 | 'Development Status :: 2 - Pre-Alpha', 74 | 'Intended Audience :: Developers', 75 | 'License :: OSI Approved :: Apache Software License', 76 | 'Natural Language :: English', 77 | 'Programming Language :: Python :: 3.6', 78 | ], 79 | cmdclass={ 80 | 'build_py': build_py, 81 | } 82 | ) 83 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackersson/gstreamer-python/c8d4e04e1cdeb3b284641b981afcf304f50480db/tests/conftest.py -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import math 4 | import typing as typ 5 | from random import randint 6 | from fractions import Fraction 7 | 8 | import numpy as np 9 | import pytest 10 | 11 | from gstreamer import GstVideo, Gst 12 | import gstreamer as gst 13 | import gstreamer.utils as utils 14 | 15 | 16 | NUM_BUFFERS = 10 17 | WIDTH, HEIGHT = 1920, 1080 18 | FPS = 15 19 | FORMAT = "RGB" 20 | 21 | Frame = typ.NamedTuple( 22 | 'Frame', [ 23 | ('buffer_format', GstVideo.VideoFormat), 24 | ('buffer', np.ndarray), 25 | ]) 26 | 27 | 28 | FRAMES = [ 29 | Frame(GstVideo.VideoFormat.RGB, np.random.randint( 30 | low=0, high=255, size=(HEIGHT, WIDTH, 3), dtype=np.uint8)), 31 | Frame(GstVideo.VideoFormat.RGBA, np.random.randint( 32 | low=0, high=255, size=(HEIGHT, WIDTH, 4), dtype=np.uint8)), 33 | Frame(GstVideo.VideoFormat.GRAY8, np.random.randint( 34 | low=0, high=255, size=(HEIGHT, WIDTH), dtype=np.uint8)), 35 | Frame(GstVideo.VideoFormat.GRAY16_BE, np.random.uniform( 36 | 0, 1, (HEIGHT, WIDTH)).astype(np.float32)) 37 | ] 38 | 39 | 40 | def test_video_sink(): 41 | num_buffers = NUM_BUFFERS 42 | 43 | command = "appsrc emit-signals=True is-live=True ! videoconvert ! fakesink sync=false" 44 | 45 | for frame in FRAMES: 46 | h, w = frame.buffer.shape[:2] 47 | with gst.GstContext(), gst.GstVideoSink(command, width=w, height=h, video_frmt=frame.buffer_format) as pipeline: 48 | assert pipeline.total_buffers_count == 0 49 | 50 | # wait pipeline to initialize 51 | max_num_tries, num_tries = 5, 0 52 | while not pipeline.is_active and num_tries <= max_num_tries: 53 | time.sleep(.1) 54 | num_tries += 1 55 | 56 | assert pipeline.is_active 57 | 58 | for _ in range(num_buffers): 59 | pipeline.push(frame.buffer) 60 | 61 | assert pipeline.total_buffers_count == num_buffers 62 | 63 | 64 | def test_video_source(): 65 | num_buffers = NUM_BUFFERS 66 | width, height = WIDTH, HEIGHT 67 | 68 | formats = [GstVideo.VideoFormat.to_string(f.buffer_format) for f in FRAMES] 69 | 70 | for fmt in formats: 71 | caps_filter = 'capsfilter caps=video/x-raw,format={},width={},height={}'.format( 72 | fmt, width, height) 73 | command = 'videotestsrc num-buffers={} ! {} ! appsink emit-signals=True sync=false'.format( 74 | num_buffers, caps_filter) 75 | with gst.GstContext(), gst.GstVideoSource(command) as pipeline: 76 | 77 | num_read = 0 78 | while num_read < num_buffers: 79 | buffer = pipeline.pop() 80 | if buffer: 81 | num_read += 1 82 | h, w = buffer.data.shape[:2] 83 | assert h == height and w == width 84 | 85 | assert pipeline.total_buffers_count == num_buffers 86 | 87 | 88 | def test_gst_pipeline(): 89 | command = "videotestsrc num-buffers=100 ! fakesink sync=false" 90 | with gst.GstContext(), gst.GstPipeline(command) as pipeline: 91 | assert isinstance(pipeline, gst.GstPipeline) 92 | 93 | 94 | # @pytest.mark.skip 95 | def test_video_src_to_source(): 96 | 97 | num_buffers = NUM_BUFFERS 98 | 99 | for frame in FRAMES: 100 | buffer = frame.buffer 101 | h, w = buffer.shape[:2] 102 | 103 | sink_cmd = "appsrc emit-signals=True is-live=True ! videoconvert ! fakesink sync=false" 104 | 105 | fmt = GstVideo.VideoFormat.to_string(frame.buffer_format) 106 | caps_filter = f'capsfilter caps=video/x-raw,format={fmt},width={w},height={h}' 107 | src_cmd = f'videotestsrc num-buffers={num_buffers} ! {caps_filter} ! appsink emit-signals=True sync=false' 108 | 109 | with gst.GstContext(), gst.GstVideoSink(sink_cmd, width=w, height=h, video_frmt=frame.buffer_format) as sink, \ 110 | gst.GstVideoSource(src_cmd) as src: 111 | assert sink.total_buffers_count == 0 112 | 113 | # wait pipeline to initialize 114 | max_num_tries, num_tries = 5, 0 115 | while not sink.is_active and num_tries <= max_num_tries: 116 | time.sleep(.1) 117 | num_tries += 1 118 | 119 | assert sink.is_active 120 | 121 | num_read = 0 122 | while num_read < num_buffers: 123 | buffer = src.pop() 124 | if buffer: 125 | num_read += 1 126 | sink.push(buffer.data, pts=buffer.pts, 127 | dts=buffer.dts, offset=buffer.offset) 128 | 129 | assert src.total_buffers_count == num_buffers 130 | assert sink.total_buffers_count == num_buffers 131 | 132 | 133 | def test_metadata(): 134 | np_buffer = np.random.randint( 135 | low=0, high=255, size=(HEIGHT, WIDTH, 3), dtype=np.uint8) 136 | 137 | gst_buffer = gst.ndarray_to_gst_buffer(np_buffer) 138 | 139 | from gstreamer.gst_objects_info_meta import gst_meta_write, gst_meta_get, gst_meta_remove 140 | 141 | objects = [ 142 | {'class_name': "person", 'bounding_box': [ 143 | 8, 10, 100, 100], 'confidence': 0.6, 'track_id': 1}, 144 | {'class_name': "person", 'bounding_box': [ 145 | 10, 9, 120, 110], 'confidence': 0.67, 'track_id': 2}, 146 | ] 147 | 148 | # no metadata at the beginning 149 | assert len(gst_meta_get(gst_buffer)) == 0 150 | 151 | # write metadata 152 | gst_meta_write(gst_buffer, objects) 153 | 154 | # read metadata 155 | meta_objects = gst_meta_get(gst_buffer) 156 | assert len(gst_meta_get(gst_buffer)) == len(objects) 157 | 158 | for gst_meta_obj, py_obj in zip(meta_objects, objects): 159 | for key, val in py_obj.items(): 160 | if isinstance(gst_meta_obj[key], float): 161 | assert math.isclose(gst_meta_obj[key], val, rel_tol=1e-07) 162 | else: 163 | assert gst_meta_obj[key] == val 164 | 165 | # remove metadata 166 | gst_meta_remove(gst_buffer) 167 | assert len(gst_meta_get(gst_buffer)) == 0 168 | 169 | 170 | def test_gst_buffer_to_ndarray(): 171 | 172 | caps = Gst.Caps.from_string( 173 | "video/x-raw,format={},width={},height={}".format(FORMAT, WIDTH, HEIGHT)) 174 | 175 | video_format = utils.gst_video_format_from_string(FORMAT) 176 | channels = utils.get_num_channels(video_format) 177 | dtype = utils.get_np_dtype(video_format) 178 | 179 | npndarray = np.random.randint(low=0, high=255, size=( 180 | HEIGHT, WIDTH, channels), dtype=dtype) 181 | gst_buffer = utils.ndarray_to_gst_buffer(npndarray) 182 | 183 | res_npndarray = utils.gst_buffer_with_caps_to_ndarray(gst_buffer, caps) 184 | 185 | assert (npndarray == res_npndarray).all() 186 | 187 | 188 | def test_gst_buffer_channels(): 189 | 190 | assert bool(8 & (1 << (4 - 1))) 191 | assert utils.is_kbit_set(2, 2) 192 | assert utils.is_kbit_set(8, 4) 193 | assert utils.is_kbit_set(8 | 1, 1) 194 | 195 | assert utils.get_num_channels(GstVideo.VideoFormat.RGB) == 3 196 | assert utils.get_num_channels(GstVideo.VideoFormat.RGBA) == 4 197 | assert utils.get_num_channels(GstVideo.VideoFormat.GRAY8) == 1 198 | assert utils.get_num_channels(GstVideo.VideoFormat.I420) == -1 199 | 200 | --------------------------------------------------------------------------------