├── LICENSE ├── README.md ├── tcp_socket ├── README.md ├── client.py ├── monarch.png ├── server.py ├── utils.py └── video_grabber.py └── udp_socket ├── README.md ├── client.py ├── monarch.png ├── server.py ├── udp_packets.py ├── utils.py └── video_grabber.py /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Video streaming through TCP/UDP sockets : experimental python scripts 2 | 3 | This repository provides a simple python implementation of video streaming through UDP sockets with JPEG compression. 4 | 5 | 6 | **Ressources** 7 | 8 | - [python3 how to on sockets](https://docs.python.org/3/howto/sockets.html) 9 | - [Beej's guide to network programming](https://beej.us/guide/bgnet/) 10 | 11 | -------------------------------------------------------------------------------- /tcp_socket/README.md: -------------------------------------------------------------------------------- 1 | This is a TCP client/server. 2 | 3 | The client sends an image to the server which applies a processing and then sends back the result 4 | 5 | The image is encoded/decoded using [libjpeg-turbo python wrapper](https://github.com/lilohuang/PyTurboJPEG.git) 6 | 7 | To use it, run the server : 8 | 9 | python3 server.py --port 1081 --jpeg_quality 10 10 | 11 | And then the client 12 | 13 | python3 client.py --host localhost --port 1081 --jpeg_quality 10 14 | 15 | 16 | [https://docs.python.org/3/howto/sockets.html](Python3 how to on sockets) 17 | 18 | -------------------------------------------------------------------------------- /tcp_socket/client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Standard modules 4 | import argparse 5 | import socket 6 | import sys 7 | import time 8 | import functools 9 | # External modules 10 | import cv2 11 | # Local modules 12 | import video_grabber 13 | import utils 14 | 15 | parser = argparse.ArgumentParser() 16 | 17 | parser.add_argument('--host', type=str, 18 | help='The IP of the echo server', 19 | required=True) 20 | parser.add_argument('--port', type=int, 21 | help='The port on which the server is listening', 22 | required=True) 23 | parser.add_argument('--jpeg_quality', type=int, 24 | help='The JPEG quality for compressing the reply', 25 | default=50) 26 | parser.add_argument('--resize', type=float, 27 | help='Resize factor of the image', 28 | default=1.0) 29 | parser.add_argument('--encoder', type=str, choices=['cv2', 'turbo'], 30 | help='Library to use to encode/decode in JPEG the images', 31 | default='cv2') 32 | parser.add_argument('--image', type=str, 33 | help='Image file to be processed', 34 | default=None) 35 | 36 | args = parser.parse_args() 37 | 38 | host = args.host 39 | port = args.port 40 | jpeg_quality = args.jpeg_quality 41 | resize_factor = args.resize 42 | 43 | cv2.namedWindow("Image") 44 | 45 | keep_running = True 46 | 47 | jpeg_handler = utils.make_jpeg_handler(args.encoder, jpeg_quality) 48 | 49 | if args.image is not None: 50 | grabber = None 51 | img = cv2.imread(args.image, cv2.IMREAD_UNCHANGED) 52 | get_buffer = functools.partial(jpeg_handler.compress, cv2_img=img) 53 | else: 54 | grabber = video_grabber.VideoGrabber(jpeg_quality, 55 | args.encoder, 56 | resize_factor) 57 | grabber.start() 58 | get_buffer = grabber.get_buffer 59 | 60 | # img = cv2.imread("monarch.png", cv2.IMREAD_UNCHANGED) 61 | # get_buffer = lambda: utils.encode_image(img, jpeg, jpeg_quality) 62 | 63 | # A temporary buffer in which the received data will be copied 64 | # this prevents creating a new buffer all the time 65 | tmp_buf = bytearray(7) 66 | # this allows to get a reference to a slice of tmp_buf 67 | tmp_view = memoryview(tmp_buf) 68 | 69 | # Creates a temporary buffer which can hold the largest image we can transmit 70 | img_buf = bytearray(9999999) 71 | img_view = memoryview(img_buf) 72 | 73 | idx = 0 74 | t0 = time.time() 75 | 76 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: 77 | sock.connect((host, port)) 78 | while keep_running: 79 | 80 | # Grab and encode the image 81 | img_buffer = get_buffer() 82 | if img_buffer is None: 83 | continue 84 | 85 | # Prepare the message with the number of bytes going to be sent 86 | msg = bytes("image{:07}".format(len(img_buffer)), "ascii") 87 | 88 | utils.send_data(sock, msg) 89 | 90 | # Send the buffer 91 | utils.send_data(sock, img_buffer) 92 | 93 | # Read the reply command 94 | utils.recv_data_into(sock, tmp_view[:5], 5) 95 | cmd = tmp_buf[:5].decode('ascii') 96 | 97 | if cmd != 'image': 98 | raise RuntimeError("Unexpected server reply") 99 | 100 | # Read the image buffer size 101 | utils.recv_data_into(sock, tmp_view, 7) 102 | img_size = int(tmp_buf.decode('ascii')) 103 | 104 | # Read the image buffer 105 | utils.recv_data_into(sock, img_view[:img_size], img_size) 106 | 107 | # Read the final handshake 108 | cmd = utils.recv_data(sock, 5).decode('ascii') 109 | if cmd != 'enod!': 110 | raise RuntimeError("Unexpected server reply. Expected 'enod!'" 111 | ", got '{}'".format(cmd)) 112 | 113 | # Transaction is done, we now process/display the received image 114 | img = jpeg_handler.decompress(img_view[:img_size]) 115 | cv2.imshow("Image", img) 116 | keep_running = not(cv2.waitKey(1) & 0xFF == ord('q')) 117 | if not keep_running: 118 | sock.sendall('quit!'.encode('ascii')) 119 | 120 | idx += 1 121 | if idx == 30: 122 | t1 = time.time() 123 | sys.stdout.write("\r {:.3} images/second ; msg size : {} ". 124 | format(30/(t1-t0), img_size)) 125 | sys.stdout.flush() 126 | t0 = t1 127 | idx = 0 128 | print() 129 | print("Closing the socket") 130 | if grabber is not None: 131 | print("Stopping the grabber") 132 | grabber.stop() 133 | -------------------------------------------------------------------------------- /tcp_socket/monarch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremyfix/udp_video_streaming/3a72d0a368514f9785d70291cc7167e91122809f/tcp_socket/monarch.png -------------------------------------------------------------------------------- /tcp_socket/server.py: -------------------------------------------------------------------------------- 1 | # Standard modules 2 | import argparse 3 | import socket 4 | # Local modules 5 | import utils 6 | 7 | 8 | def image_process(cv2_img): 9 | # For fun, we play with the image 10 | cv2_img = 255 - cv2_img 11 | return cv2_img 12 | 13 | 14 | parser = argparse.ArgumentParser() 15 | 16 | parser.add_argument('--port', type=int, 17 | help="The port on which to listen" 18 | " for incoming connections", 19 | required=True) 20 | parser.add_argument('--jpeg_quality', type=int, 21 | help='The JPEG quality for compressing the reply', 22 | default=50) 23 | parser.add_argument('--encoder', type=str, choices=['cv2', 'turbo'], 24 | help="Which library to use to encode/decode in JPEG " 25 | "the images", 26 | default='cv2') 27 | args = parser.parse_args() 28 | 29 | host = '' # any interface 30 | port = args.port 31 | jpeg_quality = args.jpeg_quality 32 | 33 | jpeg_handler = utils.make_jpeg_handler(args.encoder, jpeg_quality) 34 | 35 | # A temporary buffer in which the received data will be copied 36 | # this prevents creating a new buffer all the time 37 | tmp_buf = bytearray(7) 38 | # this allows to get a reference to a slice of tmp_buf 39 | tmp_view = memoryview(tmp_buf) 40 | 41 | # Creates a temporary buffer which can hold the largest image we can transmit 42 | img_buf = bytearray(9999999) 43 | img_view = memoryview(img_buf) 44 | 45 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: 46 | s.bind((host, port)) 47 | s.listen(1) 48 | conn, addr = s.accept() 49 | with conn: 50 | print('Connected by', addr) 51 | while True: 52 | utils.recv_data_into(conn, tmp_view[:5], 5) 53 | cmd = tmp_buf[:5].decode('ascii') 54 | if(cmd == 'image'): 55 | # Read the image buffer size 56 | utils.recv_data_into(conn, tmp_view, 7) 57 | img_size = int(tmp_buf.decode('ascii')) 58 | 59 | # Read the buffer content 60 | utils.recv_data_into(conn, img_view[:img_size], img_size) 61 | 62 | # Decode the image 63 | img = jpeg_handler.decompress(img_view[:img_size]) 64 | 65 | # Process it 66 | res = image_process(img) 67 | 68 | # Encode the image 69 | res_buffer = jpeg_handler.compress(res) 70 | 71 | # Make the reply 72 | reply = bytes("image{:07}".format(len(res_buffer)), "ascii") 73 | utils.send_data(conn, reply) 74 | utils.send_data(conn, res_buffer) 75 | utils.send_data(conn, bytes('enod!', 'ascii')) 76 | elif cmd == 'quit!': 77 | break 78 | else: 79 | print("Got something else") 80 | print("Quitting") 81 | -------------------------------------------------------------------------------- /tcp_socket/utils.py: -------------------------------------------------------------------------------- 1 | # External modules 2 | import cv2 3 | import numpy as np 4 | try: 5 | from turbojpeg import TurboJPEG 6 | except ImportError: 7 | print("Warning, failed to import turbojpeg") 8 | 9 | 10 | class CV2JpegHandler: 11 | """JPEG compression/decompression handled with CV2""" 12 | def __init__(self, jpeg_quality): 13 | self.encode_params = [int(cv2.IMWRITE_JPEG_QUALITY), jpeg_quality] 14 | 15 | def compress(self, cv2_img): 16 | _, buf = cv2.imencode('.jpg', cv2_img, self.encode_params) 17 | return buf.tobytes() 18 | 19 | def decompress(self, img_buffer): 20 | img_array = np.frombuffer(img_buffer, dtype=np.dtype('uint8')) 21 | # Decode a colored image 22 | return cv2.imdecode(img_array, flags=cv2.IMREAD_UNCHANGED) 23 | 24 | 25 | class TurboJpegHandler(object): 26 | """The object handling JPEG compression/decompression""" 27 | def __init__(self, jpeg_quality): 28 | self.jpeg_quality = jpeg_quality 29 | self.jpeg = TurboJPEG() 30 | 31 | def compress(self, cv2_img): 32 | return self.jpeg.encode(cv2_img, quality=self.jpeg_quality) 33 | 34 | def decompress(self, img_buffer): 35 | return self.jpeg.decode(img_buffer) 36 | 37 | 38 | def make_jpeg_handler(name, jpeg_quality): 39 | if name == 'cv2': 40 | return CV2JpegHandler(jpeg_quality) 41 | elif name == 'turbo': 42 | return TurboJpegHandler(jpeg_quality) 43 | else: 44 | raise ValueError("Unknow Jpeg handler {}".format(name)) 45 | 46 | 47 | def send_data(sock, msg): 48 | totalsent = 0 49 | tosend = len(msg) 50 | while totalsent < tosend: 51 | numsent = sock.send(msg[totalsent:]) 52 | if numsent == 0: 53 | raise RuntimeError("Socket connection broken") 54 | totalsent += numsent 55 | 56 | 57 | def recv_data(sock, torecv): 58 | msg = b'' 59 | while torecv > 0: 60 | chunk = sock.recv(torecv) 61 | if chunk == b'': 62 | raise RuntimeError("Socket connection broken") 63 | msg += chunk 64 | torecv -= len(chunk) 65 | return msg 66 | 67 | 68 | def recv_data_into(sock, buf_view, torecv): 69 | while torecv > 0: 70 | numrecv = sock.recv_into(buf_view[-torecv:], torecv) 71 | if numrecv == 0: 72 | raise RuntimeError("Socket connection broken") 73 | torecv -= numrecv 74 | -------------------------------------------------------------------------------- /tcp_socket/video_grabber.py: -------------------------------------------------------------------------------- 1 | # Standard modules 2 | from threading import Thread, Lock 3 | import time 4 | import sys 5 | # External modules 6 | import cv2 7 | # Local modules 8 | import utils 9 | 10 | 11 | try: 12 | from turbojpeg import TurboJPEG 13 | except ImportError as error: 14 | print("Warning, failed to import turbojpeg, " 15 | "you will not be able to use it") 16 | 17 | 18 | class VideoGrabber(Thread): 19 | """A threaded video grabber. 20 | 21 | Attributes: 22 | encode_params (): 23 | cap (str): 24 | attr2 (:obj:`int`, optional): Description of `attr2`. 25 | 26 | """ 27 | def __init__(self, jpeg_quality, jpeg_lib, resize): 28 | """Constructor. 29 | 30 | Args: 31 | jpeg_quality (:obj:`int`): Quality of JPEG encoding, in 0, 100. 32 | resize (:obj:`float'): resize factor in [0, 1] 33 | 34 | """ 35 | Thread.__init__(self) 36 | self.cap = cv2.VideoCapture(0) 37 | self.turbojpeg = TurboJPEG() 38 | self.resize_factor = resize 39 | self.running = True 40 | self.buffer = None 41 | self.lock = Lock() 42 | 43 | self.jpeg_handler = utils.make_jpeg_handler(jpeg_lib, jpeg_quality) 44 | 45 | def stop(self): 46 | self.running = False 47 | 48 | def get_buffer(self): 49 | """Method to access the encoded buffer. 50 | 51 | Returns: 52 | np.ndarray: the compressed image if one has been acquired. 53 | None otherwise. 54 | """ 55 | if self.buffer is not None: 56 | self.lock.acquire() 57 | cpy = self.buffer 58 | self.lock.release() 59 | return cpy 60 | 61 | def run(self): 62 | while self.running: 63 | success, img = self.cap.read() 64 | target_size = (int(img.shape[1] * self.resize_factor), 65 | int(img.shape[0] * self.resize_factor)) 66 | img = cv2.resize(img, target_size) 67 | if not success: 68 | continue 69 | 70 | # JPEG compression 71 | # Protected by a lock 72 | # As the main thread may asks to access the buffer 73 | self.lock.acquire() 74 | self.buffer = self.jpeg_handler.compress(img) 75 | self.lock.release() 76 | 77 | 78 | if __name__ == '__main__': 79 | 80 | jpeg_quality = 100 81 | 82 | grabber = VideoGrabber(jpeg_quality, jpeg_lib='turbo') 83 | grabber.start() 84 | time.sleep(1) 85 | 86 | turbo_jpeg = TurboJPEG() 87 | 88 | cv2.namedWindow("Image") 89 | 90 | keep_running = True 91 | idx = 0 92 | t0 = time.time() 93 | 94 | while keep_running: 95 | data = grabber.get_buffer() 96 | if data is None: 97 | time.sleep(1) 98 | continue 99 | img = turbo_jpeg.decode(data) 100 | cv2.imshow("Image", img) 101 | keep_running = not(cv2.waitKey(1) & 0xFF == ord('q')) 102 | 103 | idx += 1 104 | if idx == 100: 105 | t1 = time.time() 106 | sys.stdout.write("\r {:04} images/second ".format(100/(t1-t0))) 107 | sys.stdout.flush() 108 | t0 = t1 109 | idx = 0 110 | 111 | print() 112 | print("Quitting") 113 | grabber.stop() 114 | -------------------------------------------------------------------------------- /udp_socket/README.md: -------------------------------------------------------------------------------- 1 | # Video streaming through UDP sockets 2 | 3 | This repository provides a simple python implementation of video streaming through UDP sockets with JPEG compression. So far, it sends a single datagram. If the buffer is larger than what I think is the largest UDP datagram size, 65507, the client fails. 4 | 5 | A work in progress is to add an appropriate header to allow chunking an image into multiple datagrams. 6 | 7 | # How to use it ? 8 | 9 | In order to test it, run the server : 10 | 11 | python3 server.py --port 10080 12 | 13 | In another tab, start the client : 14 | 15 | python3 client.py --host localhost --port 10080 16 | 17 | Pressing 'q' on the client side within the CV2 window will isue a quit command to the client and server. 18 | 19 | -------------------------------------------------------------------------------- /udp_socket/client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import socket 5 | import cv2 6 | import numpy as np 7 | import sys 8 | import time 9 | import argparse 10 | 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument('--host', type=str, help='The IP at the server is listening', required=True) 13 | parser.add_argument('--port', type=int, help='The port on which the server is listening', required=True) 14 | parser.add_argument('--jpeg_quality', type=int, help='The JPEG quality for compressing the reply', default=50) 15 | parser.add_argument('--encoder', type=str, choices=['cv2','turbo'], help='Which library to use to encode/decode in JPEG the images', default='cv2') 16 | 17 | args = parser.parse_args() 18 | 19 | 20 | 21 | 22 | # Create a UDP socket 23 | sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 24 | host = args.host 25 | port = args.port 26 | server_address = (host, port) 27 | 28 | cv2.namedWindow("Image") 29 | 30 | t0 = time.time() 31 | frame_idx = 0 32 | 33 | while(True): 34 | sent = sock.sendto("get".encode('utf-8'), server_address) 35 | 36 | data, server = sock.recvfrom(65507) 37 | if len(data) == 4: 38 | # This is a message error sent back by the server 39 | if(data == "FAIL"): 40 | continue 41 | array = np.frombuffer(data, dtype=np.dtype('uint8')) 42 | img = cv2.imdecode(array, 1) 43 | cv2.imshow("Image", img) 44 | if cv2.waitKey(1) & 0xFF == ord('q'): 45 | print("Asking the server to quit") 46 | sock.sendto("quit".encode('utf-8'), server_address) 47 | print("Quitting") 48 | break 49 | frame_idx += 1 50 | 51 | if frame_idx == 30: 52 | t1 = time.time() 53 | sys.stdout.write('\r Framerate : {:.2f} frames/s. '.format(30 / (t1 - t0))) 54 | sys.stdout.flush() 55 | t0 = t1 56 | frame_idx = 0 57 | -------------------------------------------------------------------------------- /udp_socket/monarch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremyfix/udp_video_streaming/3a72d0a368514f9785d70291cc7167e91122809f/udp_socket/monarch.png -------------------------------------------------------------------------------- /udp_socket/server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import socket 5 | import cv2 6 | import sys 7 | import argparse 8 | 9 | import video_grabber 10 | 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument('--port', type=int, help='The port on which the server is listening', required=True) 13 | parser.add_argument('--jpeg_quality', type=int, help='The JPEG quality for compressing the reply', default=50) 14 | parser.add_argument('--encoder', type=str, choices=['cv2','turbo'], help='Which library to use to encode/decode in JPEG the images', default='cv2') 15 | 16 | args = parser.parse_args() 17 | 18 | jpeg_quality = args.jpeg_quality 19 | host = '' 20 | port = args.port 21 | encoder = args.encoder 22 | 23 | # The grabber of the webcam 24 | grabber = video_grabber.VideoGrabber(jpeg_quality, encoder) 25 | grabber.start() 26 | get_message = lambda: grabber.get_buffer() 27 | 28 | keep_running = True 29 | 30 | sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 31 | 32 | # Bind the socket to the port 33 | server_address = (host, port) 34 | 35 | print('starting up on %s port %s\n' % server_address) 36 | 37 | sock.bind(server_address) 38 | 39 | while(keep_running): 40 | data, address = sock.recvfrom(4) 41 | data = data.decode('utf-8') 42 | if(data == "get"): 43 | buffer = get_message() 44 | if buffer is None: 45 | continue 46 | if len(buffer) > 65507: 47 | print("The message is too large to be sent within a single UDP datagram. We do not handle splitting the message in multiple datagrams") 48 | sock.sendto("FAIL".encode('utf-8'),address) 49 | continue 50 | # We send back the buffer to the client 51 | sock.sendto(buffer, address) 52 | elif(data == "quit"): 53 | grabber.stop() 54 | keep_running = False 55 | 56 | print("Quitting..") 57 | grabber.join() 58 | sock.close() 59 | -------------------------------------------------------------------------------- /udp_socket/udp_packets.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | class UdpPacket: 4 | 5 | header_size = 16 # in bytes 6 | 7 | def __init__(self, msg_idx: int, pkt_idx: int, num_pkts: int, data: memoryview): 8 | self.msg_idx = msg_idx # The current message idx 9 | self.pkt_idx = pkt_idx # The current packet idx 10 | self.num_pkts = num_pkts # The total number of packets 11 | self.data = data # The data buffer 12 | self.header = self.msg_idx.to_bytes (4, 'big')+\ 13 | self.pkt_idx.to_bytes (4, 'big')+\ 14 | self.num_pkts.to_bytes (4, 'big')+\ 15 | len(self.data).to_bytes(4, 'big') 16 | 17 | 18 | def decode(msg: bytes): 19 | msg_idx = int.from_bytes(msg[:4], 'big') 20 | pkt_idx = int.from_bytes(msg[4:8], 'big') 21 | num_pkts = int.from_bytes(msg[8:12], 'big') 22 | data_size = int.from_bytes(msg[12:16], 'big') 23 | 24 | p = UdpPacket(msg_idx, pkt_idx, num_pkts, msg[16:16+data_size]) 25 | return p 26 | 27 | def encode(self): 28 | return self.header + self.data 29 | 30 | class UdpPacketsHandler: 31 | 32 | def __init__(self): 33 | self.current_msg_idx = None 34 | self.packets = [] 35 | self.awaited_packets = None 36 | 37 | 38 | def process_packet(self, p: UdpPacket): 39 | if (self.current_msg_idx is None) or\ 40 | (p.msg_idx > self.current_msg_idx): 41 | # This is the first time we receive a packet 42 | # or we get more recent packets and therefore drop all the 43 | # packets collected so far 44 | self.current_msg_idx = p.msg_idx 45 | self.packets = [b''] * p.num_pkts 46 | self.awaited_packets = p.num_pkts 47 | 48 | if p.msg_idx < self.current_msg_idx: 49 | # Drop the frame if too old 50 | return 51 | 52 | # We now place the current piece at the right place 53 | self.packets[p.pkt_idx] = p.data 54 | self.awaited_packets -= 1 55 | 56 | # If we collected all the packets, we can build up the full message 57 | if self.awaited_packets == 0: 58 | return b''.join(self.packets) 59 | 60 | def split_data(msg_idx: int, data: bytes, max_packet_size: int): 61 | """ 62 | return : a list of UdpPacket ready to be sent 63 | """ 64 | 65 | data_chunk_size = max_packet_size - UdpPacket.header_size 66 | num_packets = math.ceil(len(data) / data_chunk_size) 67 | packets = [] 68 | 69 | # We build a memory view to a get 0 copy 70 | dataview = memoryview(data) 71 | 72 | print("Building {} packets".format(num_packets)) 73 | 74 | for i in range(num_packets - 1): 75 | packets.append(UdpPacket(msg_idx, i, num_packets, dataview[i*data_chunk_size:(i+1)*data_chunk_size])) 76 | # The last packet 77 | packets.append(UdpPacket(msg_idx, num_packets - 1, num_packets, dataview[(num_packets-1)*data_chunk_size:])) 78 | return packets 79 | 80 | 81 | if __name__ == '__main__': 82 | 83 | import random 84 | import utils 85 | import cv2 86 | 87 | img = cv2.imread('monarch.png') 88 | 89 | # Build up a collection of messages 90 | packets = [] 91 | 92 | img0_jpeg = utils.cv2_encode_image(img, 10) 93 | img1_jpeg = utils.cv2_encode_image(img, 100) 94 | 95 | packets += UdpPacketsHandler.split_data(msg_idx=0, data=img0_jpeg, max_packet_size=2048) 96 | packets += UdpPacketsHandler.split_data(msg_idx=1, data=img1_jpeg, max_packet_size=60000) 97 | 98 | # Shuffle the packets to see if we can handle disordered packets 99 | random.shuffle(packets) 100 | 101 | print("A total of {} packets are considered sequentially".format(len(packets))) 102 | # 103 | 104 | packet_processor = UdpPacketsHandler() 105 | for p in packets: 106 | data = packet_processor.process_packet(p) 107 | if data is not None: 108 | print("Got a frame !") 109 | img = utils.cv2_decode_image_buffer(data) 110 | cv2.imshow('Image', img) 111 | cv2.waitKey() 112 | 113 | 114 | -------------------------------------------------------------------------------- /udp_socket/utils.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | 4 | def cv2_decode_image_buffer(img_buffer): 5 | img_array = np.frombuffer(img_buffer, dtype=np.dtype('uint8')) 6 | # Decode a colored image 7 | return cv2.imdecode(img_array, flags=cv2.IMREAD_UNCHANGED) 8 | 9 | def cv2_encode_image(cv2_img, jpeg_quality): 10 | encode_params = [int(cv2.IMWRITE_JPEG_QUALITY), jpeg_quality] 11 | result, buf = cv2.imencode('.jpg', cv2_img, encode_params) 12 | return buf.tobytes() 13 | 14 | def turbo_decode_image_buffer(img_buffer, jpeg): 15 | return jpeg.decode(img_buffer) 16 | 17 | def turbo_encode_image(cv2_img, jpeg, jpeg_quality): 18 | return jpeg.encode(cv2_img, quality=jpeg_quality) 19 | 20 | def send_data(sock, msg): 21 | totalsent = 0 22 | tosend = len(msg) 23 | while totalsent < tosend: 24 | numsent = sock.send(msg[totalsent:]) 25 | if numsent == 0: 26 | raise RuntimeError("Socket connection broken") 27 | totalsent += numsent 28 | 29 | def recv_data(sock, torecv): 30 | msg = b'' 31 | while torecv > 0: 32 | chunk = sock.recv(torecv) 33 | if chunk == b'': 34 | raise RuntimeError("Socket connection broken") 35 | msg += chunk 36 | torecv -= len(chunk) 37 | return msg 38 | 39 | def recv_data_into(sock, buf_view, torecv): 40 | while torecv > 0: 41 | numrecv = sock.recv_into(buf_view[-torecv:], torecv) 42 | if numrecv == 0: 43 | raise RuntimeError("Socket connection broken") 44 | torecv -= numrecv 45 | -------------------------------------------------------------------------------- /udp_socket/video_grabber.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | from threading import Thread, Lock 4 | import time 5 | import sys 6 | 7 | try: 8 | from turbojpeg import TurboJPEG 9 | except: 10 | pass 11 | 12 | import utils 13 | 14 | class VideoGrabber(Thread): 15 | """A threaded video grabber. 16 | 17 | Attributes: 18 | encode_params (): 19 | cap (str): 20 | attr2 (:obj:`int`, optional): Description of `attr2`. 21 | 22 | """ 23 | def __init__(self, jpeg_quality, jpeg_lib): 24 | """Constructor. 25 | 26 | Args: 27 | jpeg_quality (:obj:`int`): Quality of JPEG encoding, in 0, 100. 28 | 29 | """ 30 | Thread.__init__(self) 31 | self.cap = cv2.VideoCapture(0) 32 | self.turbojpeg = TurboJPEG() 33 | self.running = True 34 | self.buffer = None 35 | self.lock = Lock() 36 | 37 | if jpeg_lib == 'turbo': 38 | self.jpeg = TurboJPEG() 39 | self.jpeg_encode_func = lambda img, jpeg_quality=jpeg_quality, jpeg=self.jpeg: utils.turbo_encode_image(img, jpeg, jpeg_quality) 40 | 41 | else: 42 | self.jpeg_encode_func = lambda img, jpeg_quality=jpeg_quality: utils.cv2_encode_image(img, jpeg_quality) 43 | 44 | 45 | def stop(self): 46 | self.running = False 47 | 48 | def get_buffer(self): 49 | """Method to access the encoded buffer. 50 | 51 | Returns: 52 | np.ndarray: the compressed image if one has been acquired. None otherwise. 53 | """ 54 | if self.buffer is not None: 55 | self.lock.acquire() 56 | cpy = self.buffer 57 | self.lock.release() 58 | return cpy 59 | 60 | def run(self): 61 | while self.running: 62 | success, img = self.cap.read() 63 | if not success: 64 | continue 65 | 66 | # JPEG compression 67 | # Protected by a lock 68 | # As the main thread may asks to access the buffer 69 | self.lock.acquire() 70 | self.buffer = self.jpeg_encode_func(img) 71 | self.lock.release() 72 | 73 | 74 | if __name__ == '__main__': 75 | 76 | jpeg_quality = 100 77 | 78 | grabber = VideoGrabber(jpeg_quality, jpeg_lib='turbo') 79 | grabber.start() 80 | time.sleep(1) 81 | 82 | turbo_jpeg = TurboJPEG() 83 | 84 | cv2.namedWindow("Image") 85 | 86 | keep_running = True 87 | idx = 0 88 | t0 = time.time() 89 | 90 | while keep_running: 91 | data = grabber.get_buffer() 92 | if data is None: 93 | time.sleep(1) 94 | continue 95 | img = turbo_jpeg.decode(data) 96 | cv2.imshow("Image", img) 97 | keep_running = not(cv2.waitKey(1) & 0xFF == ord('q')) 98 | 99 | idx += 1 100 | if idx == 100: 101 | t1 = time.time() 102 | sys.stdout.write("\r {:04} images/second ".format(100/(t1-t0))) 103 | sys.stdout.flush() 104 | t0 = t1 105 | idx = 0 106 | 107 | print() 108 | print("Quitting") 109 | grabber.stop() 110 | 111 | --------------------------------------------------------------------------------