├── .gitignore ├── LICENSE ├── README.md ├── consumer.py ├── producer.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | video.mkv 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Christos Kotsis 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kafka Video Stream 2 | 3 | Visit my blog for additional info and instructions: [Kafka-Video-Streaming](https://ulfox.github.io/blog/kafka/2021-04-10/kafka-video-stream/) 4 | 5 | ## Setup and start producer 6 | 7 | Edit producer.py and change `bootstrap_servers` under __main__, then start producing by issuing: 8 | 9 | python3 producer.py myvideo 10 | 11 | 12 | ## Setup and start consumer 13 | 14 | Update bootstrap servers also in consumer.py and issue: 15 | 16 | python3 consumer.py 17 | 18 | 19 | Video should start playing in your screen. To stop, simply presh button 'q'. 20 | 21 | -------------------------------------------------------------------------------- /consumer.py: -------------------------------------------------------------------------------- 1 | from kafka import KafkaConsumer 2 | from time import sleep 3 | import cv2 4 | import numpy as np 5 | from queue import Queue 6 | from threading import Thread 7 | from threading import Event 8 | 9 | class kafkaVideoView(): 10 | def __init__(self, bootstrap_servers, topic, client_id, group_id, poll=500, frq=0.01): 11 | self.topic = topic 12 | self.client_id = client_id 13 | self.group_id = group_id 14 | self.bootstrap_servers = bootstrap_servers 15 | self.poll = poll 16 | self.frq = frq 17 | 18 | def setConsumer(self): 19 | self.consumer = KafkaConsumer( 20 | self.topic, 21 | bootstrap_servers=self.bootstrap_servers.split(','), 22 | fetch_max_bytes=52428800, 23 | fetch_max_wait_ms=1000, 24 | fetch_min_bytes=1, 25 | max_partition_fetch_bytes=1048576, 26 | value_deserializer=None, 27 | key_deserializer=None, 28 | max_in_flight_requests_per_connection=10, 29 | client_id=self.client_id, 30 | group_id=self.group_id, 31 | auto_offset_reset='earliest', 32 | max_poll_records=self.poll, 33 | max_poll_interval_ms=300000, 34 | heartbeat_interval_ms=3000, 35 | session_timeout_ms=10000, 36 | enable_auto_commit=True, 37 | auto_commit_interval_ms=5000, 38 | reconnect_backoff_ms=50, 39 | reconnect_backoff_max_ms=500, 40 | request_timeout_ms=305000, 41 | receive_buffer_bytes=32768, 42 | ) 43 | 44 | def playStream(self, queue): 45 | while self.keepPlaying: 46 | try: 47 | msg = queue.get(block=True, timeout=20) 48 | self.queue_status = True 49 | except: 50 | print("WARN: Timed out waiting for queue. Retrying...") 51 | self.queue_status = False 52 | 53 | if self.queue_status: 54 | nparr = np.frombuffer(msg, np.uint8) 55 | frame = cv2.imdecode(nparr, cv2.IMREAD_COLOR) 56 | cv2.imshow('frame', frame) 57 | 58 | if cv2.waitKey(1) & 0xFF == ord('q'): 59 | self.keepConsuming = False 60 | break 61 | 62 | sleep(self.frq) 63 | 64 | def run(self): 65 | self.keepPlaying = True 66 | self.setConsumer() 67 | self.videoQueue = Queue() 68 | self.keepConsuming = True 69 | 70 | self.playerThread = Thread(target=self.playStream, args=(self.videoQueue, ), daemon=False) 71 | self.playerThread.start() 72 | 73 | try: 74 | while self.keepConsuming: 75 | payload = self.consumer.poll(self.poll) 76 | for bucket in payload: 77 | for msg in payload[bucket]: 78 | self.videoQueue.put(msg.value) 79 | 80 | except KeyboardInterrupt: 81 | self.keepConsuming = False 82 | self.keepPlaying = False 83 | print("WARN: Keyboard Interrupt detected. Exiting...") 84 | 85 | self.playerThread.join() 86 | 87 | 88 | if __name__ == "__main__": 89 | streamVideoPlayer = kafkaVideoView( 90 | bootstrap_servers='localhost:9092', 91 | topic='KafkaVideoStream', 92 | client_id='KafkaVSClient', 93 | group_id='KafkaVideoStreamConsumer', 94 | poll=500, 95 | frq=0.025 96 | ) 97 | 98 | streamVideoPlayer.run() 99 | -------------------------------------------------------------------------------- /producer.py: -------------------------------------------------------------------------------- 1 | from sys import argv, exit 2 | from time import sleep 3 | import cv2 4 | from kafka import KafkaProducer 5 | 6 | class kafkaVideoStreaming(): 7 | def __init__(self, bootstrap_servers, topic, videoFile, client_id, batch_size=65536, frq=0.001): 8 | self.videoFile = videoFile 9 | self.topicKey = str(videoFile) 10 | self.topic = topic 11 | self.batch_size = batch_size 12 | self.client_id = client_id 13 | self.bootstrap_servers = bootstrap_servers 14 | self.frq = frq 15 | 16 | def setProducer(self): 17 | self.producer = KafkaProducer( 18 | bootstrap_servers=self.bootstrap_servers, 19 | api_version=(0,10,1), 20 | client_id=self.client_id, 21 | acks=1, 22 | value_serializer=None, 23 | key_serializer=str.encode, 24 | batch_size=self.batch_size, 25 | compression_type='gzip', 26 | linger_ms=0, 27 | buffer_memory=67108864, 28 | max_request_size=1048576, 29 | max_in_flight_requests_per_connection=1, 30 | retries=1, 31 | ) 32 | 33 | def reportCallback(self, record_metadata): 34 | print("Topic Record Metadata: ", record_metadata.topic) 35 | print("Parition Record Metadata: ", record_metadata.partition) 36 | print("Offset Record Metatada: ", record_metadata.offset) 37 | 38 | def errCallback(self, excp): 39 | print('Errback', excp) 40 | 41 | def publishFrames(self, payload): 42 | self.producer.send( 43 | topic=self.topic, key=self.topicKey, value=payload 44 | ).add_callback( 45 | self.reportCallback 46 | ).add_errback( 47 | self.errCallback 48 | ) 49 | 50 | def run(self): 51 | try: 52 | print("Opening file %s" % self.videoFile) 53 | __VIDEO_FILE = cv2.VideoCapture(self.videoFile) 54 | except: 55 | raise 56 | 57 | self.setProducer() 58 | 59 | print( 60 | "Publishing: %{v}\n\ 61 | \tBatch Size: {b},\n\ 62 | \tSleep ({t}) \n\ 63 | \tTarget Topic: {t} \n\ 64 | \tHost: {h}".format( 65 | v=self.topicKey, 66 | b=self.batch_size, 67 | t=self.topic, 68 | h=self.bootstrap_servers 69 | ) 70 | ) 71 | 72 | self.keep_processing = True 73 | try: 74 | while(__VIDEO_FILE.isOpened()) and self.keep_processing: 75 | readStat, frame = __VIDEO_FILE.read() 76 | 77 | if not readStat: 78 | self.keep_processing = False 79 | 80 | ret, buffer = cv2.imencode('.jpg', frame) 81 | self.publishFrames(buffer.tostring()) 82 | 83 | sleep(self.frq) 84 | 85 | 86 | if self.keep_processing: 87 | print('Finished processing video %s' % self.topicKey) 88 | else: 89 | print("Error while reading %s" % self.topicKey) 90 | 91 | __VIDEO_FILE.release() 92 | except KeyboardInterrupt: 93 | __VIDEO_FILE.release() 94 | print("Keyboard interrupt was detected. Exiting...") 95 | 96 | 97 | 98 | if __name__ == "__main__": 99 | videoStream = kafkaVideoStreaming( 100 | bootstrap_servers='localhost:9092', 101 | topic='KafkaVideoStream', 102 | videoFile=argv[1], 103 | client_id='KafkaVideoStreamClient', 104 | ) 105 | videoStream.run() 106 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | kafka-python==1.4.7 2 | --------------------------------------------------------------------------------