├── .gitignore
├── README.md
├── consumer
├── consumer.py
└── templates
│ └── index.html
├── producer
└── producer.py
└── videos
└── Countdown1.mp4
/.gitignore:
--------------------------------------------------------------------------------
1 | kafkaenv/
2 | env/
3 | .vscode/
4 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Distributed Streaming with Python and Kafka
2 | Source code for a project detailed [here](https://medium.com/@kevin.michael.horan/distributed-video-streaming-with-python-and-kafka-551de69fe1dd) in my blog.
3 |
--------------------------------------------------------------------------------
/consumer/consumer.py:
--------------------------------------------------------------------------------
1 | import datetime
2 | from flask import Flask, Response, render_template
3 | from kafka import KafkaConsumer
4 |
5 | # Fire up the Kafka Consumer
6 | topic = "distributed-video1"
7 |
8 | consumer = KafkaConsumer(
9 | topic,
10 | bootstrap_servers=['localhost:9092'])
11 |
12 |
13 | # Set the consumer in a Flask App
14 | app = Flask(__name__)
15 |
16 | @app.route('/')
17 | def index():
18 | return render_template('index.html')
19 |
20 | @app.route('/video_feed', methods=['GET'])
21 | def video_feed():
22 | """
23 | This is the heart of our video display. Notice we set the mimetype to
24 | multipart/x-mixed-replace. This tells Flask to replace any old images with
25 | new values streaming through the pipeline.
26 | """
27 | return Response(
28 | get_video_stream(),
29 | mimetype='multipart/x-mixed-replace; boundary=frame')
30 |
31 | def get_video_stream():
32 | """
33 | Here is where we recieve streamed images from the Kafka Server and convert
34 | them to a Flask-readable format.
35 | """
36 | for msg in consumer:
37 | yield (b'--frame\r\n'
38 | b'Content-Type: image/jpg\r\n\r\n' + msg.value + b'\r\n\r\n')
39 |
40 | if __name__ == "__main__":
41 | app.run(host='0.0.0.0', debug=True)
42 |
--------------------------------------------------------------------------------
/consumer/templates/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Video Streaming Demonstration
4 |
5 |
6 | Video Streaming Demonstration
7 |  }})
8 |
9 |
10 |
--------------------------------------------------------------------------------
/producer/producer.py:
--------------------------------------------------------------------------------
1 | import sys
2 | import time
3 | import cv2
4 | from kafka import KafkaProducer
5 |
6 | topic = "distributed-video1"
7 |
8 | def publish_video(video_file):
9 | """
10 | Publish given video file to a specified Kafka topic.
11 | Kafka Server is expected to be running on the localhost. Not partitioned.
12 |
13 | :param video_file: path to video file
14 | """
15 | # Start up producer
16 | producer = KafkaProducer(bootstrap_servers='localhost:9092')
17 |
18 | # Open file
19 | video = cv2.VideoCapture(video_file)
20 |
21 | print('publishing video...')
22 |
23 | while(video.isOpened()):
24 | success, frame = video.read()
25 |
26 | # Ensure file was read successfully
27 | if not success:
28 | print("bad read!")
29 | break
30 |
31 | # Convert image to png
32 | ret, buffer = cv2.imencode('.jpg', frame)
33 |
34 | # Convert to bytes and send to kafka
35 | producer.send(topic, buffer.tobytes())
36 |
37 | time.sleep(0.2)
38 | video.release()
39 | print('publish complete')
40 |
41 | def publish_camera():
42 | """
43 | Publish camera video stream to specified Kafka topic.
44 | Kafka Server is expected to be running on the localhost. Not partitioned.
45 | """
46 |
47 | # Start up producer
48 | producer = KafkaProducer(bootstrap_servers='localhost:9092')
49 |
50 |
51 | camera = cv2.VideoCapture(0)
52 | try:
53 | while(True):
54 | success, frame = camera.read()
55 |
56 | ret, buffer = cv2.imencode('.jpg', frame)
57 | producer.send(topic, buffer.tobytes())
58 |
59 | # Choppier stream, reduced load on processor
60 | time.sleep(0.2)
61 |
62 | except:
63 | print("\nExiting.")
64 | sys.exit(1)
65 |
66 |
67 | camera.release()
68 |
69 |
70 | if __name__ == '__main__':
71 | """
72 | Producer will publish to Kafka Server a video file given as a system arg.
73 | Otherwise it will default by streaming webcam feed.
74 | """
75 | if(len(sys.argv) > 1):
76 | video_path = sys.argv[1]
77 | publish_video(video_path)
78 | else:
79 | print("publishing feed!")
80 | publish_camera()
--------------------------------------------------------------------------------
/videos/Countdown1.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/kmhoran/DistributedStreamingWithPythonAndKafka/975ed60c95fbf6a20a34e273ea166be8939252c9/videos/Countdown1.mp4
--------------------------------------------------------------------------------