├── app ├── api │ ├── __init__.py │ ├── groups_api.py │ ├── kafka_message_api.py │ └── users_api.py ├── models │ ├── __init__.py │ ├── __pycache__ │ │ ├── otp.cpython-36.pyc │ │ ├── users.cpython-36.pyc │ │ ├── __init__.cpython-36.pyc │ │ └── messages.cpython-36.pyc │ ├── otp.py │ ├── users.py │ ├── messages.py │ └── groups.py ├── config.py ├── __init__.py └── swagger │ └── swagger.json ├── kafka_commands ├── screenshots ├── swagger.png └── WhatsApp_Kafka_Python.png ├── run.py ├── requirements.txt ├── kafka_consumer.py ├── LICENSE └── README.md /app/api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/models/__init__.py: -------------------------------------------------------------------------------- 1 | pass -------------------------------------------------------------------------------- /kafka_commands: -------------------------------------------------------------------------------- 1 | ## Start Kafka server 2 | kafka_2.12-2.6.0/bin/kafka-server-start.sh config/server.properties -------------------------------------------------------------------------------- /screenshots/swagger.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anshumanpattnaik/python-kafka-messaging/main/screenshots/swagger.png -------------------------------------------------------------------------------- /screenshots/WhatsApp_Kafka_Python.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anshumanpattnaik/python-kafka-messaging/main/screenshots/WhatsApp_Kafka_Python.png -------------------------------------------------------------------------------- /app/models/__pycache__/otp.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anshumanpattnaik/python-kafka-messaging/main/app/models/__pycache__/otp.cpython-36.pyc -------------------------------------------------------------------------------- /app/models/otp.py: -------------------------------------------------------------------------------- 1 | from .. import db 2 | 3 | class Otp(db.Document): 4 | phone_no = db.StringField(required=True) 5 | otp = db.IntField(required=True) -------------------------------------------------------------------------------- /app/models/__pycache__/users.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anshumanpattnaik/python-kafka-messaging/main/app/models/__pycache__/users.cpython-36.pyc -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | from app import initialize_app 2 | 3 | app = initialize_app() 4 | if __name__=='__main__': 5 | app.run(debug=True) # Do not use debug=True in production -------------------------------------------------------------------------------- /app/models/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anshumanpattnaik/python-kafka-messaging/main/app/models/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /app/models/__pycache__/messages.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anshumanpattnaik/python-kafka-messaging/main/app/models/__pycache__/messages.cpython-36.pyc -------------------------------------------------------------------------------- /app/models/users.py: -------------------------------------------------------------------------------- 1 | from .. import db 2 | 3 | class Users(db.Document): 4 | full_name = db.StringField(required=True) 5 | phone_no = db.StringField(required=True) 6 | photo_url = db.StringField(required=True) 7 | is_verify = db.BooleanField(request=True) -------------------------------------------------------------------------------- /app/models/messages.py: -------------------------------------------------------------------------------- 1 | from .. import db 2 | 3 | class Messages(db.Document): 4 | sender = db.StringField(required=True) 5 | receiver = db.StringField(required=True) 6 | participants = db.StringField(required=True) 7 | phone_no = db.StringField(required=True) 8 | message = db.StringField(required=True) 9 | timestamp = db.StringField(required=True) -------------------------------------------------------------------------------- /app/models/groups.py: -------------------------------------------------------------------------------- 1 | from .. import db 2 | 3 | class Participants(db.Document): 4 | full_name = db.StringField(required=True) 5 | phone_no = db.StringField(required=True) 6 | 7 | 8 | class Groups(db.Document): 9 | group_id = db.StringField(required=True) 10 | group_name = db.StringField(required=True) 11 | created_at = db.StringField(required=True) 12 | participants = db.ListField(db.ReferenceField(Participants)) -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | autopep8==1.5.4 2 | certifi==2020.11.8 3 | cffi==1.14.4 4 | chardet==3.0.4 5 | click==7.1.2 6 | cryptography==3.2.1 7 | enum34==1.1.10 8 | Flask==1.1.2 9 | Flask-JWT==0.3.2 10 | Flask-JWT-Extended==3.25.0 11 | flask-mongoengine==0.7.1 12 | flask-swagger-ui==3.36.0 13 | Flask-WTF==0.14.3 14 | gunicorn==19.10.0 15 | idna==2.10 16 | ipaddress==1.0.23 17 | itsdangerous==1.1.0 18 | Jinja2==2.11.2 19 | kafka-python==2.0.2 20 | MarkupSafe==1.1.1 21 | mongoengine==0.19.1 22 | pycodestyle==2.6.0 23 | pycparser==2.20 24 | PyJWT==1.7.1 25 | pymongo==3.11.1 26 | pyOpenSSL==20.0.0 27 | python-dotenv==0.15.0 28 | pytz==2020.4 29 | requests==2.25.0 30 | six==1.15.0 31 | toml==0.10.2 32 | twilio==6.48.0 33 | typing==3.7.4.3 34 | urllib3==1.26.2 35 | Werkzeug==1.0.1 36 | WTForms==2.3.3 37 | -------------------------------------------------------------------------------- /kafka_consumer.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from kafka import KafkaConsumer 3 | 4 | parser = argparse.ArgumentParser() 5 | 6 | # Pass user registered phone no as an argument to fetch message from producer 7 | parser.add_argument('-p', '--participant') 8 | 9 | if __name__ == '__main__': 10 | args = parser.parse_args() 11 | if args.participants: 12 | phone_no = args.participants 13 | consumer = KafkaConsumer( 14 | phone_no, 15 | bootstrap_servers=['localhost:9092'], 16 | auto_offset_reset='earliest', 17 | enable_auto_commit=True) 18 | 19 | # In case of group conversation, client needs to subscribed the consumers to a particular topic (i.e. Group Name) 20 | group_topic = 'KAFKA_TOPIC_NAME_GOES_HERE' 21 | consumer.subscribe(group_topic) 22 | 23 | for m in consumer: 24 | print('{}'.format(m.value)) 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Anshuman Pattnaik 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /app/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | class Config: 4 | # Mongo db connection url 5 | MONGO_DB_CONNECTION = 'mongodb://127.0.0.1:27017/whatsapp' 6 | 7 | # Twilio Credentials 8 | ACCOUNT_SID = os.environ.get('ACCOUNT_SID') 9 | AUTH_TOKEN = os.environ.get('AUTH_TOKEN') 10 | TWILIO_PHONE_NO = os.environ.get('TWILIO_PHONE_NO') 11 | 12 | # API Configs 13 | API_PATH = '/api/' 14 | API_VERSION = 'v1' 15 | BASE_URL = 'http://127.0.0.1:5000' 16 | 17 | # User API endpoint 18 | SIGN_IN = API_PATH+API_VERSION+'/sign_in' 19 | SIGN_UP = API_PATH+API_VERSION+'/sign_up' 20 | OTP_VERIFY = API_PATH+API_VERSION+'/otp_verify' 21 | USER_PROFILE = API_PATH+API_VERSION+'/profile/' 22 | UPDATE_PROFILE = API_PATH+API_VERSION+'/update_profile/' 23 | 24 | # Messages API endpoint 25 | SEND_MESSAGE = API_PATH+API_VERSION+'/send_message/' 26 | RECEIVE_MESSAGE = API_PATH+API_VERSION+'/receive_message//' 27 | 28 | # Groups API endpoint 29 | CREATE_GROUPS = API_PATH+API_VERSION+'/create_group/' 30 | GROUP_MESSAGES = API_PATH+API_VERSION+'/group_messages//' 31 | -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, redirect 2 | from flask.helpers import send_from_directory 3 | from .config import Config 4 | from flask_mongoengine import MongoEngine 5 | from flask_swagger_ui import get_swaggerui_blueprint 6 | from flask_jwt_extended import JWTManager 7 | 8 | db = MongoEngine() 9 | 10 | app = Flask(__name__) 11 | 12 | # Initialize jwt manager using the secret key 13 | app.config['JWT_SECRET_KEY'] = 'python-kafka-messaging' 14 | jwt = JWTManager(app) 15 | 16 | 17 | @app.route('/') 18 | def redirect_to_docs(): 19 | return redirect(Config.BASE_URL+"/api/docs") 20 | 21 | # Swagger api docs route 22 | @app.route('/swagger/') 23 | def send_static(path): 24 | return send_from_directory('swagger', path) 25 | 26 | 27 | def initialize_app(): 28 | SWAGGER_URL = '/api/docs' 29 | API_URL = '/swagger/swagger.json' 30 | 31 | swaggerui_blueprint = get_swaggerui_blueprint( 32 | SWAGGER_URL, 33 | API_URL, 34 | config={ 35 | 'app_name': "Python Kafka Messaging" 36 | }) 37 | app.register_blueprint(swaggerui_blueprint) 38 | 39 | app.config['MONGODB_SETTINGS'] = { 40 | 'host': Config.MONGO_DB_CONNECTION, 41 | } 42 | db.init_app(app) 43 | 44 | from app.api.users_api import users_bp 45 | app.register_blueprint(users_bp) 46 | 47 | from app.api.kafka_message_api import messages_bp 48 | app.register_blueprint(messages_bp) 49 | 50 | from app.api.groups_api import groups_bp 51 | app.register_blueprint(groups_bp) 52 | 53 | return app 54 | -------------------------------------------------------------------------------- /app/api/groups_api.py: -------------------------------------------------------------------------------- 1 | from kafka.admin import KafkaAdminClient, NewTopic 2 | from flask import Blueprint, make_response, jsonify, request 3 | from flask_jwt_extended import (jwt_required, get_jwt_identity) 4 | from werkzeug.exceptions import abort 5 | from ..models.messages import Messages 6 | from ..models.groups import Groups 7 | from ..config import Config 8 | 9 | groups_bp = Blueprint('groups', __name__) 10 | 11 | # Fetch group conversations using group id 12 | @groups_bp.route(Config.GROUP_MESSAGES, methods=['GET']) 13 | @jwt_required 14 | def receive_group_messages(phone_no, group_id): 15 | if get_jwt_identity() == phone_no: 16 | messages = Messages.objects(receiver=group_id) 17 | return make_response(jsonify({ 18 | 'messages': messages 19 | }), 200) 20 | else: 21 | abort(401) 22 | 23 | # Create a group using Kafka create_topics 24 | # Produce messages with subscribed consumers 25 | @groups_bp.route(Config.CREATE_GROUPS, methods=['POST']) 26 | @jwt_required 27 | def create_group(phone_no): 28 | if get_jwt_identity() == phone_no: 29 | try: 30 | group_id = request.json['group_id'] 31 | group_name = request.json['group_name'] 32 | created_at = request.json['created_at'] 33 | participants = request.json['participants'] 34 | 35 | # Instantiate kafka admin client to create a topic 36 | client = KafkaAdminClient(bootstrap_servers="localhost:9092", client_id=phone_no) 37 | 38 | topics = [] 39 | topics.append(NewTopic(name=group_id, num_partitions=1, replication_factor=1)) 40 | client.create_topics(new_topics=topics, validate_only=False) 41 | 42 | group = Groups(group_id=group_id, 43 | group_name=group_name, 44 | created_at=created_at, 45 | participants=participants) 46 | group.save() 47 | 48 | return make_response(jsonify({ 49 | 'success': 'Group created successfully' 50 | }), 200) 51 | except KeyError: 52 | abort(400) 53 | else: 54 | abort(401) 55 | 56 | 57 | @groups_bp.errorhandler(400) 58 | def invalid_request(error): 59 | return make_response(jsonify({'error': 'Invalid Request'}), 400) 60 | 61 | 62 | @groups_bp.errorhandler(401) 63 | def unauthorized(error): 64 | return make_response(jsonify({'error': 'Unauthorized Access'}), 401) 65 | -------------------------------------------------------------------------------- /app/api/kafka_message_api.py: -------------------------------------------------------------------------------- 1 | import json 2 | from flask import Blueprint, make_response, jsonify, request 3 | from flask_jwt_extended import (jwt_required, get_jwt_identity) 4 | from werkzeug.exceptions import abort 5 | from kafka import KafkaProducer 6 | from ..models.messages import Messages 7 | from ..config import Config 8 | 9 | messages_bp = Blueprint('messages', __name__) 10 | 11 | # Fetch messages for a particular consumer 12 | @messages_bp.route(Config.RECEIVE_MESSAGE, methods=['GET']) 13 | @jwt_required 14 | def receive_message(phone_no, receiver): 15 | if get_jwt_identity() == phone_no: 16 | sender_message = Messages.objects(receiver=phone_no) 17 | receiver_message = Messages.objects(receiver=receiver) 18 | 19 | return make_response(jsonify({ 20 | 'sender_message': sender_message, 21 | 'receiver_message': receiver_message 22 | }), 200) 23 | else: 24 | abort(401) 25 | 26 | # Produce messages with the consumers using KafkaProducer 27 | @messages_bp.route(Config.SEND_MESSAGE, methods=['POST']) 28 | @jwt_required 29 | def send_message(phone_no): 30 | if get_jwt_identity() == phone_no: 31 | try: 32 | message_obj = json.dumps(request.json).encode('utf-8') 33 | 34 | sender = request.json['sender'] 35 | receiver = request.json['receiver'] 36 | phone_no = request.json['phone_no'] 37 | message = request.json['message'] 38 | timestamp = request.json['timestamp'] 39 | 40 | # Removed (+) operator as Kafka doesn't allow special characters 41 | receiver = receiver.replace("+", "") 42 | participants = '{}_{}'.format(phone_no, receiver) 43 | 44 | # Instantiate kafka producer to send message to a topic 45 | producer = KafkaProducer(bootstrap_servers=['localhost:9092'], api_version=(0, 10)) 46 | producer.send(receiver, message_obj) 47 | producer.flush() 48 | 49 | message = Messages(sender=sender, 50 | receiver=receiver, 51 | participants=participants, 52 | phone_no=phone_no, 53 | message=message, 54 | timestamp=timestamp) 55 | message.save() 56 | 57 | return make_response(jsonify({'success': 'Message has been sent successfully'}), 200) 58 | except KeyError as e: 59 | print(e) 60 | abort(400) 61 | else: 62 | abort(401) 63 | 64 | 65 | @messages_bp.errorhandler(400) 66 | def invalid_request(error): 67 | return make_response(jsonify({'error': 'Invalid Request'}), 400) 68 | 69 | 70 | @messages_bp.errorhandler(401) 71 | def unauthorized(error): 72 | return make_response(jsonify({'error': 'Unauthorized Access'}), 401) 73 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Python Kafka Messaging 2 | The idea behind this project is to demonstrate the chat messaging service using [Apache Kafka](https://kafka.apache.org/) and this proof of concept follows the architecture of WhatsApp messaging, (i.e. one to one messaging and group messaging). Apache Kafka messaging framework works based on producer and consumer architecture and to implement these core concepts it provides us to create Kafka topics, send messages using KafkaProducer and receive messages using KafkaConsumer based on the subscribed topics, so the Kafka framework is quite helpful while building distributed streaming-based system. 3 | 4 | 5 | 6 | ## Technical Overview 7 | The proof of concept written using python and it follows the core & common architecture of WhatsApp Messenger and to replicate the similar messaging concepts it uses [kafka-python](https://pypi.org/project/kafka-python/) library to send/receive messages using KafkaProducer/KafkaConsumer. It also implements [Twilio](https://www.twilio.com/) messaging framework for user authentication by verifying their registered phone no using the sent OTP and to cache all the user data and chat messages it uses mongodb database for the storage. 8 | 9 | In this messaging system, there are three different types of Microservices designed using [python flask framework](https://flask.palletsprojects.com/en/1.1.x/). 10 | 11 | 1. Users 12 | 2. Kafka Messaging 13 | 3. Kafka Group Messaging 14 | 15 | To run the complete project, first of all, you need to install Kafka in your local system, I have installed [kafka_2.12-2.6.0](https://kafka.apache.org/downloads) latest version in my ubuntu-18.04 machine. 16 | 17 | ### Swagger API Documentation 18 | The Swagger API docs can be accessible via [http://127.0.0.1:5000/api/docs](http://127.0.0.1:5000/api/docs) and to test the API endpoints you need to authorize yourself using your jwt access token. 19 | 20 | 21 | 22 | ## Installation 23 | `````````````````````````````````````````````````````````````````` 24 | git clone https://github.com/anshumanpattnaik/python-kafka-messaging 25 | cd python-kafka-messaging 26 | pip install -r requirements.txt 27 | `````````````````````````````````````````````````````````````````` 28 | 29 | ## Twilio Configurations 30 | To run the Twilio messaging framework, first of all, you need to sign up an account in [Twilio developer console](https://www.twilio.com/console) and once you have the credentials then follow the below steps to configure the environment variables for secret keys. 31 | 32 | ``````````````````````````````````````````````````````````````````` 33 | $ nano ~/.bash_profile 34 | 35 | // Add the below variables to your profile 36 | export ACCOUNT_SID='TWILIO_ACCOUNT_SID' 37 | export AUTH_TOKEN='TWILIO_AUTH_TOKEN' 38 | export TWILIO_PHONE_NO='TWILIO_SENDER_PHONE_NO' 39 | 40 | $ source ~/.bash_profile 41 | ``````````````````````````````````````````````````````````````````` 42 | 43 | ## Start Kafka Server 44 | When you have Kafka set up in your local machine before starting the project first you need to start Kafka server in your localhost. 45 | 46 | ### Execute the below command in terminal-1 47 | `````````````````````````````````````````````````````````````` 48 | $ cd kafka_2.12-2.6.0 49 | $ bin/kafka-server-start.sh config/server.properties 50 | `````````````````````````````````````````````````````````````` 51 | 52 | ### Start the python kafka project in terminal-2 53 | `````````````````````````````````````````````````````````````` 54 | $ cd python-kafka-messaging 55 | $ source venv/bin/activate 56 | $ python3 run.py 57 | 58 | Open http://127.0.0.1:5000 to view in the browser. 59 | `````````````````````````````````````````````````````````````` 60 | 61 | ### Start the python kafka consumer script in terminal-3 62 | Once your Kafka server up and running then go inside the project and start the [kafka-consumer.py](https://github.com/anshumanpattnaik/python-kafka-messaging/blob/main/kafka_consumer.py) script to receive an incoming message from the producer. 63 | 64 | #### Important 65 | In case of one to one messaging you can comment out `subscribe` code at line no [21](https://github.com/anshumanpattnaik/python-kafka-messaging/blob/b651fbbdd34650408a6ae639b7ce1a536e0d121e/kafka_consumer.py#L21) because it's only required in a group conversation. 66 | 67 | ```````````````````````````````````````` 68 | # consumer.subscribe(group_topic) // in case of one to one conversation 69 | ```````````````````````````````````````` 70 | 71 | ````````````````````````````````````````````````````````````````` 72 | $ cd python-kafka-messaging 73 | $ python3 kafka_consumer.py -p YOUR_PARTICIPANT_PHONE_NO 74 | ````````````````````````````````````````````````````````````````` 75 | 76 | #### Note 77 | It's quite important to understand basic Kafka concepts (i.e Topics/Producer/Consumer) so I'll highly recommend [kafka official documentation](https://kafka.apache.org/intro) to understand the Kafka architecture in depth. 78 | 79 | ### License 80 | This project is licensed under the [MIT License](LICENSE) 81 | -------------------------------------------------------------------------------- /app/api/users_api.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint, make_response, jsonify, request 2 | from kafka.admin import KafkaAdminClient, NewTopic 3 | from flask_jwt_extended import ( 4 | jwt_required, create_access_token, get_jwt_identity) 5 | from werkzeug.exceptions import abort 6 | from twilio.rest import Client 7 | from random import randint 8 | from ..models.users import Users 9 | from ..models.otp import Otp 10 | from ..config import Config 11 | 12 | users_bp = Blueprint('users', __name__) 13 | 14 | # user verification using twilio message 15 | def twilio_msg_verify(phone_no): 16 | client = Client(Config.ACCOUNT_SID, Config.AUTH_TOKEN) 17 | 18 | random_no = randint(100000, 999999) 19 | message = '{} is your one time password (OTP)'.format(random_no) 20 | response = client.messages.create(to=phone_no, from_=Config.TWILIO_PHONE_NO, 21 | body=message) 22 | 23 | otp = Otp(phone_no=phone_no, otp=random_no) 24 | otp.save() 25 | 26 | return response 27 | 28 | # Fetch user profile with their registered phone no 29 | @users_bp.route(Config.USER_PROFILE, methods=['GET']) 30 | @jwt_required 31 | def user_profile(phone_no): 32 | if get_jwt_identity() == phone_no: 33 | user = Users.objects.get(phone_no=phone_no) 34 | return make_response(jsonify({ 35 | 'full_name': user.full_name, 36 | 'photo_url': user.photo_url 37 | }), 200) 38 | else: 39 | abort(401) 40 | 41 | # Update user details with their registered phone no 42 | @users_bp.route(Config.UPDATE_PROFILE, methods=['PUT']) 43 | @jwt_required 44 | def update_profile(phone_no): 45 | if get_jwt_identity() == phone_no: 46 | user = Users.objects(phone_no=phone_no).first() 47 | if 'full_name' in request.json: 48 | user.update(full_name=request.json['full_name']) 49 | 50 | if 'photo_url' in request.json: 51 | user.update(photo_url=request.json['photo_url']) 52 | 53 | return make_response(jsonify({ 54 | 'success': 'User profile updated successfully' 55 | }), 200) 56 | else: 57 | abort(401) 58 | 59 | # User authentication using twilio messaging 60 | @users_bp.route(Config.SIGN_IN, methods=['POST']) 61 | def sign_in(): 62 | try: 63 | phone_no = request.json['phone_no'] 64 | response = twilio_msg_verify(phone_no) 65 | 66 | # If twilio response then otp has been sent to your registered phone no 67 | if response: 68 | return make_response(jsonify({ 69 | 'success': "OTP is sent to your registered phone number" 70 | }), 200) 71 | 72 | except Users.DoesNotExist: 73 | return make_response(jsonify({ 74 | 'error': "Phone number you've entered is not registered" 75 | }), 401) 76 | 77 | # Create an account and a topic using Kafka admin client 78 | @users_bp.route(Config.SIGN_UP, methods=['POST']) 79 | def sign_up(): 80 | try: 81 | phone_no = request.json['phone_no'] 82 | try: 83 | if Users.objects.get(phone_no=phone_no): 84 | return make_response(jsonify({"phone_no": phone_no+' already exists'}), 400) 85 | except Users.DoesNotExist: 86 | pass 87 | 88 | full_name = request.json['full_name'] 89 | photo_url = request.json['photo_url'] 90 | 91 | kafka_topic_name = phone_no.replace("+", "") 92 | 93 | # To start the conversation in between 1 single user, then user's phone no 94 | # will be used as a topic to produce messages with another consumer. 95 | client = KafkaAdminClient(bootstrap_servers="localhost:9092", client_id=phone_no) 96 | 97 | topics = [] 98 | topics.append(NewTopic(name=kafka_topic_name, num_partitions=1, replication_factor=1)) 99 | client.create_topics(new_topics=topics, validate_only=False) 100 | 101 | users = Users(full_name=full_name, 102 | phone_no=phone_no, 103 | photo_url=photo_url, 104 | is_verify=False) 105 | users.save() 106 | 107 | response = twilio_msg_verify(phone_no) 108 | if response: 109 | return make_response(jsonify({ 110 | 'success': "OTP is sent to your registered phone number" 111 | }), 201) 112 | 113 | except KeyError: 114 | abort(400) 115 | 116 | # User verfication using twilio messaging system 117 | @users_bp.route(Config.OTP_VERIFY, methods=['POST']) 118 | def account_verify(): 119 | try: 120 | phone_no = request.json['phone_no'] 121 | otp = request.json['otp'] 122 | 123 | otp_obj = Otp.objects(phone_no=phone_no).first() 124 | 125 | # If none then user verification already completed 126 | if otp_obj == None: 127 | return make_response(jsonify({'error': 'User verification already completed'}), 200) 128 | 129 | if(otp_obj.otp == otp): 130 | # Delete the otp collection once it verifies 131 | otp_obj.delete() 132 | 133 | user = Users.objects(phone_no=phone_no).first() 134 | 135 | # If it's already verified then it will generate the JWT token 136 | if(user.is_verify == True): 137 | access_token = create_access_token(identity=phone_no) 138 | 139 | return make_response(jsonify({ 140 | 'access_token': access_token, 141 | 'full_name': user.full_name, 142 | 'photo_url': user.photo_url 143 | }), 200) 144 | else: 145 | user.update(is_verify=True) 146 | return make_response(jsonify({'message': 'User verification successful'}), 200) 147 | else: 148 | return make_response(jsonify({'error': 'Wrong One Time Password (OTP)'}), 200) 149 | except KeyError: 150 | abort(400) 151 | 152 | 153 | @users_bp.errorhandler(400) 154 | def invalid_request(error): 155 | return make_response(jsonify({'error': 'Invalid Request'}), 400) 156 | -------------------------------------------------------------------------------- /app/swagger/swagger.json: -------------------------------------------------------------------------------- 1 | { 2 | "openapi": "3.0.0", 3 | "info": { 4 | "version": "0.0.1", 5 | "title": "Python Kafka Messaging", 6 | "description": "The idea behind this project is to demostrate the chat messaging service using Apache Kafka and the proof of concept follows the architecture of whatsapp", 7 | "contact": { 8 | "name": "Anshuman Pattnaik", 9 | "url": "https://github.com/anshumanpattnaik/python-kafka-messaging" 10 | } 11 | }, 12 | "servers": [ 13 | { 14 | "url": "http://127.0.0.1:5000/api/v1" 15 | } 16 | ], 17 | "tags": [ 18 | { 19 | "name": "User", 20 | "description": "Create your profile" 21 | }, 22 | { 23 | "name": "Kafka Messaging", 24 | "description": "Create topics and produce message to the consumers" 25 | }, 26 | { 27 | "name": "Kafka Group Messaging", 28 | "description": "Create topics and produce message to the subscribed consumers" 29 | } 30 | ], 31 | "paths": { 32 | "/sign_up": { 33 | "post": { 34 | "tags": [ 35 | "User" 36 | ], 37 | "summary": "Sign up and connect with millions of friends over messaging", 38 | "requestBody": { 39 | "description": "", 40 | "required": true, 41 | "content": { 42 | "application/json": { 43 | "schema": { 44 | "$ref": "#/definitions/User" 45 | } 46 | } 47 | } 48 | }, 49 | "responses": { 50 | "201": { 51 | "description": "OTP is sent to your registered phone number" 52 | }, 53 | "400": { 54 | "description": "Invalid Request" 55 | } 56 | } 57 | } 58 | }, 59 | "/sign_in": { 60 | "post": { 61 | "tags": [ 62 | "User" 63 | ], 64 | "summary": "Authenticate yourself by using your phone number", 65 | "requestBody": { 66 | "description": "", 67 | "required": true, 68 | "content": { 69 | "application/json": { 70 | "schema": { 71 | "$ref": "#/definitions/UserLogin" 72 | } 73 | } 74 | } 75 | }, 76 | "responses": { 77 | "200": { 78 | "description": "OTP is sent to your registered phone number" 79 | }, 80 | "401": { 81 | "description": "Phone number you've entered is not registered" 82 | } 83 | } 84 | } 85 | }, 86 | "/otp_verify": { 87 | "post": { 88 | "tags": [ 89 | "User" 90 | ], 91 | "summary": "User verfication using twilio otp", 92 | "requestBody": { 93 | "description": "", 94 | "required": true, 95 | "content": { 96 | "application/json": { 97 | "schema": { 98 | "$ref": "#/definitions/OTP" 99 | } 100 | } 101 | } 102 | }, 103 | "responses": { 104 | "200": { 105 | "description": "User verification successful" 106 | }, 107 | "401": { 108 | "description": "Phone number you've entered is not registered" 109 | } 110 | } 111 | } 112 | }, 113 | "/update_profile/{phone_no}": { 114 | "put": { 115 | "tags": [ 116 | "User" 117 | ], 118 | "security": [ 119 | { 120 | "bearerAuth": [] 121 | } 122 | ], 123 | "summary": "Update your profile", 124 | "requestBody": { 125 | "description": "", 126 | "required": true, 127 | "content": { 128 | "application/json": { 129 | "schema": { 130 | "$ref": "#/definitions/Profile" 131 | } 132 | } 133 | } 134 | }, 135 | "parameters": [ 136 | { 137 | "name": "phone_no", 138 | "in": "path", 139 | "description": "Your registered phone no", 140 | "required": true, 141 | "type": "string" 142 | } 143 | ], 144 | "responses": { 145 | "200": { 146 | "description": "User profile updated successfully" 147 | }, 148 | "400": { 149 | "description": "Invalid Request" 150 | } 151 | } 152 | } 153 | }, 154 | "/profile/{phone_no}": { 155 | "put": { 156 | "tags": [ 157 | "User" 158 | ], 159 | "security": [ 160 | { 161 | "bearerAuth": [] 162 | } 163 | ], 164 | "summary": "Find your profile", 165 | "requestBody": { 166 | "description": "", 167 | "required": true, 168 | "content": { 169 | "application/json": { 170 | "schema": { 171 | "$ref": "#/definitions/Profile" 172 | } 173 | } 174 | } 175 | }, 176 | "parameters": [ 177 | { 178 | "name": "phone_no", 179 | "in": "path", 180 | "description": "Your registered phone no", 181 | "required": true, 182 | "type": "string" 183 | } 184 | ], 185 | "responses": { 186 | "200": { 187 | "schema": { 188 | "$ref": "#/definitions/Profile" 189 | } 190 | }, 191 | "400": { 192 | "description": "Invalid Request" 193 | } 194 | } 195 | } 196 | }, 197 | "/send_message/{phone_no}": { 198 | "post": { 199 | "tags": [ 200 | "Kafka Messaging" 201 | ], 202 | "security": [ 203 | { 204 | "bearerAuth": [] 205 | } 206 | ], 207 | "summary": "Send Message to the consumers", 208 | "requestBody": { 209 | "description": "", 210 | "required": true, 211 | "content": { 212 | "application/json": { 213 | "schema": { 214 | "$ref": "#/definitions/Message" 215 | } 216 | } 217 | } 218 | }, 219 | "parameters": [ 220 | { 221 | "name": "phone_no", 222 | "in": "path", 223 | "description": "Your registered phone no", 224 | "required": true, 225 | "type": "string" 226 | } 227 | ], 228 | "responses": { 229 | "200": { 230 | "schema": { 231 | "$ref": "#/definitions/MessageResponse" 232 | } 233 | }, 234 | "400": { 235 | "description": "Invalid Request" 236 | } 237 | } 238 | } 239 | }, 240 | "/receive_message/{phone_no}/{receiver}": { 241 | "get": { 242 | "tags": [ 243 | "Kafka Messaging" 244 | ], 245 | "security": [ 246 | { 247 | "bearerAuth": [] 248 | } 249 | ], 250 | "summary": "Receive Message from the consumers", 251 | "parameters": [ 252 | { 253 | "name": "phone_no", 254 | "in": "path", 255 | "description": "Your registered phone no", 256 | "required": true, 257 | "type": "string" 258 | }, 259 | { 260 | "name": "receiver", 261 | "in": "path", 262 | "description": "Receiver topic name", 263 | "required": true, 264 | "type": "string" 265 | } 266 | ], 267 | "responses": { 268 | "200": { 269 | "schema": { 270 | "$ref": "#/definitions/ReceiverMessage" 271 | } 272 | }, 273 | "400": { 274 | "description": "Invalid Request" 275 | } 276 | } 277 | } 278 | }, 279 | "/create_group/{phone_no}": { 280 | "post": { 281 | "tags": [ 282 | "Kafka Group Messaging" 283 | ], 284 | "security": [ 285 | { 286 | "bearerAuth": [] 287 | } 288 | ], 289 | "summary": "Create a group to start producing message to subscribed consumers", 290 | "requestBody": { 291 | "description": "", 292 | "required": true, 293 | "content": { 294 | "application/json": { 295 | "schema": { 296 | "$ref": "#/definitions/Group" 297 | } 298 | } 299 | } 300 | }, 301 | "parameters": [ 302 | { 303 | "name": "phone_no", 304 | "in": "path", 305 | "description": "Your registered phone no", 306 | "required": true, 307 | "type": "string" 308 | } 309 | ], 310 | "responses": { 311 | "200": { 312 | "description": "Group created successfully" 313 | }, 314 | "400": { 315 | "description": "Invalid Request" 316 | } 317 | } 318 | } 319 | }, 320 | "/group_messages/{phone_no}/{group_id}": { 321 | "get": { 322 | "tags": [ 323 | "Kafka Group Messaging" 324 | ], 325 | "security": [ 326 | { 327 | "bearerAuth": [] 328 | } 329 | ], 330 | "summary": "Fetch the messages from group", 331 | "requestBody": { 332 | "description": "", 333 | "required": true, 334 | "content": { 335 | "application/json": { 336 | "schema": { 337 | "$ref": "#/definitions/Group" 338 | } 339 | } 340 | } 341 | }, 342 | "parameters": [ 343 | { 344 | "name": "phone_no", 345 | "in": "path", 346 | "description": "Your registered phone no", 347 | "required": true, 348 | "type": "string" 349 | }, 350 | { 351 | "name": "group_id", 352 | "in": "path", 353 | "description": "Unique group id", 354 | "required": true, 355 | "type": "string" 356 | } 357 | ], 358 | "responses": { 359 | "200": { 360 | "schema": { 361 | "$ref": "#/definitions/MessageResponse" 362 | } 363 | }, 364 | "400": { 365 | "description": "Invalid Request" 366 | } 367 | } 368 | } 369 | } 370 | }, 371 | "components": { 372 | "securitySchemes": { 373 | "bearerAuth": { 374 | "type": "http", 375 | "scheme": "bearer", 376 | "bearerFormat": "JWT" 377 | } 378 | } 379 | }, 380 | "definitions": { 381 | "User": { 382 | "type": "object", 383 | "properties": { 384 | "full_name": { 385 | "type": "string" 386 | }, 387 | "phone_no": { 388 | "type": "string" 389 | }, 390 | "photo_url": { 391 | "type": "string" 392 | }, 393 | "is_verify": { 394 | "type": "boolean" 395 | } 396 | } 397 | }, 398 | "UserLogin": { 399 | "type": "object", 400 | "properties": { 401 | "phone_no": { 402 | "type": "string" 403 | } 404 | } 405 | }, 406 | "OTP": { 407 | "type": "object", 408 | "properites": { 409 | "phone_no": { 410 | "type": "string" 411 | }, 412 | "otp": { 413 | "type": "string" 414 | } 415 | } 416 | }, 417 | "UserLoggedIn": { 418 | "type": "object", 419 | "properties": { 420 | "access_token": { 421 | "type": "string" 422 | }, 423 | "name": { 424 | "type": "string" 425 | }, 426 | "email": { 427 | "type": "string" 428 | }, 429 | "dob": { 430 | "type": "string" 431 | } 432 | } 433 | }, 434 | "Profile": { 435 | "type": "object", 436 | "properties": { 437 | "full_name": { 438 | "type": "string" 439 | }, 440 | "phone_no": { 441 | "type": "string" 442 | } 443 | } 444 | }, 445 | "Message": { 446 | "type": "object", 447 | "properties": { 448 | "sender": { 449 | "type": "string" 450 | }, 451 | "receiver": { 452 | "type": "string" 453 | }, 454 | "phone_no": { 455 | "type": "string" 456 | }, 457 | "message": { 458 | "type": "string" 459 | }, 460 | "timestamp": { 461 | "type": "string" 462 | } 463 | } 464 | }, 465 | "MessageResponse": { 466 | "type": "object", 467 | "properties": { 468 | "sender": { 469 | "type": "string" 470 | }, 471 | "receiver": { 472 | "type": "string" 473 | }, 474 | "phone_no": { 475 | "type": "string" 476 | }, 477 | "message": { 478 | "type": "string" 479 | }, 480 | "timestamp": { 481 | "type": "string" 482 | }, 483 | "participants": { 484 | "type": "string" 485 | } 486 | } 487 | }, 488 | "ReceiverMessage": { 489 | "type": "object", 490 | "properties": { 491 | "sender_message": { 492 | "type": "string" 493 | }, 494 | "receiver_message": { 495 | "type": "string" 496 | } 497 | } 498 | }, 499 | "Group": { 500 | "type": "object", 501 | "properties": { 502 | "group_id": { 503 | "type": "string" 504 | }, 505 | "group_name": { 506 | "type": "string" 507 | }, 508 | "created_at": { 509 | "type": "string" 510 | }, 511 | "participants": { 512 | "type": "string" 513 | } 514 | } 515 | } 516 | } 517 | } --------------------------------------------------------------------------------