├── ActiveMQ-code ├── ActiveMQ.py ├── Listeners.py ├── Load_Consumer.py ├── Universal_Consumer.py └── __pycache__ │ └── main.cpython-37.pyc ├── IBM_MQ-Code ├── receive.py └── send.py ├── Images ├── Consumer.png ├── Producer.png ├── Zookeeper.png ├── activemq-monitoring.png ├── activemq-server.png ├── async-message-queue.png ├── exchanges.png ├── kafka.png ├── message-queue-architecture.jpg ├── p_to_s.png ├── point_to_point.png ├── rabbitmq-monitoring.png └── sync-req-res.png ├── Kafka ├── consumer.py └── producer.py ├── README.md └── RabbitMQ-Code ├── rabbitmq-monitoring.py ├── receive.py └── send.py /ActiveMQ-code/ActiveMQ.py: -------------------------------------------------------------------------------- 1 | import time 2 | import sys 3 | import os 4 | import stomp 5 | from Listeners import MessageListener ,LoadListener 6 | 7 | 8 | 9 | 10 | class JMS_broker: 11 | def __init__(self): 12 | self.user = "admin" 13 | self.password = "password" 14 | self.host = "localhost" 15 | self.port = 61613 16 | 17 | def create_connection(self): 18 | conn = stomp.Connection(host_and_ports = [(self.host, self.port)]) 19 | conn.connect(login=self.user,passcode=self.password) 20 | return conn 21 | 22 | def send_msg(self,conn,destination_type,destination): 23 | 24 | p=1 25 | while p==1: 26 | msg= input("Producer :-- ") 27 | if(msg=="Shutdown"): 28 | conn.disconnect() 29 | break 30 | else: 31 | conn.send(body=msg, destination="/"+destination_type+"/"+destination, persistent='false',id=1) # id not added to parameter 32 | 33 | 34 | 35 | def receive_msg(self,destination_type,destination): 36 | conn = stomp.Connection(host_and_ports = [(self.host, self.port)]) 37 | conn.set_listener('', MessageListener(conn)) 38 | conn.connect(login=self.user,passcode=self.password) 39 | conn.subscribe(destination="/"+destination_type+"/"+destination, id=1, ack='auto') ##here for set ack tag in multiple ways 40 | print("Waiting for messages...") 41 | p=1 42 | while p==1: 43 | time.sleep(5) 44 | 45 | def load_send_msg(self,conn,destination_type,destination): 46 | print("Sending messages") 47 | msg="load testing" 48 | p=0 49 | while p!=10000: 50 | conn.send(body=msg, destination="/"+destination_type+"/"+destination, persistent='false',id=1) # id not added to parameter 51 | p+=1 52 | 53 | 54 | def load_receive_msg(self,destination_type,destination): 55 | conn = stomp.Connection(host_and_ports = [(self.host, self.port)]) 56 | conn.set_listener('', LoadListener(conn)) 57 | conn.connect(login=self.user,passcode=self.password) 58 | conn.subscribe(destination="/"+destination_type+"/"+destination, id=1, ack='auto') ##here for set ack tag in multiple ways 59 | print("Waiting for messages...") 60 | p=1 61 | while p==1: 62 | time.sleep(5) 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | if __name__ == "__main__": 71 | i=1 72 | while i==1: 73 | print("--1) JMS queue based messaging \n"+ 74 | "--2) JMS topic based broadcast \n" 75 | "--3)JMS Topic Load testing ") 76 | n=int(input()) 77 | if(n==1): 78 | print("Enter the queue") 79 | destination=input() 80 | jms=JMS_broker() 81 | conn=jms.create_connection() 82 | jms.send_msg(conn,"queue",destination) 83 | elif(n==2): 84 | print("Enter the Topic") 85 | destination=input() 86 | jms=JMS_broker() 87 | conn=jms.create_connection() 88 | jms.send_msg(conn,"topic",destination) 89 | elif(n==3): 90 | print("Enter the Topic") 91 | destination=input() 92 | jms=JMS_broker() 93 | conn=jms.create_connection() 94 | jms.load_send_msg(conn,"topic",destination) 95 | else: 96 | print("--Enter Correct Option") 97 | 98 | -------------------------------------------------------------------------------- /ActiveMQ-code/Listeners.py: -------------------------------------------------------------------------------- 1 | import time 2 | import sys 3 | import os 4 | import stomp 5 | class MessageListener(object): #Listener has function defined which are provoked when certain task is done 6 | 7 | def __init__(self, conn): 8 | self.conn = conn 9 | self.count = 0 10 | self.start = time.time() 11 | def on_error(self, headers, message): 12 | print('received an error %s' % message) 13 | def on_message(self, headers, message): 14 | self.start = time.time() 15 | self.count += 1 16 | print("Received :--"+message) 17 | if message == "SHUTDOWN": 18 | diff = time.time() - self.start 19 | print("Received %s in %f seconds" % (self.count, diff)) 20 | self.conn.disconnect() 21 | sys.exit(0) 22 | else: 23 | if self.count%1000 == 0: 24 | print("Received %s messages." % self.count) 25 | class LoadListener(object): #Listener has function defined which are provoked when certain task is done 26 | def __init__(self, conn): 27 | self.conn = conn 28 | self.count = 0 29 | self.start = time.time() 30 | self.flag = 0 31 | def on_error(self, headers, message): 32 | print('received an error %s' % message) 33 | def on_message(self, headers, message): 34 | if(self.flag==0): 35 | self.flag=1 36 | self.start=time.time() 37 | self.count += 1 38 | diff = time.time() - self.start 39 | if self.count==10000: 40 | print("Received %s in %f seconds" % (self.count, diff)) 41 | rate=1000/diff 42 | print(str(rate)+"per sec") 43 | self.conn.disconnect() 44 | sys.exit(0) 45 | -------------------------------------------------------------------------------- /ActiveMQ-code/Load_Consumer.py: -------------------------------------------------------------------------------- 1 | import time 2 | import sys 3 | import os 4 | import stomp 5 | 6 | from ActiveMQ import JMS_broker 7 | 8 | 9 | if __name__ == "__main__": 10 | jms=JMS_broker() 11 | print("Enter the Topic") 12 | destination = input() 13 | jms.load_receive_msg("topic",destination) 14 | 15 | -------------------------------------------------------------------------------- /ActiveMQ-code/Universal_Consumer.py: -------------------------------------------------------------------------------- 1 | import time 2 | import sys 3 | import os 4 | import stomp 5 | from ActiveMQ import JMS_broker 6 | 7 | 8 | if __name__ == "__main__": 9 | jms=JMS_broker() 10 | print("Enter the desination") 11 | destination = input() 12 | print("Enter the destination type") 13 | destination_type = input() 14 | jms.receive_msg(destination_type,destination) 15 | 16 | -------------------------------------------------------------------------------- /ActiveMQ-code/__pycache__/main.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prakhar308/message-queues-in-python/c9c380bb3e3983663d70b8bb0dd37718189f7433/ActiveMQ-code/__pycache__/main.cpython-37.pyc -------------------------------------------------------------------------------- /IBM_MQ-Code/receive.py: -------------------------------------------------------------------------------- 1 | import pymqi 2 | 3 | queue_manager = 'QM1' 4 | channel = 'DEV.APP.SVRCONN' 5 | host = '127.0.0.1' 6 | port = '1414' 7 | queue_name = 'TEST.1' 8 | conn_info = '%s(%s)' % (host, port) 9 | user = 'app' 10 | password = 'passw0rd' 11 | 12 | qmgr = pymqi.connect(queue_manager, channel, conn_info, user, password) 13 | 14 | queue = pymqi.Queue(qmgr, queue_name) 15 | message = queue.get() 16 | queue.close() 17 | 18 | qmgr.disconnect() -------------------------------------------------------------------------------- /IBM_MQ-Code/send.py: -------------------------------------------------------------------------------- 1 | import pymqi 2 | 3 | queue_manager = 'QM1' 4 | channel = 'DEV.APP.SVRCONN' 5 | host = '127.0.0.1' 6 | port = '1414' 7 | queue_name = 'TEST.1' 8 | message = 'Hello from Python!' 9 | conn_info = '%s(%s)' % (host, port) 10 | user = 'app' 11 | password = 'passw0rd' 12 | 13 | qmgr = pymqi.connect(queue_manager, channel, conn_info, user, password) 14 | 15 | queue = pymqi.Queue(qmgr, queue_name) 16 | queue.put(message) 17 | queue.close() 18 | 19 | qmgr.disconnect() -------------------------------------------------------------------------------- /Images/Consumer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prakhar308/message-queues-in-python/c9c380bb3e3983663d70b8bb0dd37718189f7433/Images/Consumer.png -------------------------------------------------------------------------------- /Images/Producer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prakhar308/message-queues-in-python/c9c380bb3e3983663d70b8bb0dd37718189f7433/Images/Producer.png -------------------------------------------------------------------------------- /Images/Zookeeper.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prakhar308/message-queues-in-python/c9c380bb3e3983663d70b8bb0dd37718189f7433/Images/Zookeeper.png -------------------------------------------------------------------------------- /Images/activemq-monitoring.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prakhar308/message-queues-in-python/c9c380bb3e3983663d70b8bb0dd37718189f7433/Images/activemq-monitoring.png -------------------------------------------------------------------------------- /Images/activemq-server.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prakhar308/message-queues-in-python/c9c380bb3e3983663d70b8bb0dd37718189f7433/Images/activemq-server.png -------------------------------------------------------------------------------- /Images/async-message-queue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prakhar308/message-queues-in-python/c9c380bb3e3983663d70b8bb0dd37718189f7433/Images/async-message-queue.png -------------------------------------------------------------------------------- /Images/exchanges.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prakhar308/message-queues-in-python/c9c380bb3e3983663d70b8bb0dd37718189f7433/Images/exchanges.png -------------------------------------------------------------------------------- /Images/kafka.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prakhar308/message-queues-in-python/c9c380bb3e3983663d70b8bb0dd37718189f7433/Images/kafka.png -------------------------------------------------------------------------------- /Images/message-queue-architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prakhar308/message-queues-in-python/c9c380bb3e3983663d70b8bb0dd37718189f7433/Images/message-queue-architecture.jpg -------------------------------------------------------------------------------- /Images/p_to_s.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prakhar308/message-queues-in-python/c9c380bb3e3983663d70b8bb0dd37718189f7433/Images/p_to_s.png -------------------------------------------------------------------------------- /Images/point_to_point.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prakhar308/message-queues-in-python/c9c380bb3e3983663d70b8bb0dd37718189f7433/Images/point_to_point.png -------------------------------------------------------------------------------- /Images/rabbitmq-monitoring.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prakhar308/message-queues-in-python/c9c380bb3e3983663d70b8bb0dd37718189f7433/Images/rabbitmq-monitoring.png -------------------------------------------------------------------------------- /Images/sync-req-res.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prakhar308/message-queues-in-python/c9c380bb3e3983663d70b8bb0dd37718189f7433/Images/sync-req-res.png -------------------------------------------------------------------------------- /Kafka/consumer.py: -------------------------------------------------------------------------------- 1 | from kafka import KafkaConsumer 2 | 3 | 4 | consumer = KafkaConsumer('Hello-everyone') 5 | 6 | for message in consumer: 7 | msg = str(message.value.decode()) 8 | print("Writing message %s " % msg) 9 | 10 | metric = consumer.metrics(raw=False) 11 | print(metric) 12 | print('\n\nThe Consumer Fetch Manager metrics : ') 13 | print('Fetch latency max : ') 14 | print(metric['consumer-fetch-manager-metrics']['fetch-latency-max']) 15 | print('Bytes-consumed-rate : ') 16 | print(metric['consumer-fetch-manager-metrics']['bytes-consumed-rate']) 17 | print('Fetch-size-max: ') 18 | print(metric['consumer-fetch-manager-metrics']['fetch-size-max']) 19 | print('\nKafka-metrics-count : ') 20 | print(metric['kafka-metrics-count']) 21 | print('\nThe Consumer metrics : ') 22 | print('Consumer-metric Request-size-max: ') 23 | print(metric['consumer-metrics']['request-size-max']) 24 | 25 | print(metric.keys()) 26 | -------------------------------------------------------------------------------- /Kafka/producer.py: -------------------------------------------------------------------------------- 1 | from kafka import KafkaProducer 2 | producer = KafkaProducer(bootstrap_servers = 'localhost:9092') 3 | 4 | while True: 5 | print("\n\nType \"quit\" to exit") 6 | print("Enter message to be sent:") 7 | msg = input() 8 | if msg == "quit": 9 | print("Exiting") 10 | break 11 | producer.send('Hello-everyone', msg.encode('utf-8')) 12 | print("Sending msg \"{}\"".format(msg)) 13 | 14 | metric = producer.metrics(raw=False) 15 | print(metric) 16 | print('\nThe Producer metrics : ') 17 | print('Request size max : ') 18 | print(metric['producer-metrics']['request-size-max']) 19 | print('\nKafka-metrics-count : ') 20 | print(metric['kafka-metrics-count']) 21 | 22 | print(metric.keys()) 23 | 24 | print("Message sent!") 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # message-queues-in-python 2 | 3 | Table of Contents 4 | ----------------- 5 | 6 | - [Synchronous Request Response Model](#Synchronous-Request-Response-Model) 7 | - [Asynchronous Publish Subscribe Model](#Asynchronous-Publish-Subscribe-Model) 8 | - [What are Message Queues](#What-are-Message-Queues) 9 | - Different Message Queues: 10 | + [RabbitMQ](#RabbitMQ) 11 | + [Kafka](#Kafka) 12 | + [ActiveMQ](#ActiveMQ) 13 | + [IBM MQ](#IBM-MQ) 14 | 15 | 16 | Synchronous Request Response Model 17 | --------- 18 | 19 | ![Synchronous Request Response Model](Images/sync-req-res.png) 20 | The client makes a request to the API and has to wait for the response until all the processing has been completed. 21 | 22 | #### Disadvantages 23 | - If the task is **time consuming** and if the API also performs another task like sending an **email** then the client has to wait for a longer time. 24 | - The task and email will be **coupled** even when they can be performed separately. 25 | 26 | Asynchronous Publish Subscribe Model 27 | ---------- 28 | 29 | ![Asynchronous Publish Subscribe Model](Images/async-message-queue.png) 30 | 31 | Inorder to overcome the disadvantages of Synchronous Request Response Model we do the following: 32 | 33 | - Add two more applications: 34 | + one application will be responsible for the time consuming task. 35 | + the other will be responsible for sending email. Thus we have **decoupled** the system. 36 | - Add two **message queues** each for the two applications **subscribed** to it. 37 | - Client will send request to the web app. The web app will validate the request and if everything is ok it will **publish** the message to both the queues and send and immediate 'OK' response to the client. 38 | - Thus the client won't have to wait and meanwhile the other two applications carry on **consuming** and processing the tasks of the queue. 39 | 40 | 41 | What are Message Queues 42 | -------- 43 | 44 | Message queuing allows applications to communicate by sending messages to each other.The message queue provides temporary message storage when the destination program is busy or not connected. 45 | 46 | ![Message Queue Architecture](Images/message-queue-architecture.jpg) 47 | 48 | The basic architecture of a message queue is simple; there are **client** applications called **producers** that create messages and deliver them to the message **queue**. Another application, called a **consumer**, connects to the queue and gets the messages to be processed. Messages placed onto the queue are stored until the consumer retrieves them. 49 | 50 | A message queue provides an **asynchronous communications protocol**, which is a system that puts a message onto a message queue and does not require an immediate response to continuing processing. **Email** is probably the best example of asynchronous communication. When an email is sent, the sender continues to process other things without needing an immediate response from the receiver. This way of handling messages **decouples** the producer from the consumer so that they do not need to interact with the message queue at the same time. 51 | 52 | 53 | [RabbitMQ](https://www.rabbitmq.com/) 54 | -------- 55 | 56 | RabbitMQ is a message broker: it accepts and forwards messages. 57 | 58 | RabbitMQ, and messaging in general, uses some jargon. 59 | 60 | - **Producing** means nothing more than sending. A program that sends messages is a **producer** 61 | - A **queue** is the name for a post box which lives inside RabbitMQ. Although messages flow through RabbitMQ and your applications, they can only be stored inside a queue. A queue is only bound by the host's memory & disk limits, it's essentially a large message buffer. 62 | - **Consuming** has a similar meaning to receiving. A **consumer** is a program that mostly waits to receive messages 63 | 64 | #### Quick start (using the Pika Python client): 65 | 66 | [Install RabbitMQ](https://www.rabbitmq.com/download.html) 67 | 68 | Install Pika - RabbitMQ Python Client: 69 | ```bash 70 | python -m pip install pika --upgrade 71 | ``` 72 | 73 | #### Code 74 | 75 | A producer (sender) that sends a single message, and a consumer (receiver) that receives messages and prints them out. It's a "Hello World" of messaging. 76 | 77 | You can find the code [here](RabbitMQ-Code) 78 | 79 | [Tutorial: "Hello World!"](https://www.rabbitmq.com/tutorial-one-python.html): 80 | 81 | ```bash 82 | python send.py 83 | python receive.py 84 | ``` 85 | 86 | #### RabbitMQ Features: 87 | 88 | - **Round-robin dispatching**: [Tutorial](https://www.rabbitmq.com/tutorials/tutorial-two-python.html) 89 | + By default, RabbitMQ will send each message to the next consumer, in sequence. On average every consumer will get the same number of messages. This way of distributing messages is called round-robin. 90 | + This helps is easily parallelise work. If we are building up a backlog of work, we can just add more consumers and that way, scale easily. 91 | 92 | - **Message acknowledgement**: [Tutorial](https://www.rabbitmq.com/tutorials/tutorial-two-python.html) 93 | + An **ack(nowledgement)** is sent back by the consumer to tell RabbitMQ that a particular message had been received, processed and that RabbitMQ is free to delete it. 94 | + If a consumer dies without sending an ack, RabbitMQ will re-queue it. If there are other consumers online at the same time, it will then quickly redeliver it to another consumer. 95 | 96 | - **Message Durability**: [Tutorial](https://www.rabbitmq.com/tutorials/tutorial-two-python.html) 97 | + we can mark both the queue and messages as durable. 98 | 99 | - **Fair dispatch**: [Tutorial](https://www.rabbitmq.com/tutorials/tutorial-two-python.html) 100 | + In a situation with two workers, when all odd messages are heavy and even messages are light, one worker will be constantly busy and the other one will do hardly any work. 101 | + In order to defeat RabbitMQ allows us to use the `Channel#basic_qos` channel method with the `prefetch_count=1` setting. This uses the `basic.qos` protocol method to tell RabbitMQ not to give more than one message to a worker at a time 102 | 103 | - **Exchanges**: 104 | + The core idea in the messaging model in RabbitMQ is that the producer never sends any messages directly to a queue. 105 | + Instead, the producer can only send messages to an exchange. 106 | + An exchange receives messages from producers and the other side it pushes the to the queue. 107 | 108 | ![Exchange](Images/exchanges.png) 109 | + Here X is an exchange 110 | + It allows us to send message to multiple consumers 111 | 112 | + Types: 113 | * fanout - it broadcast all the messages it receives to all the queues [Tutorial](https://www.rabbitmq.com/tutorials/tutorial-three-python.html) 114 | * direct - send message to a particular queue [Tutorial](https://www.rabbitmq.com/tutorials/tutorial-four-python.html) 115 | * topic - send message to all the queues which matches the topic [Tutorial](https://www.rabbitmq.com/tutorials/tutorial-five-python.html) 116 | 117 | ### Monitoring 118 | 119 | - Run [rabbitmq-monitoring.py](RabbitMQ-Code/rabbitmq-monitoring.py) 120 | ```bash 121 | python rabbitmq-monitoring.py 122 | ``` 123 | - The default username/password is guest/guest 124 | - Monitoring is performed by accessing RabbitMQ's HTTP API 125 | - Output Example: 126 | 127 | ![RabbitMQ-monitoring](Images/rabbitmq-monitoring.png) 128 | 129 | - It displays: 130 | + Overview: queue name and state(idle/running) 131 | + No.of messages : ready, unacknowledged and total 132 | + Rate: incoming/publish 133 | - Every 5 seconds the monitoring output is updated 134 | 135 | [KAFKA](https://kafka.apache.org/) 136 | --------- 137 | 138 | 139 | **INTRODUCTION** 140 | 141 | **Apache Kafka** is a distributed streaming platform. 142 | 143 | A streaming platform has **three key capabilities ** : 144 | 145 | 1. Publish and subscribe to streams of records, similar to a message queue or enterprise messaging system. 146 | 2. Store streams of records in a fault-tolerant durable way. 147 | 3. Process streams of records as they occur. 148 | 149 | Kafka runs as a cluster on one or more servers that can span multiple datacenters.The Kafka cluster stores streams of records in categories called topics.Each record consists of a key, a value, and a timestamp. 150 | 151 | Kafka is generally used for two broad classes of applications: 152 | 153 | 1. Building real-time streaming data pipelines that reliably get data between systems or applications. 154 | 2. Building real-time streaming applications that transform or react to the streams of data. 155 | 156 | Kafka has **five core APIs **: 157 | 158 | 1. The [Producer API](https://kafka.apache.org/documentation/#api) allows an application to publish a stream of records to one or more Kafka topics. 159 | 2. The [Consumer API](https://kafka.apache.org/documentation/#api) allows an application to subscribe to one or more topics and process the stream of records produced to them. 160 | 3. The [Streams API](https://kafka.apache.org/documentation/#api) allows an application to act as a stream processor, consuming an input stream from one or more topics and producing an output stream to one or more output topics, effectively transforming the input streams to output streams. 161 | 4. The [Connector API](https://kafka.apache.org/documentation/#api) allows building and running reusable producers or consumers that connect Kafka topics to existing applications or data systems. For example, a connector to a relational database might capture every change to a table. 162 | 5. The [Admin API](https://kafka.apache.org/documentation/#api) allows managing and inspecting topics, brokers and other Kafka objects. 163 | 164 | In Kafka the communication between the clients and the servers is done with a simple, high-performance, language agnostic TCP protocol. This protocol is versioned and maintains backwards compatibility with older versions. 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | **INSTALLATION STEPS :** 175 | 176 | [Installation]() 177 | 178 | 1. **Verifying Java Installation** 179 | java –version 180 | 181 | 2. **Install Zookeeper Framework** 182 | To [install ZooKeeper](http://zookeeper.apache.org/releases.html) framework on your machine, visit the following link and download the latest version of ZooKeeper. 183 | 184 | As of now, latest version of ZooKeeper is 3.4.6 (ZooKeeper-3.4.6.tar.gz). 185 | Open terminal on ubuntu : 186 | ```bash 187 | tar -zxf zookeeper-3.4.6.tar.gz 188 | cd zookeeper-3.4.6 189 | ``` 190 | 191 | 192 | 193 | 3.**Install Apache Kafka** 194 | Visit the official and download the latest version and then extract it on terminal 195 | 196 | tar -zxf kafka_2.11.0.9.0.0 tar.gz 197 | cd folder_name 198 | 199 | **START** 200 | 201 | 1. New terminal 202 | 203 | cd zookeeper 204 | 205 | start zookeeper : 206 | ```bash 207 | bin/zkServer.sh start 208 | bin/zkCli.sh 209 | ``` 210 | 211 | ![Zookeeper](Images/Zookeeper.png) 212 | 213 | To stop zookeeper : bin/zkServer.sh stop 214 | 215 | 2. New terminal 216 | 217 | cd kafka 218 | 219 | Start kafka : 220 | ```bash 221 | bin/kafka-server-start.sh config/server.properties 222 | ``` 223 | 224 | ![Kafka](Images/kafka.png) 225 | 226 | To stop kafka : 227 | ```bash 228 | bin/kafka-server-stop.sh config/server.properties 229 | ``` 230 | 231 | 3.New terminal 232 | 233 | cd kafka 234 | 235 | **CREATING A KAFKA TOPIC** 236 | ```bash 237 | bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 238 | --partitions 1 --topic Hello-Kafka 239 | ``` 240 | 241 | 242 | IF trying to send messages through terminal : 243 | 244 | START PRODUCER TO SEND MESSAGES 245 | 246 | ```bash 247 | bin/kafka-console-producer.sh --broker-list localhost:9092 --topic Hello-Kafka 248 | //start typing messages on terminal 249 | ``` 250 | //type messages 251 | 252 | 253 | START CONSUMER TO RECEIVE MESSAGES 254 | ```bash 255 | bin/kafka-console-consumer.sh --zookeeper localhost:2181 —topic Hello-Kafka 256 | --from-beginning 257 | ``` 258 | 259 | //you will see messages on this terminal 260 | 261 | ### WITH PYTHON CLIENT : 262 | 263 | **Install Python-kafka** : 264 | 265 | ```bash 266 | pip install kafka- python 267 | ``` 268 | 269 | Keep apache kafka and zookeeper running in background 270 | Run consumer.py and producer.py on the new terminals 271 | 272 | 273 | #### Code 274 | 275 | A producer (sender) that sends a message, and a consumer (receiver) that receives messages and prints them out. 276 | 277 | You can find the code [here](Kafka) 278 | 279 | [Kafka-Python](https://kafka-python.readthedocs.io/en/master/apidoc/modules.html) 280 | 281 | ```bash 282 | python consumer.py 283 | python producer.py 284 | ``` 285 | 286 | #### Monitoring 287 | 288 | It can be done by metric function when called in the program. 289 | 290 | [Prodcuer Metrics](https://kafka.apache.org/documentation/#producer_sender_monitoring): 291 | 292 | ![Producer Metrics](Images/Producer.png) 293 | 294 | [Consumer Metrics](https://kafka.apache.org/documentation/#consumer_monitoring): 295 | 296 | ![Consumer Metrics](Images/Consumer.png) 297 | 298 | 299 | 300 | 301 | [ACTIVEMQ](https://activemq.apache.org/) 302 | -------- 303 | 304 | **INTRODUCTION** 305 | 306 | **Apache ActiveMQ** is open source, multi-protocol, Java-based messaging server.It has 2 messaging broker servers named as ActiveMQ 5 Classic and ActiveMQ Artemis. 307 | Here we have used ActiveMQ 5 Classic. 308 | 309 | It is mostly used for send messages between different applications.It has own JMS-Api to send, receive,acknowledge for messsage between server and client.JMS provides Java based connectivity to server.For connectiing server with other languages diffrent protocols are used externally. 310 | 311 | **Connectivity** 312 | 313 | 314 | ActiveMQ provides a wide range of connectivity options, including support for protocols such as HTTP/S, IP multicast, SSL, STOMP, TCP, UDP, XMPP, and more. Support for such a wide range of protocols equates to more flexibility. Many existing systems utilize a particular protocol and don’t have the option to change, so a messaging platform that supports many protocols lowers the barrier to adoption. 315 | 316 | For Python Connectivity Stomp protocol is used to message and acknowledge server. 317 | 318 | 319 | **JMS Broker** 320 | 321 | There is a common misunderstanding that JMS messages are sent directly from one application to another.Between the sender and receiver (aka producers and consumers) is a broker – an application server – that receives messages from producers and sends messages to consumers. The producers and consumers are completely decoupled. In principle, the producer has no idea who is getting its messages and the consumer has no idea who is sending them. 322 | 323 | It’s the broker to which producers are sending the messages using a kind of address called a Topic or Queue. After receiving a message, the broker forwards the message to consumers who have subscribed to the Topic or Queue. Transactions with producers and consumers are transactions between the producer and the broker or between the broker and the consumers, but never between the producer and the consumer. 324 | 325 | **JMS Topic vs. Queues** 326 | 327 | With JMS you have the option of publishing messages to a Topic or Queue. There is a fundamental difference between the two which is illustrated below. A Topic forwards a message from the producer to many consumers at once. It’s a broadcast. This is often called Publish-and-Subscribe (Pub/Sub) messaging. A Queue may also have many consumers, but it forwards each message to only one consumer. The consumers wait in line, in a queue, taking turns getting new messages from the Queue. This often called Point-to-Point (P2P) messaging 328 | 329 | ![pointtopoint](Images/point_to_point.png) 330 | 331 | 332 | 333 | 334 | **INSTALLATION STEPS :** 335 | 336 | 1. **Installing ActiveMQ Server** 337 | 338 | 1. Download Apache ActiveMQ Classic from [here](https://activemq.apache.org/components/classic/download/). 339 | 2. Open command prompt in C:\Program Files\active mq\apache-activemq-5.16.0\bin 340 | 3. Run this command to start server 341 | ```bash 342 | activemq start 343 | ``` 344 | ![ActiveMq-server](Images/activemq-server.png) 345 | 346 | 347 | 3. **Installing Client Connectivity protocol Library stomp.py** 348 | ```bash 349 | pip install stomp.py 350 | ``` 351 | 352 | 353 | 354 | #### Code 355 | 356 | A publisher (sender) that sends a message, and a listner (receiver) that receives messages and prints them out. 357 | At first run Listener and then run publisher in seperate command prompt. 358 | 359 | ```bash 360 | python publisher.py 361 | python listener.py 362 | ``` 363 | ### Monitoring 364 | For monitoring ActiveMQ server provides its own web application.It is started along with the message server at local host. 365 | Here is snap of it. 366 | 367 | 368 | ![ActiveMQ-monitoring](Images/activemq-monitoring.png) 369 | 370 | 371 | 372 | [IBM MQ](https://developer.ibm.com/components/ibm-mq/) 373 | -------- 374 | 375 | IBM MQ is a robust, reliable, and secure messaging solution. IBM MQ enables applications to communicate and exchange data in a reliable and scalable way, that decouples one application from another. 376 | 377 | ### The Basics of IBM MQ 378 | 379 | **Messages** are packages of data that are produced and consumed by applications. 380 | 381 | **Queues** are addressable locations to deliver messages to and store them reliably until they need to be consumed. 382 | 383 | **Queue managers** are the MQ servers that host the queues. 384 | 385 | **Channels** are the way queue managers communicate with each other and with the applications. 386 | 387 | ### Messaging Styles 388 | - Point-to-point messaging 389 | - Publish/subscribe messaging 390 | 391 | 392 | [Install IBM MQ](https://developer.ibm.com/articles/mq-downloads/) 393 | 394 | [PyMQI](https://dsuch.github.io/pymqi/) - python client fro IBM MQ 395 | 396 | Install 397 | ```bash 398 | pip install pymqi 399 | ``` 400 | 401 | ### Code 402 | 403 | A producer (sender) that sends a single message, and a consumer (receiver) that receives messages and prints them out. It's a "Hello World" of messaging. 404 | 405 | You can find the code [here](IBM_MQ-Code) 406 | 407 | 408 | -------------------------------------------------------------------------------- /RabbitMQ-Code/rabbitmq-monitoring.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import requests 3 | import base64 4 | import schedule 5 | from tabulate import tabulate 6 | 7 | # credentials 8 | username = "" 9 | password = "" 10 | req_headers = { 11 | "Content-Type": "application/json", 12 | "Authorization": "" 13 | } 14 | 15 | URL = "http://localhost:15672/api/queues/%2F" 16 | 17 | def take_credentials(): 18 | username = input("Enter username: ") 19 | password = input("Enter password: ") 20 | # Since RabbitMQ API uses HTTP Basic authentication 21 | # we encode username and password into base64 strings 22 | # and then add it to request header 23 | credentials = f"{username}:{password}" 24 | credentials_bytes = credentials.encode("ascii") 25 | base64_credentials = base64.b64encode(credentials_bytes) 26 | base64_credentials_string = base64_credentials.decode("ascii") 27 | req_headers["Authorization"] = f"Basic {base64_credentials_string}" 28 | 29 | # monitoring 30 | def monitor(): 31 | print("Monitoring:") 32 | # send api request to get queue details 33 | res = requests.get( 34 | url=URL, 35 | headers=req_headers) 36 | # convert res to json 37 | queues = res.json() 38 | 39 | 40 | table_headers = ['name', 'state', 'ready', 'unacknowledged', 'total', 'incoming'] 41 | rows = [] 42 | 43 | for queue in queues: 44 | # Overview 45 | name = queue['name'] 46 | state = 'idle' if 'idle_since' in list(queue.keys()) else 'running' 47 | 48 | # Messages 49 | ready = queue['messages_ready'] 50 | unacknowledged = queue['messages_unacknowledged'] 51 | total = queue['messages'] 52 | 53 | # Message rates 54 | if 'message_stats' in list(queue.keys()): 55 | if 'publish_details' in list(queue['message_stats'].keys()): 56 | incoming = queue['message_stats']['publish_details']['rate'] 57 | else: 58 | incoming = 0.0 59 | 60 | # create a row for a queue 61 | rows.append([name, state, ready, unacknowledged, total, str(incoming)+'/s']) 62 | 63 | # print queue details 64 | table = tabulate(rows, headers=table_headers, tablefmt='orgtbl') 65 | print(table) 66 | 67 | 68 | def main(): 69 | # take credentials from user 70 | take_credentials() 71 | 72 | # call monitor every 5 seconds 73 | schedule.every(5).seconds.do(monitor) 74 | 75 | while True: 76 | schedule.run_pending() 77 | 78 | if __name__ == '__main__': 79 | main() -------------------------------------------------------------------------------- /RabbitMQ-Code/receive.py: -------------------------------------------------------------------------------- 1 | import pika 2 | 3 | connection = pika.BlockingConnection( 4 | pika.ConnectionParameters(host='localhost')) 5 | channel = connection.channel() 6 | 7 | channel.queue_declare(queue='hello') 8 | 9 | def callback(ch, method, properties, body): 10 | print("Received %r" % body) 11 | 12 | channel.basic_consume( 13 | queue='hello', on_message_callback=callback, auto_ack=True) 14 | 15 | print(' [*] Waiting for messages. To exit press CTRL+C') 16 | channel.start_consuming() -------------------------------------------------------------------------------- /RabbitMQ-Code/send.py: -------------------------------------------------------------------------------- 1 | import pika 2 | 3 | connection = pika.BlockingConnection( 4 | pika.ConnectionParameters(host='localhost')) 5 | channel = connection.channel() 6 | 7 | channel.queue_declare(queue='hello') 8 | 9 | channel.basic_publish( 10 | exchange='', 11 | routing_key='hello', 12 | body='Hello World') 13 | 14 | print("Sent Hello World") 15 | connection.close() 16 | --------------------------------------------------------------------------------