├── producer ├── api │ ├── gateway │ │ ├── __init__.py │ │ └── rabbitmq.py │ ├── services │ │ ├── __init__.py │ │ └── handler.py │ ├── __main__.py │ ├── enum.py │ └── __init__.py ├── app.py ├── requirements.txt ├── .flake8 └── Dockerfile ├── receiver ├── api │ ├── gateway │ │ ├── __init__.py │ │ └── rabbitmq.py │ ├── __main__.py │ ├── enums.py │ └── __init__.py ├── requirements.txt ├── app.py ├── .flake8 └── Dockerfile ├── docs └── images │ └── architecture.png ├── docker-compose.yml ├── .gitignore └── README.md /producer/api/gateway/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /producer/api/services/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /receiver/api/gateway/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /producer/api/__main__.py: -------------------------------------------------------------------------------- 1 | from api import main 2 | 3 | if __name__ == '__main__': 4 | main() 5 | -------------------------------------------------------------------------------- /receiver/api/__main__.py: -------------------------------------------------------------------------------- 1 | from api import main 2 | 3 | if __name__ == '__main__': 4 | main() 5 | -------------------------------------------------------------------------------- /docs/images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FernandoBLima/python-rabbitmq-docker/HEAD/docs/images/architecture.png -------------------------------------------------------------------------------- /receiver/requirements.txt: -------------------------------------------------------------------------------- 1 | pika==1.1.0 2 | flake8==3.8.4 3 | autopep8==1.5.4 4 | flake8-import-order==0.18.1 5 | python-dotenv==0.19.0 -------------------------------------------------------------------------------- /producer/app.py: -------------------------------------------------------------------------------- 1 | from api import main 2 | from dotenv import load_dotenv 3 | 4 | if __name__ == '__main__': 5 | load_dotenv() 6 | main() -------------------------------------------------------------------------------- /receiver/app.py: -------------------------------------------------------------------------------- 1 | from api import main 2 | 3 | from dotenv import load_dotenv 4 | 5 | if __name__ == '__main__': 6 | load_dotenv() 7 | main() 8 | -------------------------------------------------------------------------------- /producer/requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.7.4 2 | requests==2.24.0 3 | pika==1.1.0 4 | flake8==3.8.4 5 | autopep8==1.5.4 6 | flake8-import-order==0.18.1 7 | python-dotenv==0.19.0 -------------------------------------------------------------------------------- /producer/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = D203 3 | exclude = 4 | .git, 5 | __pycache__, 6 | docs/source/conf.py, 7 | old, 8 | build, 9 | dist 10 | max-complexity = 10 11 | max-line-length = 120 -------------------------------------------------------------------------------- /receiver/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = D203 3 | exclude = 4 | .git, 5 | __pycache__, 6 | docs/source/conf.py, 7 | old, 8 | build, 9 | dist 10 | max-complexity = 10 11 | max-line-length = 120 -------------------------------------------------------------------------------- /producer/api/services/handler.py: -------------------------------------------------------------------------------- 1 | from aiohttp import web 2 | 3 | 4 | class Handler(): 5 | 6 | def __init__(self, rabbitmq) -> None: 7 | self.rabbitmq = rabbitmq 8 | 9 | async def publish(self, request): 10 | body = await request.json() 11 | self.rabbitmq.publish(message={"data": body}) 12 | return web.json_response({'message': body}) 13 | -------------------------------------------------------------------------------- /producer/api/enum.py: -------------------------------------------------------------------------------- 1 | import os 2 | from enum import Enum 3 | 4 | 5 | class EnvironmentVariables(str, Enum): 6 | RABBITMQ_USERNAME = 'RABBITMQ_USERNAME' 7 | RABBITMQ_PASSSWORD = 'RABBITMQ_PASSSWORD' 8 | RABBITMQ_HOST = 'RABBITMQ_HOST' 9 | RABBITMQ_QUEUE = 'RABBITMQ_QUEUE' 10 | RABBITMQ_ROUTING_KEY = 'RABBITMQ_ROUTING_KEY' 11 | RABBITMQ_EXCHANGE = 'RABBITMQ_EXCHANGE' 12 | SERVER_PORT = 'SERVER_PORT' 13 | SERVER_HOST = 'SERVER_HOST' 14 | 15 | def get_env(self, variable=None): 16 | return os.environ.get(self, variable) 17 | -------------------------------------------------------------------------------- /receiver/api/enums.py: -------------------------------------------------------------------------------- 1 | import os 2 | from enum import Enum 3 | 4 | 5 | class EnvironmentVariables(str, Enum): 6 | RABBITMQ_USERNAME = 'RABBITMQ_USERNAME' 7 | RABBITMQ_PASSSWORD = 'RABBITMQ_PASSSWORD' 8 | RABBITMQ_HOST = 'RABBITMQ_HOST' 9 | RABBITMQ_QUEUE = 'RABBITMQ_QUEUE' 10 | RABBITMQ_ROUTING_KEY = 'RABBITMQ_ROUTING_KEY' 11 | RABBITMQ_EXCHANGE = 'RABBITMQ_EXCHANGE' 12 | SERVER_PORT = 'SERVER_PORT' 13 | SERVER_HOST = 'SERVER_HOST' 14 | 15 | def get_env(self, variable=None): 16 | return os.environ.get(self, variable) 17 | -------------------------------------------------------------------------------- /receiver/Dockerfile: -------------------------------------------------------------------------------- 1 | # Set arguments 2 | ARG BASE_CONTAINER=python:3.8 3 | 4 | # Set the base image. 5 | FROM --platform=linux/amd64 $BASE_CONTAINER 6 | 7 | # Adds metadata to image. 8 | LABEL maintainer="Fernando" 9 | 10 | # Sets the user name to use when running the image. 11 | USER root 12 | RUN apt-get update && apt-get install -y && apt-get clean 13 | 14 | # Make a directory for our app 15 | WORKDIR /receiver 16 | 17 | # Install dependencies 18 | COPY requirements.txt . 19 | RUN pip install --upgrade pip && pip install -r requirements.txt 20 | 21 | # Copy our source code 22 | COPY /api ./api 23 | 24 | # Run the application 25 | CMD ["python", "-m", "api"] -------------------------------------------------------------------------------- /producer/Dockerfile: -------------------------------------------------------------------------------- 1 | # Set arguments 2 | ARG BASE_CONTAINER=python:3.8 3 | 4 | # Set the base image. 5 | FROM --platform=linux/amd64 $BASE_CONTAINER 6 | 7 | # Adds metadata to image. 8 | LABEL maintainer="Fernando" 9 | 10 | # Sets the user name to use when running the image. 11 | USER root 12 | RUN apt-get update && apt-get install -y && apt-get clean 13 | 14 | # Make a directory for our app 15 | WORKDIR /producer 16 | 17 | # Install dependencies 18 | COPY requirements.txt . 19 | COPY app.py . 20 | 21 | RUN pip install --upgrade pip && pip install -r requirements.txt 22 | 23 | # Copy source code 24 | COPY ./api ./api 25 | 26 | # Run the application 27 | CMD ["python", "-m", "api"] -------------------------------------------------------------------------------- /receiver/api/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from api.enums import EnvironmentVariables 4 | from api.gateway.rabbitmq import rabbitMQServer 5 | 6 | 7 | def main(): 8 | logging.basicConfig( 9 | format='%(asctime)s %(message)s', 10 | datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.INFO 11 | ) 12 | 13 | server = rabbitMQServer( 14 | queue=EnvironmentVariables.RABBITMQ_QUEUE.get_env(), 15 | host=EnvironmentVariables.RABBITMQ_HOST.get_env(), 16 | routing_key=EnvironmentVariables.RABBITMQ_ROUTING_KEY.get_env(), 17 | username=EnvironmentVariables.RABBITMQ_USERNAME.get_env(), 18 | password=EnvironmentVariables.RABBITMQ_PASSSWORD.get_env(), 19 | exchange=EnvironmentVariables.RABBITMQ_EXCHANGE.get_env(), 20 | ) 21 | server.get_messages() 22 | -------------------------------------------------------------------------------- /producer/api/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from aiohttp import web 4 | 5 | from api.enum import EnvironmentVariables 6 | from api.gateway.rabbitmq import RabbitMQ 7 | from api.services.handler import Handler 8 | 9 | 10 | async def health_status(request): 11 | return web.Response(text='UP') 12 | 13 | 14 | def main(): 15 | logging.basicConfig( 16 | format='%(asctime)s %(message)s', 17 | datefmt='%m/%d/%Y %I:%M:%S %p', 18 | level=logging.INFO 19 | ) 20 | 21 | rabbitMQ_instance = RabbitMQ( 22 | queue=EnvironmentVariables.RABBITMQ_QUEUE.get_env(), 23 | host=EnvironmentVariables.RABBITMQ_HOST.get_env(), 24 | routing_key=EnvironmentVariables.RABBITMQ_ROUTING_KEY.get_env(), 25 | username=EnvironmentVariables.RABBITMQ_USERNAME.get_env(), 26 | password=EnvironmentVariables.RABBITMQ_PASSSWORD.get_env(), 27 | exchange=EnvironmentVariables.RABBITMQ_EXCHANGE.get_env() 28 | ) 29 | 30 | app = web.Application() 31 | handler = Handler(rabbitMQ_instance) 32 | 33 | app.add_routes([ 34 | web.get('/', health_status), 35 | web.post('/', handler.publish), 36 | ]) 37 | 38 | web.run_app( 39 | app, 40 | port=7000 41 | ) 42 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | rabbitmqServer: 5 | container_name: rabbitmqServer 6 | image: "rabbitmq:3.6-management-alpine" 7 | hostname: "rabbitmq-host" 8 | ports: 9 | - "5672:5672" 10 | - "15672:15672" 11 | environment: 12 | RABBITMQ_DEFAULT_USER: "admin" 13 | RABBITMQ_DEFAULT_PASS: "pass" 14 | networks: 15 | - rabbitnetwork 16 | 17 | producer: 18 | build: 19 | context: ./producer 20 | dockerfile: Dockerfile 21 | container_name: producer 22 | depends_on: 23 | - rabbitmqServer 24 | - receiver 25 | ports: 26 | - 7000:7000 27 | environment: 28 | RABBITMQ_USERNAME: "admin" 29 | RABBITMQ_PASSSWORD: "pass" 30 | RABBITMQ_HOST: rabbitmqServer 31 | RABBITMQ_QUEUE: dataqueue 32 | RABBITMQ_ROUTING_KEY: dataqueue 33 | RABBITMQ_EXCHANGE: exchange_test 34 | SERVER_PORT: 7000 35 | SERVER_HOST: receiver 36 | networks: 37 | - rabbitnetwork 38 | restart: always 39 | # healthcheck: 40 | # interval: 10s 41 | # timeout: 5s 42 | 43 | receiver: 44 | container_name: receiver 45 | build: 46 | context: ./receiver 47 | dockerfile: Dockerfile 48 | restart: always 49 | depends_on: 50 | - rabbitmqServer 51 | ports: 52 | - 8000:8000 53 | environment: 54 | RABBITMQ_USERNAME: admin 55 | RABBITMQ_PASSSWORD: pass 56 | RABBITMQ_HOST: rabbitmqServer 57 | RABBITMQ_QUEUE: dataqueue 58 | RABBITMQ_ROUTING_KEY: dataqueue 59 | RABBITMQ_EXCHANGE: exchange_test 60 | networks: 61 | - rabbitnetwork 62 | 63 | networks: 64 | rabbitnetwork: 65 | driver: bridge -------------------------------------------------------------------------------- /producer/api/gateway/rabbitmq.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | import pika 5 | 6 | 7 | class RabbitMQ(): 8 | 9 | def __init__(self, queue, host, routing_key, username, password, exchange=''): 10 | self._queue = queue 11 | self._host = host 12 | self._routing_key = routing_key 13 | self._exchange = exchange 14 | self._username = username 15 | self._password = password 16 | self.start_server() 17 | 18 | def start_server(self): 19 | self.create_channel() 20 | self.create_exchange() 21 | self.create_bind() 22 | logging.info("Channel created...") 23 | 24 | def create_channel(self): 25 | credentials = pika.PlainCredentials(username=self._username, password=self._password) 26 | parameters = pika.ConnectionParameters(self._host, credentials=credentials) 27 | self._connection = pika.BlockingConnection(parameters) 28 | self._channel = self._connection.channel() 29 | 30 | def create_exchange(self): 31 | self._channel.exchange_declare( 32 | exchange=self._exchange, 33 | exchange_type='direct', 34 | passive=False, 35 | durable=True, 36 | auto_delete=False 37 | ) 38 | self._channel.queue_declare(queue=self._queue, durable=False) 39 | 40 | def create_bind(self): 41 | self._channel.queue_bind( 42 | queue=self._queue, 43 | exchange=self._exchange, 44 | routing_key=self._routing_key 45 | ) 46 | 47 | def publish(self, message={}): 48 | """ 49 | :param message: message to be publish in JSON format 50 | """ 51 | 52 | self._channel.basic_publish( 53 | exchange=self._exchange, 54 | routing_key=self._routing_key, 55 | body=json.dumps(message), 56 | properties=pika.BasicProperties(content_type='application/json') 57 | ) 58 | logging.info("Published Message: {}".format(message)) 59 | -------------------------------------------------------------------------------- /receiver/api/gateway/rabbitmq.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import pika 4 | 5 | 6 | class rabbitMQServer(): 7 | """ 8 | Producer component that will publish message and handle 9 | connection and channel interactions with RabbitMQ. 10 | """ 11 | 12 | def __init__(self, queue, host, routing_key, username, password, exchange=''): 13 | self._queue = queue 14 | self._host = host 15 | self._routing_key = routing_key 16 | self._exchange = exchange 17 | self._username = username 18 | self._password = password 19 | self.start_server() 20 | 21 | def start_server(self): 22 | self.create_channel() 23 | self.create_exchange() 24 | self.create_bind() 25 | logging.info("Channel created...") 26 | 27 | def create_channel(self): 28 | credentials = pika.PlainCredentials(username=self._username, password=self._password) 29 | parameters = pika.ConnectionParameters(self._host, credentials=credentials) 30 | self._connection = pika.BlockingConnection(parameters) 31 | self._channel = self._connection.channel() 32 | 33 | def create_exchange(self): 34 | self._channel.exchange_declare( 35 | exchange=self._exchange, 36 | exchange_type='direct', 37 | passive=False, 38 | durable=True, 39 | auto_delete=False 40 | ) 41 | self._channel.queue_declare(queue=self._queue, durable=False) 42 | 43 | def create_bind(self): 44 | self._channel.queue_bind( 45 | queue=self._queue, 46 | exchange=self._exchange, 47 | routing_key=self._routing_key 48 | ) 49 | self._channel.basic_qos(prefetch_count=1) 50 | 51 | @staticmethod 52 | def callback(channel, method, properties, body): 53 | logging.info(f'Consumed message {body.decode()} from queue') 54 | 55 | def get_messages(self): 56 | try: 57 | logging.info("Starting the server...") 58 | self._channel.basic_consume( 59 | queue=self._queue, 60 | on_message_callback=rabbitMQServer.callback, 61 | auto_ack=True 62 | ) 63 | self._channel.start_consuming() 64 | except Exception as e: 65 | logging.debug(f'Exception: {e}') 66 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 98 | __pypackages__/ 99 | 100 | # Celery stuff 101 | celerybeat-schedule 102 | celerybeat.pid 103 | 104 | # SageMath parsed files 105 | *.sage.py 106 | 107 | # Environments 108 | .env 109 | .venv 110 | env/ 111 | venv/ 112 | ENV/ 113 | env.bak/ 114 | venv.bak/ 115 | 116 | # Spyder project settings 117 | .spyderproject 118 | .spyproject 119 | 120 | # Rope project settings 121 | .ropeproject 122 | 123 | # mkdocs documentation 124 | /site 125 | 126 | # mypy 127 | .mypy_cache/ 128 | .dmypy.json 129 | dmypy.json 130 | 131 | # Pyre type checker 132 | .pyre/ 133 | 134 | # pytype static type analyzer 135 | .pytype/ 136 | 137 | # Cython debug symbols 138 | cython_debug/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # python-rabbitmq-docker 2 | 3 | In this project, the main objective was to learn how to create an application that sends and receives a message from a RabbitMQ message broker, using Docker and docker-compose tools. 4 | 5 | ## Stack 6 | 7 | - aiohttp 8 | - Docker 9 | - RabbitMQ 10 | - Pika 11 | - Python 3.8 12 | 13 | ## How to use 14 | 15 | ### Using Docker Compose 16 | You will need Docker installed to follow the next step. To create and run the image use the following command: 17 | ```bash 18 | docker-compose up --build 19 | ``` 20 | 21 | The configuration will create a cluster with 3 containers: 22 | 23 | - Producer container 24 | - Receiver container 25 | - RabbitMQ container 26 | 27 | The Producer container will create a simple RESTful API application that sends data to the message broker (RabbitMQ). It will take a few seconds to come up, then the server will be accessible at `http://localhost:7000`. 28 | 29 | The Receiver container is a script that aims to wait and receive messages from Message Broker. 30 | 31 | And the RabbitMQ container is where messages flow through RabbitMQ and applications, stored inside a queue. A web browser access to the Dashboard is also provided for RabbitMQ message management and monitoring which can be accessed at `http://localhost:15672`. 32 | 33 | 34 | ### API 35 | - Send Message 36 | Request to send message to Message Broker. Below is an example request: 37 | ```json 38 | POST http://localhost:7000/ 39 | Accept: application/json 40 | Content-Type: application/json 41 | Body: 42 | { 43 | "key": "value", 44 | } 45 | ``` 46 | 47 | - Health check 48 | Request to check if the application is available. 49 | ```json 50 | GET http://localhost:7000/ 51 | Accept: application/json 52 | Content-Type: application/json 53 | ``` 54 | 55 | ## Project Structure 56 | Below is a project structure created: 57 | 58 | ```cmd 59 | . 60 | ├── README.md 61 | ├── docker-compose.yml 62 | ├── producer 63 | │ ├── Dockerfile 64 | │ ├── api 65 | │ │ ├── __init__.py 66 | │ │ ├── __main__.py 67 | │ │ ├── enum.py 68 | │ │ ├── gateway 69 | │ │ │ ├── __init__.py 70 | │ │ │ └── rabbitmq.py 71 | │ │ └── services 72 | │ │ ├── __init__.py 73 | │ │ └── handler.py 74 | │ ├── app.py 75 | │ └── requirements.txt 76 | └── receiver 77 | ├── Dockerfile 78 | ├── api 79 | │ ├── __init__.py 80 | │ ├── __main__.py 81 | │ ├── enums.py 82 | │ └── gateway 83 | │ ├── __init__.py 84 | │ └── rabbitmq.py 85 | ├── app.py 86 | └── requirements.txt 87 | ``` 88 | 89 | ## Architecture 90 | 91 | ![image](./docs/images/architecture.png) 92 | 93 | 94 | ## Environment Variables 95 | Listed below are the environment variables needed to run the application. They can be included in docker-compose or to run locally, you need to create an `.env` file in the root of the Producer and Receiver service folders. 96 | 97 | - Receiver: 98 | ```cmd 99 | RABBITMQ_USERNAME: 100 | RABBITMQ_PASSSWORD: 101 | RABBITMQ_HOST: 102 | RABBITMQ_QUEUE: 103 | RABBITMQ_ROUTING_KEY: 104 | RABBITMQ_EXCHANGE: 105 | ``` 106 | 107 | - Producer: 108 | ```cmd 109 | RABBITMQ_USERNAME: 110 | RABBITMQ_PASSSWORD: 111 | RABBITMQ_HOST: 112 | RABBITMQ_QUEUE: 113 | RABBITMQ_ROUTING_KEY: 114 | RABBITMQ_EXCHANGE: 115 | SERVER_PORT: 116 | SERVER_HOST: 117 | ``` 118 | 119 | - RabbitMQ: 120 | ```cmd 121 | RABBITMQ_DEFAULT_USER: 122 | RABBITMQ_DEFAULT_PASS: 123 | ``` 124 | 125 | ## Help and Resources 126 | You can read more about the tools documentation: 127 | 128 | - [aiohttp](https://docs.aiohttp.org/en/stable/index.html) 129 | - [Docker](https://docs.docker.com/get-started/overview/) 130 | - [RabbitMQ](https://www.rabbitmq.com) 131 | - [Pika](https://pika.readthedocs.io/en/stable/#) --------------------------------------------------------------------------------