├── README.md ├── docker-compose.yml └── kafka_app ├── consumer ├── .dockerignore ├── Dockerfile ├── app │ ├── __init__.py │ ├── config.py │ └── main.py ├── entrypoint.sh └── requirements.txt └── producer ├── .dockerignore ├── Dockerfile ├── app ├── __init__.py ├── config.py └── main.py ├── entrypoint.sh ├── pyproject.toml ├── requirements.txt └── setup.cfg /README.md: -------------------------------------------------------------------------------- 1 | # Kafka PoC using FastAPI 2 | Both the producer and the consumer wait for Kafka to accept connections using kafkacat. 3 | ## Run and test 4 | 5 | To start the application just run 6 | ``` 7 | docker-compose up -d 8 | ``` 9 | 10 | To post messages for the producer access swagger 11 | ``` 12 | http://localhost:8080/kafka_producer/docs 13 | ``` 14 | 15 | To see the messages consumed by the consumer 16 | ``` 17 | docker-compose logs consumer 18 | ``` 19 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | 3 | services: 4 | zookeeper: 5 | image: bitnami/zookeeper:latest 6 | ports: 7 | - 2181:2181 8 | environment: 9 | - ALLOW_ANONYMOUS_LOGIN=yes 10 | 11 | kafka: 12 | image: bitnami/kafka:latest 13 | ports: 14 | - 9092:9092 15 | - 9093:9093 16 | environment: 17 | - KAFKA_BROKER_ID=1 18 | - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092 19 | - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092 20 | - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 21 | - ALLOW_PLAINTEXT_LISTENER=yes 22 | - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT 23 | - KAFKA_CFG_LISTENERS=CLIENT://:9092 24 | - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka:9092 25 | - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT 26 | depends_on: 27 | - zookeeper 28 | 29 | producer: 30 | build: 31 | context: ./kafka_app/producer 32 | dockerfile: Dockerfile 33 | command: uvicorn app.main:app --reload --workers 1 --host 0.0.0.0 --port 8000 34 | ports: 35 | - 8080:8000 36 | volumes: 37 | - ./kafka_app/producer:/usr/src/app 38 | environment: 39 | - KAFKA_HOST=kafka 40 | - KAFKA_PORT=9092 41 | depends_on: 42 | - kafka 43 | - zookeeper 44 | 45 | consumer: 46 | build: 47 | context: ./kafka_app/consumer 48 | dockerfile: Dockerfile 49 | command: uvicorn app.main:app --reload --workers 1 --host 0.0.0.0 --port 8000 50 | volumes: 51 | - ./kafka_app/consumer:/usr/src/app 52 | environment: 53 | - KAFKA_HOST=kafka 54 | - KAFKA_PORT=9092 55 | - KAFKA_TOPICS=jobs 56 | depends_on: 57 | - kafka 58 | - zookeeper -------------------------------------------------------------------------------- /kafka_app/consumer/.dockerignore: -------------------------------------------------------------------------------- 1 | **/env/ 2 | **/__pycache__/ 3 | **/.pytest_cache/ 4 | **/.vscode/ 5 | **/htmlcov/ 6 | **/.coverage 7 | -------------------------------------------------------------------------------- /kafka_app/consumer/Dockerfile: -------------------------------------------------------------------------------- 1 | # pull official base image 2 | FROM python:3.9.6-slim-buster 3 | 4 | # set working directory 5 | WORKDIR /usr/src/app 6 | 7 | # set environment variables 8 | ENV PYTHONDONTWRITEBYTECODE 1 9 | ENV PYTHONUNBUFFERED 1 10 | 11 | # install system dependencies 12 | RUN apt-get update \ 13 | && apt-get -y install kafkacat\ 14 | && apt-get clean \ 15 | && rm -rf /var/lib/apt/lists/* 16 | 17 | # install python dependencies 18 | RUN pip install --upgrade pip 19 | COPY ./requirements.txt . 20 | RUN pip install -r requirements.txt --no-cache-dir 21 | 22 | # add entrypoint.sh 23 | COPY ./entrypoint.sh . 24 | RUN chmod +x /usr/src/app/entrypoint.sh 25 | 26 | # run entrypoint.sh 27 | ENTRYPOINT ["sh", "/usr/src/app/entrypoint.sh"] 28 | -------------------------------------------------------------------------------- /kafka_app/consumer/app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GavriloviciEduard/fastapi-kafka/fa39aa5b3f3b92ba369b7b8065abc0d2adfc7f91/kafka_app/consumer/app/__init__.py -------------------------------------------------------------------------------- /kafka_app/consumer/app/config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from functools import lru_cache 4 | 5 | from pydantic import BaseSettings 6 | 7 | log = logging.getLogger("uvicorn") 8 | 9 | 10 | class Settings(BaseSettings): 11 | """Class for storing settings.""" 12 | 13 | kafka_host: str = os.getenv("KAFKA_HOST") 14 | kafka_port: str = os.getenv("KAFKA_PORT") 15 | kafka_topics: str = os.getenv("KAFKA_TOPICS") 16 | kafka_instance = f"{kafka_host}:{kafka_port}" 17 | file_encoding: str = "utf-8" 18 | 19 | 20 | @lru_cache() 21 | def get_settings() -> BaseSettings: 22 | """Get application settings usually stored as environment variables. 23 | 24 | Returns: 25 | Settings: Application settings. 26 | """ 27 | log.info("Loading config settings from the environment...") 28 | return Settings() 29 | -------------------------------------------------------------------------------- /kafka_app/consumer/app/main.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import brotli 4 | from aiokafka import AIOKafkaConsumer 5 | from fastapi import FastAPI 6 | 7 | from app.config import get_settings 8 | 9 | log = logging.getLogger("uvicorn") 10 | 11 | 12 | def create_application() -> FastAPI: 13 | """Create FastAPI application and set routes. 14 | 15 | Returns: 16 | FastAPI: The created FastAPI instance. 17 | """ 18 | 19 | return FastAPI() 20 | 21 | 22 | def create_consumer() -> AIOKafkaConsumer: 23 | 24 | return AIOKafkaConsumer( 25 | get_settings().kafka_topics, 26 | bootstrap_servers=get_settings().kafka_instance, 27 | ) 28 | 29 | 30 | app = create_application() 31 | consumer = create_consumer() 32 | 33 | 34 | async def decompress(file_bytes: bytes) -> str: 35 | return str( 36 | brotli.decompress(file_bytes), 37 | get_settings().file_encoding, 38 | ) 39 | 40 | 41 | async def consume(): 42 | while True: 43 | async for msg in consumer: 44 | print( 45 | "consumed: ", 46 | f"topic: {msg.topic},", 47 | f"partition: {msg.partition},", 48 | f"offset: {msg.offset},", 49 | f"key: {msg.key},", 50 | f"value: {await decompress(msg.value)},", 51 | f"timestamp: {msg.timestamp}", 52 | ) 53 | 54 | 55 | @app.on_event("startup") 56 | async def startup_event(): 57 | """Start up event for FastAPI application.""" 58 | 59 | log.info("Starting up...") 60 | await consumer.start() 61 | await consume() 62 | 63 | 64 | @app.on_event("shutdown") 65 | async def shutdown_event(): 66 | """Shutdown event for FastAPI application.""" 67 | 68 | log.info("Shutting down...") 69 | await consumer.stop() 70 | -------------------------------------------------------------------------------- /kafka_app/consumer/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "Waiting for kafka..." 4 | 5 | while ! kafkacat -b $KAFKA_HOST:$KAFKA_PORT -L; do 6 | sleep 0.1 7 | done 8 | 9 | echo "Kafka started" 10 | 11 | exec "$@" -------------------------------------------------------------------------------- /kafka_app/consumer/requirements.txt: -------------------------------------------------------------------------------- 1 | asyncpg==0.23.0 2 | fastapi==0.65.3 3 | requests==2.25.1 4 | uvicorn==0.14.0 5 | aiokafka 6 | brotli 7 | -------------------------------------------------------------------------------- /kafka_app/producer/.dockerignore: -------------------------------------------------------------------------------- 1 | **/env/ 2 | **/__pycache__/ 3 | **/.pytest_cache/ 4 | **/.vscode/ 5 | **/htmlcov/ 6 | **/.coverage 7 | -------------------------------------------------------------------------------- /kafka_app/producer/Dockerfile: -------------------------------------------------------------------------------- 1 | # pull official base image 2 | FROM python:3.7.12-slim-buster 3 | 4 | # set working directory 5 | WORKDIR /usr/src/app 6 | 7 | # set environment variables 8 | ENV PYTHONDONTWRITEBYTECODE 1 9 | ENV PYTHONUNBUFFERED 1 10 | 11 | # install system dependencies 12 | RUN apt-get update \ 13 | && apt-get -y install kafkacat \ 14 | && apt-get clean \ 15 | && rm -rf /var/lib/apt/lists/* 16 | 17 | # install python dependencies 18 | RUN pip install --upgrade pip 19 | COPY ./requirements.txt . 20 | RUN pip install -r requirements.txt --no-cache-dir 21 | 22 | # add entrypoint.sh 23 | COPY ./entrypoint.sh . 24 | RUN chmod +x /usr/src/app/entrypoint.sh 25 | 26 | # run entrypoint.sh 27 | ENTRYPOINT ["sh", "/usr/src/app/entrypoint.sh"] 28 | -------------------------------------------------------------------------------- /kafka_app/producer/app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GavriloviciEduard/fastapi-kafka/fa39aa5b3f3b92ba369b7b8065abc0d2adfc7f91/kafka_app/producer/app/__init__.py -------------------------------------------------------------------------------- /kafka_app/producer/app/config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from functools import lru_cache 4 | 5 | from pydantic import BaseSettings 6 | 7 | log = logging.getLogger("uvicorn") 8 | 9 | 10 | class Settings(BaseSettings): 11 | """Class for storing settings.""" 12 | 13 | kafka_host: str = os.getenv("KAFKA_HOST") 14 | kafka_port: str = os.getenv("KAFKA_PORT") 15 | kafka_instance = f"{kafka_host}:{kafka_port}" 16 | file_encoding: str = "utf-8" 17 | file_compression_quality: int = 1 18 | 19 | 20 | @lru_cache() 21 | def get_settings() -> BaseSettings: 22 | """Get application settings usually stored as environment variables. 23 | 24 | Returns: 25 | Settings: Application settings. 26 | """ 27 | 28 | log.info("Loading config settings from the environment...") 29 | return Settings() 30 | -------------------------------------------------------------------------------- /kafka_app/producer/app/main.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import brotli 4 | from aiokafka import AIOKafkaProducer 5 | from fastapi import FastAPI, APIRouter, Query 6 | from app.config import get_settings 7 | 8 | log = logging.getLogger("uvicorn") 9 | 10 | router = APIRouter(prefix="/kafka_producer") 11 | 12 | 13 | async def compress(message: str) -> bytes: 14 | 15 | return brotli.compress( 16 | bytes(message, get_settings().file_encoding), 17 | quality=get_settings().file_compression_quality, 18 | ) 19 | 20 | 21 | @router.post("/") 22 | async def produce_message(message: str = Query(...)) -> dict: 23 | return await producer.send_and_wait("jobs", await compress(message)) 24 | 25 | 26 | def create_application() -> FastAPI: 27 | """Create FastAPI application and set routes. 28 | 29 | Returns: 30 | FastAPI: The created FastAPI instance. 31 | """ 32 | 33 | application = FastAPI(openapi_url="/kafka_producer/openapi.json", docs_url="/kafka_producer/docs") 34 | application.include_router(router, tags=["producer"]) 35 | return application 36 | 37 | 38 | def create_producer() -> AIOKafkaProducer: 39 | 40 | return AIOKafkaProducer( 41 | bootstrap_servers=get_settings().kafka_instance, 42 | ) 43 | 44 | 45 | app = create_application() 46 | producer = create_producer() 47 | 48 | 49 | @app.on_event("startup") 50 | async def startup_event(): 51 | """Start up event for FastAPI application.""" 52 | log.info("Starting up...") 53 | await producer.start() 54 | 55 | 56 | @app.on_event("shutdown") 57 | async def shutdown_event(): 58 | """Shutdown event for FastAPI application.""" 59 | 60 | log.info("Shutting down...") 61 | await producer.stop() 62 | -------------------------------------------------------------------------------- /kafka_app/producer/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "Waiting for kafka..." 4 | 5 | while ! kafkacat -b $KAFKA_HOST:$KAFKA_PORT -L; do 6 | sleep 0.1 7 | done 8 | 9 | echo "Kafka started" 10 | 11 | exec "$@" -------------------------------------------------------------------------------- /kafka_app/producer/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 120 3 | -------------------------------------------------------------------------------- /kafka_app/producer/requirements.txt: -------------------------------------------------------------------------------- 1 | asyncpg==0.23.0 2 | fastapi==0.65.3 3 | requests==2.25.1 4 | uvicorn==0.14.0 5 | aiokafka 6 | kafka-python 7 | brotli 8 | -------------------------------------------------------------------------------- /kafka_app/producer/setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | extend-ignore = E203, CF009 4 | --------------------------------------------------------------------------------