├── Dockerfile ├── README.md ├── app.py ├── assets └── temp.txt ├── confs └── supervisord.conf ├── docker-compose.yaml ├── requirements.txt └── tasks.py /Dockerfile: -------------------------------------------------------------------------------- 1 | # Dockerfile for building the inference application and Celery Worker 2 | FROM continuumio/miniconda3:latest 3 | COPY ./assets /assets 4 | COPY app.py . 5 | COPY tasks.py . 6 | COPY requirements.txt . 7 | RUN pip3 install -r requirements.txt 8 | RUN apt-get update && apt-get install -y supervisor 9 | COPY confs/supervisord.conf supervisord.conf 10 | EXPOSE 8080 11 | CMD [ "supervisord" ] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Asyncronous Processing pipelines in Python 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 |
Celeryrabbitmqfastapimongodbdocker
10 | Set up an async pipeline in python using Celery, RabbitMQ and MongoDB. This repo covers the end to end deployment of an async pipeline for your projects using Docker 11 | 12 | # Quickstart 13 | 14 | 1. Clone the repo 15 | 2. Make sure Docker and Docker Compose are installed 16 | ```bash 17 | $ pip install docker-compose 18 | ``` 19 | 3. Start the docker containers 20 | ```bash 21 | $ docker compose up --build 22 | ``` 23 | 4. Check the pipeline using FastAPI endpoints 24 | ```bash 25 | # To start processing a task 26 | $ curl -X POST http://localhost:8080/process 27 | # To check the progress of a task 28 | $ curl -X POST http://localhost:8080/check_progress/ 29 | ``` 30 | # Setting up your tasks 31 | You can use this pipline to run your custom tasks. You just have to modify the [tasks.py](https://github.com/aarunjith/async-demo/blob/main/tasks.py) and connect it to a fastapi endpoint for trigger 32 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | # Lets create a simple FastAPI service which listens to our API calls 2 | # to start processing our task 3 | 4 | from fastapi import FastAPI 5 | from celery.result import AsyncResult 6 | from tasks import start_processing 7 | from loguru import logger 8 | from pymongo import MongoClient 9 | import uvicorn 10 | 11 | # Lets create a connection to our backend where celery stores the results 12 | client = MongoClient("mongodb://mongodb:27017") 13 | 14 | # Default database and collection names that Celery create 15 | db = client['task_results'] 16 | coll = db["celery_taskmeta"] 17 | 18 | app = FastAPI() 19 | 20 | 21 | @app.post('/process') 22 | async def process_text_file(): 23 | ''' 24 | Process endpoint to trigger the start of a process 25 | ''' 26 | try: 27 | result = start_processing.delay() 28 | logger.info(f'Started processing the task with id {result.id}') 29 | return { 30 | "status": result.state, 31 | 'id': result.id, 32 | 'error': '' 33 | } 34 | except Exception as e: 35 | logger.info(f'Task Execution failed: {e}') 36 | return { 37 | "status": "FAILURE", 38 | 'id': None, 39 | 'error': e 40 | } 41 | 42 | 43 | @app.post('/check_progress/{task_id}') 44 | async def check_async_progress(task_id: str): 45 | ''' 46 | Endpoint to check the task progress and fetch the results if the task is 47 | complete. 48 | ''' 49 | try: 50 | result = AsyncResult(task_id) 51 | if result.ready(): 52 | data = coll.find({'_id': task_id})[0] 53 | return {'status': 'SUCEESS', 'data': data['result']} 54 | else: 55 | return {"status": result.state, "error": ''} 56 | except Exception as e: 57 | data = coll.find({'_id': task_id})[0] 58 | if data: 59 | return {'status': 'SUCEESS', 'data': data['result']} 60 | return {'status': 'Task ID invalid', 'error': e} 61 | 62 | if __name__ == "__main__": 63 | uvicorn.run("app:app", host='0.0.0.0', port='8080') 64 | -------------------------------------------------------------------------------- /assets/temp.txt: -------------------------------------------------------------------------------- 1 | hello -------------------------------------------------------------------------------- /confs/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon=true 3 | 4 | [program:fastapi] 5 | command=python3 app.py 6 | 7 | [program:celery] 8 | command=celery -A tasks worker --loglevel=info -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | services: 3 | api: 4 | build: . 5 | ports: 6 | - "8080:8080" 7 | depends_on: 8 | - mongodb 9 | - rabbitmq 10 | networks: 11 | - app-network 12 | mongodb: 13 | image: mongo:latest 14 | ports: 15 | - "27017:27017" 16 | networks: 17 | - app-network 18 | rabbitmq: 19 | image: rabbitmq:latest 20 | ports: 21 | - "5672:5672" 22 | networks: 23 | - app-network 24 | 25 | networks: 26 | app-network: 27 | driver: 'bridge' -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | celery 2 | fastapi 3 | loguru 4 | pymongo 5 | pymongo[srv] 6 | uvicorn -------------------------------------------------------------------------------- /tasks.py: -------------------------------------------------------------------------------- 1 | from celery import Celery 2 | import time 3 | from loguru import logger 4 | 5 | app = Celery('ROFL', broker="amqp://rabbitmq:5672", 6 | backend="mongodb://mongodb:27017/task_results") 7 | 8 | 9 | # This is the task that celery tries to execute. bind=True specifies the task is 10 | # bound and are required for task retires and accessing the task status 11 | 12 | 13 | @app.task(bind=True) 14 | def start_processing(self): 15 | # Lets simulate a long running task here. By long running we are 16 | # talking even tasks that take just a minute even. Lets pretend to 17 | # read a file that takes 1 minute to be read, and it contains just 18 | # a "Hello World" 19 | logger.info('Reading a book :|') 20 | with open('assets/temp.txt', 'r') as file: 21 | result = file.read() 22 | time.sleep(60) 23 | logger.info('Done reading a book :)') 24 | return result 25 | --------------------------------------------------------------------------------