├── .gitignore ├── README.md ├── api.Dockerfile ├── celery.Dockerfile ├── docker-compose.yml ├── flower.Dockerfile ├── rabbitmq.conf ├── requirements-api.txt ├── requirements-celery.txt └── shopping ├── __init__.py ├── api ├── __init__.py └── product.py ├── celery ├── __init__.py ├── tasks.py └── worker.py ├── config.py └── main.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | .idea 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | *.db 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fastapi - celery - rabbitmq - redis -> Docker 2 | 3 | 4 | A simple docker-compose app for orchestrating a fastapi application, a celery queue with rabbitmq(broker) and redis(backend) 5 | 6 | 7 | ## build and run containers 8 | 9 | ```bash 10 | docker-compose up -d --build 11 | ``` 12 | 13 | This will expose fastapi application on 5000 and celery flower on 5555 14 | 15 | swagger docs - `http://localhost:5000/` 16 | 17 | redoc - `http://localhost:5000/redoc` 18 | 19 | celery flower - `http://localhost:5555` 20 | -------------------------------------------------------------------------------- /api.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8 2 | 3 | COPY ./shopping /usr/src/shopping 4 | 5 | COPY ./requirements-api.txt /usr/src/ 6 | 7 | RUN pip3 install --upgrade pip 8 | 9 | RUN pip3 install -r /usr/src/requirements-api.txt 10 | 11 | WORKDIR /usr/src 12 | 13 | CMD gunicorn --bind 0.0.0.0:5000 shopping.main:app -w 4 -k uvicorn.workers.UvicornWorker --access-logfile - --error-logfile - --log-level info 14 | -------------------------------------------------------------------------------- /celery.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8 2 | 3 | COPY ./shopping/celery /usr/src/shopping/celery 4 | 5 | COPY ./shopping/__init__.py /usr/src/shopping/ 6 | 7 | COPY ./shopping/config.py /usr/src/shopping/ 8 | 9 | COPY ./requirements-celery.txt /usr/src/ 10 | 11 | RUN pip3 install --upgrade pip 12 | 13 | RUN pip3 install -r /usr/src/requirements-celery.txt 14 | 15 | WORKDIR /usr/src 16 | 17 | CMD celery worker -A shopping.celery.tasks --loglevel=info 18 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | krabbitmq: 4 | hostname: krabbitmq 5 | image: rabbitmq:latest 6 | volumes: 7 | - "./rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf" 8 | ports: 9 | - "5672:5672" 10 | worker: 11 | image: karthikasasanka/shopping-celery:v1 12 | environment: 13 | - REDIS_HOST=kredis 14 | - REDIS_PORT=6379 15 | - REDIS_CELERY_DB_INDEX=10 16 | - REDIS_STORE_DB_INDEX=0 17 | - RABBITMQ_HOST=krabbitmq 18 | - RABBITMQ_USERNAME=guest 19 | - RABBITMQ_PASSWORD=guest 20 | - RABBITMQ_PORT=5672 21 | depends_on: 22 | - krabbitmq 23 | - kredis 24 | kredis: 25 | hostname: kredis 26 | image: redis:latest 27 | ports: 28 | - "6379:6379" 29 | api: 30 | image: karthikasasanka/shopping-api:v1 31 | environment: 32 | - REDIS_HOST=kredis 33 | - REDIS_PORT=6379 34 | - REDIS_CELERY_DB_INDEX=10 35 | - REDIS_STORE_DB_INDEX=0 36 | - RABBITMQ_HOST=krabbitmq 37 | - RABBITMQ_USERNAME=guest 38 | - RABBITMQ_PASSWORD=guest 39 | - RABBITMQ_PORT=5672 40 | volumes: 41 | - ./shopping:/usr/src/shopping/ 42 | depends_on: 43 | - worker 44 | ports: 45 | - "5000:5000" 46 | flower: 47 | image: karthikasasanka/shopping-celery-flower:v1 48 | environment: 49 | - REDIS_HOST=kredis 50 | - REDIS_PORT=6379 51 | - REDIS_CELERY_DB_INDEX=10 52 | - REDIS_STORE_DB_INDEX=0 53 | - RABBITMQ_HOST=krabbitmq 54 | - RABBITMQ_USERNAME=guest 55 | - RABBITMQ_PASSWORD=guest 56 | - RABBITMQ_PORT=5672 57 | ports: 58 | - 5555:5555 59 | depends_on: 60 | - krabbitmq 61 | - kredis -------------------------------------------------------------------------------- /flower.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8 2 | 3 | COPY ./shopping/celery /usr/src/shopping/celery 4 | 5 | COPY ./shopping/__init__.py /usr/src/shopping/ 6 | 7 | COPY ./shopping/config.py /usr/src/shopping/ 8 | 9 | COPY ./requirements-celery.txt /usr/src/ 10 | 11 | RUN pip3 install --upgrade pip 12 | 13 | RUN pip3 install -r /usr/src/requirements-celery.txt 14 | 15 | RUN pip3 install flower 16 | 17 | WORKDIR /usr/src 18 | 19 | CMD celery flower -A shopping.celery.tasks --broker=amqp://${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}@${RABBITMQ_HOST}:${RABBITMQ_PORT}// 20 | 21 | -------------------------------------------------------------------------------- /rabbitmq.conf: -------------------------------------------------------------------------------- 1 | default_user = guest 2 | default_pass = guest -------------------------------------------------------------------------------- /requirements-api.txt: -------------------------------------------------------------------------------- 1 | amqp==2.6.0 2 | billiard==3.6.3.0 3 | celery==5.2.2 4 | click==7.1.2 5 | fastapi==0.65.2 6 | future==0.18.2 7 | gunicorn==20.0.4 8 | h11==0.9.0 9 | httptools==0.1.1 10 | kombu==4.6.11 11 | pydantic==1.6.2 12 | pytz==2020.1 13 | redis==3.5.3 14 | starlette==0.13.4 15 | uvicorn==0.11.7 16 | uvloop==0.14.0 17 | vine==1.3.0 18 | websockets==9.1 19 | -------------------------------------------------------------------------------- /requirements-celery.txt: -------------------------------------------------------------------------------- 1 | amqp==2.6.0 2 | billiard==3.6.3.0 3 | celery==5.2.2 4 | future==0.18.2 5 | kombu==4.6.11 6 | pytz==2020.1 7 | redis==3.5.3 8 | vine==1.3.0 9 | -------------------------------------------------------------------------------- /shopping/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karthikasasanka/fastapi-celery-redis-rabbitmq/a4660c87225155d8f4c13af3c36ce48cc5136da2/shopping/__init__.py -------------------------------------------------------------------------------- /shopping/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/karthikasasanka/fastapi-celery-redis-rabbitmq/a4660c87225155d8f4c13af3c36ce48cc5136da2/shopping/api/__init__.py -------------------------------------------------------------------------------- /shopping/api/product.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter 2 | import redis 3 | 4 | from shopping.config import stages, REDIS_STORE_CONN_URI, STAGING_TIME 5 | from shopping.celery.tasks import move_to_next_stage 6 | 7 | redis_store = redis.Redis.from_url(REDIS_STORE_CONN_URI) 8 | 9 | 10 | product_router = APIRouter() 11 | 12 | 13 | @product_router.get("/buy/{name}") 14 | async def buy(name: str): 15 | for i in range(0, 5): 16 | move_to_next_stage.apply_async((name, stages[i]), countdown=i*STAGING_TIME) 17 | return True 18 | 19 | 20 | @product_router.get("/status/{name}") 21 | async def status(name: str): 22 | return redis_store.get(name) 23 | -------------------------------------------------------------------------------- /shopping/celery/__init__.py: -------------------------------------------------------------------------------- 1 | # broker_url = 'amqp://myuser:mypassword@localhost:5672/myvhost' 2 | -------------------------------------------------------------------------------- /shopping/celery/tasks.py: -------------------------------------------------------------------------------- 1 | import redis 2 | 3 | from .worker import celery 4 | from shopping.config import REDIS_STORE_CONN_URI 5 | 6 | redis_store = redis.Redis.from_url(REDIS_STORE_CONN_URI) 7 | 8 | 9 | @celery.task 10 | def move_to_next_stage(name, stage): 11 | redis_store.set(name, stage) 12 | return stage 13 | -------------------------------------------------------------------------------- /shopping/celery/worker.py: -------------------------------------------------------------------------------- 1 | 2 | from celery import Celery 3 | from shopping.config import BACKEND_CONN_URI, BROKER_CONN_URI 4 | 5 | 6 | celery = Celery('hello', broker=BROKER_CONN_URI, backend=BACKEND_CONN_URI) 7 | 8 | -------------------------------------------------------------------------------- /shopping/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | REDIS_HOST = os.environ.get('REDIS_HOST') 4 | REDIS_PORT = os.environ.get('REDIS_PORT') 5 | REDIS_CELERY_DB_INDEX = os.environ.get('REDIS_CELERY_DB_INDEX') 6 | REDIS_STORE_DB_INDEX = os.environ.get('REDIS_STORE_DB_INDEX') 7 | 8 | RABBITMQ_HOST = os.environ.get('RABBITMQ_HOST') 9 | RABBITMQ_USERNAME = os.environ.get('RABBITMQ_USERNAME') 10 | RABBITMQ_PASSWORD = os.environ.get('RABBITMQ_PASSWORD') 11 | RABBITMQ_PORT = os.environ.get('RABBITMQ_PORT') 12 | 13 | BROKER_CONN_URI = f"amqp://{RABBITMQ_USERNAME}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}" 14 | BACKEND_CONN_URI = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB_INDEX}" 15 | REDIS_STORE_CONN_URI = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_STORE_DB_INDEX}" 16 | 17 | stages = ["confirmed", "shipped", "in transit", "arrived", "delivered"] 18 | STAGING_TIME = 15 # seconds 19 | -------------------------------------------------------------------------------- /shopping/main.py: -------------------------------------------------------------------------------- 1 | 2 | from fastapi import FastAPI 3 | from shopping.api.product import product_router 4 | 5 | 6 | app = FastAPI(title="Shop API", docs_url="/", version="1.0.0") 7 | 8 | app.include_router( 9 | product_router, 10 | prefix="/product", 11 | tags=["Product"], 12 | ) 13 | 14 | 15 | --------------------------------------------------------------------------------