├── .gitignore ├── README.md ├── api ├── Dockerfile ├── __init__.py ├── crud.py ├── database.py ├── logs │ └── celery.log ├── main.py ├── models.py ├── requirements.txt ├── schemas.py ├── tasks.py └── tests │ ├── __init__.py │ └── tests.py ├── docker-compose.yml └── img └── Summary.png /.gitignore: -------------------------------------------------------------------------------- 1 | # Editors 2 | .vscode/ 3 | .idea/ 4 | 5 | # Vagrant 6 | .vagrant/ 7 | 8 | # Mac/OSX 9 | .DS_Store 10 | 11 | # Windows 12 | Thumbs.db 13 | 14 | # Source for the following rules: https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore 15 | # Byte-compiled / optimized / DLL files 16 | __pycache__/ 17 | *.py[cod] 18 | *$py.class 19 | 20 | # C extensions 21 | *.so 22 | 23 | # Distribution / packaging 24 | .Python 25 | build/ 26 | develop-eggs/ 27 | dist/ 28 | downloads/ 29 | eggs/ 30 | .eggs/ 31 | lib/ 32 | lib64/ 33 | parts/ 34 | sdist/ 35 | var/ 36 | wheels/ 37 | *.egg-info/ 38 | .installed.cfg 39 | *.egg 40 | MANIFEST 41 | 42 | # PyInstaller 43 | # Usually these files are written by a python script from a template 44 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 45 | *.manifest 46 | *.spec 47 | 48 | # Installer logs 49 | pip-log.txt 50 | pip-delete-this-directory.txt 51 | 52 | # Unit test / coverage reports 53 | htmlcov/ 54 | .tox/ 55 | .nox/ 56 | .coverage 57 | .coverage.* 58 | .cache 59 | nosetests.xml 60 | coverage.xml 61 | *.cover 62 | .hypothesis/ 63 | .pytest_cache/ 64 | 65 | # Translations 66 | *.mo 67 | *.pot 68 | 69 | # Django stuff: 70 | local_settings.py 71 | db.sqlite3 72 | 73 | # Flask stuff: 74 | instance/ 75 | .webassets-cache 76 | 77 | # Scrapy stuff: 78 | .scrapy 79 | 80 | # Sphinx documentation 81 | docs/_build/ 82 | 83 | # PyBuilder 84 | target/ 85 | 86 | # Jupyter Notebook 87 | .ipynb_checkpoints 88 | 89 | # IPython 90 | profile_default/ 91 | ipython_config.py 92 | 93 | # pyenv 94 | .python-version 95 | 96 | # celery beat schedule file 97 | celerybeat-schedule 98 | 99 | # SageMath parsed files 100 | *.sage.py 101 | 102 | # Environments 103 | .env 104 | .venv 105 | env/ 106 | venv/ 107 | ENV/ 108 | env.bak/ 109 | venv.bak/ 110 | 111 | # Spyder project settings 112 | .spyderproject 113 | .spyproject 114 | 115 | # Rope project settings 116 | .ropeproject 117 | 118 | # mkdocs documentation 119 | /site 120 | 121 | # mypy 122 | .mypy_cache/ 123 | .dmypy.json 124 | dmypy.json -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FastAPI Celery Redis Postgres Docker REST API 2 | 3 | ### Summary: 4 | 5 | This is simple REST API project using a modern stack with **[FastAPI](https://fastapi.tiangolo.com/)**. 6 | **[Celery](http://www.celeryproject.org/)** for background tasks 7 | **[Redis](https://redis.io/)** for the message broker 8 | **[PostgreSQL](https://www.postgresql.org/)** for the database 9 | **[SqlAlchemy](https://www.sqlalchemy.org/)** for ORM 10 | **[Docker](https://docs.docker.com/)** for containerization 11 | **[Docker Compose](https://docs.docker.com/compose/)** for defining and running multi-container 12 | 13 | ![Summary](img/Summary.png) 14 | 15 | --- 16 | 17 | ### Endpoints Table: 18 | 19 | | Request URL | Description | HTTP | 20 | | ------------------------ | --------------------------------------------------------------------------------------------------- | ---- | 21 | | /users/{count} | Get random user data from randomuser.me/api and add database using Celery. (Delay = 10 sec) | `POST` | 22 | | /users/{count}/{delay} | Get random user data from randomuser.me/api and add database using Celery. | `POST` | 23 | | /users/{user\_id} | Get user from database. | `GET` | 24 | | /weathers/{city} | Get weather data from api.collectapi.com/weather and add database using Celery. (Delay = 10 sec) | `POST` | 25 | | /weathers/{city}/{delay} | Get weather data from api.collectapi.com/weather and add database using Celery. | `POST` | 26 | | /weathers/{city} | Get weather from database. | `GET` | 27 | | /tasks/{task\_id} | Get task status. | `GET` | 28 | 29 | --- 30 | 31 | ### Requirements: 32 | * Docker and Docker Compose 33 | 34 | ### How to Run: 35 | 36 | ``` 37 | docker-compose up --build 38 | ``` 39 | 40 | ### Example Requests: 41 | 42 | --- 43 | 44 | #### Request: 45 | ```http request 46 | POST /users/10 47 | ``` 48 | 49 | #### Response: 50 | ```json 51 | { 52 | "task_id": "44178ce4-6f7a-4a6b-97fd-0de72a055360" 53 | } 54 | ``` 55 | 56 | --- 57 | 58 | #### Request: 59 | ```http request 60 | GET /users/5 61 | ``` 62 | 63 | #### Response: 64 | ```json 65 | { 66 | "first_name": "Lorenzo", 67 | "last_name": "Domínguez" 68 | } 69 | ``` 70 | 71 | --- 72 | 73 | #### Request: 74 | ```http request 75 | POST /weathers/erzincan 76 | ``` 77 | 78 | #### Response: 79 | ```json 80 | { 81 | "task_id": "46f5f77a-5fd7-41dd-898b-235d5def4a70" 82 | } 83 | ``` 84 | 85 | --- 86 | 87 | #### Request: 88 | ```http request 89 | GET /weathers/erzincan 90 | ``` 91 | 92 | #### Response: 93 | ```json 94 | { 95 | "erzincan": [ 96 | { 97 | "date": "08.10.2022", 98 | "day": "Cumartesi", 99 | "description": "orta şiddetli yağmur", 100 | "degree": 26.02 101 | }, 102 | { 103 | "date": "09.10.2022", 104 | "day": "Pazar", 105 | "description": "hafif yağmur", 106 | "degree": 18.59 107 | }, 108 | { 109 | "date": "10.10.2022", 110 | "day": "Pazartesi", 111 | "description": "açık", 112 | "degree": 17.85 113 | }, 114 | { 115 | "date": "11.10.2022", 116 | "day": "Salı", 117 | "description": "açık", 118 | "degree": 17.49 119 | }, 120 | { 121 | "date": "12.10.2022", 122 | "day": "Çarşamba", 123 | "description": "kapalı", 124 | "degree": 17.42 125 | }, 126 | { 127 | "date": "13.10.2022", 128 | "day": "Perşembe", 129 | "description": "hafif yağmur", 130 | "degree": 19.42 131 | }, 132 | { 133 | "date": "14.10.2022", 134 | "day": "Cuma", 135 | "description": "hafif yağmur", 136 | "degree": 16.37 137 | } 138 | ] 139 | } 140 | ``` 141 | 142 | --- 143 | 144 | #### Request: 145 | ```http request 146 | GET /tasks/46f5f77a-5fd7-41dd-898b-235d5def4a70 147 | ``` 148 | 149 | #### Response: 150 | ```json 151 | { 152 | "state": "SUCCESS" 153 | } 154 | ``` 155 | 156 | --- 157 | 158 | **Alperen Cubuk** -------------------------------------------------------------------------------- /api/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | WORKDIR /usr/src/app 4 | 5 | ENV PYTHONDONTWRITEBYTECODE 1 6 | ENV PYTHONUNBUFFERED 1 7 | 8 | COPY . . 9 | RUN pip install --upgrade pip 10 | RUN pip install --upgrade -r requirements.txt 11 | -------------------------------------------------------------------------------- /api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alperencubuk/fastapi-celery-redis-postgres-docker-rest-api/3326587862bd4ff310edf4a50eddb4c2d9b6154f/api/__init__.py -------------------------------------------------------------------------------- /api/crud.py: -------------------------------------------------------------------------------- 1 | from database import db_context 2 | from models import User, Weather 3 | from schemas import UserIn, UserOut, WeatherIn, WeatherOut 4 | 5 | 6 | def crud_add_user(user: UserIn): 7 | db_user = User(**user.dict()) 8 | with db_context() as db: 9 | db.add(db_user) 10 | db.commit() 11 | db.refresh(db_user) 12 | return db_user 13 | 14 | 15 | def crud_get_user(user_id: int): 16 | with db_context() as db: 17 | user = db.query(User).filter(User.id == user_id).first() 18 | if user: 19 | return UserOut(**user.__dict__) 20 | return None 21 | 22 | 23 | def crud_add_weather(weather: WeatherIn): 24 | db_weather = Weather(**weather.dict()) 25 | with db_context() as db: 26 | exist = ( 27 | db.query(Weather) 28 | .filter(Weather.city == weather.city, Weather.date == weather.date) 29 | .first() 30 | ) 31 | if exist: 32 | return None 33 | db.add(db_weather) 34 | db.commit() 35 | db.refresh(db_weather) 36 | return db_weather 37 | 38 | 39 | def crud_get_weather(city: str): 40 | with db_context() as db: 41 | weather = ( 42 | db.query(Weather) 43 | .filter(Weather.city == city) 44 | .order_by(Weather.date.desc()) 45 | .limit(7) 46 | .all() 47 | ) 48 | if weather: 49 | result = [] 50 | for item in weather: 51 | result.append(WeatherOut(**item.__dict__)) 52 | return {city: result[::-1]} 53 | return None 54 | 55 | 56 | def crud_error_message(message): 57 | return {"error": message} 58 | -------------------------------------------------------------------------------- /api/database.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import create_engine 2 | from sqlalchemy.orm import declarative_base, sessionmaker 3 | from sqlalchemy.util.compat import contextmanager 4 | 5 | DATABASE_URL = "postgresql://user:password@database:5432/alpha" 6 | 7 | engine = create_engine(DATABASE_URL) 8 | SessionLocal = sessionmaker(bind=engine) 9 | Base = declarative_base() 10 | 11 | 12 | def get_db_session(): 13 | session = SessionLocal() 14 | try: 15 | yield session 16 | finally: 17 | session.close() 18 | 19 | 20 | db_context = contextmanager(get_db_session) 21 | -------------------------------------------------------------------------------- /api/logs/celery.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alperencubuk/fastapi-celery-redis-postgres-docker-rest-api/3326587862bd4ff310edf4a50eddb4c2d9b6154f/api/logs/celery.log -------------------------------------------------------------------------------- /api/main.py: -------------------------------------------------------------------------------- 1 | from celery.result import AsyncResult 2 | from crud import crud_error_message, crud_get_user, crud_get_weather 3 | from database import engine 4 | from fastapi import FastAPI, HTTPException 5 | from models import Base 6 | from tasks import task_add_user, task_add_weather 7 | 8 | Base.metadata.create_all(bind=engine) 9 | 10 | app = FastAPI() 11 | 12 | 13 | @app.get("/") 14 | def read_root(): 15 | """ 16 | Developer: Alperen Cubuk 17 | """ 18 | return {"Alperen": "Cubuk"} 19 | 20 | 21 | @app.post("/users/{count}/{delay}", status_code=201) 22 | def add_user(count: int, delay: int): 23 | """ 24 | Get random user data from randomuser.me/api and 25 | add database using Celery. Uses Redis as Broker 26 | and Postgres as Backend. 27 | """ 28 | task = task_add_user.delay(count, delay) 29 | return {"task_id": task.id} 30 | 31 | 32 | @app.post("/users/{count}", status_code=201) 33 | def add_user_default_delay(count: int): 34 | """ 35 | Get random user data from randomuser.me/api add 36 | database using Celery. Uses Redis as Broker 37 | and Postgres as Backend. (Delay = 10 sec) 38 | """ 39 | return add_user(count, 10) 40 | 41 | 42 | @app.get("/users/{user_id}") 43 | def get_user(user_id: int): 44 | """ 45 | Get user from database. 46 | """ 47 | user = crud_get_user(user_id) 48 | if user: 49 | return user 50 | else: 51 | raise HTTPException(404, crud_error_message(f"No user found for id: {user_id}")) 52 | 53 | 54 | @app.post("/weathers/{city}/{delay}", status_code=201) 55 | def add_weather(city: str, delay: int): 56 | """ 57 | Get weather data from api.collectapi.com/weather 58 | and add database using Celery. Uses Redis as Broker 59 | and Postgres as Backend. 60 | """ 61 | task = task_add_weather.delay(city, delay) 62 | return {"task_id": task.id} 63 | 64 | 65 | @app.post("/weathers/{city}", status_code=201) 66 | def add_weather_default_delay(city: str): 67 | """ 68 | Get weather data from api.collectapi.com/weather 69 | and add database using Celery. Uses Redis as Broker 70 | and Postgres as Backend. (Delay = 10 sec) 71 | """ 72 | return add_weather(city, 10) 73 | 74 | 75 | @app.get("/weathers/{city}") 76 | def get_weather(city: str): 77 | """ 78 | Get weather from database. 79 | """ 80 | weather = crud_get_weather(city.lower()) 81 | if weather: 82 | return weather 83 | else: 84 | raise HTTPException( 85 | 404, crud_error_message(f"No weather found for city: {city}") 86 | ) 87 | 88 | 89 | @app.get("/tasks/{task_id}") 90 | def task_status(task_id: str): 91 | """ 92 | Get task status. 93 | PENDING (waiting for execution or unknown task id) 94 | STARTED (task has been started) 95 | SUCCESS (task executed successfully) 96 | FAILURE (task execution resulted in exception) 97 | RETRY (task is being retried) 98 | REVOKED (task has been revoked) 99 | """ 100 | task = AsyncResult(task_id) 101 | state = task.state 102 | 103 | if state == "FAILURE": 104 | error = str(task.result) 105 | response = { 106 | "state": state, 107 | "error": error, 108 | } 109 | else: 110 | response = { 111 | "state": state, 112 | } 113 | return response 114 | -------------------------------------------------------------------------------- /api/models.py: -------------------------------------------------------------------------------- 1 | from database import Base 2 | from sqlalchemy import Column, Float, Integer, String 3 | 4 | 5 | class User(Base): 6 | __tablename__ = "User" 7 | id = Column("id", Integer, primary_key=True, autoincrement=True) 8 | first_name = Column("first_name", String) 9 | last_name = Column("last_name", String) 10 | mail = Column("mail", String) 11 | age = Column("age", Integer) 12 | 13 | 14 | class Weather(Base): 15 | __tablename__ = "Weather" 16 | id = Column("id", Integer, primary_key=True, autoincrement=True) 17 | city = Column("city", String, index=True) 18 | date = Column("date", String) 19 | day = Column("day", String) 20 | description = Column("description", String) 21 | degree = Column("degree", Float) 22 | -------------------------------------------------------------------------------- /api/requirements.txt: -------------------------------------------------------------------------------- 1 | celery==5.2.7 2 | fastapi==0.87.0 3 | psycopg2==2.9.5 4 | redis==4.3.4 5 | requests==2.28.1 6 | SQLAlchemy==1.4.44 7 | uvicorn==0.19.0 8 | -------------------------------------------------------------------------------- /api/schemas.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class UserIn(BaseModel): 5 | first_name: str 6 | last_name: str 7 | mail: str 8 | age: int 9 | 10 | 11 | class UserOut(BaseModel): 12 | first_name: str 13 | last_name: str 14 | 15 | 16 | class WeatherIn(BaseModel): 17 | city: str 18 | date: str 19 | day: str 20 | description: str 21 | degree: float 22 | 23 | 24 | class WeatherOut(BaseModel): 25 | date: str 26 | day: str 27 | description: str 28 | degree: float 29 | -------------------------------------------------------------------------------- /api/tasks.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import requests 4 | from celery import Celery 5 | from crud import crud_add_user, crud_add_weather 6 | from schemas import UserIn, WeatherIn 7 | 8 | app = Celery( 9 | "tasks", 10 | broker="redis://localhost:6379/0", 11 | backend="sqla+postgresql://user:password@database:5432/alpha", 12 | ) 13 | 14 | 15 | @app.task 16 | def task_add_user(count: int, delay: int): 17 | url = "https://randomuser.me/api" 18 | response = requests.get(f"{url}?results={count}").json()["results"] 19 | time.sleep(delay) 20 | result = [] 21 | for item in response: 22 | user = UserIn( 23 | first_name=item["name"]["first"], 24 | last_name=item["name"]["last"], 25 | mail=item["email"], 26 | age=item["dob"]["age"], 27 | ) 28 | if crud_add_user(user): 29 | result.append(user.dict()) 30 | return {"success": result} 31 | 32 | 33 | @app.task 34 | def task_add_weather(city: str, delay: int): 35 | url = "https://api.collectapi.com/weather/getWeather?data.lang=tr&data.city=" 36 | headers = { 37 | "content-type": "application/json", 38 | "authorization": "apikey 4HKS8SXTYAsGz45l4yIo9P:0NVczbcuJfjQb8PW7hQV48", 39 | } 40 | response = requests.get(f"{url}{city}", headers=headers).json()["result"] 41 | time.sleep(delay) 42 | result = [] 43 | for item in response: 44 | weather = WeatherIn( 45 | city=city.lower(), 46 | date=item["date"], 47 | day=item["day"], 48 | description=item["description"], 49 | degree=item["degree"], 50 | ) 51 | if crud_add_weather(weather): 52 | result.append(weather.dict()) 53 | return {"success": result} 54 | -------------------------------------------------------------------------------- /api/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alperencubuk/fastapi-celery-redis-postgres-docker-rest-api/3326587862bd4ff310edf4a50eddb4c2d9b6154f/api/tests/__init__.py -------------------------------------------------------------------------------- /api/tests/tests.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from api.main import app 3 | 4 | client = TestClient(app) 5 | 6 | 7 | def test_read_root(): 8 | response = client.get("/") 9 | assert response.status_code == 200 10 | 11 | 12 | def test_task_add_user(): 13 | response = client.post("/users/1") 14 | content = response.json() 15 | task_id = content["task_id"] 16 | assert task_id 17 | 18 | response = client.get(f"tasks/{task_id}") 19 | content = response.json() 20 | assert content == {"state": "PENDING"} 21 | assert response.status_code == 200 22 | 23 | while content["state"] == "PENDING": 24 | response = client.get(f"tasks/{task_id}") 25 | content = response.json() 26 | assert content == {"state": "SUCCESS"} 27 | 28 | 29 | def test_task_add_weather(): 30 | response = client.post("/weathers/erzincan") 31 | content = response.json() 32 | task_id = content["task_id"] 33 | assert task_id 34 | 35 | response = client.get(f"tasks/{task_id}") 36 | content = response.json() 37 | assert content == {"state": "PENDING"} 38 | assert response.status_code == 200 39 | 40 | while content["state"] == "PENDING": 41 | response = client.get(f"tasks/{task_id}") 42 | content = response.json() 43 | assert content == {"state": "SUCCESS"} 44 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | 3 | services: 4 | 5 | api: 6 | build: ./api 7 | ports: 8 | - "80:8000" 9 | command: uvicorn main:app --host 0.0.0.0 --reload 10 | volumes: 11 | - ./api:/usr/src/app 12 | environment: 13 | - CELERY_BROKER_URL=redis://redis:6379/0 14 | - CELERY_RESULT_BACKEND=db+postgresql://user:password@database:5432/alpha 15 | depends_on: 16 | - redis 17 | - database 18 | 19 | worker: 20 | build: ./api 21 | command: celery -A tasks worker --loglevel=info --logfile=logs/celery.log 22 | volumes: 23 | - ./api:/usr/src/app 24 | environment: 25 | - CELERY_BROKER_URL=redis://redis:6379/0 26 | - CELERY_RESULT_BACKEND=db+postgresql://user:password@database:5432/alpha 27 | depends_on: 28 | - api 29 | - redis 30 | - database 31 | 32 | redis: 33 | image: redis:latest 34 | 35 | database: 36 | image: postgres:latest 37 | volumes: 38 | - postgres_data:/var/lib/postgresql/data/ 39 | environment: 40 | - POSTGRES_USER=user 41 | - POSTGRES_PASSWORD=password 42 | - POSTGRES_DB=alpha 43 | ports: 44 | - "5432:5432" 45 | user: postgres 46 | 47 | volumes: 48 | postgres_data: 49 | -------------------------------------------------------------------------------- /img/Summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alperencubuk/fastapi-celery-redis-postgres-docker-rest-api/3326587862bd4ff310edf4a50eddb4c2d9b6154f/img/Summary.png --------------------------------------------------------------------------------