├── .dockerignore ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── app ├── __init__.py ├── app.py ├── routes.py └── tasks.py ├── celery_workers.yaml ├── complete_flow_load_test.py ├── flask_server.yaml ├── generate_flow_load_test.py ├── keda.yaml ├── main.py ├── postgres.yaml ├── rabbitmq.yaml └── requirements.txt /.dockerignore: -------------------------------------------------------------------------------- 1 | *.yaml 2 | *.pyc 3 | */*.pyc 4 | .env 5 | .DS_Store 6 | .git/* 7 | venv/* 8 | .git 9 | venv -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /venv/* 2 | *.pyc 3 | */__pycache__/* 4 | .env 5 | .DS_Store -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim 2 | 3 | 4 | # Set work directory 5 | WORKDIR /app 6 | 7 | # Copy requirements.txt 8 | COPY requirements.txt /app/ 9 | 10 | # Install Requirements 11 | RUN pip install -r requirements.txt 12 | 13 | # Copy code 14 | COPY . /app/ 15 | 16 | CMD gunicorn --workers 10 --bind 0.0.0.0:5000 --log-level DEBUG main:app 17 | 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 S Santhosh Nagaraj 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # flask-celery-microservice 2 | 3 | Contains code samples for [Scaling Celery workers with RabbitMQ on Kubernetes](https://learnk8s.io/scaling-celery-rabbitmq-kubernetes) 4 | -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- 1 | from . import routes, tasks 2 | from .app import app,celery -------------------------------------------------------------------------------- /app/app.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | from celery import Celery 3 | import os 4 | 5 | app = Flask(__name__) 6 | app.config['CELERY_BROKER_URL'] = os.getenv("CELERY_BROKER_URL") 7 | app.config['RESULT_BACKEND'] = os.getenv("CELERY_RESULT_BACKEND") 8 | app.config['SECRET_KEY'] = os.getenv("SECRET_KEY") 9 | 10 | celery = Celery(app.import_name, 11 | backend=app.config['RESULT_BACKEND'], 12 | broker=app.config['CELERY_BROKER_URL']) 13 | celery.conf.update(app.config) 14 | -------------------------------------------------------------------------------- /app/routes.py: -------------------------------------------------------------------------------- 1 | from app.app import app 2 | from flask import request, render_template, jsonify 3 | from celery.result import AsyncResult 4 | from app.tasks import * 5 | 6 | 7 | @app.route('/') 8 | def default(): 9 | return "Welcome to Report Service" 10 | 11 | @app.route('/health') 12 | def health(): 13 | return jsonify({"state":"healthy"}) 14 | 15 | @app.route('/report', methods=['POST']) 16 | def generate_report(): 17 | async_result = report.delay() 18 | return jsonify({"report_id":async_result.id}) 19 | 20 | 21 | @app.route('/report/') 22 | def get_report(report_id): 23 | res = AsyncResult(report_id,app=celery) 24 | return jsonify({"id":res.id,"result":res.result}) 25 | -------------------------------------------------------------------------------- /app/tasks.py: -------------------------------------------------------------------------------- 1 | from app.app import celery 2 | import time 3 | import random 4 | 5 | 6 | @celery.task(name="report", acks_late=True) 7 | def report(): 8 | print("Generating report") 9 | time.sleep(60) 10 | return {"state":"completed"} 11 | -------------------------------------------------------------------------------- /celery_workers.yaml: -------------------------------------------------------------------------------- 1 | # namespace 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: celery-workers 6 | --- 7 | # secret 8 | apiVersion: v1 9 | kind: Secret 10 | metadata: 11 | name: celery-workers-secret 12 | namespace: celery-workers 13 | type: Opaque 14 | data: 15 | CELERY_BROKER_URL: YW1xcDovL2FkbWluOnNlY3JldHBhc3N3b3JkQHJhYmJpdG1xLnJhYmJpdG1xOjU2NzI= 16 | CELERY_RESULT_BACKEND: ZGIrcG9zdGdyZXNxbDovL3Rlc3Q6dGVzdEAxMjNAcG9zdGdyZXNxbC5wb3N0Z3Jlczo1NDMyL2ZsYXNrLXNlcnZpY2U= 17 | SECRET_KEY: dGVzdA== 18 | --- 19 | # deployment 20 | apiVersion: apps/v1 21 | kind: Deployment 22 | metadata: 23 | name: celery-worker 24 | namespace: celery-workers 25 | spec: 26 | replicas: 2 27 | selector: 28 | matchLabels: 29 | name: celery-worker 30 | template: 31 | metadata: 32 | labels: 33 | name: celery-worker 34 | spec: 35 | containers: 36 | - name: celery-worker 37 | image: xasag94215/flask-celery-microservice 38 | command: 39 | - "bash" 40 | - "-c" 41 | - "celery -A main.celery worker -l info" 42 | envFrom: 43 | - secretRef: 44 | name: celery-workers-secret 45 | --- 46 | 47 | -------------------------------------------------------------------------------- /complete_flow_load_test.py: -------------------------------------------------------------------------------- 1 | import gevent 2 | import requests 3 | import time 4 | import locust 5 | from locust import HttpUser, task, constant 6 | 7 | def async_success(name, start_time, resp): 8 | locust.events.request_success.fire( 9 | request_type=resp.request.method, 10 | name=name, 11 | response_time=int((time.monotonic() - start_time) * 1000), 12 | response_length=len(resp.content), 13 | ) 14 | 15 | def async_failure(name, start_time, resp, message): 16 | locust.events.request_failure.fire( 17 | request_type=resp.request.method, 18 | name=name, 19 | response_time=int((time.monotonic() - start_time) * 1000), 20 | exception=Exception(message), 21 | ) 22 | 23 | class reportService(HttpUser): 24 | 25 | wait_time = constant(1) 26 | 27 | def _do_async_thing_handler(self, timeout=600): 28 | post_resp = requests.post(self.host + 'report') 29 | if not post_resp.status_code == 200: 30 | return 31 | id = post_resp.json()['report_id'] 32 | print(id) 33 | 34 | # Now poll for an ACTIVE status 35 | start_time = time.monotonic() 36 | end_time = start_time + timeout 37 | while time.monotonic() < end_time: 38 | r = requests.get(self.host + 'report/' + id) 39 | if r.status_code == 200 and r.json()['result'] != None: 40 | async_success('POST /report/ID - async', start_time, post_resp) 41 | return 42 | 43 | # IMPORTANT: Sleep must be monkey-patched by gevent (typical), or else 44 | # use gevent.sleep to avoid blocking the world. 45 | time.sleep(1) 46 | async_failure('POST /report/ID - async', start_time, post_resp, 47 | 'Failed - timed out after %s seconds' % timeout) 48 | 49 | @task 50 | def do_async_thing(self): 51 | gevent.spawn(self._do_async_thing_handler) -------------------------------------------------------------------------------- /flask_server.yaml: -------------------------------------------------------------------------------- 1 | # namespace 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: flask-backend 6 | --- 7 | # secret 8 | apiVersion: v1 9 | kind: Secret 10 | metadata: 11 | name: flask-secret 12 | namespace: flask-backend 13 | type: Opaque 14 | data: 15 | CELERY_BROKER_URL: YW1xcDovL2FkbWluOnNlY3JldHBhc3N3b3JkQHJhYmJpdG1xLnJhYmJpdG1xOjU2NzI= 16 | CELERY_RESULT_BACKEND: ZGIrcG9zdGdyZXNxbDovL3Rlc3Q6dGVzdEAxMjNAcG9zdGdyZXNxbC5wb3N0Z3Jlczo1NDMyL2ZsYXNrLXNlcnZpY2U= 17 | SECRET_KEY: dGVzdA== 18 | --- 19 | # deployment 20 | apiVersion: apps/v1 21 | kind: Deployment 22 | metadata: 23 | name: flask-server 24 | namespace: flask-backend 25 | spec: 26 | replicas: 1 27 | selector: 28 | matchLabels: 29 | name: flask-server 30 | template: 31 | metadata: 32 | labels: 33 | name: flask-server 34 | spec: 35 | containers: 36 | - name: flask-server 37 | image: xasag94215/flask-celery-microservice 38 | envFrom: 39 | - secretRef: 40 | name: flask-secret 41 | ports: 42 | - containerPort: 5000 43 | name: rest 44 | --- 45 | # service 46 | apiVersion: v1 47 | kind: Service 48 | metadata: 49 | name: flask-server 50 | namespace: flask-backend 51 | spec: 52 | selector: 53 | name: flask-server 54 | ports: 55 | - port: 80 56 | targetPort: rest -------------------------------------------------------------------------------- /generate_flow_load_test.py: -------------------------------------------------------------------------------- 1 | from locust import HttpUser, task, constant 2 | 3 | 4 | class reportService(HttpUser): 5 | 6 | wait_time = constant(1) 7 | 8 | def generate_report(self): 9 | resp = self.client.post("report") 10 | if resp.status_code != 200: 11 | print("Error generating report") 12 | return resp 13 | 14 | @task 15 | def generate_flow(self): 16 | self.generate_report() 17 | 18 | 19 | -------------------------------------------------------------------------------- /keda.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: keda.sh/v1alpha1 2 | kind: ScaledObject 3 | metadata: 4 | name: celery-worker-scaler 5 | namespace: celery-workers 6 | spec: 7 | scaleTargetRef: 8 | name: celery-worker 9 | pollingInterval: 3 10 | minReplicaCount: 2 11 | maxReplicaCount: 30 12 | triggers: 13 | - type: rabbitmq 14 | metadata: 15 | queueName: celery 16 | queueLength: "10" 17 | authenticationRef: 18 | name: rabbitmq-worker-trigger 19 | --- 20 | apiVersion: keda.sh/v1alpha1 21 | kind: TriggerAuthentication 22 | metadata: 23 | name: rabbitmq-worker-trigger 24 | namespace: celery-workers 25 | spec: 26 | secretTargetRef: 27 | - parameter: host 28 | name: celery-workers-secret 29 | key: CELERY_BROKER_URL 30 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from app import app,celery 2 | 3 | 4 | if __name__ == "__main__": 5 | app.run(debug=False) 6 | -------------------------------------------------------------------------------- /postgres.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # namespace 3 | apiVersion: v1 4 | kind: Namespace 5 | metadata: 6 | name: postgres 7 | --- 8 | # secret 9 | apiVersion: v1 10 | kind: Secret 11 | metadata: 12 | name: postgres-secret 13 | namespace: postgres 14 | type: Opaque 15 | data: 16 | POSTGRES_USER: dGVzdA== 17 | POSTGRES_DB: Zmxhc2stc2VydmljZQ== 18 | POSTGRES_PASSWORD: dGVzdEAxMjM= 19 | --- 20 | # deploy 21 | apiVersion: apps/v1 22 | kind: Deployment 23 | metadata: 24 | name: postgresql 25 | namespace: postgres 26 | labels: 27 | name: postgresql 28 | spec: 29 | replicas: 1 30 | selector: 31 | matchLabels: 32 | name: postgresql 33 | template: 34 | metadata: 35 | labels: 36 | name: postgresql 37 | spec: 38 | containers: 39 | - image: postgres:9.6.2-alpine 40 | name: postgresql 41 | envFrom: 42 | - secretRef: 43 | name: postgres-secret 44 | ports: 45 | - containerPort: 5432 46 | name: postgresql 47 | --- 48 | # service 49 | apiVersion: v1 50 | kind: Service 51 | metadata: 52 | name: postgresql 53 | namespace: postgres 54 | labels: 55 | name: postgresql 56 | spec: 57 | ports: 58 | - port: 5432 59 | selector: 60 | name: postgresql 61 | --- 62 | -------------------------------------------------------------------------------- /rabbitmq.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # namespace 3 | apiVersion: v1 4 | kind: Namespace 5 | metadata: 6 | name: rabbitmq 7 | --- 8 | # secret 9 | apiVersion: v1 10 | kind: Secret 11 | metadata: 12 | name: rabbitmq-secret 13 | namespace: rabbitmq 14 | type: Opaque 15 | data: 16 | RABBITMQ_DEFAULT_USER: YWRtaW4= 17 | RABBITMQ_DEFAULT_PASS: c2VjcmV0cGFzc3dvcmQ= 18 | --- 19 | # deployment 20 | apiVersion: apps/v1 21 | kind: Deployment 22 | metadata: 23 | name: rabbitmq 24 | namespace: rabbitmq 25 | labels: 26 | name: rabbitmq 27 | spec: 28 | replicas: 1 29 | selector: 30 | matchLabels: 31 | name: rabbitmq 32 | template: 33 | metadata: 34 | labels: 35 | name: rabbitmq 36 | spec: 37 | containers: 38 | - name: rabbitmq 39 | image: rabbitmq:3.6.8-management 40 | envFrom: 41 | - secretRef: 42 | name: rabbitmq-secret 43 | ports: 44 | - containerPort: 15672 45 | name: management 46 | - containerPort: 5672 47 | name: rabbitmq 48 | --- 49 | # service 50 | apiVersion: v1 51 | kind: Service 52 | metadata: 53 | name: rabbitmq 54 | namespace: rabbitmq 55 | labels: 56 | name: rabbitmq 57 | spec: 58 | selector: 59 | name: rabbitmq 60 | ports: 61 | - port: 5672 62 | name: rabbitmq 63 | targetPort: rabbitmq 64 | protocol: TCP 65 | - port: 15672 66 | name: rabbitmq-mgmt 67 | targetPort: management 68 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | amqp==5.0.1 2 | billiard==3.6.3.0 3 | celery==5.0.1 4 | click==7.1.2 5 | click-didyoumean==0.0.3 6 | click-repl==0.1.6 7 | Flask==1.1.2 8 | gunicorn==20.0.4 9 | itsdangerous==1.1.0 10 | Jinja2==2.11.2 11 | kombu==5.0.2 12 | MarkupSafe==1.1.1 13 | prompt-toolkit==3.0.8 14 | psycopg2-binary==2.8.6 15 | pytz==2020.1 16 | six==1.15.0 17 | SQLAlchemy==1.3.20 18 | vine==5.0.0 19 | wcwidth==0.2.5 20 | Werkzeug==1.0.1 21 | yapf==0.30.0 22 | --------------------------------------------------------------------------------