├── .gitignore
├── README.md
├── celery_main
    ├── __init__.py
    ├── celery.py
    ├── task_receiver.py
    └── task_submitter.py
├── docker-compose.yml
├── dockerfile
└── requirements.txt


/.gitignore:
--------------------------------------------------------------------------------
1 | **/*.pyc
2 | .idea
3 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | ### Boilerplate code for celery rabbitmq docker cluster
 2 | 
 3 | -------------------------
 4 | 
 5 | ##### Version
 6 | 
 7 | - Docker-compose : `docker-compose version 1.8.0, build unknown`
 8 | - Docker : `Docker version 17.12.0-ce, build c97c6d6`
 9 | - Python: Python:3 docker image being used in dockerfile
10 | -----------------------
11 | 
12 | 
13 | ##### How to run
14 | 
15 | - Run command `sudo docker-compose up`
16 | 
17 | - Above command will start 1 container for each worker and rabbit
18 | 
19 | - Now go inside one worker container and run `python -m celery_main.task_submitter`
20 | 
21 | - this will start pushing tasks in rabitmq and workers
22 | ---------------------
23 | 
24 | 
25 | ##### Speedup the process
26 | 
27 | - To run 5 workers and 1 rabbitmq:
28 | 
29 |   `sudo docker-compose up --scale worker=5`
30 | 
31 | - Do not increase concurrency to too much in dockerfile as machine might not be able to handle it
32 | 
33 |   `ENTRYPOINT celery -A test_celery worker --concurrency=10 --loglevel=info`
34 | 
35 | 
36 | 


--------------------------------------------------------------------------------
/celery_main/__init__.py:
--------------------------------------------------------------------------------
1 | 
2 | 


--------------------------------------------------------------------------------
/celery_main/celery.py:
--------------------------------------------------------------------------------
1 | from celery import Celery
2 | 
3 | app = Celery(
4 |     'celery_main',
5 |     broker='amqp://myuser:mypassword@rabbit:5672',
6 |     backend='rpc://',
7 |     include=['celery_main.task_receiver']
8 | )
9 | 


--------------------------------------------------------------------------------
/celery_main/task_receiver.py:
--------------------------------------------------------------------------------
 1 | from celery_main.celery import app
 2 | import time
 3 | import random
 4 | 
 5 | 
 6 | @app.task(bind=True,default_retry_delay=10)
 7 | def do_work(self, item):
 8 |     print('Task received ' + str(item))
 9 |     # sleep for random seconds to simulate a really long task
10 |     time.sleep(random.randint(1, 3))
11 | 
12 |     result = item + item
13 |     return result
14 | 
15 | 
16 | 
17 | 


--------------------------------------------------------------------------------
/celery_main/task_submitter.py:
--------------------------------------------------------------------------------
1 | from .task_receiver import do_work
2 | 
3 | 
4 | if __name__ == '__main__':
5 |     for i in range(10):
6 |         result = do_work.delay(i)
7 |         print('task submitted' + str(i))
8 | 


--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
 1 | version: '3.2'
 2 | services:
 3 |     worker:
 4 |         build:
 5 |             context: .
 6 |             dockerfile: dockerfile
 7 |         volumes:
 8 |             - type: bind
 9 |               source: .
10 |               target: /app
11 |         links:
12 |             - rabbit
13 |         depends_on:
14 |             - rabbit
15 |     rabbit:
16 |         hostname: rabbit
17 |         image: rabbitmq:latest
18 |         environment:
19 |             - RABBITMQ_DEFAULT_USER=myuser
20 |             - RABBITMQ_DEFAULT_PASS=mypassword
21 |         ports:
22 |             - "5672:5672"
23 | 


--------------------------------------------------------------------------------
/dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3
2 | ADD requirements.txt /app/requirements.txt
3 | ADD ./celery_main/ /app/
4 | WORKDIR /app/
5 | RUN pip install -r requirements.txt
6 | ENTRYPOINT celery -A celery_main worker --concurrency=5 --loglevel=info
7 | 
8 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
 1 | amqp==2.2.2
 2 | billiard==3.5.0.3
 3 | celery==4.0.2
 4 | certifi==2018.1.18
 5 | chardet==3.0.4
 6 | idna==2.6
 7 | kombu==4.1.0
 8 | pytz==2018.3
 9 | requests==2.18.4
10 | urllib3==1.22
11 | vine==1.1.4
12 | psycopg2==2.7.4
13 | beautifulsoup4==4.6.0
14 | bs4==0.0.1
15 | lxml==4.1.1
16 | urllib3==1.22


--------------------------------------------------------------------------------