├── app ├── tasks │ ├── __init__.py │ └── test.py └── __init__.py ├── .gitignore ├── config.py ├── requirements.txt ├── scripts ├── startup │ └── ubuntu_docker_setup.sh ├── setup_server_before_docker_build.sh └── install_redis.sh ├── manage.py ├── celeryconfig.py ├── configs ├── conf.d │ ├── redisd.conf │ ├── celerybeatd.conf │ └── celeryd.conf └── supervisord.conf ├── LICENSE ├── Dockerfile └── README.md /app/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | redis*/ 2 | celerybeat-schedule 3 | **/*.pyc 4 | .DS_Store -------------------------------------------------------------------------------- /app/tasks/test.py: -------------------------------------------------------------------------------- 1 | import celery 2 | 3 | 4 | @celery.task() 5 | def print_hello(): 6 | logger = print_hello.get_logger() 7 | logger.info("Hello") 8 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | from os import environ 2 | 3 | 4 | REDIS_HOST = "0.0.0.0" 5 | REDIS_PORT = 6379 6 | BROKER_URL = environ.get('REDIS_URL', "redis://{host}:{port}/0".format( 7 | host=REDIS_HOST, port=str(REDIS_PORT))) 8 | CELERY_RESULT_BACKEND = BROKER_URL 9 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | amqp==2.2.2 2 | billiard==3.5.0.3 3 | celery==4.1.0 4 | click==6.7 5 | Flask==0.12.2 6 | Flask-Script==2.0.6 7 | itsdangerous==0.24 8 | Jinja2==2.9.6 9 | kombu==4.1.0 10 | MarkupSafe==1.0 11 | pytz==2017.2 12 | redis==2.10.6 13 | six==1.11.0 14 | vine==1.1.4 15 | Werkzeug==0.12.2 -------------------------------------------------------------------------------- /scripts/startup/ubuntu_docker_setup.sh: -------------------------------------------------------------------------------- 1 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 2 | sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" 3 | apt-get update 4 | apt-cache policy docker-ce 5 | apt-get install -y docker-ce -------------------------------------------------------------------------------- /scripts/setup_server_before_docker_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # From local, run: 4 | # scp Dockerfile ubuntu@server_address:/home/ubuntu 5 | 6 | sudo docker build -t celery-scheduler . 7 | sudo docker run -p 3020:80 -d celery-scheduler /usr/bin/supervisord --nodaemon 8 | # sudo docker stop $(sudo docker ps -f ancestor=celery-scheduler --format "{{.ID}}") 9 | -------------------------------------------------------------------------------- /scripts/install_redis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script downloads redis-server 3 | # if redis has not already been downloaded 4 | if [ ! -d redis-3.2.1/src ]; then 5 | wget http://download.redis.io/releases/redis-3.2.1.tar.gz 6 | tar xzf redis-3.2.1.tar.gz 7 | rm redis-3.2.1.tar.gz 8 | cd redis-3.2.1 9 | make 10 | else 11 | cd redis-3.2.1 12 | fi 13 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | from app import app 2 | from flask_script import Manager, Server 3 | 4 | 5 | manager = Manager(app) 6 | manager.add_command("runserver", Server(host="0.0.0.0", port=8889)) 7 | 8 | 9 | @manager.command 10 | def test(): 11 | from app.tasks.test import print_hello 12 | 13 | print_hello() 14 | 15 | 16 | if __name__ == '__main__': 17 | manager.run() 18 | -------------------------------------------------------------------------------- /celeryconfig.py: -------------------------------------------------------------------------------- 1 | from celery.schedules import crontab 2 | 3 | 4 | CELERY_IMPORTS = ('app.tasks.test') 5 | CELERY_TASK_RESULT_EXPIRES = 30 6 | CELERY_TIMEZONE = 'UTC' 7 | 8 | CELERY_ACCEPT_CONTENT = ['json', 'msgpack', 'yaml'] 9 | CELERY_TASK_SERIALIZER = 'json' 10 | CELERY_RESULT_SERIALIZER = 'json' 11 | 12 | CELERYBEAT_SCHEDULE = { 13 | 'test-celery': { 14 | 'task': 'app.tasks.test.print_hello', 15 | # Every minute 16 | 'schedule': crontab(minute="*"), 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /configs/conf.d/redisd.conf: -------------------------------------------------------------------------------- 1 | ; ================================ 2 | ; redis supervisor 3 | ; ================================ 4 | 5 | [program:redis] 6 | command=/home/ubuntu/celery-scheduler/redis-3.2.1/src/redis-server 7 | directory=/home/ubuntu/celery-scheduler/redis-3.2.1 8 | 9 | user=root 10 | numprocs=1 11 | stdout_logfile=/var/log/redis/redis.log 12 | stderr_logfile=/var/log/redis/redis_err.log 13 | autostart=true 14 | autorestart=true 15 | startsecs=10 16 | 17 | ; Causes supervisor to send the termination signal (SIGTERM) to the whole process group. 18 | stopasgroup=true 19 | 20 | ; if rabbitmq is supervised, set its priority higher 21 | ; so it starts first 22 | priority=998 23 | -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | from celery import Celery 3 | import celeryconfig 4 | 5 | 6 | app = Flask(__name__) 7 | app.config.from_object('config') 8 | 9 | 10 | def make_celery(app): 11 | # create context tasks in celery 12 | celery = Celery(app.import_name, broker=app.config['BROKER_URL']) 13 | celery.config_from_object(celeryconfig) 14 | # celery.conf.update(app.config) 15 | TaskBase = celery.Task 16 | 17 | class ContextTask(TaskBase): 18 | abstract = True 19 | 20 | def __call__(self, *args, **kwargs): 21 | with app.app_context(): 22 | return TaskBase.__call__(self, *args, **kwargs) 23 | 24 | celery.Task = ContextTask 25 | 26 | return celery 27 | 28 | celery = make_celery(app) 29 | -------------------------------------------------------------------------------- /configs/conf.d/celerybeatd.conf: -------------------------------------------------------------------------------- 1 | ; ================================ 2 | ; celery beat supervisor 3 | ; ================================ 4 | 5 | [program:celerybeat] 6 | command=/home/ubuntu/.virtualenvs/celery_env/bin/celery beat -A app.celery --schedule=/tmp/celerybeat-schedule --loglevel=INFO --pidfile=/tmp/celerybeat.pid 7 | directory=/home/ubuntu/celery-scheduler 8 | 9 | user=root 10 | numprocs=1 11 | stdout_logfile=/var/log/celery/beat.log 12 | stderr_logfile=/var/log/celery/beat.log 13 | autostart=true 14 | autorestart=true 15 | startsecs=10 16 | 17 | ; Causes supervisor to send the termination signal (SIGTERM) to the whole process group. 18 | stopasgroup=true 19 | 20 | ; if rabbitmq is supervised, set its priority higher 21 | ; so it starts first 22 | priority=999 23 | -------------------------------------------------------------------------------- /configs/conf.d/celeryd.conf: -------------------------------------------------------------------------------- 1 | ; ================================== 2 | ; celery worker supervisor 3 | ; ================================== 4 | 5 | [program:celery] 6 | command=/home/ubuntu/.virtualenvs/celery_env/bin/celery worker -A app.celery --loglevel=INFO 7 | directory=/home/ubuntu/celery-scheduler 8 | 9 | user=root 10 | numprocs=1 11 | stdout_logfile=/var/log/celery/worker.log 12 | stderr_logfile=/var/log/celery/worker.log 13 | autostart=true 14 | autorestart=true 15 | startsecs=10 16 | 17 | ; Need to wait for currently executing tasks to finish at shutdown. 18 | ; Increase this if you have very long running tasks. 19 | stopwaitsecs = 600 20 | 21 | ; Causes supervisor to send the termination signal (SIGTERM) to the whole process group. 22 | stopasgroup=true 23 | 24 | ; Set Celery priority higher than default (999) 25 | ; so, if rabbitmq is supervised, it will start first. 26 | priority=1000 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Shannon Chan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /configs/supervisord.conf: -------------------------------------------------------------------------------- 1 | ; ================================== 2 | ; supervisor config file 3 | ; ================================== 4 | 5 | [unix_http_server] 6 | file=/var/run/supervisor.sock ; (the path to the socket file) 7 | chmod=0700 ; sockef file mode (default 0700) 8 | 9 | [supervisord] 10 | logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log) 11 | pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) 12 | childlogdir=/var/log/supervisor ; ('AUTO' child log dir, default $TEMP) 13 | 14 | ; the below section must remain in the config file for RPC 15 | ; (supervisorctl/web interface) to work, additional interfaces may be 16 | ; added by defining them in separate rpcinterface: sections 17 | [rpcinterface:supervisor] 18 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 19 | 20 | [supervisorctl] 21 | serverurl=unix:///var/run/supervisor.sock ; use a unix:// URL for a unix socket 22 | 23 | ; The [include] section can just contain the "files" setting. This 24 | ; setting can list multiple files (separated by whitespace or 25 | ; newlines). It can also contain wildcards. The filenames are 26 | ; interpreted as relative to this file. Included files *cannot* 27 | ; include files themselves. 28 | 29 | [include] 30 | files = /etc/supervisor/conf.d/*.conf 31 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Pull base image. 2 | FROM ubuntu 3 | 4 | # Install Supervisor. 5 | RUN \ 6 | mkdir /var/log/celery && \ 7 | mkdir /var/log/redis && \ 8 | mkdir /home/ubuntu && \ 9 | apt-get update && \ 10 | apt-get install -y supervisor python-pip wget vim git && \ 11 | rm -rf /var/lib/apt/lists/* && \ 12 | sed -i 's/^\(\[supervisord\]\)$/\1\nnodaemon=true/' /etc/supervisor/supervisord.conf 13 | 14 | # needs to be set else Celery gives an error (because docker runs commands inside container as root) 15 | # https://github.com/pm990320/docker-flask-celery/blob/master/Dockerfile 16 | ENV C_FORCE_ROOT=1 17 | 18 | # expose port 80 of the container (HTTP port, change to 443 for HTTPS) 19 | EXPOSE 80 20 | 21 | # Create virtualenv. 22 | RUN \ 23 | pip install --upgrade pip && \ 24 | pip install --upgrade virtualenv && \ 25 | virtualenv -p /usr/bin/python2.7 /home/ubuntu/.virtualenvs/celery_env 26 | 27 | # Setup for ssh onto github, clone and define working directory 28 | ADD https://api.github.com/repos/channeng/celery-scheduler/git/refs/heads/master repo_version.json 29 | RUN git clone https://github.com/channeng/celery-scheduler.git /home/ubuntu/celery-scheduler 30 | 31 | WORKDIR /home/ubuntu/celery-scheduler 32 | 33 | # Install app requirements 34 | RUN \ 35 | . /home/ubuntu/.virtualenvs/celery_env/bin/activate && \ 36 | pip install -r requirements.txt && \ 37 | . scripts/install_redis.sh 38 | 39 | # Copy supervisor configs 40 | RUN \ 41 | cp configs/supervisord.conf /etc/supervisor/supervisord.conf && \ 42 | cp configs/conf.d/*.conf /etc/supervisor/conf.d/ 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Celery Scheduler 2 | 3 | A Docker implementation of Celery running on Flask, managed with supervisord. 4 | 5 | A walkthrough of this setup is documented at this [Medium article](https://medium.com/@channeng/setting-up-a-task-scheduler-application-with-celery-flask-part-1-8652265050dc). 6 | 7 | ## Why do I need this? 8 | 9 | Celery Scheduler allows you to setup a powerful, distributed and fuss-free application task scheduler. Once you set it up on a server, it can reliably run scheduled tasks at regular defined intervals. 10 | 11 | All you need to do is to [define your task method](app/tasks/test.py), and the [task schedule](celeryconfig.py), and Celery Scheduler will handle the rest for you. 12 | 13 | Some interesting uses for your own task scheduler include: 14 | - Home automation (IOT projects) 15 | - Data Workflow management for Business Intelligence (BI) 16 | - Triggering Email campaigns 17 | - Any other routine, periodic tasks 18 | 19 | ## How does it work? 20 | 21 | This is a scheduler application powered by [Celery](http://docs.celeryproject.org/en/latest/index.html) running on a minimal python web framework, [Flask](http://flask.pocoo.org/). 22 | 23 | The application is process-managed by [Supervisord](http://supervisord.org/) which takes care of managing celery task workers, celerybeat and Redis as the message broker. 24 | 25 | The deployment of the application is handled through [Docker](https://www.docker.com/what-docker) which isolates the application environment. It allows the application to run the same, whether locally, in staging or when deployed within a server. 26 | 27 | # Getting Started 28 | 29 | ## Running this setup 30 | 31 | This setup is built for deployment with Docker. You may also choose to run this setup without Docker however no script is provided. Setup instructions can be interpreted from the given Dockerfile. 32 | 33 | Deployment with Docker is recommended for consistency of application environment. 34 | 35 | 1. Clone the repository 36 | ```bash 37 | cd ~ 38 | git clone https://github.com/channeng/celery-scheduler.git 39 | cd celery-scheduler 40 | ``` 41 | 42 | 2. Install Docker 43 | - [Mac or Windows](https://docs.docker.com/engine/installation/) 44 | - [Ubuntu server](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-16-04) 45 | - To install docker in Ubuntu, you may run the install script: 46 | ```bash 47 | sudo bash scripts/startup/ubuntu_docker_setup.sh 48 | ``` 49 | 50 | *Note*: You may need to run the following docker commands with `sudo` prefix if docker was set up to run with root. 51 | 52 | 3. Build docker image 53 | ```bash 54 | docker build -t celery-scheduler . 55 | ``` 56 | 4. (Optional) Stop any containers running on existing docker image 57 | ```bash 58 | docker stop $(docker ps -f ancestor=celery-scheduler --format "{{.ID}}") 59 | ``` 60 | 5. Run supervisord with docker container 61 | ```bash 62 | docker run -p 3020:80 -d celery-scheduler /usr/bin/supervisord --nodaemon 63 | ``` 64 | 65 | ## Checking successful deployment 66 | - Enter bash terminal of running Docker container 67 | ```bash 68 | docker exec -i -t $(docker ps -f ancestor=celery-scheduler --format "{{.ID}}") /bin/bash 69 | ``` 70 | - Check all required processes are running: 71 | ```bash 72 | ps aux 73 | ``` 74 | 75 | You shoud see something like the following: 76 | ``` 77 | USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND 78 | root 1 0.0 2.5 56492 12728 ? Ss Oct21 0:05 /usr/bin/python /usr/bin/supervisord --nodaemon 79 | root 7 0.0 0.5 31468 2616 ? Sl Oct21 0:30 /home/ubuntu/celery-scheduler/redis-3.2.1/src/redis-server *:6379 80 | root 8 0.0 8.3 98060 41404 ? S Oct21 0:00 /home/ubuntu/.virtualenvs/celery_env/bin/python2.7 /home/ubuntu/.virtualenvs/celery_env/bin/celery beat -A ap 81 | root 9 0.1 8.4 91652 41900 ? S Oct21 0:42 /home/ubuntu/.virtualenvs/celery_env/bin/python2.7 /home/ubuntu/.virtualenvs/celery_env/bin/celery worker -A 82 | root 20 0.0 9.3 99540 46820 ? S Oct21 0:00 /home/ubuntu/.virtualenvs/celery_env/bin/python2.7 /home/ubuntu/.virtualenvs/celery_env/bin/celery worker -A 83 | ``` 84 | - Retrieving logs 85 | ```bash 86 | tail /var/log/redis/redis.log 87 | tail /var/log/celery/beat.log 88 | tail /var/log/celery/worker.log 89 | tail /var/log/supervisor/supervisord.log 90 | ``` 91 | - If successfully deployed, supervisor logs should display: 92 | ```bash 93 | INFO success: redis entered RUNNING state, process has stayed up for > than 10 seconds (startsecs) 94 | INFO success: celerybeat entered RUNNING state, process has stayed up for > than 10 seconds (startsecs) 95 | INFO success: celery entered RUNNING state, process has stayed up for > than 10 seconds (startsecs) 96 | ``` 97 | - You should also see the task print_hello running every minute in your worker.log 98 | ```bash 99 | tail -f /var/log/celery/worker.log 100 | ``` 101 | Output: 102 | ```bash 103 | [2017-10-22 03:18:00,050: INFO/MainProcess] Received task: app.tasks.test.print_hello[aa1b7700-1665-4751-ada2-35aba5670d40] 104 | [2017-10-22 03:18:00,051: INFO/ForkPoolWorker-1] app.tasks.test.print_hello[aa1b7700-1665-4751-ada2-35aba5670d40]: Hello 105 | [2017-10-22 03:18:00,052: INFO/ForkPoolWorker-1] Task app.tasks.test.print_hello[aa1b7700-1665-4751-ada2-35aba5670d40] succeeded in 0.000455291003163s: None 106 | ``` 107 | 108 | # Adding tasks to Celery 109 | 110 | - Task scripts should be written and stored in app/tasks. 111 | - Update `celeryconfig.py` for new tasks and trigger times. 112 | - Remember to rebuild the docker image after updating for new tasks. 113 | 114 | ## Running adhoc tasks 115 | 116 | - Update `manage.py` for a manager command for the task to run on trigger 117 | - Run: ```python manage.py ``` 118 | - Eg. 119 | ```bash 120 | source /home/ubuntu/.virtualenvs/celery_env/bin/activate 121 | python manage.py test 122 | ``` 123 | 124 | ## Terminating Supervisor within container 125 | 126 | ```bash 127 | supervisorctl stop all 128 | ``` 129 | 130 | ## Contributing 131 | Feel free to submit Pull Requests. 132 | For any other enquiries, you may contact me at channeng@gmail.com. 133 | --------------------------------------------------------------------------------