├── uploads └── .gitignore ├── .gitignore ├── requrements.txt ├── generate_thumbnail.py ├── generate_thumbnail_celery.py ├── supervisord.conf ├── Dockerfile ├── LICENSE ├── README.md ├── server.py └── server_celery.py /uploads/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | env 2 | env/* 3 | *.swo 4 | *.swp 5 | *.pyc 6 | -------------------------------------------------------------------------------- /requrements.txt: -------------------------------------------------------------------------------- 1 | flask 2 | celery 3 | supervisor-stdout 4 | -------------------------------------------------------------------------------- /generate_thumbnail.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | import Image 3 | 4 | size = 128, 128 5 | 6 | 7 | app= Celery() 8 | 9 | def generate(src_img,dest_img): 10 | im = Image.open(src_img) 11 | im.thumbnail(size) 12 | im.save(dest_img, "JPEG") 13 | -------------------------------------------------------------------------------- /generate_thumbnail_celery.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | import Image 3 | 4 | size = 128, 128 5 | 6 | from celery import Celery 7 | 8 | app= Celery() 9 | 10 | @app.task 11 | def generate(src_img,dest_img): 12 | im = Image.open(src_img) 13 | im.thumbnail(size) 14 | im.save(dest_img, "JPEG") 15 | -------------------------------------------------------------------------------- /supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon=true 3 | 4 | [program:rabbitmq] 5 | command=service rabbitmq-server start 6 | 7 | [program:upload_image_server] 8 | command=python /build/server_celery.py 9 | stdout_events_enabled = true 10 | stderr_events_enabled = true 11 | stopwaitsecs = 600 12 | 13 | [program:celery_task] 14 | command=celery worker -A generate_thumbnail_celery -l INFO 15 | directory=/build 16 | stdout_events_enabled = true 17 | stderr_events_enabled = true 18 | stopwaitsecs = 600 19 | 20 | [eventlistener:stdout] 21 | command = supervisor_stdout 22 | buffer_size = 100 23 | events = PROCESS_LOG 24 | result_handler = supervisor_stdout:event_handler 25 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # on afternoon, weekend, hanoi, vietnam 2 | FROM ubuntu:14.04 3 | MAINTAINER Nguyen Sy Thanh Son 4 | 5 | RUN \ 6 | apt-get update && \ 7 | apt-get install -y python-pip rabbitmq-server python-dev 8 | 9 | RUN apt-get install -y supervisor 10 | RUN apt-get install -y libjpeg8-dev python-imaging 11 | 12 | # To solve "JPEG support not available" in PIL 13 | RUN ln -s /usr/lib/x86_64-linux-gnu/libjpeg.so /usr/lib 14 | 15 | # Supervisort supports run multiprocess in Docker Image 16 | COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf 17 | WORKDIR /build 18 | ADD . /build 19 | RUN \ 20 | pip install PIL --allow-external PIL --allow-unverified PIL && \ 21 | sudo pip install -r requrements.txt 22 | 23 | # Run Celery with Root Account 24 | ENV C_FORCE_ROOT "true" 25 | CMD ["/usr/bin/supervisord"] 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Nguyen Sy Thanh Son 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | If you are learning the way how to work with RABBITMQ + CELERY. This source code may help you. 2 | 3 | This source code supports 2 ways to run it. 4 | 5 | At the first, clone this source code to your local: 6 | ``` 7 | git clone https://github.com/thanhson1085/flask-celery-rabbitmq-generate-thumbnail.git 8 | ``` 9 | 10 | ### Using Docker 11 | Build from Dockerfile 12 | ``` 13 | sudo docker build -t thanhson1085/flask-celery-rabbitmq-example . 14 | ``` 15 | Or pull from Docker Repo 16 | ``` 17 | sudo docker pull thanhson1085/flask-celery-rabbitmq-example 18 | ``` 19 | Run Docker image 20 | ``` 21 | sudo docker run -i -t -p 5000:5000 thanhson1085/flask-celery-rabbitmq-example 22 | ``` 23 | And waiting for the output as below 24 | ``` 25 | celery_task stderr | [2015-07-29 06:04:52,580: INFO/MainProcess] Connected to amqp://guest:**@127.0.0.1:5672// 26 | ``` 27 | 28 | ### Install packets normally (Ubuntu 14.04) 29 | I will show you how to run this source code from scratch. And i am using ubuntu server 14.04, installed virtualenv, pip. 30 | 31 | Install RabbitMQ Server 32 | ``` 33 | sudo apt-get install rabbitmq-server libjpeg8-dev python-imaging 34 | ``` 35 | 36 | Fix the issue of PIL 37 | ``` 38 | ln -s /usr/lib/x86_64-linux-gnu/libjpeg.so /usr/lib 39 | ``` 40 | 41 | Create environment to run the application with virtualenv 42 | ``` 43 | cd flask-celery-rabbitmq-generate-thumbnail 44 | virtualenv env 45 | source env/bin/activate 46 | ``` 47 | 48 | Install all packets required 49 | ``` 50 | pip install -r requirement.txt 51 | pip install PIL --allow-external PIL --allow-unverified PIL 52 | ``` 53 | 54 | Run web server to upload files 55 | ``` 56 | python ./server_celery.py 57 | ``` 58 | Run the "generate thumbnail" task in Celery 59 | ``` 60 | celery worker -A generate_thumbnail_celery -l INFO 61 | ``` 62 | 63 | Now, it is ready for testing. 64 | (http://localhost:5000) 65 | 66 | The page should be: 67 | 68 | ![alt tag](https://sonnguyen.ws/wp-content/uploads/2015/06/29-07-2015-1-07-40-CH.png) 69 | 70 | ### License 71 | 72 | The MIT License (MIT) 73 | 74 | Copyright (c) 2015 Nguyen Sy Thanh Son 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /server.py: -------------------------------------------------------------------------------- 1 | import os 2 | from flask import Flask, request, redirect, url_for 3 | from werkzeug import secure_filename 4 | 5 | import generate_thumbnail 6 | from os.path import basename 7 | 8 | 9 | UPLOAD_FOLDER = 'uploads' 10 | ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif']) 11 | 12 | app = Flask(__name__) 13 | app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER 14 | 15 | 16 | def allowed_file(filename): 17 | return '.' in filename and \ 18 | filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS 19 | 20 | @app.route('/', methods=['GET', 'POST']) 21 | def upload_file(): 22 | if request.method == 'POST': 23 | import time 24 | start_time = time.time() 25 | file = request.files['file'] 26 | 27 | if file and allowed_file(file.filename): 28 | filename = secure_filename(file.filename) 29 | 30 | _,ext=os.path.splitext(filename) 31 | 32 | 33 | file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename)) 34 | generate_thumbnail.generate(os.path.join(app.config['UPLOAD_FOLDER'], filename),os.path.join(app.config['UPLOAD_FOLDER'], "facedetect-"+filename)) 35 | print "--- %s seconds ---" % str (time.time() - start_time) 36 | return redirect("/") 37 | return redirect(url_for('uploaded_file', 38 | filename="facedetect-"+filename)) 39 | 40 | from os import listdir 41 | from os.path import isfile, join 42 | htmlpic="" 43 | for f in listdir(UPLOAD_FOLDER): 44 | if isfile(join(UPLOAD_FOLDER,f)): 45 | print f 46 | htmlpic+=""" 47 | 48 | 49 | 50 | """ 51 | 52 | return ''' 53 | 54 | 55 | 56 | 57 | Upload new File 58 |

Upload new File

59 |
60 |

61 | 62 |

63 | '''+htmlpic 64 | 65 | from flask import send_from_directory 66 | 67 | @app.route('/uploads/') 68 | def uploaded_file(filename): 69 | return send_from_directory(app.config['UPLOAD_FOLDER'], 70 | filename) 71 | 72 | from werkzeug import SharedDataMiddleware 73 | app.add_url_rule('/uploads/', 'uploaded_file', 74 | build_only=True) 75 | app.wsgi_app = SharedDataMiddleware(app.wsgi_app, { 76 | '/uploads': app.config['UPLOAD_FOLDER'] 77 | }) 78 | 79 | 80 | 81 | if __name__ == "__main__": 82 | app.debug=True 83 | app.run() 84 | -------------------------------------------------------------------------------- /server_celery.py: -------------------------------------------------------------------------------- 1 | import os 2 | from flask import Flask, request, redirect, url_for 3 | from werkzeug import secure_filename 4 | 5 | import generate_thumbnail_celery as generate_thumbnail 6 | from os.path import basename 7 | 8 | 9 | UPLOAD_FOLDER = 'uploads' 10 | ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif']) 11 | 12 | app = Flask(__name__) 13 | app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER 14 | 15 | 16 | def allowed_file(filename): 17 | return '.' in filename and filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS 18 | 19 | @app.route('/', methods=['GET', 'POST']) 20 | def upload_file(): 21 | if request.method == 'POST': 22 | import time 23 | start_time = time.time() 24 | file = request.files['file'] 25 | 26 | if file and allowed_file(file.filename): 27 | filename = secure_filename(file.filename) 28 | 29 | base,ext=os.path.splitext(filename) 30 | 31 | 32 | file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename)) 33 | generate_thumbnail.generate.delay(os.path.join(app.config['UPLOAD_FOLDER'], filename),os.path.join(app.config['UPLOAD_FOLDER'], base+"-face"+ext)) 34 | 35 | print "--- %s seconds ---" % str (time.time() - start_time) 36 | return redirect("/") 37 | return redirect(url_for('uploaded_file', 38 | filename="facedetect-"+filename)) 39 | 40 | from os import listdir 41 | from os.path import isfile, join 42 | htmlpic="" 43 | for f in sorted(listdir(UPLOAD_FOLDER)): 44 | if isfile(join(UPLOAD_FOLDER,f)): 45 | print f 46 | htmlpic+=""" 47 | 48 | 49 | 50 | """ 51 | 52 | return ''' 53 | 54 | 55 | 56 | 57 | Upload new File 58 |

Upload new File

59 |
60 |

61 | 62 |

63 | '''+htmlpic 64 | 65 | from flask import send_from_directory 66 | 67 | @app.route('/uploads/') 68 | def uploaded_file(filename): 69 | return send_from_directory(app.config['UPLOAD_FOLDER'], 70 | filename) 71 | 72 | from werkzeug import SharedDataMiddleware 73 | app.add_url_rule('/uploads/', 'uploaded_file', 74 | build_only=True) 75 | app.wsgi_app = SharedDataMiddleware(app.wsgi_app, { 76 | '/uploads': app.config['UPLOAD_FOLDER'] 77 | }) 78 | 79 | 80 | 81 | if __name__ == "__main__": 82 | app.debug=True 83 | app.run(host='0.0.0.0', port=5000) 84 | --------------------------------------------------------------------------------