├── .gitignore ├── LICENSE ├── MAINTAINERS.md ├── MANIFEST.in ├── README.md ├── celery_progress ├── __init__.py ├── backend.py ├── static │ └── celery_progress │ │ ├── celery_progress.js │ │ └── websockets.js ├── tasks.py ├── urls.py ├── views.py └── websockets │ ├── __init__.py │ ├── backend.py │ ├── consumers.py │ ├── routing.py │ └── tasks.py ├── pyproject.toml └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | # PyCharm files 104 | .idea/ 105 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Cory Zue 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # Maintainers Guide 2 | 3 | The process for publishing new versions of this library is the following: 4 | 5 | 1. Update the version number in `setup.py` 6 | 2. [Generate distribution archives](https://packaging.python.org/en/latest/tutorials/packaging-projects/#generating-distribution-archives) 7 | 1. `pip install --upgrade build` 8 | 2. `python -m build` 9 | 3. [Upload distribution archives](https://packaging.python.org/en/latest/tutorials/packaging-projects/#uploading-the-distribution-archives) 10 | 1. `pip install --upgrade twine` 11 | 2. `twine upload dist/*` 12 | 4. Publish the tag 13 | 1. git tag -a "0.X" -m "Release 0.X" 14 | 2. git push --tags 15 | 5. Publish the release 16 | 1. Go to [https://github.com/czue/celery-progress/releases/new](https://github.com/czue/celery-progress/releases/new) 17 | 2. Create a release from the tag you just added. 18 | 3. Click "generate release notes" 19 | 4. Click "publish" 20 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.md 3 | recursive-include celery_progress/static * 4 | recursive-include docs * 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Celery Progress Bars for Django 2 | 3 | Drop in, dependency-free progress bars for your Django/Celery applications. 4 | 5 | Super simple setup. Lots of customization available. 6 | 7 | ## Demo 8 | 9 | [Celery Progress Bar demo on SaaS Pegasus](https://www.saaspegasus.com/guides/celery-progress-demo/) 10 | 11 | ### Github demo application: build a download progress bar for Django 12 | Starting with Celery can be challenging, [eeintech](https://github.com/eeintech) built a complete [Django demo application](https://github.com/eeintech/django-celery-progress-demo) along with a [step-by-step guide](https://eeinte.ch/stream/progress-bar-django-using-celery/) to get you started on building your own progress bar! 13 | 14 | ## Installation 15 | 16 | If you haven't already, make sure you have properly [set up celery in your project](https://docs.celeryq.dev/en/stable/getting-started/first-steps-with-celery.html#first-steps). 17 | 18 | Then install this library: 19 | 20 | ```bash 21 | pip install celery-progress 22 | ``` 23 | 24 | ## Usage 25 | 26 | ### Prerequisites 27 | 28 | First add `celery_progress` to your `INSTALLED_APPS` in `settings.py`. 29 | 30 | Then add the following url config to your main `urls.py`: 31 | 32 | ```python 33 | from django.urls import path, include 34 | 35 | urlpatterns = [ 36 | # your project's patterns here 37 | ... 38 | path(r'^celery-progress/', include('celery_progress.urls')), # add this line (the endpoint is configurable) 39 | ] 40 | ``` 41 | 42 | ### Recording Progress 43 | 44 | In your task you should add something like this: 45 | 46 | ```python 47 | from celery import shared_task 48 | from celery_progress.backend import ProgressRecorder 49 | import time 50 | 51 | @shared_task(bind=True) 52 | def my_task(self, seconds): 53 | progress_recorder = ProgressRecorder(self) 54 | result = 0 55 | for i in range(seconds): 56 | time.sleep(1) 57 | result += i 58 | progress_recorder.set_progress(i + 1, seconds) 59 | return result 60 | ``` 61 | 62 | You can add an optional progress description like this: 63 | 64 | ```python 65 | progress_recorder.set_progress(i + 1, seconds, description='my progress description') 66 | ``` 67 | 68 | ### Displaying progress 69 | 70 | In the view where you call the task you need to get the task ID like so: 71 | 72 | **views.py** 73 | ```python 74 | def progress_view(request): 75 | result = my_task.delay(10) 76 | return render(request, 'display_progress.html', context={'task_id': result.task_id}) 77 | ``` 78 | 79 | Then in the page you want to show the progress bar you just do the following. 80 | 81 | #### Add the following HTML wherever you want your progress bar to appear: 82 | 83 | **display_progress.html** 84 | ```html 85 |
86 |
 
87 |
88 |
Waiting for progress to start...
89 | ``` 90 | 91 | #### Import the javascript file. 92 | 93 | **display_progress.html** 94 | ```html 95 | 96 | ``` 97 | 98 | #### Initialize the progress bar: 99 | 100 | ```javascript 101 | // vanilla JS version 102 | document.addEventListener("DOMContentLoaded", function () { 103 | var progressUrl = "{% url 'celery_progress:task_status' task_id %}"; 104 | CeleryProgressBar.initProgressBar(progressUrl); 105 | }); 106 | ``` 107 | 108 | or 109 | 110 | ```javascript 111 | // JQuery 112 | $(function () { 113 | var progressUrl = "{% url 'celery_progress:task_status' task_id %}"; 114 | CeleryProgressBar.initProgressBar(progressUrl) 115 | }); 116 | ``` 117 | 118 | ### Displaying the result of a task 119 | 120 | If you'd like you can also display the result of your task on the front end. 121 | 122 | To do that follow the steps below. Result handling can also be customized. 123 | 124 | #### Initialize the result block: 125 | 126 | This is all that's needed to render the result on the page. 127 | 128 | **display_progress.html** 129 | ```html 130 |
131 | ``` 132 | 133 | But more likely you will want to customize how the result looks, which can be done as below: 134 | 135 | ```javascript 136 | // JQuery 137 | var progressUrl = "{% url 'celery_progress:task_status' task_id %}"; 138 | 139 | function customResult(resultElement, result) { 140 | $( resultElement ).append( 141 | $('

').text('Sum of all seconds is ' + result) 142 | ); 143 | } 144 | 145 | $(function () { 146 | CeleryProgressBar.initProgressBar(progressUrl, { 147 | onResult: customResult, 148 | }) 149 | }); 150 | ``` 151 | 152 | ### Working with Groups 153 | 154 | This library includes experimental support for working with [Celery groups](https://docs.celeryq.dev/en/stable/userguide/canvas.html#groups). 155 | You can use the `"group_status"` URL endpoint for this. Here is a basic example: 156 | 157 | **Example task:** 158 | 159 | ```python 160 | @shared_task(bind=True) 161 | def add(self, x, y): 162 | return x + y 163 | ``` 164 | 165 | **Calling view:** 166 | 167 | ```python 168 | from celery import group 169 | from .tasks import add 170 | 171 | def progress_view(request): 172 | task_group = group(add.s(i, i) for i in range(100)) 173 | group_result = task_group.apply_async() 174 | # you must explicitly call the save function on the group_result after calling the tasks 175 | group_result.save() 176 | return render(request, 'display_progress.html', context={'task_id': group_result.id}) 177 | 178 | ``` 179 | 180 | **Template:** 181 | 182 | ```html 183 | document.addEventListener("DOMContentLoaded", function () { 184 | var progressUrl = "{% url 'celery_progress:group_status' task_id %}"; 185 | CeleryProgressBar.initProgressBar(progressUrl); 186 | }); 187 | ``` 188 | 189 | ## Customization 190 | 191 | The `initProgressBar` function takes an optional object of options. The following options are supported: 192 | 193 | | Option | What it does | Default Value | 194 | |--------|--------------|---------------| 195 | | pollInterval | How frequently to poll for progress (in milliseconds) | 500 | 196 | | progressBarId | Override the ID used for the progress bar | 'progress-bar' | 197 | | progressBarMessageId | Override the ID used for the progress bar message | 'progress-bar-message' | 198 | | progressBarElement | Override the *element* used for the progress bar. If specified, progressBarId will be ignored. | document.getElementById(progressBarId) | 199 | | progressBarMessageElement | Override the *element* used for the progress bar message. If specified, progressBarMessageId will be ignored. | document.getElementById(progressBarMessageId) | 200 | | resultElementId | Override the ID used for the result | 'celery-result' | 201 | | resultElement | Override the *element* used for the result. If specified, resultElementId will be ignored. | document.getElementById(resultElementId) | 202 | | onProgress | function to call when progress is updated | onProgressDefault | 203 | | onSuccess | function to call when progress successfully completes | onSuccessDefault | 204 | | onError | function to call on a known error with no specified handler | onErrorDefault | 205 | | onRetry | function to call when a task attempts to retry | onRetryDefault | 206 | | onIgnored | function to call when a task result is ignored | onIgnoredDefault | 207 | | onTaskError | function to call when progress completes with an error | onError | 208 | | onNetworkError | function to call on a network error (ignored by WebSocket) | onError | 209 | | onHttpError | function to call on a non-200 response (ignored by WebSocket) | onError | 210 | | onDataError | function to call on a response that's not JSON or has invalid schema due to a programming error | onError | 211 | | onResult | function to call when returned non empty result | CeleryProgressBar.onResultDefault | 212 | | barColors | dictionary containing color values for various progress bar states. Colors that are not specified will defer to defaults | barColorsDefault | 213 | | defaultMessages | dictionary containing default messages that can be overridden | see below | 214 | 215 | The `barColors` option allows you to customize the color of each progress bar state by passing a dictionary of key-value pairs of `state: #hexcode`. The defaults are shown below. 216 | 217 | | State | Hex Code | Image Color | 218 | |-------|----------|:-------------:| 219 | | success | #76ce60 | ![#76ce60](https://via.placeholder.com/15/76ce60/000000?text=+) | 220 | | error | #dc4f63 | ![#dc4f63](https://via.placeholder.com/15/dc4f63/000000?text=+) | 221 | | progress | #68a9ef | ![#68a9ef](https://via.placeholder.com/15/68a9ef/000000?text=+) | 222 | | ignored | #7a7a7a | ![#7a7a7a](https://via.placeholder.com/15/7a7a7a/000000?text=+) | 223 | 224 | The `defaultMessages` option allows you to override some default messages in the UI. At the moment these are: 225 | 226 | | Message Id | When Shown | Default Value | 227 | |-------|----------|:-------------:| 228 | | waiting | Task is waiting to start | 'Waiting for task to start...' 229 | | started | Task has started but reports no progress | 'Task started...' 230 | 231 | # WebSocket Support 232 | 233 | Additionally, this library offers WebSocket support using [Django Channels](https://channels.readthedocs.io/en/latest/) 234 | courtesy of [EJH2](https://github.com/EJH2/). 235 | 236 | A working example project leveraging WebSockets is [available here](https://github.com/EJH2/cp_ws-example). 237 | 238 | To use WebSockets, install with `pip install celery-progress[websockets,redis]` or 239 | `pip install celery-progress[websockets,rabbitmq]` (depending on broker dependencies). 240 | 241 | See `WebSocketProgressRecorder` and `websockets.js` for details. 242 | 243 | # Securing the get_progress endpoint 244 | By default, anyone can see the status and result of any task by accessing `/celery-progress/` 245 | 246 | To limit access, you need to wrap `get_progress()` in a view of your own which implements the permissions check, and create a new url routing to point to your view. Make sure to remove any existing (unprotected) celery progress urls from your root urlconf at the same time. 247 | 248 | 249 | For example, requiring login with a class-based view: 250 | ```python 251 | 252 | # views.py 253 | from celery_progress.views import get_progress 254 | from django.contrib.auth.mixins import LoginRequiredMixin 255 | from django.views.generic import View 256 | 257 | class TaskStatus(LoginRequiredMixin, View): 258 | def get(self, request, task_id, *args, **kwargs): 259 | # Other checks could go here 260 | return get_progress(request, task_id=task_id) 261 | ``` 262 | 263 | ```python 264 | # urls.py 265 | from django.urls import path 266 | from . import views 267 | 268 | urlpatterns = [ 269 | ... 270 | path('task-status/', views.TaskStatus.as_view(), name='task_status'), 271 | ... 272 | ] 273 | ``` 274 | 275 | Requiring login with a function-based view: 276 | ```python 277 | 278 | # views.py 279 | from celery_progress.views import get_progress 280 | from django.contrib.auth.decorators import login_required 281 | 282 | @login_required 283 | def task_status(request, task_id): 284 | # Other checks could go here 285 | return get_progress(request, task_id) 286 | ``` 287 | 288 | ```python 289 | # urls.py 290 | from django.urls import path 291 | 292 | from . import views 293 | 294 | urlpatterns = [ 295 | ... 296 | path('task-status/', views.task_status, name='task_status'), 297 | ... 298 | ] 299 | ``` 300 | 301 | 302 | Any links to `'celery_progress:task_status'` will need to be changed to point to your new endpoint. 303 | 304 | -------------------------------------------------------------------------------- /celery_progress/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/czue/celery-progress/54f85be40d41143d2c5447de58a70a33b27c39ee/celery_progress/__init__.py -------------------------------------------------------------------------------- /celery_progress/backend.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | import re 4 | from abc import ABCMeta, abstractmethod 5 | from decimal import Decimal 6 | 7 | from celery.result import EagerResult, allow_join_result, AsyncResult 8 | from celery.backends.base import DisabledBackend 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | PROGRESS_STATE = 'PROGRESS' 13 | 14 | 15 | class AbstractProgressRecorder(object): 16 | __metaclass__ = ABCMeta 17 | 18 | @abstractmethod 19 | def set_progress(self, current, total, description=""): 20 | pass 21 | 22 | @abstractmethod 23 | def increment_progress(self, by=1, description=""): 24 | pass 25 | 26 | 27 | class BaseProgressRecorder(AbstractProgressRecorder): 28 | current = 0 29 | total = 0 30 | description = "" 31 | 32 | 33 | def set_progress(self, current, total, description=""): 34 | self.current = current 35 | self.total = total 36 | if description: 37 | self.description = description 38 | 39 | def increment_progress(self, by=1, description=""): 40 | """ 41 | Increments progress by one, with an optional description. Useful if the caller doesn't know the total. 42 | """ 43 | self.set_progress(self.current + by, self.total, description) 44 | 45 | 46 | 47 | 48 | class ConsoleProgressRecorder(BaseProgressRecorder): 49 | 50 | def set_progress(self, current, total, description=""): 51 | super().set_progress(current, total, description) 52 | print('processed {} items of {}. {}'.format(current, total, description)) 53 | 54 | 55 | 56 | class ProgressRecorder(BaseProgressRecorder): 57 | 58 | def __init__(self, task): 59 | self.task = task 60 | 61 | def set_progress(self, current, total, description=""): 62 | super().set_progress(current, total, description) 63 | percent = 0 64 | if total > 0: 65 | percent = (Decimal(current) / Decimal(total)) * Decimal(100) 66 | percent = float(round(percent, 2)) 67 | state = PROGRESS_STATE 68 | meta = { 69 | 'pending': False, 70 | 'current': current, 71 | 'total': total, 72 | 'percent': percent, 73 | 'description': description 74 | } 75 | self.task.update_state( 76 | state=state, 77 | meta=meta 78 | ) 79 | return state, meta 80 | 81 | 82 | class Progress(object): 83 | 84 | def __init__(self, result): 85 | """ 86 | result: 87 | an AsyncResult or an object that mimics it to a degree 88 | """ 89 | self.result = result 90 | 91 | def get_info(self): 92 | task_meta = self.result._get_task_meta() 93 | state = task_meta["status"] 94 | info = task_meta["result"] 95 | response = {'state': state} 96 | if state in ['SUCCESS', 'FAILURE']: 97 | success = self.result.successful() 98 | with allow_join_result(): 99 | response.update({ 100 | 'complete': True, 101 | 'success': success, 102 | 'progress': _get_completed_progress(), 103 | 'result': self.result.get(self.result.id) if success else str(info), 104 | }) 105 | elif state in ['RETRY', 'REVOKED']: 106 | if state == 'RETRY': 107 | # in a retry sceneario, result is the exception, and 'traceback' has the details 108 | # https://docs.celeryq.dev/en/stable/userguide/tasks.html#retry 109 | traceback = task_meta.get("traceback") 110 | seconds_re = re.search(r"Retry in \d{1,10}s", traceback) 111 | if seconds_re: 112 | next_retry_seconds = int(seconds_re.group()[9:-1]) 113 | else: 114 | next_retry_seconds = "Unknown" 115 | 116 | result = {"next_retry_seconds": next_retry_seconds, "message": f"{str(task_meta['result'])[0:50]}..."} 117 | else: 118 | result = 'Task ' + str(info) 119 | response.update({ 120 | 'complete': True, 121 | 'success': False, 122 | 'progress': _get_completed_progress(), 123 | 'result': result, 124 | }) 125 | elif state == 'IGNORED': 126 | response.update({ 127 | 'complete': True, 128 | 'success': None, 129 | 'progress': _get_completed_progress(), 130 | 'result': str(info) 131 | }) 132 | elif state == PROGRESS_STATE: 133 | response.update({ 134 | 'complete': False, 135 | 'success': None, 136 | 'progress': info, 137 | }) 138 | elif state in ['PENDING', 'STARTED']: 139 | response.update({ 140 | 'complete': False, 141 | 'success': None, 142 | 'progress': _get_unknown_progress(state), 143 | }) 144 | else: 145 | logger.error('Task %s has unknown state %s with metadata %s', self.result.id, state, info) 146 | response.update({ 147 | 'complete': True, 148 | 'success': False, 149 | 'progress': _get_unknown_progress(state), 150 | 'result': 'Unknown state {}'.format(state), 151 | }) 152 | return response 153 | 154 | @property 155 | def is_failed(self): 156 | info = self.get_info() 157 | return info["complete"] and info["success"] is False 158 | 159 | 160 | class KnownResult(EagerResult): 161 | """Like EagerResult but supports non-ready states.""" 162 | def __init__(self, id, ret_value, state, traceback=None): 163 | """ 164 | ret_value: 165 | result, exception, or progress metadata 166 | """ 167 | # set backend to get state groups (like READY_STATES in ready()) 168 | self.backend = DisabledBackend 169 | super().__init__(id, ret_value, state, traceback) 170 | 171 | def ready(self): 172 | return super(EagerResult, self).ready() 173 | 174 | def __del__(self): 175 | # throws an exception if not overridden 176 | pass 177 | 178 | 179 | def _get_completed_progress(): 180 | return { 181 | 'pending': False, 182 | 'current': 100, 183 | 'total': 100, 184 | 'percent': 100, 185 | } 186 | 187 | 188 | def _get_unknown_progress(state): 189 | return { 190 | 'pending': state == 'PENDING', 191 | 'current': 0, 192 | 'total': 100, 193 | 'percent': 0, 194 | } 195 | 196 | 197 | class GroupProgress: 198 | 199 | def __init__(self, group_result): 200 | """ 201 | group_result: 202 | a GroupResult or an object that mimics it to a degree 203 | """ 204 | self.group_result = group_result 205 | 206 | def get_info(self): 207 | if not self.group_result.children: 208 | raise Exception("There were no tasks to track in the group!") 209 | else: 210 | child_progresses = [Progress(child) for child in self.group_result.children] 211 | child_infos = [cp.get_info() for cp in child_progresses] 212 | child_progress_dicts = [ci["progress"] for ci in child_infos] 213 | total = sum(cp["total"] for cp in child_progress_dicts) 214 | current = sum(cp["current"] for cp in child_progress_dicts) 215 | percent = float(round(100 * current / total, 2)) 216 | info = { 217 | "complete": all(ci["complete"] for ci in child_infos), 218 | "success": all(ci["success"] for ci in child_infos), 219 | "progress": { 220 | "total": total, 221 | "current": current, 222 | "percent": percent, 223 | } 224 | } 225 | return info 226 | -------------------------------------------------------------------------------- /celery_progress/static/celery_progress/celery_progress.js: -------------------------------------------------------------------------------- 1 | class CeleryProgressBar { 2 | 3 | constructor(progressUrl, options) { 4 | this.progressUrl = progressUrl; 5 | options = options || {}; 6 | let progressBarId = options.progressBarId || 'progress-bar'; 7 | let progressBarMessage = options.progressBarMessageId || 'progress-bar-message'; 8 | this.progressBarElement = options.progressBarElement || document.getElementById(progressBarId); 9 | this.progressBarMessageElement = options.progressBarMessageElement || document.getElementById(progressBarMessage); 10 | this.onProgress = options.onProgress || this.onProgressDefault; 11 | this.onSuccess = options.onSuccess || this.onSuccessDefault; 12 | this.onError = options.onError || this.onErrorDefault; 13 | this.onTaskError = options.onTaskError || this.onTaskErrorDefault; 14 | this.onDataError = options.onDataError || this.onError; 15 | this.onRetry = options.onRetry || this.onRetryDefault; 16 | this.onIgnored = options.onIgnored || this.onIgnoredDefault; 17 | let resultElementId = options.resultElementId || 'celery-result'; 18 | this.resultElement = options.resultElement || document.getElementById(resultElementId); 19 | this.onResult = options.onResult || this.onResultDefault; 20 | // HTTP options 21 | this.onNetworkError = options.onNetworkError || this.onError; 22 | this.onHttpError = options.onHttpError || this.onError; 23 | this.pollInterval = options.pollInterval || 500; 24 | this.maxNetworkRetryAttempts = options.maxNetworkRetryAttempts | 5; 25 | // Other options 26 | this.barColors = Object.assign({}, this.constructor.getBarColorsDefault(), options.barColors); 27 | 28 | let defaultMessages = { 29 | waiting: 'Waiting for task to start...', 30 | started: 'Task started...', 31 | } 32 | this.messages = Object.assign({}, defaultMessages, options.defaultMessages); 33 | } 34 | 35 | onSuccessDefault(progressBarElement, progressBarMessageElement, result) { 36 | result = this.getMessageDetails(result); 37 | if (progressBarElement) { 38 | progressBarElement.style.backgroundColor = this.barColors.success; 39 | } 40 | if (progressBarMessageElement) { 41 | progressBarMessageElement.textContent = "Success! " + result; 42 | } 43 | } 44 | 45 | onResultDefault(resultElement, result) { 46 | if (resultElement) { 47 | resultElement.textContent = result; 48 | } 49 | } 50 | 51 | /** 52 | * Default handler for all errors. 53 | * @param data - A Response object for HTTP errors, undefined for other errors 54 | */ 55 | onErrorDefault(progressBarElement, progressBarMessageElement, excMessage, data) { 56 | progressBarElement.style.backgroundColor = this.barColors.error; 57 | excMessage = excMessage || ''; 58 | progressBarMessageElement.textContent = "Uh-Oh, something went wrong! " + excMessage; 59 | } 60 | 61 | onTaskErrorDefault(progressBarElement, progressBarMessageElement, excMessage) { 62 | let message = this.getMessageDetails(excMessage); 63 | this.onError(progressBarElement, progressBarMessageElement, message); 64 | } 65 | 66 | onRetryDefault(progressBarElement, progressBarMessageElement, excMessage, retrySeconds) { 67 | let message = 'Retrying after ' + retrySeconds + 's: ' + excMessage; 68 | progressBarElement.style.backgroundColor = this.barColors.error; 69 | progressBarMessageElement.textContent = message; 70 | } 71 | 72 | onIgnoredDefault(progressBarElement, progressBarMessageElement, result) { 73 | progressBarElement.style.backgroundColor = this.barColors.ignored; 74 | progressBarMessageElement.textContent = result || 'Task result ignored!' 75 | } 76 | 77 | onProgressDefault(progressBarElement, progressBarMessageElement, progress) { 78 | progressBarElement.style.backgroundColor = this.barColors.progress; 79 | progressBarElement.style.width = progress.percent + "%"; 80 | var description = progress.description || ""; 81 | if (progress.current == 0) { 82 | if (progress.pending === true) { 83 | progressBarMessageElement.textContent = this.messages.waiting; 84 | } else { 85 | progressBarMessageElement.textContent = this.messages.started; 86 | } 87 | } else { 88 | progressBarMessageElement.textContent = progress.current + ' of ' + progress.total + ' processed. ' + description; 89 | } 90 | } 91 | 92 | getMessageDetails(result) { 93 | if (this.resultElement) { 94 | return '' 95 | } else { 96 | return result || ''; 97 | } 98 | } 99 | 100 | /** 101 | * Process update message data. 102 | * @return true if the task is complete, false if it's not, undefined if `data` is invalid 103 | */ 104 | onData(data) { 105 | let done = false; 106 | if (data.progress) { 107 | this.onProgress(this.progressBarElement, this.progressBarMessageElement, data.progress); 108 | } 109 | if (data.complete === true) { 110 | done = true; 111 | if (data.success === true) { 112 | this.onSuccess(this.progressBarElement, this.progressBarMessageElement, data.result); 113 | } else if (data.success === false) { 114 | if (data.state === 'RETRY') { 115 | this.onRetry(this.progressBarElement, this.progressBarMessageElement, data.result.message, data.result.next_retry_seconds); 116 | done = false; 117 | delete data.result; 118 | } else { 119 | this.onTaskError(this.progressBarElement, this.progressBarMessageElement, data.result); 120 | } 121 | } else { 122 | if (data.state === 'IGNORED') { 123 | this.onIgnored(this.progressBarElement, this.progressBarMessageElement, data.result); 124 | delete data.result; 125 | } else { 126 | done = undefined; 127 | this.onDataError(this.progressBarElement, this.progressBarMessageElement, "Data Error"); 128 | } 129 | } 130 | if (data.hasOwnProperty('result')) { 131 | this.onResult(this.resultElement, data.result); 132 | } 133 | } else if (data.complete === undefined) { 134 | done = undefined; 135 | this.onDataError(this.progressBarElement, this.progressBarMessageElement, "Data Error"); 136 | } 137 | return done; 138 | } 139 | 140 | async connect() { 141 | let response; 142 | let success = false; 143 | let error = null; 144 | let attempts = 0; 145 | while(!success && attempts < this.maxNetworkRetryAttempts) { 146 | try { 147 | response = await fetch(this.progressUrl); 148 | success = true; 149 | } catch (networkError) { 150 | error = networkError; 151 | this.onNetworkError(this.progressBarElement, this.progressBarMessageElement, "Network Error"); 152 | attempts++; 153 | await new Promise(r => setTimeout(r, 1000)); 154 | } 155 | } 156 | 157 | if (!success) { 158 | throw(error) 159 | } 160 | 161 | if (response.status === 200) { 162 | let data; 163 | try { 164 | data = await response.json(); 165 | } catch (parsingError) { 166 | this.onDataError(this.progressBarElement, this.progressBarMessageElement, "Parsing Error") 167 | throw parsingError; 168 | } 169 | 170 | const complete = this.onData(data); 171 | 172 | if (complete === false) { 173 | setTimeout(this.connect.bind(this), this.pollInterval); 174 | } 175 | } else { 176 | this.onHttpError(this.progressBarElement, this.progressBarMessageElement, "HTTP Code " + response.status, response); 177 | } 178 | } 179 | 180 | static getBarColorsDefault() { 181 | return { 182 | success: '#76ce60', 183 | error: '#dc4f63', 184 | progress: '#68a9ef', 185 | ignored: '#7a7a7a' 186 | }; 187 | } 188 | 189 | static initProgressBar(progressUrl, options) { 190 | const bar = new this(progressUrl, options); 191 | bar.connect(); 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /celery_progress/static/celery_progress/websockets.js: -------------------------------------------------------------------------------- 1 | class CeleryWebSocketProgressBar extends CeleryProgressBar { 2 | 3 | constructor(progressUrl, options) { 4 | super(progressUrl, options); 5 | } 6 | 7 | async connect() { 8 | var ProgressSocket = new WebSocket((location.protocol === 'https:' ? 'wss' : 'ws') + '://' + 9 | window.location.host + this.progressUrl); 10 | 11 | ProgressSocket.onopen = function (event) { 12 | ProgressSocket.send(JSON.stringify({'type': 'check_task_completion'})); 13 | }; 14 | 15 | const bar = this; 16 | ProgressSocket.onmessage = function (event) { 17 | let data; 18 | try { 19 | data = JSON.parse(event.data); 20 | } catch (parsingError) { 21 | bar.onDataError(bar.progressBarElement, bar.progressBarMessageElement, "Parsing Error") 22 | throw parsingError; 23 | } 24 | 25 | const complete = bar.onData(data); 26 | 27 | if (complete === true || complete === undefined) { 28 | ProgressSocket.close(); 29 | } 30 | }; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /celery_progress/tasks.py: -------------------------------------------------------------------------------- 1 | from celery.signals import task_postrun 2 | 3 | 4 | @task_postrun.connect(retry=True) 5 | def task_postrun_handler(**kwargs): 6 | """Runs after a task has finished. This will update the result backend to include the IGNORED result state. 7 | 8 | Necessary for HTTP to properly receive ignored task event.""" 9 | if kwargs.pop('state') == 'IGNORED': 10 | task = kwargs.pop('task') 11 | task.update_state(state='IGNORED', meta=str(kwargs.pop('retval'))) 12 | -------------------------------------------------------------------------------- /celery_progress/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import re_path 2 | from . import views 3 | 4 | app_name = 'celery_progress' 5 | urlpatterns = [ 6 | re_path(r'^(?P[\w-]+)/$', views.get_progress, name='task_status'), 7 | re_path(r'^g/(?P[\w-]+)/$', views.get_group_progress, name='group_status') 8 | ] 9 | -------------------------------------------------------------------------------- /celery_progress/views.py: -------------------------------------------------------------------------------- 1 | import json 2 | from django.http import HttpResponse 3 | from celery.result import AsyncResult, GroupResult 4 | from celery_progress.backend import Progress, GroupProgress 5 | from django.views.decorators.cache import never_cache 6 | 7 | @never_cache 8 | def get_progress(request, task_id): 9 | progress = Progress(AsyncResult(task_id)) 10 | return HttpResponse(json.dumps(progress.get_info()), content_type='application/json') 11 | 12 | 13 | 14 | @never_cache 15 | def get_group_progress(request, group_id): 16 | group_progress = GroupProgress(GroupResult.restore(group_id)) 17 | return HttpResponse(json.dumps(group_progress.get_info()), content_type='application/json') 18 | -------------------------------------------------------------------------------- /celery_progress/websockets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/czue/celery-progress/54f85be40d41143d2c5447de58a70a33b27c39ee/celery_progress/websockets/__init__.py -------------------------------------------------------------------------------- /celery_progress/websockets/backend.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from celery_progress.backend import ProgressRecorder, Progress, KnownResult 4 | 5 | try: 6 | from asgiref.sync import async_to_sync 7 | from channels.layers import get_channel_layer 8 | except ImportError: 9 | channel_layer = None 10 | else: 11 | channel_layer = get_channel_layer() 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | async def closing_group_send(channel_layer, channel, message): 16 | await channel_layer.group_send(channel, message) 17 | await channel_layer.close_pools() 18 | 19 | class WebSocketProgressRecorder(ProgressRecorder): 20 | 21 | def __init__(self, *args, **kwargs): 22 | super().__init__(*args, **kwargs) 23 | 24 | if not channel_layer: 25 | logger.warning( 26 | 'Tried to use websocket progress bar, but dependencies were not installed / configured. ' 27 | 'Use pip install celery-progress[websockets] and set up channels to enable this feature. ' 28 | 'See: https://channels.readthedocs.io/en/latest/ for more details.' 29 | ) 30 | 31 | @staticmethod 32 | def push_update(task_id, data, final=False): 33 | try: 34 | async_to_sync(closing_group_send)( 35 | channel_layer, 36 | task_id, 37 | {'type': 'update_task_progress', 'data': data} 38 | ) 39 | except AttributeError: # No channel layer to send to, so ignore it 40 | pass 41 | except RuntimeError as e: # We're sending messages too fast for asgiref to handle, drop it 42 | if final and channel_layer: # Send error back to post-run handler for a retry 43 | raise e 44 | 45 | def set_progress(self, current, total, description=""): 46 | state, meta = super().set_progress(current, total, description) 47 | result = KnownResult(self.task.request.id, meta, state) 48 | data = Progress(result).get_info() 49 | self.push_update(self.task.request.id, data) 50 | -------------------------------------------------------------------------------- /celery_progress/websockets/consumers.py: -------------------------------------------------------------------------------- 1 | from channels.generic.websocket import AsyncWebsocketConsumer 2 | import json 3 | 4 | from celery.result import AsyncResult 5 | from celery_progress.backend import Progress 6 | 7 | 8 | class ProgressConsumer(AsyncWebsocketConsumer): 9 | async def connect(self): 10 | self.task_id = self.scope['url_route']['kwargs']['task_id'] 11 | 12 | await self.channel_layer.group_add( 13 | self.task_id, 14 | self.channel_name 15 | ) 16 | 17 | await self.accept() 18 | 19 | async def disconnect(self, close_code): 20 | await self.channel_layer.group_discard( 21 | self.task_id, 22 | self.channel_name 23 | ) 24 | 25 | async def receive(self, text_data): 26 | text_data_json = json.loads(text_data) 27 | task_type = text_data_json['type'] 28 | 29 | if task_type == 'check_task_completion': 30 | await self.channel_layer.group_send( 31 | self.task_id, 32 | { 33 | 'type': 'update_task_progress', 34 | 'data': Progress(AsyncResult(self.task_id)).get_info() 35 | } 36 | ) 37 | 38 | async def update_task_progress(self, event): 39 | data = event['data'] 40 | 41 | await self.send(text_data=json.dumps(data)) 42 | -------------------------------------------------------------------------------- /celery_progress/websockets/routing.py: -------------------------------------------------------------------------------- 1 | try: 2 | from django.conf.urls import url 3 | re_path = url 4 | except ImportError: 5 | from django.urls import re_path 6 | 7 | from celery_progress.websockets import consumers 8 | 9 | try: 10 | progress_consumer = consumers.ProgressConsumer.as_asgi() # New in Channels 3, works similar to Django's .as_view() 11 | except AttributeError: 12 | progress_consumer = consumers.ProgressConsumer # Channels 3 not installed, revert to Channels 2 behavior 13 | 14 | urlpatterns = [ 15 | re_path(r'^ws/progress/(?P[\w-]+)/?$', progress_consumer), 16 | ] 17 | -------------------------------------------------------------------------------- /celery_progress/websockets/tasks.py: -------------------------------------------------------------------------------- 1 | from celery.signals import task_postrun, task_revoked 2 | 3 | from .backend import WebSocketProgressRecorder 4 | from celery_progress.backend import KnownResult, Progress 5 | 6 | 7 | @task_postrun.connect(retry=True) 8 | def task_postrun_handler(task_id, **kwargs): 9 | """Runs after a task has finished. This will be used to push a websocket update for completed events. 10 | 11 | If the websockets version of this package is not installed, this will fail silently.""" 12 | result = KnownResult(task_id, kwargs.pop('retval'), kwargs.pop('state')) 13 | data = Progress(result).get_info() 14 | WebSocketProgressRecorder.push_update(task_id, data=data, final=True) 15 | 16 | 17 | @task_revoked.connect(retry=True) 18 | def task_revoked_handler(request, **kwargs): 19 | """Runs if a task has been revoked. This will be used to push a websocket update for revoked events. 20 | 21 | If the websockets version of this package is not installed, this will fail silently.""" 22 | _result = ('terminated' if kwargs.pop('terminated') else None) or ('expired' if kwargs.pop('expired') else None) \ 23 | or 'revoked' 24 | result = KnownResult(request.id, _result, 'REVOKED') 25 | data = Progress(result).get_info() 26 | WebSocketProgressRecorder.push_update(request.id, data=data, final=True) 27 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools >= 40.6.0", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import find_packages, setup 3 | 4 | from glob import glob 5 | 6 | readme_name = os.path.join(os.path.dirname(__file__), 'README.md') 7 | 8 | with open(readme_name, 'r') as readme: 9 | long_description = readme.read() 10 | 11 | # allow setup.py to be run from any path 12 | os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) 13 | 14 | setup( 15 | name='celery-progress', 16 | version='0.5', 17 | packages=find_packages(), 18 | include_package_data=True, 19 | license='MIT License', 20 | description='Drop in, configurable, dependency-free progress bars for your Django/Celery applications.', 21 | long_description=long_description, 22 | long_description_content_type="text/markdown", 23 | url='https://github.com/czue/celery-progress', 24 | author='Cory Zue', 25 | author_email='cory@coryzue.com', 26 | classifiers=[ 27 | 'Environment :: Web Environment', 28 | 'Framework :: Django', 29 | 'Framework :: Django :: 3.2', 30 | 'Framework :: Django :: 4.0', 31 | 'Framework :: Django :: 4.1', 32 | 'Framework :: Django :: 4.2', 33 | 'Framework :: Django :: 5.0', 34 | 'Framework :: Django :: 5.1', 35 | 'Intended Audience :: Developers', 36 | 'License :: OSI Approved :: MIT License', 37 | 'Operating System :: OS Independent', 38 | 'Programming Language :: Python', 39 | 'Programming Language :: Python :: 3.5', 40 | 'Programming Language :: Python :: 3.6', 41 | 'Programming Language :: Python :: 3.7', 42 | 'Programming Language :: Python :: 3.8', 43 | 'Programming Language :: Python :: 3.9', 44 | 'Programming Language :: Python :: 3.10', 45 | 'Programming Language :: Python :: 3.11', 46 | 'Programming Language :: Python :: 3.12', 47 | 'Topic :: Internet :: WWW/HTTP', 48 | 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 49 | ], 50 | data_files=[ 51 | ('static/celery_progress', glob('celery_progress/static/celery_progress/*', recursive=True)), 52 | ], 53 | extras_require={ 54 | 'websockets': ['channels'], 55 | 'redis': ['channels_redis'], 56 | 'rabbitmq': ['channels_rabbitmq'] 57 | } 58 | ) 59 | --------------------------------------------------------------------------------