├── .gitignore
├── LICENSE
├── MAINTAINERS.md
├── MANIFEST.in
├── README.md
├── celery_progress
├── __init__.py
├── backend.py
├── static
│ └── celery_progress
│ │ ├── celery_progress.js
│ │ └── websockets.js
├── tasks.py
├── urls.py
├── views.py
└── websockets
│ ├── __init__.py
│ ├── backend.py
│ ├── consumers.py
│ ├── routing.py
│ └── tasks.py
├── pyproject.toml
└── setup.py
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | env/
12 | build/
13 | develop-eggs/
14 | dist/
15 | downloads/
16 | eggs/
17 | .eggs/
18 | lib/
19 | lib64/
20 | parts/
21 | sdist/
22 | var/
23 | wheels/
24 | *.egg-info/
25 | .installed.cfg
26 | *.egg
27 |
28 | # PyInstaller
29 | # Usually these files are written by a python script from a template
30 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 | *.manifest
32 | *.spec
33 |
34 | # Installer logs
35 | pip-log.txt
36 | pip-delete-this-directory.txt
37 |
38 | # Unit test / coverage reports
39 | htmlcov/
40 | .tox/
41 | .coverage
42 | .coverage.*
43 | .cache
44 | nosetests.xml
45 | coverage.xml
46 | *.cover
47 | .hypothesis/
48 |
49 | # Translations
50 | *.mo
51 | *.pot
52 |
53 | # Django stuff:
54 | *.log
55 | local_settings.py
56 |
57 | # Flask stuff:
58 | instance/
59 | .webassets-cache
60 |
61 | # Scrapy stuff:
62 | .scrapy
63 |
64 | # Sphinx documentation
65 | docs/_build/
66 |
67 | # PyBuilder
68 | target/
69 |
70 | # Jupyter Notebook
71 | .ipynb_checkpoints
72 |
73 | # pyenv
74 | .python-version
75 |
76 | # celery beat schedule file
77 | celerybeat-schedule
78 |
79 | # SageMath parsed files
80 | *.sage.py
81 |
82 | # dotenv
83 | .env
84 |
85 | # virtualenv
86 | .venv
87 | venv/
88 | ENV/
89 |
90 | # Spyder project settings
91 | .spyderproject
92 | .spyproject
93 |
94 | # Rope project settings
95 | .ropeproject
96 |
97 | # mkdocs documentation
98 | /site
99 |
100 | # mypy
101 | .mypy_cache/
102 |
103 | # PyCharm files
104 | .idea/
105 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2018 Cory Zue
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/MAINTAINERS.md:
--------------------------------------------------------------------------------
1 | # Maintainers Guide
2 |
3 | The process for publishing new versions of this library is the following:
4 |
5 | 1. Update the version number in `setup.py`
6 | 2. [Generate distribution archives](https://packaging.python.org/en/latest/tutorials/packaging-projects/#generating-distribution-archives)
7 | 1. `pip install --upgrade build`
8 | 2. `python -m build`
9 | 3. [Upload distribution archives](https://packaging.python.org/en/latest/tutorials/packaging-projects/#uploading-the-distribution-archives)
10 | 1. `pip install --upgrade twine`
11 | 2. `twine upload dist/*`
12 | 4. Publish the tag
13 | 1. git tag -a "0.X" -m "Release 0.X"
14 | 2. git push --tags
15 | 5. Publish the release
16 | 1. Go to [https://github.com/czue/celery-progress/releases/new](https://github.com/czue/celery-progress/releases/new)
17 | 2. Create a release from the tag you just added.
18 | 3. Click "generate release notes"
19 | 4. Click "publish"
20 |
--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include LICENSE
2 | include README.md
3 | recursive-include celery_progress/static *
4 | recursive-include docs *
5 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Celery Progress Bars for Django
2 |
3 | Drop in, dependency-free progress bars for your Django/Celery applications.
4 |
5 | Super simple setup. Lots of customization available.
6 |
7 | ## Demo
8 |
9 | [Celery Progress Bar demo on SaaS Pegasus](https://www.saaspegasus.com/guides/celery-progress-demo/)
10 |
11 | ### Github demo application: build a download progress bar for Django
12 | Starting with Celery can be challenging, [eeintech](https://github.com/eeintech) built a complete [Django demo application](https://github.com/eeintech/django-celery-progress-demo) along with a [step-by-step guide](https://eeinte.ch/stream/progress-bar-django-using-celery/) to get you started on building your own progress bar!
13 |
14 | ## Installation
15 |
16 | If you haven't already, make sure you have properly [set up celery in your project](https://docs.celeryq.dev/en/stable/getting-started/first-steps-with-celery.html#first-steps).
17 |
18 | Then install this library:
19 |
20 | ```bash
21 | pip install celery-progress
22 | ```
23 |
24 | ## Usage
25 |
26 | ### Prerequisites
27 |
28 | First add `celery_progress` to your `INSTALLED_APPS` in `settings.py`.
29 |
30 | Then add the following url config to your main `urls.py`:
31 |
32 | ```python
33 | from django.urls import path, include
34 |
35 | urlpatterns = [
36 | # your project's patterns here
37 | ...
38 | path(r'^celery-progress/', include('celery_progress.urls')), # add this line (the endpoint is configurable)
39 | ]
40 | ```
41 |
42 | ### Recording Progress
43 |
44 | In your task you should add something like this:
45 |
46 | ```python
47 | from celery import shared_task
48 | from celery_progress.backend import ProgressRecorder
49 | import time
50 |
51 | @shared_task(bind=True)
52 | def my_task(self, seconds):
53 | progress_recorder = ProgressRecorder(self)
54 | result = 0
55 | for i in range(seconds):
56 | time.sleep(1)
57 | result += i
58 | progress_recorder.set_progress(i + 1, seconds)
59 | return result
60 | ```
61 |
62 | You can add an optional progress description like this:
63 |
64 | ```python
65 | progress_recorder.set_progress(i + 1, seconds, description='my progress description')
66 | ```
67 |
68 | ### Displaying progress
69 |
70 | In the view where you call the task you need to get the task ID like so:
71 |
72 | **views.py**
73 | ```python
74 | def progress_view(request):
75 | result = my_task.delay(10)
76 | return render(request, 'display_progress.html', context={'task_id': result.task_id})
77 | ```
78 |
79 | Then in the page you want to show the progress bar you just do the following.
80 |
81 | #### Add the following HTML wherever you want your progress bar to appear:
82 |
83 | **display_progress.html**
84 | ```html
85 |
88 | Waiting for progress to start...
89 | ```
90 |
91 | #### Import the javascript file.
92 |
93 | **display_progress.html**
94 | ```html
95 |
96 | ```
97 |
98 | #### Initialize the progress bar:
99 |
100 | ```javascript
101 | // vanilla JS version
102 | document.addEventListener("DOMContentLoaded", function () {
103 | var progressUrl = "{% url 'celery_progress:task_status' task_id %}";
104 | CeleryProgressBar.initProgressBar(progressUrl);
105 | });
106 | ```
107 |
108 | or
109 |
110 | ```javascript
111 | // JQuery
112 | $(function () {
113 | var progressUrl = "{% url 'celery_progress:task_status' task_id %}";
114 | CeleryProgressBar.initProgressBar(progressUrl)
115 | });
116 | ```
117 |
118 | ### Displaying the result of a task
119 |
120 | If you'd like you can also display the result of your task on the front end.
121 |
122 | To do that follow the steps below. Result handling can also be customized.
123 |
124 | #### Initialize the result block:
125 |
126 | This is all that's needed to render the result on the page.
127 |
128 | **display_progress.html**
129 | ```html
130 |
131 | ```
132 |
133 | But more likely you will want to customize how the result looks, which can be done as below:
134 |
135 | ```javascript
136 | // JQuery
137 | var progressUrl = "{% url 'celery_progress:task_status' task_id %}";
138 |
139 | function customResult(resultElement, result) {
140 | $( resultElement ).append(
141 | $('').text('Sum of all seconds is ' + result)
142 | );
143 | }
144 |
145 | $(function () {
146 | CeleryProgressBar.initProgressBar(progressUrl, {
147 | onResult: customResult,
148 | })
149 | });
150 | ```
151 |
152 | ### Working with Groups
153 |
154 | This library includes experimental support for working with [Celery groups](https://docs.celeryq.dev/en/stable/userguide/canvas.html#groups).
155 | You can use the `"group_status"` URL endpoint for this. Here is a basic example:
156 |
157 | **Example task:**
158 |
159 | ```python
160 | @shared_task(bind=True)
161 | def add(self, x, y):
162 | return x + y
163 | ```
164 |
165 | **Calling view:**
166 |
167 | ```python
168 | from celery import group
169 | from .tasks import add
170 |
171 | def progress_view(request):
172 | task_group = group(add.s(i, i) for i in range(100))
173 | group_result = task_group.apply_async()
174 | # you must explicitly call the save function on the group_result after calling the tasks
175 | group_result.save()
176 | return render(request, 'display_progress.html', context={'task_id': group_result.id})
177 |
178 | ```
179 |
180 | **Template:**
181 |
182 | ```html
183 | document.addEventListener("DOMContentLoaded", function () {
184 | var progressUrl = "{% url 'celery_progress:group_status' task_id %}";
185 | CeleryProgressBar.initProgressBar(progressUrl);
186 | });
187 | ```
188 |
189 | ## Customization
190 |
191 | The `initProgressBar` function takes an optional object of options. The following options are supported:
192 |
193 | | Option | What it does | Default Value |
194 | |--------|--------------|---------------|
195 | | pollInterval | How frequently to poll for progress (in milliseconds) | 500 |
196 | | progressBarId | Override the ID used for the progress bar | 'progress-bar' |
197 | | progressBarMessageId | Override the ID used for the progress bar message | 'progress-bar-message' |
198 | | progressBarElement | Override the *element* used for the progress bar. If specified, progressBarId will be ignored. | document.getElementById(progressBarId) |
199 | | progressBarMessageElement | Override the *element* used for the progress bar message. If specified, progressBarMessageId will be ignored. | document.getElementById(progressBarMessageId) |
200 | | resultElementId | Override the ID used for the result | 'celery-result' |
201 | | resultElement | Override the *element* used for the result. If specified, resultElementId will be ignored. | document.getElementById(resultElementId) |
202 | | onProgress | function to call when progress is updated | onProgressDefault |
203 | | onSuccess | function to call when progress successfully completes | onSuccessDefault |
204 | | onError | function to call on a known error with no specified handler | onErrorDefault |
205 | | onRetry | function to call when a task attempts to retry | onRetryDefault |
206 | | onIgnored | function to call when a task result is ignored | onIgnoredDefault |
207 | | onTaskError | function to call when progress completes with an error | onError |
208 | | onNetworkError | function to call on a network error (ignored by WebSocket) | onError |
209 | | onHttpError | function to call on a non-200 response (ignored by WebSocket) | onError |
210 | | onDataError | function to call on a response that's not JSON or has invalid schema due to a programming error | onError |
211 | | onResult | function to call when returned non empty result | CeleryProgressBar.onResultDefault |
212 | | barColors | dictionary containing color values for various progress bar states. Colors that are not specified will defer to defaults | barColorsDefault |
213 | | defaultMessages | dictionary containing default messages that can be overridden | see below |
214 |
215 | The `barColors` option allows you to customize the color of each progress bar state by passing a dictionary of key-value pairs of `state: #hexcode`. The defaults are shown below.
216 |
217 | | State | Hex Code | Image Color |
218 | |-------|----------|:-------------:|
219 | | success | #76ce60 |  |
220 | | error | #dc4f63 |  |
221 | | progress | #68a9ef |  |
222 | | ignored | #7a7a7a |  |
223 |
224 | The `defaultMessages` option allows you to override some default messages in the UI. At the moment these are:
225 |
226 | | Message Id | When Shown | Default Value |
227 | |-------|----------|:-------------:|
228 | | waiting | Task is waiting to start | 'Waiting for task to start...'
229 | | started | Task has started but reports no progress | 'Task started...'
230 |
231 | # WebSocket Support
232 |
233 | Additionally, this library offers WebSocket support using [Django Channels](https://channels.readthedocs.io/en/latest/)
234 | courtesy of [EJH2](https://github.com/EJH2/).
235 |
236 | A working example project leveraging WebSockets is [available here](https://github.com/EJH2/cp_ws-example).
237 |
238 | To use WebSockets, install with `pip install celery-progress[websockets,redis]` or
239 | `pip install celery-progress[websockets,rabbitmq]` (depending on broker dependencies).
240 |
241 | See `WebSocketProgressRecorder` and `websockets.js` for details.
242 |
243 | # Securing the get_progress endpoint
244 | By default, anyone can see the status and result of any task by accessing `/celery-progress/`
245 |
246 | To limit access, you need to wrap `get_progress()` in a view of your own which implements the permissions check, and create a new url routing to point to your view. Make sure to remove any existing (unprotected) celery progress urls from your root urlconf at the same time.
247 |
248 |
249 | For example, requiring login with a class-based view:
250 | ```python
251 |
252 | # views.py
253 | from celery_progress.views import get_progress
254 | from django.contrib.auth.mixins import LoginRequiredMixin
255 | from django.views.generic import View
256 |
257 | class TaskStatus(LoginRequiredMixin, View):
258 | def get(self, request, task_id, *args, **kwargs):
259 | # Other checks could go here
260 | return get_progress(request, task_id=task_id)
261 | ```
262 |
263 | ```python
264 | # urls.py
265 | from django.urls import path
266 | from . import views
267 |
268 | urlpatterns = [
269 | ...
270 | path('task-status/', views.TaskStatus.as_view(), name='task_status'),
271 | ...
272 | ]
273 | ```
274 |
275 | Requiring login with a function-based view:
276 | ```python
277 |
278 | # views.py
279 | from celery_progress.views import get_progress
280 | from django.contrib.auth.decorators import login_required
281 |
282 | @login_required
283 | def task_status(request, task_id):
284 | # Other checks could go here
285 | return get_progress(request, task_id)
286 | ```
287 |
288 | ```python
289 | # urls.py
290 | from django.urls import path
291 |
292 | from . import views
293 |
294 | urlpatterns = [
295 | ...
296 | path('task-status/', views.task_status, name='task_status'),
297 | ...
298 | ]
299 | ```
300 |
301 |
302 | Any links to `'celery_progress:task_status'` will need to be changed to point to your new endpoint.
303 |
304 |
--------------------------------------------------------------------------------
/celery_progress/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/czue/celery-progress/54f85be40d41143d2c5447de58a70a33b27c39ee/celery_progress/__init__.py
--------------------------------------------------------------------------------
/celery_progress/backend.py:
--------------------------------------------------------------------------------
1 | import datetime
2 | import logging
3 | import re
4 | from abc import ABCMeta, abstractmethod
5 | from decimal import Decimal
6 |
7 | from celery.result import EagerResult, allow_join_result, AsyncResult
8 | from celery.backends.base import DisabledBackend
9 |
10 | logger = logging.getLogger(__name__)
11 |
12 | PROGRESS_STATE = 'PROGRESS'
13 |
14 |
15 | class AbstractProgressRecorder(object):
16 | __metaclass__ = ABCMeta
17 |
18 | @abstractmethod
19 | def set_progress(self, current, total, description=""):
20 | pass
21 |
22 | @abstractmethod
23 | def increment_progress(self, by=1, description=""):
24 | pass
25 |
26 |
27 | class BaseProgressRecorder(AbstractProgressRecorder):
28 | current = 0
29 | total = 0
30 | description = ""
31 |
32 |
33 | def set_progress(self, current, total, description=""):
34 | self.current = current
35 | self.total = total
36 | if description:
37 | self.description = description
38 |
39 | def increment_progress(self, by=1, description=""):
40 | """
41 | Increments progress by one, with an optional description. Useful if the caller doesn't know the total.
42 | """
43 | self.set_progress(self.current + by, self.total, description)
44 |
45 |
46 |
47 |
48 | class ConsoleProgressRecorder(BaseProgressRecorder):
49 |
50 | def set_progress(self, current, total, description=""):
51 | super().set_progress(current, total, description)
52 | print('processed {} items of {}. {}'.format(current, total, description))
53 |
54 |
55 |
56 | class ProgressRecorder(BaseProgressRecorder):
57 |
58 | def __init__(self, task):
59 | self.task = task
60 |
61 | def set_progress(self, current, total, description=""):
62 | super().set_progress(current, total, description)
63 | percent = 0
64 | if total > 0:
65 | percent = (Decimal(current) / Decimal(total)) * Decimal(100)
66 | percent = float(round(percent, 2))
67 | state = PROGRESS_STATE
68 | meta = {
69 | 'pending': False,
70 | 'current': current,
71 | 'total': total,
72 | 'percent': percent,
73 | 'description': description
74 | }
75 | self.task.update_state(
76 | state=state,
77 | meta=meta
78 | )
79 | return state, meta
80 |
81 |
82 | class Progress(object):
83 |
84 | def __init__(self, result):
85 | """
86 | result:
87 | an AsyncResult or an object that mimics it to a degree
88 | """
89 | self.result = result
90 |
91 | def get_info(self):
92 | task_meta = self.result._get_task_meta()
93 | state = task_meta["status"]
94 | info = task_meta["result"]
95 | response = {'state': state}
96 | if state in ['SUCCESS', 'FAILURE']:
97 | success = self.result.successful()
98 | with allow_join_result():
99 | response.update({
100 | 'complete': True,
101 | 'success': success,
102 | 'progress': _get_completed_progress(),
103 | 'result': self.result.get(self.result.id) if success else str(info),
104 | })
105 | elif state in ['RETRY', 'REVOKED']:
106 | if state == 'RETRY':
107 | # in a retry sceneario, result is the exception, and 'traceback' has the details
108 | # https://docs.celeryq.dev/en/stable/userguide/tasks.html#retry
109 | traceback = task_meta.get("traceback")
110 | seconds_re = re.search(r"Retry in \d{1,10}s", traceback)
111 | if seconds_re:
112 | next_retry_seconds = int(seconds_re.group()[9:-1])
113 | else:
114 | next_retry_seconds = "Unknown"
115 |
116 | result = {"next_retry_seconds": next_retry_seconds, "message": f"{str(task_meta['result'])[0:50]}..."}
117 | else:
118 | result = 'Task ' + str(info)
119 | response.update({
120 | 'complete': True,
121 | 'success': False,
122 | 'progress': _get_completed_progress(),
123 | 'result': result,
124 | })
125 | elif state == 'IGNORED':
126 | response.update({
127 | 'complete': True,
128 | 'success': None,
129 | 'progress': _get_completed_progress(),
130 | 'result': str(info)
131 | })
132 | elif state == PROGRESS_STATE:
133 | response.update({
134 | 'complete': False,
135 | 'success': None,
136 | 'progress': info,
137 | })
138 | elif state in ['PENDING', 'STARTED']:
139 | response.update({
140 | 'complete': False,
141 | 'success': None,
142 | 'progress': _get_unknown_progress(state),
143 | })
144 | else:
145 | logger.error('Task %s has unknown state %s with metadata %s', self.result.id, state, info)
146 | response.update({
147 | 'complete': True,
148 | 'success': False,
149 | 'progress': _get_unknown_progress(state),
150 | 'result': 'Unknown state {}'.format(state),
151 | })
152 | return response
153 |
154 | @property
155 | def is_failed(self):
156 | info = self.get_info()
157 | return info["complete"] and info["success"] is False
158 |
159 |
160 | class KnownResult(EagerResult):
161 | """Like EagerResult but supports non-ready states."""
162 | def __init__(self, id, ret_value, state, traceback=None):
163 | """
164 | ret_value:
165 | result, exception, or progress metadata
166 | """
167 | # set backend to get state groups (like READY_STATES in ready())
168 | self.backend = DisabledBackend
169 | super().__init__(id, ret_value, state, traceback)
170 |
171 | def ready(self):
172 | return super(EagerResult, self).ready()
173 |
174 | def __del__(self):
175 | # throws an exception if not overridden
176 | pass
177 |
178 |
179 | def _get_completed_progress():
180 | return {
181 | 'pending': False,
182 | 'current': 100,
183 | 'total': 100,
184 | 'percent': 100,
185 | }
186 |
187 |
188 | def _get_unknown_progress(state):
189 | return {
190 | 'pending': state == 'PENDING',
191 | 'current': 0,
192 | 'total': 100,
193 | 'percent': 0,
194 | }
195 |
196 |
197 | class GroupProgress:
198 |
199 | def __init__(self, group_result):
200 | """
201 | group_result:
202 | a GroupResult or an object that mimics it to a degree
203 | """
204 | self.group_result = group_result
205 |
206 | def get_info(self):
207 | if not self.group_result.children:
208 | raise Exception("There were no tasks to track in the group!")
209 | else:
210 | child_progresses = [Progress(child) for child in self.group_result.children]
211 | child_infos = [cp.get_info() for cp in child_progresses]
212 | child_progress_dicts = [ci["progress"] for ci in child_infos]
213 | total = sum(cp["total"] for cp in child_progress_dicts)
214 | current = sum(cp["current"] for cp in child_progress_dicts)
215 | percent = float(round(100 * current / total, 2))
216 | info = {
217 | "complete": all(ci["complete"] for ci in child_infos),
218 | "success": all(ci["success"] for ci in child_infos),
219 | "progress": {
220 | "total": total,
221 | "current": current,
222 | "percent": percent,
223 | }
224 | }
225 | return info
226 |
--------------------------------------------------------------------------------
/celery_progress/static/celery_progress/celery_progress.js:
--------------------------------------------------------------------------------
1 | class CeleryProgressBar {
2 |
3 | constructor(progressUrl, options) {
4 | this.progressUrl = progressUrl;
5 | options = options || {};
6 | let progressBarId = options.progressBarId || 'progress-bar';
7 | let progressBarMessage = options.progressBarMessageId || 'progress-bar-message';
8 | this.progressBarElement = options.progressBarElement || document.getElementById(progressBarId);
9 | this.progressBarMessageElement = options.progressBarMessageElement || document.getElementById(progressBarMessage);
10 | this.onProgress = options.onProgress || this.onProgressDefault;
11 | this.onSuccess = options.onSuccess || this.onSuccessDefault;
12 | this.onError = options.onError || this.onErrorDefault;
13 | this.onTaskError = options.onTaskError || this.onTaskErrorDefault;
14 | this.onDataError = options.onDataError || this.onError;
15 | this.onRetry = options.onRetry || this.onRetryDefault;
16 | this.onIgnored = options.onIgnored || this.onIgnoredDefault;
17 | let resultElementId = options.resultElementId || 'celery-result';
18 | this.resultElement = options.resultElement || document.getElementById(resultElementId);
19 | this.onResult = options.onResult || this.onResultDefault;
20 | // HTTP options
21 | this.onNetworkError = options.onNetworkError || this.onError;
22 | this.onHttpError = options.onHttpError || this.onError;
23 | this.pollInterval = options.pollInterval || 500;
24 | this.maxNetworkRetryAttempts = options.maxNetworkRetryAttempts | 5;
25 | // Other options
26 | this.barColors = Object.assign({}, this.constructor.getBarColorsDefault(), options.barColors);
27 |
28 | let defaultMessages = {
29 | waiting: 'Waiting for task to start...',
30 | started: 'Task started...',
31 | }
32 | this.messages = Object.assign({}, defaultMessages, options.defaultMessages);
33 | }
34 |
35 | onSuccessDefault(progressBarElement, progressBarMessageElement, result) {
36 | result = this.getMessageDetails(result);
37 | if (progressBarElement) {
38 | progressBarElement.style.backgroundColor = this.barColors.success;
39 | }
40 | if (progressBarMessageElement) {
41 | progressBarMessageElement.textContent = "Success! " + result;
42 | }
43 | }
44 |
45 | onResultDefault(resultElement, result) {
46 | if (resultElement) {
47 | resultElement.textContent = result;
48 | }
49 | }
50 |
51 | /**
52 | * Default handler for all errors.
53 | * @param data - A Response object for HTTP errors, undefined for other errors
54 | */
55 | onErrorDefault(progressBarElement, progressBarMessageElement, excMessage, data) {
56 | progressBarElement.style.backgroundColor = this.barColors.error;
57 | excMessage = excMessage || '';
58 | progressBarMessageElement.textContent = "Uh-Oh, something went wrong! " + excMessage;
59 | }
60 |
61 | onTaskErrorDefault(progressBarElement, progressBarMessageElement, excMessage) {
62 | let message = this.getMessageDetails(excMessage);
63 | this.onError(progressBarElement, progressBarMessageElement, message);
64 | }
65 |
66 | onRetryDefault(progressBarElement, progressBarMessageElement, excMessage, retrySeconds) {
67 | let message = 'Retrying after ' + retrySeconds + 's: ' + excMessage;
68 | progressBarElement.style.backgroundColor = this.barColors.error;
69 | progressBarMessageElement.textContent = message;
70 | }
71 |
72 | onIgnoredDefault(progressBarElement, progressBarMessageElement, result) {
73 | progressBarElement.style.backgroundColor = this.barColors.ignored;
74 | progressBarMessageElement.textContent = result || 'Task result ignored!'
75 | }
76 |
77 | onProgressDefault(progressBarElement, progressBarMessageElement, progress) {
78 | progressBarElement.style.backgroundColor = this.barColors.progress;
79 | progressBarElement.style.width = progress.percent + "%";
80 | var description = progress.description || "";
81 | if (progress.current == 0) {
82 | if (progress.pending === true) {
83 | progressBarMessageElement.textContent = this.messages.waiting;
84 | } else {
85 | progressBarMessageElement.textContent = this.messages.started;
86 | }
87 | } else {
88 | progressBarMessageElement.textContent = progress.current + ' of ' + progress.total + ' processed. ' + description;
89 | }
90 | }
91 |
92 | getMessageDetails(result) {
93 | if (this.resultElement) {
94 | return ''
95 | } else {
96 | return result || '';
97 | }
98 | }
99 |
100 | /**
101 | * Process update message data.
102 | * @return true if the task is complete, false if it's not, undefined if `data` is invalid
103 | */
104 | onData(data) {
105 | let done = false;
106 | if (data.progress) {
107 | this.onProgress(this.progressBarElement, this.progressBarMessageElement, data.progress);
108 | }
109 | if (data.complete === true) {
110 | done = true;
111 | if (data.success === true) {
112 | this.onSuccess(this.progressBarElement, this.progressBarMessageElement, data.result);
113 | } else if (data.success === false) {
114 | if (data.state === 'RETRY') {
115 | this.onRetry(this.progressBarElement, this.progressBarMessageElement, data.result.message, data.result.next_retry_seconds);
116 | done = false;
117 | delete data.result;
118 | } else {
119 | this.onTaskError(this.progressBarElement, this.progressBarMessageElement, data.result);
120 | }
121 | } else {
122 | if (data.state === 'IGNORED') {
123 | this.onIgnored(this.progressBarElement, this.progressBarMessageElement, data.result);
124 | delete data.result;
125 | } else {
126 | done = undefined;
127 | this.onDataError(this.progressBarElement, this.progressBarMessageElement, "Data Error");
128 | }
129 | }
130 | if (data.hasOwnProperty('result')) {
131 | this.onResult(this.resultElement, data.result);
132 | }
133 | } else if (data.complete === undefined) {
134 | done = undefined;
135 | this.onDataError(this.progressBarElement, this.progressBarMessageElement, "Data Error");
136 | }
137 | return done;
138 | }
139 |
140 | async connect() {
141 | let response;
142 | let success = false;
143 | let error = null;
144 | let attempts = 0;
145 | while(!success && attempts < this.maxNetworkRetryAttempts) {
146 | try {
147 | response = await fetch(this.progressUrl);
148 | success = true;
149 | } catch (networkError) {
150 | error = networkError;
151 | this.onNetworkError(this.progressBarElement, this.progressBarMessageElement, "Network Error");
152 | attempts++;
153 | await new Promise(r => setTimeout(r, 1000));
154 | }
155 | }
156 |
157 | if (!success) {
158 | throw(error)
159 | }
160 |
161 | if (response.status === 200) {
162 | let data;
163 | try {
164 | data = await response.json();
165 | } catch (parsingError) {
166 | this.onDataError(this.progressBarElement, this.progressBarMessageElement, "Parsing Error")
167 | throw parsingError;
168 | }
169 |
170 | const complete = this.onData(data);
171 |
172 | if (complete === false) {
173 | setTimeout(this.connect.bind(this), this.pollInterval);
174 | }
175 | } else {
176 | this.onHttpError(this.progressBarElement, this.progressBarMessageElement, "HTTP Code " + response.status, response);
177 | }
178 | }
179 |
180 | static getBarColorsDefault() {
181 | return {
182 | success: '#76ce60',
183 | error: '#dc4f63',
184 | progress: '#68a9ef',
185 | ignored: '#7a7a7a'
186 | };
187 | }
188 |
189 | static initProgressBar(progressUrl, options) {
190 | const bar = new this(progressUrl, options);
191 | bar.connect();
192 | }
193 | }
194 |
--------------------------------------------------------------------------------
/celery_progress/static/celery_progress/websockets.js:
--------------------------------------------------------------------------------
1 | class CeleryWebSocketProgressBar extends CeleryProgressBar {
2 |
3 | constructor(progressUrl, options) {
4 | super(progressUrl, options);
5 | }
6 |
7 | async connect() {
8 | var ProgressSocket = new WebSocket((location.protocol === 'https:' ? 'wss' : 'ws') + '://' +
9 | window.location.host + this.progressUrl);
10 |
11 | ProgressSocket.onopen = function (event) {
12 | ProgressSocket.send(JSON.stringify({'type': 'check_task_completion'}));
13 | };
14 |
15 | const bar = this;
16 | ProgressSocket.onmessage = function (event) {
17 | let data;
18 | try {
19 | data = JSON.parse(event.data);
20 | } catch (parsingError) {
21 | bar.onDataError(bar.progressBarElement, bar.progressBarMessageElement, "Parsing Error")
22 | throw parsingError;
23 | }
24 |
25 | const complete = bar.onData(data);
26 |
27 | if (complete === true || complete === undefined) {
28 | ProgressSocket.close();
29 | }
30 | };
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/celery_progress/tasks.py:
--------------------------------------------------------------------------------
1 | from celery.signals import task_postrun
2 |
3 |
4 | @task_postrun.connect(retry=True)
5 | def task_postrun_handler(**kwargs):
6 | """Runs after a task has finished. This will update the result backend to include the IGNORED result state.
7 |
8 | Necessary for HTTP to properly receive ignored task event."""
9 | if kwargs.pop('state') == 'IGNORED':
10 | task = kwargs.pop('task')
11 | task.update_state(state='IGNORED', meta=str(kwargs.pop('retval')))
12 |
--------------------------------------------------------------------------------
/celery_progress/urls.py:
--------------------------------------------------------------------------------
1 | from django.urls import re_path
2 | from . import views
3 |
4 | app_name = 'celery_progress'
5 | urlpatterns = [
6 | re_path(r'^(?P[\w-]+)/$', views.get_progress, name='task_status'),
7 | re_path(r'^g/(?P[\w-]+)/$', views.get_group_progress, name='group_status')
8 | ]
9 |
--------------------------------------------------------------------------------
/celery_progress/views.py:
--------------------------------------------------------------------------------
1 | import json
2 | from django.http import HttpResponse
3 | from celery.result import AsyncResult, GroupResult
4 | from celery_progress.backend import Progress, GroupProgress
5 | from django.views.decorators.cache import never_cache
6 |
7 | @never_cache
8 | def get_progress(request, task_id):
9 | progress = Progress(AsyncResult(task_id))
10 | return HttpResponse(json.dumps(progress.get_info()), content_type='application/json')
11 |
12 |
13 |
14 | @never_cache
15 | def get_group_progress(request, group_id):
16 | group_progress = GroupProgress(GroupResult.restore(group_id))
17 | return HttpResponse(json.dumps(group_progress.get_info()), content_type='application/json')
18 |
--------------------------------------------------------------------------------
/celery_progress/websockets/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/czue/celery-progress/54f85be40d41143d2c5447de58a70a33b27c39ee/celery_progress/websockets/__init__.py
--------------------------------------------------------------------------------
/celery_progress/websockets/backend.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 | from celery_progress.backend import ProgressRecorder, Progress, KnownResult
4 |
5 | try:
6 | from asgiref.sync import async_to_sync
7 | from channels.layers import get_channel_layer
8 | except ImportError:
9 | channel_layer = None
10 | else:
11 | channel_layer = get_channel_layer()
12 |
13 | logger = logging.getLogger(__name__)
14 |
15 | async def closing_group_send(channel_layer, channel, message):
16 | await channel_layer.group_send(channel, message)
17 | await channel_layer.close_pools()
18 |
19 | class WebSocketProgressRecorder(ProgressRecorder):
20 |
21 | def __init__(self, *args, **kwargs):
22 | super().__init__(*args, **kwargs)
23 |
24 | if not channel_layer:
25 | logger.warning(
26 | 'Tried to use websocket progress bar, but dependencies were not installed / configured. '
27 | 'Use pip install celery-progress[websockets] and set up channels to enable this feature. '
28 | 'See: https://channels.readthedocs.io/en/latest/ for more details.'
29 | )
30 |
31 | @staticmethod
32 | def push_update(task_id, data, final=False):
33 | try:
34 | async_to_sync(closing_group_send)(
35 | channel_layer,
36 | task_id,
37 | {'type': 'update_task_progress', 'data': data}
38 | )
39 | except AttributeError: # No channel layer to send to, so ignore it
40 | pass
41 | except RuntimeError as e: # We're sending messages too fast for asgiref to handle, drop it
42 | if final and channel_layer: # Send error back to post-run handler for a retry
43 | raise e
44 |
45 | def set_progress(self, current, total, description=""):
46 | state, meta = super().set_progress(current, total, description)
47 | result = KnownResult(self.task.request.id, meta, state)
48 | data = Progress(result).get_info()
49 | self.push_update(self.task.request.id, data)
50 |
--------------------------------------------------------------------------------
/celery_progress/websockets/consumers.py:
--------------------------------------------------------------------------------
1 | from channels.generic.websocket import AsyncWebsocketConsumer
2 | import json
3 |
4 | from celery.result import AsyncResult
5 | from celery_progress.backend import Progress
6 |
7 |
8 | class ProgressConsumer(AsyncWebsocketConsumer):
9 | async def connect(self):
10 | self.task_id = self.scope['url_route']['kwargs']['task_id']
11 |
12 | await self.channel_layer.group_add(
13 | self.task_id,
14 | self.channel_name
15 | )
16 |
17 | await self.accept()
18 |
19 | async def disconnect(self, close_code):
20 | await self.channel_layer.group_discard(
21 | self.task_id,
22 | self.channel_name
23 | )
24 |
25 | async def receive(self, text_data):
26 | text_data_json = json.loads(text_data)
27 | task_type = text_data_json['type']
28 |
29 | if task_type == 'check_task_completion':
30 | await self.channel_layer.group_send(
31 | self.task_id,
32 | {
33 | 'type': 'update_task_progress',
34 | 'data': Progress(AsyncResult(self.task_id)).get_info()
35 | }
36 | )
37 |
38 | async def update_task_progress(self, event):
39 | data = event['data']
40 |
41 | await self.send(text_data=json.dumps(data))
42 |
--------------------------------------------------------------------------------
/celery_progress/websockets/routing.py:
--------------------------------------------------------------------------------
1 | try:
2 | from django.conf.urls import url
3 | re_path = url
4 | except ImportError:
5 | from django.urls import re_path
6 |
7 | from celery_progress.websockets import consumers
8 |
9 | try:
10 | progress_consumer = consumers.ProgressConsumer.as_asgi() # New in Channels 3, works similar to Django's .as_view()
11 | except AttributeError:
12 | progress_consumer = consumers.ProgressConsumer # Channels 3 not installed, revert to Channels 2 behavior
13 |
14 | urlpatterns = [
15 | re_path(r'^ws/progress/(?P[\w-]+)/?$', progress_consumer),
16 | ]
17 |
--------------------------------------------------------------------------------
/celery_progress/websockets/tasks.py:
--------------------------------------------------------------------------------
1 | from celery.signals import task_postrun, task_revoked
2 |
3 | from .backend import WebSocketProgressRecorder
4 | from celery_progress.backend import KnownResult, Progress
5 |
6 |
7 | @task_postrun.connect(retry=True)
8 | def task_postrun_handler(task_id, **kwargs):
9 | """Runs after a task has finished. This will be used to push a websocket update for completed events.
10 |
11 | If the websockets version of this package is not installed, this will fail silently."""
12 | result = KnownResult(task_id, kwargs.pop('retval'), kwargs.pop('state'))
13 | data = Progress(result).get_info()
14 | WebSocketProgressRecorder.push_update(task_id, data=data, final=True)
15 |
16 |
17 | @task_revoked.connect(retry=True)
18 | def task_revoked_handler(request, **kwargs):
19 | """Runs if a task has been revoked. This will be used to push a websocket update for revoked events.
20 |
21 | If the websockets version of this package is not installed, this will fail silently."""
22 | _result = ('terminated' if kwargs.pop('terminated') else None) or ('expired' if kwargs.pop('expired') else None) \
23 | or 'revoked'
24 | result = KnownResult(request.id, _result, 'REVOKED')
25 | data = Progress(result).get_info()
26 | WebSocketProgressRecorder.push_update(request.id, data=data, final=True)
27 |
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [build-system]
2 | requires = ["setuptools >= 40.6.0", "wheel"]
3 | build-backend = "setuptools.build_meta"
4 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | import os
2 | from setuptools import find_packages, setup
3 |
4 | from glob import glob
5 |
6 | readme_name = os.path.join(os.path.dirname(__file__), 'README.md')
7 |
8 | with open(readme_name, 'r') as readme:
9 | long_description = readme.read()
10 |
11 | # allow setup.py to be run from any path
12 | os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir)))
13 |
14 | setup(
15 | name='celery-progress',
16 | version='0.5',
17 | packages=find_packages(),
18 | include_package_data=True,
19 | license='MIT License',
20 | description='Drop in, configurable, dependency-free progress bars for your Django/Celery applications.',
21 | long_description=long_description,
22 | long_description_content_type="text/markdown",
23 | url='https://github.com/czue/celery-progress',
24 | author='Cory Zue',
25 | author_email='cory@coryzue.com',
26 | classifiers=[
27 | 'Environment :: Web Environment',
28 | 'Framework :: Django',
29 | 'Framework :: Django :: 3.2',
30 | 'Framework :: Django :: 4.0',
31 | 'Framework :: Django :: 4.1',
32 | 'Framework :: Django :: 4.2',
33 | 'Framework :: Django :: 5.0',
34 | 'Framework :: Django :: 5.1',
35 | 'Intended Audience :: Developers',
36 | 'License :: OSI Approved :: MIT License',
37 | 'Operating System :: OS Independent',
38 | 'Programming Language :: Python',
39 | 'Programming Language :: Python :: 3.5',
40 | 'Programming Language :: Python :: 3.6',
41 | 'Programming Language :: Python :: 3.7',
42 | 'Programming Language :: Python :: 3.8',
43 | 'Programming Language :: Python :: 3.9',
44 | 'Programming Language :: Python :: 3.10',
45 | 'Programming Language :: Python :: 3.11',
46 | 'Programming Language :: Python :: 3.12',
47 | 'Topic :: Internet :: WWW/HTTP',
48 | 'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
49 | ],
50 | data_files=[
51 | ('static/celery_progress', glob('celery_progress/static/celery_progress/*', recursive=True)),
52 | ],
53 | extras_require={
54 | 'websockets': ['channels'],
55 | 'redis': ['channels_redis'],
56 | 'rabbitmq': ['channels_rabbitmq']
57 | }
58 | )
59 |
--------------------------------------------------------------------------------