├── .gitignore ├── LICENSE ├── README.rst ├── celerybeatmongo ├── __init__.py ├── models.py └── schedulers.py ├── requirements.txt ├── requirements_dev.txt ├── setup.py └── tests ├── __init__.py ├── test_models.py └── test_scheduler.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .* 3 | *.egg-info 4 | /env 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | celerybeat-mongo 2 | ################ 3 | 4 | This is a `Celery Beat Scheduler `_ 5 | that stores both the schedules themselves and their status 6 | information in a backend Mongo database. It can be installed by 7 | installing the celerybeat-mongo Python egg:: 8 | 9 | # pip install celerybeat-mongo 10 | 11 | And specifying the scheduler when running Celery Beat, e.g.:: 12 | 13 | $ celery beat -S celerybeatmongo.schedulers.MongoScheduler 14 | 15 | Settings 16 | ######## 17 | 18 | The settings for the scheduler are defined in your celery configuration file 19 | similar to how other aspects of Celery are configured: 20 | 21 | * mongodb_scheduler_url: The mongodb `url `_ connection used to store task results. 22 | * mongodb_scheduler_db: The Mongodb database name 23 | * mongodb_scheduler_collection (optional): the collection name used by model. If no value are specified, the default value will be used: **schedules**. 24 | 25 | Usage 26 | =================== 27 | Celerybeat-mongo just supports Interval and Crontab schedules. 28 | Schedules easily can be manipulated using the mongoengine models in celerybeat mongo.models module. 29 | 30 | Example creating interval-based periodic task 31 | --------------------------------------------- 32 | 33 | To create a periodic task executing at an interval you must first 34 | create the interval object:: 35 | 36 | from celery import Celery 37 | 38 | config = { 39 | "mongodb_scheduler_db": "my_project", 40 | "mongodb_scheduler_url": "mongodb://localhost:27017", 41 | } 42 | 43 | app = Celery('hello', broker='redis://localhost//') 44 | app.conf.update(**config) 45 | 46 | from celerybeatmongo.models import PeriodicTask 47 | 48 | periodic = PeriodicTask( 49 | name='Importing contacts', 50 | task="proj.import_contacts" 51 | interval=PeriodicTask.Interval(every=10, period="seconds") # executes every 10 seconds. 52 | ) 53 | periodic.save() 54 | 55 | .. note:: 56 | 57 | You should import celerybeat-mongo just after celery initialization. 58 | 59 | 60 | Example creating crontab periodic task 61 | --------------------------------------------- 62 | 63 | A crontab schedule has the fields: minute, hour, day_of_week, day_of_month and month_of_year, so if you want the equivalent of a 30 7 * * 1 (Executes every Monday morning at 7:30 a.m) crontab entry you specify:: 64 | 65 | 66 | from celery import Celery 67 | 68 | config = { 69 | "mongodb_scheduler_db": "my_project", 70 | "mongodb_scheduler_url": "mongodb://localhost:27017", 71 | } 72 | 73 | app = Celery('hello', broker='redis://localhost//') 74 | app.conf.update(**config) 75 | 76 | from celerybeatmongo.models import PeriodicTask 77 | 78 | periodic = PeriodicTask(name="Send Email Notification", task="proj.notify_customers") 79 | periodic.crontab = PeriodicTask.Crontab(minute="30", hour="7", day_of_week="1", 80 | day_of_month="0", month_of_year="*") 81 | periodic.save() 82 | -------------------------------------------------------------------------------- /celerybeatmongo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zmap/celerybeat-mongo/bde083eba90326d320d66f99a6c732fed7aeeca8/celerybeatmongo/__init__.py -------------------------------------------------------------------------------- /celerybeatmongo/models.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Regents of the University of Michigan 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy 5 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | from datetime import datetime, timedelta 8 | 9 | from mongoengine import * 10 | from celery import current_app 11 | import celery.schedules 12 | 13 | 14 | def get_periodic_task_collection(): 15 | if hasattr(current_app.conf, "mongodb_scheduler_collection"): 16 | return current_app.conf.get("mongodb_scheduler_collection") 17 | elif hasattr(current_app.conf, "CELERY_MONGODB_SCHEDULER_COLLECTION"): 18 | return current_app.conf.CELERY_MONGODB_SCHEDULER_COLLECTION 19 | return "schedules" 20 | 21 | 22 | #: Authorized values for PeriodicTask.Interval.period 23 | PERIODS = ('days', 'hours', 'minutes', 'seconds', 'microseconds') 24 | 25 | 26 | class PeriodicTask(DynamicDocument): 27 | """MongoDB model that represents a periodic task""" 28 | 29 | meta = {'collection': get_periodic_task_collection(), 30 | 'allow_inheritance': True} 31 | 32 | class Interval(EmbeddedDocument): 33 | """Schedule executing on a regular interval. 34 | 35 | Example: execute every 4 days 36 | every=4, period="days" 37 | """ 38 | every = IntField(min_value=0, default=0, required=True) 39 | period = StringField(choices=PERIODS) 40 | 41 | meta = {'allow_inheritance': True} 42 | 43 | @property 44 | def schedule(self): 45 | return celery.schedules.schedule(timedelta(**{self.period: self.every})) 46 | 47 | @property 48 | def period_singular(self): 49 | return self.period[:-1] 50 | 51 | def __unicode__(self): 52 | if self.every == 1: 53 | return 'every {0.period_singular}'.format(self) 54 | return 'every {0.every} {0.period}'.format(self) 55 | 56 | class Crontab(EmbeddedDocument): 57 | """Crontab-like schedule. 58 | 59 | Example: Run every hour at 0 minutes for days of month 10-15 60 | minute="0", hour="*", day_of_week="*", day_of_month="10-15", month_of_year="*" 61 | """ 62 | minute = StringField(default='*', required=True) 63 | hour = StringField(default='*', required=True) 64 | day_of_week = StringField(default='*', required=True) 65 | day_of_month = StringField(default='*', required=True) 66 | month_of_year = StringField(default='*', required=True) 67 | 68 | meta = {'allow_inheritance': True} 69 | 70 | @property 71 | def schedule(self): 72 | return celery.schedules.crontab(minute=self.minute, 73 | hour=self.hour, 74 | day_of_week=self.day_of_week, 75 | day_of_month=self.day_of_month, 76 | month_of_year=self.month_of_year) 77 | 78 | def __unicode__(self): 79 | rfield = lambda f: f and str(f).replace(' ', '') or '*' 80 | return '{0} {1} {2} {3} {4} (m/h/d/dM/MY)'.format( 81 | rfield(self.minute), rfield(self.hour), rfield(self.day_of_week), 82 | rfield(self.day_of_month), rfield(self.month_of_year), 83 | ) 84 | 85 | name = StringField(unique=True) 86 | task = StringField(required=True) 87 | 88 | interval = EmbeddedDocumentField(Interval) 89 | crontab = EmbeddedDocumentField(Crontab) 90 | 91 | args = ListField() 92 | kwargs = DictField() 93 | 94 | queue = StringField() 95 | exchange = StringField() 96 | routing_key = StringField() 97 | soft_time_limit = IntField() 98 | 99 | expires = DateTimeField() 100 | start_after = DateTimeField() 101 | enabled = BooleanField(default=True) 102 | 103 | last_run_at = DateTimeField() 104 | 105 | total_run_count = IntField(min_value=0, default=0) 106 | max_run_count = IntField(min_value=0, default=0) 107 | 108 | date_changed = DateTimeField() 109 | date_creation = DateTimeField() 110 | description = StringField() 111 | 112 | run_immediately = BooleanField() 113 | no_changes = False 114 | 115 | def save(self, force_insert=False, validate=True, clean=True, 116 | write_concern=None, cascade=None, cascade_kwargs=None, 117 | _refs=None, save_condition=None, signal_kwargs=None, **kwargs): 118 | if not self.date_creation: 119 | self.date_creation = datetime.now() 120 | self.date_changed = datetime.now() 121 | super(PeriodicTask, self).save(force_insert, validate, clean, 122 | write_concern, cascade, cascade_kwargs, _refs, 123 | save_condition, signal_kwargs, **kwargs) 124 | 125 | def clean(self): 126 | """validation by mongoengine to ensure that you only have 127 | an interval or crontab schedule, but not both simultaneously""" 128 | if self.interval and self.crontab: 129 | msg = 'Cannot define both interval and crontab schedule.' 130 | raise ValidationError(msg) 131 | if not (self.interval or self.crontab): 132 | msg = 'Must defined either interval or crontab schedule.' 133 | raise ValidationError(msg) 134 | 135 | @property 136 | def schedule(self): 137 | if self.interval: 138 | return self.interval.schedule 139 | elif self.crontab: 140 | return self.crontab.schedule 141 | else: 142 | raise Exception("must define interval or crontab schedule") 143 | 144 | def __unicode__(self): 145 | fmt = '{0.name}: {{no schedule}}' 146 | if self.interval: 147 | fmt = '{0.name}: {0.interval}' 148 | elif self.crontab: 149 | fmt = '{0.name}: {0.crontab}' 150 | else: 151 | raise Exception("must define interval or crontab schedule") 152 | return fmt.format(self) 153 | -------------------------------------------------------------------------------- /celerybeatmongo/schedulers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Regents of the University of Michigan 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy 5 | # of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | import mongoengine 8 | import traceback 9 | import datetime 10 | 11 | from celery import schedules 12 | from celerybeatmongo.models import PeriodicTask 13 | from celery.beat import Scheduler, ScheduleEntry 14 | from celery.utils.log import get_logger 15 | from celery import current_app 16 | 17 | 18 | logger = get_logger(__name__) 19 | 20 | 21 | class MongoScheduleEntry(ScheduleEntry): 22 | 23 | def __init__(self, task): 24 | self._task = task 25 | 26 | self.app = current_app._get_current_object() 27 | self.name = self._task.name 28 | self.task = self._task.task 29 | 30 | self.schedule = self._task.schedule 31 | 32 | self.args = self._task.args 33 | self.kwargs = self._task.kwargs 34 | self.options = { 35 | 'queue': self._task.queue, 36 | 'exchange': self._task.exchange, 37 | 'routing_key': self._task.routing_key, 38 | 'expires': self._task.expires, 39 | 'soft_time_limit': self._task.soft_time_limit, 40 | 'enabled': self._task.enabled 41 | } 42 | if self._task.total_run_count is None: 43 | self._task.total_run_count = 0 44 | self.total_run_count = self._task.total_run_count 45 | 46 | if not self._task.last_run_at: 47 | self._task.last_run_at = self._default_now() 48 | self.last_run_at = self._task.last_run_at 49 | 50 | def _default_now(self): 51 | return self.app.now() 52 | 53 | def next(self): 54 | self._task.last_run_at = self.app.now() 55 | self._task.total_run_count += 1 56 | self._task.run_immediately = False 57 | return self.__class__(self._task) 58 | 59 | __next__ = next 60 | 61 | def is_due(self): 62 | if not self._task.enabled: 63 | return schedules.schedstate(False, 5.0) # 5 second delay for re-enable. 64 | if hasattr(self._task, 'start_after') and self._task.start_after: 65 | if datetime.datetime.now() < self._task.start_after: 66 | return schedules.schedstate(False, 5.0) 67 | if hasattr(self._task, 'max_run_count') and self._task.max_run_count: 68 | if (self._task.total_run_count or 0) >= self._task.max_run_count: 69 | self._task.enabled = False 70 | self._task.save() 71 | # Don't recheck 72 | return schedules.schedstate(False, None) 73 | if self._task.run_immediately: 74 | # figure out when the schedule would run next anyway 75 | _, n = self.schedule.is_due(self.last_run_at) 76 | return True, n 77 | return self.schedule.is_due(self.last_run_at) 78 | 79 | def __repr__(self): 80 | return (u'<{0} ({1} {2}(*{3}, **{4}) {{5}})>'.format( 81 | self.__class__.__name__, 82 | self.name, self.task, self.args, 83 | self.kwargs, self.schedule, 84 | )) 85 | 86 | def reserve(self, entry): 87 | new_entry = Scheduler.reserve(self, entry) 88 | return new_entry 89 | 90 | def save(self): 91 | if self.total_run_count > self._task.total_run_count: 92 | self._task.total_run_count = self.total_run_count 93 | if self.last_run_at and self._task.last_run_at and self.last_run_at > self._task.last_run_at: 94 | self._task.last_run_at = self.last_run_at 95 | self._task.run_immediately = False 96 | try: 97 | self._task.save(save_condition={}) 98 | except Exception: 99 | logger.error(traceback.format_exc()) 100 | 101 | 102 | class MongoScheduler(Scheduler): 103 | 104 | #: how often should we sync in schedule information 105 | #: from the backend mongo database 106 | UPDATE_INTERVAL = datetime.timedelta(seconds=5) 107 | 108 | Entry = MongoScheduleEntry 109 | 110 | Model = PeriodicTask 111 | 112 | def __init__(self, app, *args, **kwargs): 113 | if hasattr(app.conf, "mongodb_scheduler_db"): 114 | db = app.conf.get("mongodb_scheduler_db") 115 | elif hasattr(app.conf, "CELERY_MONGODB_SCHEDULER_DB"): 116 | db = app.conf.CELERY_MONGODB_SCHEDULER_DB 117 | else: 118 | db = "celery" 119 | if hasattr(app.conf, "mongodb_scheduler_connection_alias"): 120 | alias = app.conf.get('mongodb_scheduler_connection_alias') 121 | elif hasattr(app.conf, "CELERY_MONGODB_SCHEDULER_CONNECTION_ALIAS"): 122 | alias = app.conf.CELERY_MONGODB_SCHEDULER_CONNECTION_ALIAS 123 | else: 124 | alias = "default" 125 | 126 | if hasattr(app.conf, "mongodb_scheduler_url"): 127 | host = app.conf.get('mongodb_scheduler_url') 128 | elif hasattr(app.conf, "CELERY_MONGODB_SCHEDULER_URL"): 129 | host = app.conf.CELERY_MONGODB_SCHEDULER_URL 130 | else: 131 | host = None 132 | 133 | self._mongo = mongoengine.connect(db, host=host, alias=alias) 134 | 135 | if host: 136 | logger.info("backend scheduler using %s/%s:%s", 137 | host, db, self.Model._get_collection().name) 138 | else: 139 | logger.info("backend scheduler using %s/%s:%s", 140 | "mongodb://localhost", db, self.Model._get_collection().name) 141 | self._schedule = {} 142 | self._last_updated = None 143 | Scheduler.__init__(self, app, *args, **kwargs) 144 | self.max_interval = (kwargs.get('max_interval') 145 | or self.app.conf.CELERYBEAT_MAX_LOOP_INTERVAL or 5) 146 | 147 | def setup_schedule(self): 148 | pass 149 | 150 | def requires_update(self): 151 | """check whether we should pull an updated schedule 152 | from the backend database""" 153 | if not self._last_updated: 154 | return True 155 | return self._last_updated + self.UPDATE_INTERVAL < datetime.datetime.now() 156 | 157 | def get_from_database(self): 158 | self.sync() 159 | d = {} 160 | for doc in self.Model.objects.filter(enabled=True): 161 | d[doc.name] = self.Entry(doc) 162 | return d 163 | 164 | @property 165 | def schedule(self): 166 | if self.requires_update(): 167 | self._schedule = self.get_from_database() 168 | self._last_updated = datetime.datetime.now() 169 | return self._schedule 170 | 171 | def sync(self): 172 | logger.debug('Writing entries...') 173 | for entry in self._schedule.values(): 174 | entry.save() 175 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | celery 2 | mongoengine 3 | pymongo -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | mongomock -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | setup( 5 | name="celerybeat-mongo", 6 | description="A Celery Beat Scheduler that uses MongoDB to store both schedule definitions and status information", 7 | version="0.2.0", 8 | license="Apache License, Version 2.0", 9 | author="Zakir Durumeric", 10 | author_email="zakird@gmail.com", 11 | maintainer="Zakir Durumeric", 12 | maintainer_email="zakird@gmail.com", 13 | keywords="python celery beat periodic task mongodb", 14 | packages=[ 15 | "celerybeatmongo" 16 | ], 17 | install_requires=[ 18 | 'setuptools', 19 | 'pymongo', 20 | 'mongoengine', 21 | 'celery', 22 | 'blinker' 23 | ], 24 | classifiers=[ 25 | 'Development Status :: 2 - Production/Stable', 26 | 'License :: Apache License 2.0', 27 | 'Programming Language :: Python', 28 | 'Programming Language :: Python :: 2.7', 29 | 'Programming Language :: Python :: 3', 30 | 'Programming Language :: Python :: 3.5', 31 | 'Programming Language :: Python :: 3.6', 32 | 'Programming Language :: Python :: 3.7', 33 | 'Programming Language :: Python :: 3.8', 34 | 'Programming Language :: Python :: Implementation :: CPython', 35 | 'Topic :: Communications', 36 | 'Topic :: System :: Distributed Computing', 37 | 'Topic :: Software Development :: Libraries :: Python Modules', 38 | 'Operating System :: OS Independent', 39 | ] 40 | ) 41 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | import unittest 3 | from mongoengine import connect, disconnect 4 | 5 | 6 | class BeatMongoCase(unittest.TestCase): 7 | 8 | @classmethod 9 | def setUpClass(cls): 10 | connect('mongoenginetest', host='mongomock://localhost') 11 | 12 | @classmethod 13 | def tearDownClass(cls): 14 | disconnect() -------------------------------------------------------------------------------- /tests/test_models.py: -------------------------------------------------------------------------------- 1 | 2 | import unittest 3 | from mongoengine import ValidationError 4 | 5 | from celerybeatmongo.models import PeriodicTask 6 | from tests import BeatMongoCase 7 | 8 | 9 | class IntervalScheduleTest(BeatMongoCase): 10 | 11 | def test_cannot_save_interval_schduler_with_a_invalid_period(self): 12 | periodic = PeriodicTask(task="foo") 13 | with self.assertRaises(ValidationError): 14 | periodic.interval = PeriodicTask.Interval(every=1, period="days111") 15 | periodic.save() 16 | 17 | def test_scheduler(self): 18 | periodic = PeriodicTask(task="foo") 19 | periodic.interval = PeriodicTask.Interval(every=1, period="days") 20 | periodic.save() 21 | self.assertIsNotNone(periodic.schedule) 22 | 23 | def test_str(self): 24 | periodic = PeriodicTask(task="foo") 25 | periodic.interval = PeriodicTask.Interval(every=1, period="days") 26 | self.assertEqual("every day", str(periodic.interval)) 27 | 28 | periodic.interval = PeriodicTask.Interval(every=2, period="days") 29 | self.assertEqual('every 2 days', str(periodic.interval)) 30 | 31 | 32 | class CrontabScheduleTest(unittest.TestCase): 33 | 34 | def test_str(self): 35 | periodic = PeriodicTask(task="foo") 36 | periodic.crontab = PeriodicTask.Crontab(minute="0", hour="*", day_of_week="*", 37 | day_of_month="10-15", month_of_year="*") 38 | self.assertEqual("0 * * 10-15 * (m/h/d/dM/MY)", str(periodic.crontab)) 39 | 40 | 41 | class PeriodicTaskTest(BeatMongoCase): 42 | 43 | def test_must_define_interval_or_crontab(self): 44 | with self.assertRaises(ValidationError) as err: 45 | periodic = PeriodicTask(task="foo") 46 | periodic.save() 47 | self.assertTrue("Must defined either interval or crontab schedule." in err.exception.message) 48 | 49 | def test_cannot_define_both_interval_and_contrab(self): 50 | periodic = PeriodicTask(task="foo") 51 | periodic.interval = PeriodicTask.Interval(every=1, period="days") 52 | periodic.crontab = PeriodicTask.Crontab(minute="0", hour="*", day_of_week="*", 53 | day_of_month="10-15", month_of_year="*") 54 | with self.assertRaises(ValidationError) as err: 55 | periodic.save() 56 | self.assertTrue("Cannot define both interval and crontab schedule." in err.exception.message) 57 | 58 | def test_creation_date(self): 59 | periodic = PeriodicTask(task="foo") 60 | periodic.interval = PeriodicTask.Interval(every=1, period="days") 61 | self.assertIsNone(periodic.date_creation, 62 | "date_creation should be none on the first object instantion") 63 | periodic.save() 64 | self.assertIsNotNone(periodic.date_creation) 65 | date_creation = periodic.date_creation 66 | 67 | periodic.name = "I'm changing" 68 | periodic.save() 69 | self.assertEqual(date_creation, periodic.date_creation, 70 | "Update object should not change date_creation value") 71 | 72 | def test_date_changed(self): 73 | periodic = PeriodicTask(task="foo") 74 | periodic.interval = PeriodicTask.Interval(every=1, period="days") 75 | 76 | self.assertIsNone(periodic.date_changed) 77 | periodic.save() 78 | self.assertIsNotNone(periodic.date_changed) 79 | 80 | date_changed = periodic.date_changed 81 | periodic.name = "I'm changing now" 82 | periodic.save() 83 | self.assertGreater(periodic.date_changed, date_changed) 84 | -------------------------------------------------------------------------------- /tests/test_scheduler.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from mongoengine import disconnect 4 | from celerybeatmongo.schedulers import MongoScheduler 5 | from celery import Celery 6 | 7 | 8 | class MongoSchedulerTest(unittest.TestCase): 9 | 10 | def setUp(self): 11 | conf = {"mongodb_scheduler_url": "mongomock://localhost"} 12 | self.app = Celery(**conf) 13 | self.app.conf.update(**conf) 14 | self.scheduler = MongoScheduler(app=self.app) 15 | 16 | def tearDown(self): 17 | disconnect() 18 | 19 | def test_get_from_database(self): 20 | from celerybeatmongo.models import PeriodicTask 21 | PeriodicTask.objects.create(name="a1", task="foo", enabled=True, interval=PeriodicTask.Interval(every=1, period="days")) 22 | PeriodicTask.objects.create(name="b1", task="foo", enabled=True, interval=PeriodicTask.Interval(every=2, period="days")) 23 | PeriodicTask.objects.create(name="c2", task="foo", enabled=False, interval=PeriodicTask.Interval(every=3, period="days")) 24 | 25 | scheduler = MongoScheduler(app=self.app) 26 | self.assertEqual(2, len(scheduler.get_from_database()) 27 | , "get_from_database should return just enabled tasks") 28 | 29 | def test_max_interval(self): 30 | scheduler = MongoScheduler(self.app, max_interval=600, sync_every_tasks=10) 31 | self.assertEqual(600, scheduler.max_interval) 32 | 33 | def test_should_create_mongo_db_connection(self): 34 | scheduler = MongoScheduler(self.app, max_interval=600, sync_every_tasks=10) 35 | self.assertIsNotNone(scheduler._mongo) 36 | self.assertEqual(0, scheduler._mongo.db.test.count({})) 37 | --------------------------------------------------------------------------------