├── .github └── FUNDING.yml ├── .gitignore ├── .npmignore ├── .travis.yml ├── README.md ├── giphy.gif ├── nodemon.json ├── package-lock.json ├── package.json ├── src ├── cron.ts ├── index.ts ├── scripts │ ├── example.ts │ └── speedtest.ts └── tests │ ├── cron.test.ts │ └── index.test.ts ├── tsconfig.json └── tslint.json /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [xpepermint] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .vscode 3 | .nyc* 4 | node_modules 5 | dist 6 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .vscode 3 | node_modules 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: node_js 3 | node_js: 4 | - 11 5 | - 10 6 | services: 7 | - mongodb 8 | addons: 9 | apt: 10 | sources: 11 | - mongodb-3.2-precise 12 | packages: 13 | - mongodb-org-server 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/xpepermint/mongodb-cron.svg?branch=master)](https://travis-ci.org/xpepermint/mongodb-cron) [![NPM Version](https://badge.fury.io/js/mongodb-cron.svg)](https://badge.fury.io/js/mongodb-cron) 2 | 3 | # [mongodb](https://docs.mongodb.com/ecosystem/drivers/node-js/)-cron 4 | 5 | > MongoDB collection as crontab 6 | 7 | This package offers a simple API for scheduling tasks and running recurring jobs on [MongoDB](https://www.mongodb.org) collections. Any collection can be converted into a job queue or crontab list. It uses the officially supported [Node.js driver for MongoDB](https://docs.mongodb.com/ecosystem/drivers/node-js/). It's fast, minimizes processing overhead and it uses atomic commands to ensure safe job executions even in cluster environments. 8 | 9 | This is a light weight open source package for NodeJS written with [TypeScript](https://www.typescriptlang.org). It's actively maintained, well tested and already used in production environments. The source code is available on [GitHub](https://github.com/xpepermint/mongodb-cron) where you can also find our [issue tracker](https://github.com/xpepermint/mongodb-cron/issues). 10 | 11 | 12 | 13 | ## Installation 14 | 15 | This is a module for [Node.js](http://nodejs.org/) and can be installed via [npm](https://www.npmjs.com). It depends on the [mongodb](https://docs.mongodb.com/ecosystem/drivers/node-js/) package and uses promises. 16 | 17 | ``` 18 | $ npm install --save mongodb mongodb-cron 19 | ``` 20 | 21 | ## Example 22 | 23 | Below, is a simple example to show the benefit of using this package in your Node.js projects. 24 | 25 | Let's start by initializing the database connection. 26 | 27 | ```js 28 | import { MongoClient } from 'mongodb'; 29 | 30 | const mongo = await MongoClient.connect('mongodb://localhost:27017'); 31 | const db = mongo.db('test'); 32 | ``` 33 | 34 | Continue by initializing and starting a the worker. 35 | 36 | ```js 37 | import { MongoCron } from 'mongodb-cron'; 38 | 39 | const collection = db.collection('jobs'); 40 | const cron = new MongoCron({ 41 | collection, // a collection where jobs are stored 42 | onDocument: async (doc) => console.log(doc), // triggered on job processing 43 | onError: async (err) => console.log(err), // triggered on error 44 | }); 45 | 46 | cron.start(); // start processing 47 | ``` 48 | 49 | We can now create our first job. 50 | 51 | ```js 52 | const job = await collection.insert({ 53 | sleepUntil: new Date('2016-01-01'), // ISO 8601 format (can include timezone) 54 | }); 55 | ``` 56 | 57 | When the processing starts the `onDocument` handler (defined earlier) is triggered. We have a very basic example here so please continue reading. 58 | 59 | ## Documentation 60 | 61 | The `MongoCron` class converts a collection into a job queue. Jobs are represented by the documents stored in a MongoDB collection. When cron is started it loops through the collection and processes available jobs one by one. 62 | 63 | A job should have at least the `sleepUntil` field. Cron processes only documents where this field exists, other documents are ignored. 64 | 65 | ### One-time Jobs 66 | 67 | To create a one-time job we only need to define the required field `sleepUntil`. When this filed is set to some date in the past, the processing starts immediately. 68 | 69 | ```js 70 | const job = await collection.insert({ 71 | sleepUntil: new Date(), 72 | }); 73 | ``` 74 | 75 | When the processing of a document starts the `sleepUntil` field is updated to a new date in the future. This locks the document for a certain amount of time in which the processing must complete (lock duration is configurable). This mechanism prevents possible race conditions and ensures that a job is always processed by only one process at a time. 76 | 77 | When the processing ends, the `sleepUntil` field is set to `null`. 78 | 79 | If cron is unexpectedly interrupted during the processing of a job (e.g. server shutdown), the system automatically recovers and transparently restarts. 80 | 81 | ## Deferred Execution 82 | 83 | We can schedule job execution for a particular time in the future by setting the `sleepUntil` field to a future date. 84 | 85 | ```js 86 | const job = await collection.insert({ 87 | ... 88 | sleepUntil: new Date('2016-01-01'), // start on 2016-01-01 89 | }); 90 | ``` 91 | 92 | ## Recurring Jobs 93 | 94 | By setting the `interval` field we define a recurring job. 95 | 96 | ```js 97 | const job = await collection.insert({ 98 | ... 99 | interval: '* * * * * *', // every second 100 | }); 101 | ``` 102 | 103 | The interval above consists of 6 values. 104 | 105 | ``` 106 | * * * * * * 107 | ┬ ┬ ┬ ┬ ┬ ┬ 108 | │ │ │ │ │ └── day of week (0 - 7) (0 or 7 is Sun) 109 | │ │ │ │ └──── month (1 - 12) 110 | │ │ │ └────── day of month (1 - 31) 111 | │ │ └──────── hour (0 - 23) 112 | │ └────────── minute (0 - 59) 113 | └──────────── second (0 - 59) 114 | ``` 115 | 116 | A recurring job will repeat endlessly unless we limit that by setting the `repeatUntil` field. When a job expires it stops repeating by removing the `processable` field. 117 | 118 | ```js 119 | const job = await collection.insert({ 120 | ... 121 | interval: '* * * * * *', 122 | repeatUntil: new Date('2020-01-01'), 123 | }); 124 | ``` 125 | 126 | ## Auto-removable Jobs 127 | 128 | A job can automatically remove itself from the collection when the processing completes. To configure that, we need to set the `autoRemove` field to `true`. 129 | 130 | ```js 131 | const job = await collection.insert({ 132 | ... 133 | autoRemove: true, 134 | }); 135 | ``` 136 | 137 | ## API 138 | 139 | **new MongoCron({ collection, condition, onStart, onStop, onDocument, onError, nextDelay, reprocessDelay, idleDelay, lockDuration, sleepUntilFieldPath, intervalFieldPath, repeatUntilFieldPath, autoRemoveFieldPath })** 140 | > The core class for converting a MongoDB collection into a job queue. 141 | 142 | | Option | Type | Required | Default | Description 143 | |--------|------|----------|---------|------------ 144 | | autoRemoveFieldPath | String | No | autoRemove | The `autoRemove` field path. 145 | | collection | Object | Yes | - | MongoDB collection object. 146 | | condition | Object | No | null | Additional query condition. 147 | | idleDelay | Integer | No | 0 | A variable which tells how many milliseconds the worker should wait before checking for new jobs after all jobs has been processed. 148 | | intervalFieldPath | String | No | interval | The `interval` field path. 149 | | lockDuration | Integer | No | 600000 | A number of milliseconds for which each job gets locked for (we have to make sure that the job completes in that time frame). 150 | | nextDelay | Integer | No | 0 | A variable which tells how fast the next job can be processed. 151 | | onDocument | Function/Promise | No | - | A method which is triggered when a document should be processed. 152 | | onError | Function/Promise | No | - | A method which is triggered in case of an error. 153 | | onIdle | Function/Promise | No | - | A method which is triggered when all jobs in a collection have been processed. 154 | | onStart | Function/Promise | No | - | A method which is triggered when the cron is started. 155 | | onStop | Function/Promise | No | - | A method which is triggered when the cron is stopped. 156 | | repeatUntilFieldPath | String | No | repeatUntil | The `repeatUntil` field path. 157 | | reprocessDelay | Integer | No | 0 | A variable which tells how many milliseconds the worker should wait before processing the same job again in case the job is a recurring job. 158 | | sleepUntilFieldPath | String | No | sleepUntil | The `sleepUntil` field path. 159 | 160 | ```js 161 | import { MongoClient } from 'mongodb'; 162 | 163 | const mongo = await MongoClient.connect('mongodb://localhost:27017/test'); 164 | 165 | const cron = new MongoCron({ 166 | collection: db.collection('jobs'), 167 | onStart: async () => {}, 168 | onStop: async () => {}, 169 | onDocument: async (doc) => {}, 170 | onIdle: async (doc) => {}, 171 | onError: async (err) => {}, 172 | nextDelay: 1000, 173 | reprocessDelay: 1000, 174 | idleDelay: 10000, 175 | lockDuration: 600000, 176 | sleepUntilFieldPath: 'cron.sleepUntil', 177 | intervalFieldPath: 'cron.interval', 178 | repeatUntilFieldPath: 'cron.repeatUntil', 179 | autoRemoveFieldPath: 'cron.autoRemove', 180 | }); 181 | ``` 182 | 183 | **cron.start()**:Promise 184 | > Starts the cron processor. 185 | 186 | **cron.stop()**:Promise 187 | > Stops the cron processor. 188 | 189 | **cron.isRunning()**:Boolean 190 | > Returns true if the cron is started. 191 | 192 | **cron.isProcessing()**:Boolean 193 | > Returns true if cron is processing a document. 194 | 195 | **cron.isIdle()**:Boolean 196 | > Returns true if the cron is in idle state. 197 | 198 | ## Processing Speed 199 | 200 | Processing speed can be reduced when more and more documents are added into the collection. We can maintain the speed by creating indexes. 201 | 202 | ```js 203 | await collection.createIndex({ 204 | sleepUntil: 1, // the `sleepUntil` field path, set by the sleepUntilFieldPath 205 | }, { 206 | sparse: true, 207 | }); 208 | ``` 209 | 210 | Don't forget to adjust the index definition when using your custom query `condition`. 211 | 212 | ## Best Practice 213 | 214 | * Make your jobs idempotent and transactional. [Idempotency](https://en.wikipedia.org/wiki/Idempotence) means that your job can safely execute multiple times. 215 | * Run this package in cluster mode. Design your jobs in a way that you can run lots of them in parallel. 216 | 217 | ## Licence 218 | 219 | ``` 220 | Copyright (c) 2016 Kristijan Sedlak 221 | 222 | Permission is hereby granted, free of charge, to any person obtaining a copy 223 | of this software and associated documentation files (the "Software"), to deal 224 | in the Software without restriction, including without limitation the rights 225 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 226 | copies of the Software, and to permit persons to whom the Software is 227 | furnished to do so, subject to the following conditions: 228 | 229 | The above copyright notice and this permission notice shall be included in 230 | all copies or substantial portions of the Software. 231 | 232 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 233 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 234 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 235 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 236 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 237 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 238 | THE SOFTWARE. 239 | ``` 240 | -------------------------------------------------------------------------------- /giphy.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xpepermint/mongodb-cron/116f3ab3da5a44444931322ab13721d8c468c2f9/giphy.gif -------------------------------------------------------------------------------- /nodemon.json: -------------------------------------------------------------------------------- 1 | { 2 | "ignore": ["dist/*"], 3 | "ext": "js,ts" 4 | } 5 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mongodb-cron", 3 | "version": "1.9.0", 4 | "description": "MongoDB collection as crontab", 5 | "main": "./dist/index.js", 6 | "types": "./dist/index.d.ts", 7 | "scripts": { 8 | "clean": "rm -Rf ./dist", 9 | "build": "npm run clean && npx tsc", 10 | "example": "npm run build && npx ts-node ./src/scripts/example.ts", 11 | "lint": "npx tslint 'src/**/*.ts?(x)'", 12 | "prepublish": "npm run build", 13 | "test": "npm run lint && npx nyc npx hayspec test", 14 | "speedtest": "npx ts-node ./src/scripts/speedtest.ts" 15 | }, 16 | "hayspec": { 17 | "require": [ 18 | "ts-node/register" 19 | ], 20 | "match": [ 21 | "./src/tests/**/*.test.ts" 22 | ] 23 | }, 24 | "nyc": { 25 | "extension": [ 26 | ".ts" 27 | ], 28 | "require": [ 29 | "ts-node/register" 30 | ], 31 | "exclude": [ 32 | "src/tests" 33 | ] 34 | }, 35 | "repository": { 36 | "type": "git", 37 | "url": "git+https://github.com/xpepermint/mongodb-cron.git" 38 | }, 39 | "keywords": [ 40 | "mongo", 41 | "mongodb", 42 | "database", 43 | "nosql", 44 | "cron", 45 | "schedule", 46 | "scheduling", 47 | "queue", 48 | "job", 49 | "jobs", 50 | "collection", 51 | "collections", 52 | "capped" 53 | ], 54 | "author": "Xpepermint (Kristijan Sedlak)", 55 | "license": "MIT", 56 | "bugs": { 57 | "url": "https://github.com/xpepermint/mongodb-cron/issues" 58 | }, 59 | "homepage": "https://github.com/xpepermint/mongodb-cron#readme", 60 | "peerDependencies": { 61 | "mongodb": "^6.1.0" 62 | }, 63 | "devDependencies": { 64 | "@hayspec/cli": "0.10.2", 65 | "@hayspec/spec": "0.10.2", 66 | "@types/dot-object": "2.1.6", 67 | "@types/node": "20.11.20", 68 | "mongodb": "6.3.0", 69 | "nyc": "15.1.0", 70 | "ts-node": "10.9.2", 71 | "tslint": "6.1.3", 72 | "typescript": "4.7.4" 73 | }, 74 | "dependencies": { 75 | "cron-parser": "4.9.0", 76 | "dot-object": "2.1.4", 77 | "es6-sleep": "2.0.2", 78 | "moment": "2.30.1" 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/cron.ts: -------------------------------------------------------------------------------- 1 | import * as parser from 'cron-parser'; 2 | import * as dot from 'dot-object'; 3 | import { promise as sleep } from 'es6-sleep'; 4 | import * as moment from 'moment'; 5 | import { Collection } from 'mongodb'; 6 | 7 | /** 8 | * Configuration object interface. 9 | */ 10 | export interface MongoCronCfg { 11 | collection: Collection | (() => Collection); 12 | condition?: any; 13 | nextDelay?: number; // wait before processing next job 14 | reprocessDelay?: number; // wait before processing the same job again 15 | idleDelay?: number; // when there is no jobs for processing, wait before continue 16 | lockDuration?: number; // the time of milliseconds that each job gets locked (we have to make sure that the job completes in that time frame) 17 | sleepUntilFieldPath?: string; 18 | intervalFieldPath?: string; 19 | repeatUntilFieldPath?: string; 20 | autoRemoveFieldPath?: string; 21 | onDocument?(doc: any): (any | Promise); 22 | onStart?(doc: any): (any | Promise); 23 | onStop?(): (any | Promise); 24 | onIdle?(): (any | Promise); 25 | onError?(err: any): (any | Promise); 26 | } 27 | 28 | /** 29 | * Main class for converting a collection into cron. 30 | */ 31 | export class MongoCron { 32 | protected running = false; 33 | protected processing = false; 34 | protected idle = false; 35 | protected readonly config: MongoCronCfg; 36 | 37 | /** 38 | * Class constructor. 39 | * @param config Configuration object. 40 | */ 41 | public constructor(config: MongoCronCfg) { 42 | this.config = { 43 | onDocument: (doc) => doc, 44 | onError: console.error, 45 | nextDelay: 0, 46 | reprocessDelay: 0, 47 | idleDelay: 0, 48 | lockDuration: 600000, 49 | sleepUntilFieldPath: 'sleepUntil', 50 | intervalFieldPath: 'interval', 51 | repeatUntilFieldPath: 'repeatUntil', 52 | autoRemoveFieldPath: 'autoRemove', 53 | ...config, 54 | }; 55 | } 56 | 57 | /** 58 | * Returns the collection instance (the collection can be provided in 59 | * the config as an instance or a function). 60 | */ 61 | protected getCollection(): Collection { 62 | return typeof this.config.collection === 'function' 63 | ? this.config.collection() 64 | : this.config.collection; 65 | } 66 | 67 | /** 68 | * Tells if the process is started. 69 | */ 70 | public isRunning() { 71 | return this.running; 72 | } 73 | 74 | /** 75 | * Tells if a document is processing. 76 | */ 77 | public isProcessing() { 78 | return this.processing; 79 | } 80 | 81 | /** 82 | * Tells if the process is idle. 83 | */ 84 | public isIdle() { 85 | return this.idle; 86 | } 87 | 88 | /** 89 | * Starts the heartbit. 90 | */ 91 | public async start() { 92 | if (!this.running) { 93 | this.running = true; 94 | 95 | if (this.config.onStart) { 96 | await this.config.onStart.call(this, this); 97 | } 98 | 99 | process.nextTick(this.tick.bind(this)); 100 | } 101 | } 102 | 103 | /** 104 | * Stops the heartbit. 105 | */ 106 | public async stop() { 107 | this.running = false; 108 | 109 | if (this.processing) { 110 | await sleep(300); 111 | return process.nextTick(this.stop.bind(this)); // wait until processing is complete 112 | } 113 | 114 | if (this.config.onStop) { 115 | await this.config.onStop.call(this, this); 116 | } 117 | } 118 | 119 | /** 120 | * Private method which runs the heartbit tick. 121 | */ 122 | protected async tick() { 123 | if (!this.running) { return; } 124 | await sleep(this.config.nextDelay); 125 | if (!this.running) { return; } 126 | 127 | this.processing = true; 128 | try { 129 | const doc = await this.lockNext(); // locking next job 130 | if (!doc) { 131 | this.processing = false; 132 | if (!this.idle) { 133 | this.idle = true; 134 | if (this.config.onIdle) { 135 | await this.config.onIdle.call(this, this); 136 | } 137 | } 138 | await sleep(this.config.idleDelay); 139 | } else { 140 | this.idle = false; 141 | if (this.config.onDocument) { 142 | await this.config.onDocument.call(this, doc, this); 143 | } 144 | await this.reschedule(doc); 145 | this.processing = false; 146 | } 147 | } catch (err) { 148 | await this.config.onError.call(this, err, this); 149 | } 150 | 151 | process.nextTick(() => this.tick()); 152 | } 153 | 154 | /** 155 | * Locks the next job document for processing and returns it. 156 | */ 157 | protected async lockNext() { 158 | const sleepUntil = moment().add(this.config.lockDuration, 'milliseconds').toDate(); 159 | const currentDate = moment().toDate(); 160 | 161 | const res = await this.getCollection().findOneAndUpdate({ 162 | $and: [ 163 | { [this.config.sleepUntilFieldPath]: { $exists: true, $ne: null }}, 164 | { [this.config.sleepUntilFieldPath]: { $not: { $gt: currentDate } } }, 165 | this.config.condition, 166 | ].filter((c) => !!c), 167 | }, { 168 | $set: { [this.config.sleepUntilFieldPath]: sleepUntil }, 169 | }, { 170 | returnDocument: 'before', // return original document to calculate next start based on the original value 171 | includeResultMetadata: true, 172 | }); 173 | return res.value; 174 | } 175 | 176 | /** 177 | * Returns the next date when a job document can be processed or `null` if the 178 | * job has expired. 179 | * @param doc Mongo document. 180 | */ 181 | protected getNextStart(doc: any): Date { 182 | if (!dot.pick(this.config.intervalFieldPath, doc)) { // not recurring job 183 | return null; 184 | } 185 | 186 | const available = moment(dot.pick(this.config.sleepUntilFieldPath, doc)); // first available next date 187 | const future = moment(available).add(this.config.reprocessDelay, 'milliseconds'); // date when the next start is possible 188 | 189 | try { 190 | const interval = parser.parseExpression(dot.pick(this.config.intervalFieldPath, doc), { 191 | currentDate: future.toDate(), 192 | endDate: dot.pick(this.config.repeatUntilFieldPath, doc), 193 | }); 194 | const next = interval.next().toDate(); 195 | const now = moment().toDate(); 196 | return next < now ? now : next; // process old recurring jobs only once 197 | } catch (err) { 198 | return null; 199 | } 200 | } 201 | 202 | /** 203 | * Tries to reschedule a job document, to mark it as expired or to delete a job 204 | * if `autoRemove` is set to `true`. 205 | * @param doc Mongo document. 206 | */ 207 | public async reschedule(doc: any): Promise { 208 | const nextStart = this.getNextStart(doc); 209 | const _id = doc._id; 210 | 211 | if (!nextStart && dot.pick(this.config.autoRemoveFieldPath, doc)) { // remove if auto-removable and not recuring 212 | await this.getCollection().deleteOne({ _id }); 213 | } else if (!nextStart) { // stop execution 214 | await this.getCollection().updateOne({ _id }, { 215 | $set: { [this.config.sleepUntilFieldPath]: null }, 216 | }); 217 | } else { // reschedule for reprocessing in the future (recurring) 218 | await this.getCollection().updateOne({ _id }, { 219 | $set: { [this.config.sleepUntilFieldPath]: nextStart }, 220 | }); 221 | } 222 | } 223 | 224 | } 225 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './cron'; 2 | -------------------------------------------------------------------------------- /src/scripts/example.ts: -------------------------------------------------------------------------------- 1 | import { promise as sleep } from 'es6-sleep'; 2 | import * as moment from 'moment'; 3 | import { MongoClient } from 'mongodb'; 4 | import { MongoCron } from '..'; 5 | 6 | (async function() { 7 | const mongo = await MongoClient.connect('mongodb://localhost:27017'); 8 | const db = mongo.db('test'); 9 | const collection = db.collection('jobs'); 10 | 11 | const cron = new MongoCron({ 12 | collection, 13 | onDocument: (doc) => console.log('onDocument', doc), 14 | onError: (err) => console.log(err), 15 | onStart: () => console.log('started ...'), 16 | onStop: () => console.log('stopped'), 17 | nextDelay: 1000, 18 | reprocessDelay: 1000, 19 | idleDelay: 10000, 20 | lockDuration: 600000, 21 | }); 22 | 23 | await collection.insertMany([ 24 | { name: 'Job #3', 25 | sleepUntil: moment().add(3, 'seconds').toDate(), 26 | }, 27 | { name: 'Job #1', 28 | sleepUntil: null, 29 | }, 30 | { name: 'Job #2', 31 | sleepUntil: moment().add(2, 'seconds').toDate(), 32 | }, 33 | { name: 'Job #4', 34 | sleepUntil: moment().add(8, 'seconds').toDate(), 35 | }, 36 | ]); 37 | 38 | cron.start(); 39 | await sleep(30000); 40 | cron.stop(); 41 | 42 | process.exit(0); 43 | 44 | })().catch(console.error); 45 | -------------------------------------------------------------------------------- /src/scripts/speedtest.ts: -------------------------------------------------------------------------------- 1 | import { MongoClient } from 'mongodb'; 2 | import { MongoCron } from '..'; 3 | 4 | /** 5 | * Number of documents. 6 | */ 7 | 8 | const SAMPLE_SIZE = process.argv[2] ? parseInt(process.argv[2]) : 1000; 9 | 10 | /** 11 | * TEST: One-time jobs. 12 | */ 13 | 14 | async function testOneTimeJobs(mongo) { 15 | let time = 0; 16 | const collection = mongo.collection('jobs'); 17 | 18 | try { await collection.drop(); } catch (e) { /** */ } 19 | 20 | console.log(`> Creating ${SAMPLE_SIZE} documents ...`); 21 | 22 | time = Date.now(); 23 | for (let i = 0; i < SAMPLE_SIZE; i++) { 24 | await collection.insertOne({ 25 | sleepUntil: null, 26 | }); 27 | } 28 | console.log(`> Done (${Date.now() - time}ms)`); 29 | 30 | console.log('> Processing ...'); 31 | 32 | time = Date.now(); 33 | await new Promise((resolve, reject) => { 34 | const cron = new MongoCron({ 35 | collection, 36 | onError: (err) => console.log(err), 37 | onIdle: () => { 38 | cron.stop().then(() => { 39 | console.log(`> Done (${Date.now() - time}ms)`); 40 | resolve(null); 41 | }); 42 | }, 43 | nextDelay: 0, 44 | reprocessDelay: 0, 45 | idleDelay: 0, 46 | lockDuration: 600000, 47 | }); 48 | cron.start(); 49 | }); 50 | } 51 | 52 | /** 53 | * Starts testing. 54 | */ 55 | 56 | (async function() { 57 | const mongo = await MongoClient.connect('mongodb://localhost:27017/test'); 58 | await testOneTimeJobs(mongo.db('test')); 59 | await mongo.close(); 60 | })().catch(console.error); 61 | -------------------------------------------------------------------------------- /src/tests/cron.test.ts: -------------------------------------------------------------------------------- 1 | import { Spec } from '@hayspec/spec'; 2 | import { promise as sleep } from 'es6-sleep'; 3 | import * as moment from 'moment'; 4 | import { Collection, Db, MongoClient } from 'mongodb'; 5 | import { MongoCron } from '..'; 6 | 7 | const spec = new Spec<{ 8 | db: Db; 9 | mongo: MongoClient; 10 | collection: Collection; 11 | }>(); 12 | 13 | spec.before(async (stage) => { 14 | const mongo = await MongoClient.connect('mongodb://localhost:27017'); 15 | const db = mongo.db('test'); 16 | const collection = db.collection('jobs'); 17 | stage.set('mongo', mongo); 18 | stage.set('db', db); 19 | stage.set('collection', collection); 20 | }); 21 | 22 | spec.beforeEach(async (ctx) => { 23 | const collection = ctx.get('collection'); 24 | await collection.drop().catch(() => { /** does not exist */ }); 25 | }); 26 | 27 | spec.after(async (stage) => { 28 | const mongo = stage.get('mongo'); 29 | await mongo.close(); 30 | }); 31 | 32 | spec.test('document with `sleepUntil` should be processed', async (ctx) => { 33 | let times = 0; 34 | const collection = ctx.get('collection'); 35 | const cron = new MongoCron({ 36 | collection, 37 | lockDuration: 0, 38 | onDocument: () => times++, 39 | }); 40 | await collection.insertMany([ 41 | { sleepUntil: new Date() }, 42 | { sleepUntil: new Date() }, 43 | { sleepUntil: null }, 44 | { sleepUntil: new Date() }, 45 | ]); 46 | await cron.start(); 47 | await sleep(3000); 48 | await cron.stop(); 49 | ctx.is(times, 3); 50 | ctx.is(await collection.countDocuments({ sleepUntil: { $ne: null }}), 0); 51 | }); 52 | 53 | spec.test('cron should trigger event methods', async (ctx) => { 54 | let onStart = false; 55 | let onStop = false; 56 | let onDocument = false; 57 | const collection = ctx.get('collection'); 58 | const cron = new MongoCron({ 59 | collection, 60 | lockDuration: 0, 61 | onStart: async () => onStart = true, 62 | onStop: async () => onStop = true, 63 | onDocument: async (doc) => onDocument = true, 64 | }); 65 | await collection.insertOne({ 66 | sleepUntil: new Date(), 67 | }); 68 | await cron.start(); 69 | await sleep(300); 70 | await cron.stop(); 71 | await sleep(100); 72 | ctx.is(onStart, true); 73 | ctx.is(onStop, true); 74 | ctx.is(onDocument, true); 75 | }); 76 | 77 | spec.test('cron should trigger the `onIdle` handler only once', async (ctx) => { 78 | let count = 0; 79 | const collection = ctx.get('collection'); 80 | const cron = new MongoCron({ 81 | collection, 82 | lockDuration: 0, 83 | onIdle: () => count++, 84 | }); 85 | await cron.start(); 86 | await sleep(1000); 87 | await cron.stop(); 88 | ctx.is(count, 1); 89 | }); 90 | 91 | spec.test('locked documents should not be available for locking', async (ctx) => { 92 | let processed = false; 93 | const future = moment().add(5000, 'milliseconds'); 94 | const collection = ctx.get('collection'); 95 | const cron = new MongoCron({ 96 | collection, 97 | lockDuration: 5000, 98 | onDocument: () => processed = true, 99 | }); 100 | await collection.insertOne({ 101 | sleepUntil: future.toDate(), 102 | }); 103 | await cron.start(); 104 | await sleep(500); 105 | await cron.stop(); 106 | ctx.is(processed, false); 107 | }); 108 | 109 | spec.test('recurring documents should be unlocked when prossed', async (ctx) => { 110 | let processed = 0; 111 | const now = moment(); 112 | const collection = ctx.get('collection'); 113 | const cron = new MongoCron({ 114 | collection, 115 | lockDuration: 60000, 116 | onDocument: () => { 117 | processed++; 118 | return sleep(2000); 119 | }, 120 | }); 121 | await collection.insertOne({ 122 | sleepUntil: now.toDate(), 123 | interval: '* * * * * *', 124 | }); 125 | await cron.start(); 126 | await sleep(6000); 127 | await cron.stop(); 128 | ctx.is(processed, 3); 129 | }); 130 | 131 | spec.test('recurring documents should process from current date', async (ctx) => { 132 | let processed = 0; 133 | const past = moment().subtract(10, 'days'); 134 | const collection = ctx.get('collection'); 135 | const cron = new MongoCron({ 136 | collection, 137 | onDocument: () => processed++, 138 | }); 139 | await collection.insertOne({ 140 | sleepUntil: past.toDate(), // should be treated as now() date 141 | interval: '* * * * * *', 142 | }); 143 | await cron.start(); 144 | await sleep(2000); 145 | await cron.stop(); 146 | ctx.true(processed <= 4); 147 | }); 148 | 149 | spec.test('condition should filter lockable documents', async (ctx) => { 150 | let count = 0; 151 | const collection = ctx.get('collection'); 152 | const cron = new MongoCron({ 153 | collection, 154 | lockDuration: 0, 155 | condition: { handle: true }, 156 | onDocument: () => count++, 157 | }); 158 | await collection.insertOne({ 159 | handle: true, 160 | sleepUntil: new Date(), 161 | }); 162 | await collection.insertOne({ 163 | sleepUntil: new Date(), 164 | }); 165 | await cron.start(); 166 | await sleep(4000); 167 | await cron.stop(); 168 | ctx.is(count, 1); 169 | }); 170 | 171 | spec.test('document processing should not start before `sleepUntil`', async (ctx) => { 172 | let ranInFuture = false; 173 | const future = moment().add(3000, 'milliseconds'); 174 | const collection = ctx.get('collection'); 175 | const cron = new MongoCron({ 176 | collection, 177 | lockDuration: 0, 178 | onDocument: async (doc) => ranInFuture = moment() >= future, 179 | }); 180 | await cron.start(); 181 | await collection.insertOne({ 182 | sleepUntil: future.toDate(), 183 | }); 184 | await sleep(4000); 185 | await cron.stop(); 186 | ctx.is(ranInFuture, true); 187 | }); 188 | 189 | spec.test('document with `interval` should run repeatedly', async (ctx) => { 190 | let repeated = 0; 191 | const collection = ctx.get('collection'); 192 | const cron = new MongoCron({ 193 | collection, 194 | lockDuration: 0, 195 | onDocument: async (doc) => { 196 | repeated++; 197 | }, 198 | }); 199 | await cron.start(); 200 | await collection.insertOne({ 201 | sleepUntil: new Date(), 202 | interval: '* * * * * *', 203 | }); 204 | await sleep(3100); 205 | await cron.stop(); 206 | ctx.is(repeated >= 3, true); 207 | }); 208 | 209 | spec.test('document should stop recurring at `repeatUntil`', async (ctx) => { 210 | let repeated = moment(); 211 | const stop = moment().add(2500, 'milliseconds'); 212 | const collection = ctx.get('collection'); 213 | const cron = new MongoCron({ 214 | collection, 215 | lockDuration: 0, 216 | onDocument: async (doc) => repeated = moment(), 217 | reprocessDelay: 1000, 218 | }); 219 | await cron.start(); 220 | await collection.insertOne({ 221 | sleepUntil: new Date(), 222 | interval: '* * * * * *', 223 | repeatUntil: stop.toDate(), 224 | }); 225 | await sleep(6000); 226 | await cron.stop(); 227 | ctx.is(repeated.isAfter(stop), false); 228 | }); 229 | 230 | spec.test('document with `autoRemove` should be deleted when completed', async (ctx) => { 231 | const collection = ctx.get('collection'); 232 | const cron = new MongoCron({ 233 | collection, 234 | lockDuration: 0, 235 | }); 236 | await cron.start(); 237 | await collection.insertOne({ 238 | sleepUntil: new Date(), 239 | autoRemove: true, 240 | }); 241 | await sleep(2000); 242 | await cron.stop(); 243 | ctx.is(await collection.countDocuments(), 0); 244 | }); 245 | 246 | export default spec; 247 | -------------------------------------------------------------------------------- /src/tests/index.test.ts: -------------------------------------------------------------------------------- 1 | import { Spec } from '@hayspec/spec'; 2 | import * as all from '..'; 3 | 4 | const spec = new Spec(); 5 | 6 | spec.test('exposes objects', (ctx) => { 7 | ctx.true(!!all.MongoCron); 8 | }); 9 | 10 | export default spec; 11 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "es6", 5 | "noImplicitAny": false, 6 | "removeComments": true, 7 | "sourceMap": true, 8 | "outDir": "dist", 9 | "declaration": true 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | "no-unnecessary-class": [ 4 | true, 5 | "allow-constructor-only", 6 | "allow-static-only", 7 | "allow-empty-class" 8 | ], 9 | "member-access": [ 10 | true, 11 | "check-accessor", 12 | "check-constructor" 13 | ], 14 | "adjacent-overload-signatures": true, 15 | "prefer-function-over-method": [ 16 | true, 17 | "allow-public", 18 | "allow-protected" 19 | ], 20 | "no-invalid-this": [ 21 | true, 22 | "check-function-in-method" 23 | ], 24 | "no-this-assignment": true, 25 | "unnecessary-constructor": true, 26 | "no-duplicate-super": true, 27 | "new-parens": true, 28 | "no-misused-new": true, 29 | "no-construct": true, 30 | "prefer-method-signature": true, 31 | "interface-over-type-literal": true, 32 | "function-constructor": true, 33 | "no-arg": true, 34 | "arrow-parens": [ 35 | true 36 | ], 37 | "arrow-return-shorthand": [ 38 | false, 39 | "multiline" 40 | ], 41 | "unnecessary-bind": true, 42 | "no-return-await": true, 43 | "prefer-const": true, 44 | "no-var-keyword": true, 45 | "one-variable-per-declaration": [ 46 | true, 47 | "ignore-for-loop" 48 | ], 49 | "no-duplicate-variable": [ 50 | true, 51 | "check-parameters" 52 | ], 53 | "no-unnecessary-initializer": true, 54 | "no-implicit-dependencies": [ 55 | true, 56 | "dev" 57 | ], 58 | "no-import-side-effect": [ 59 | true, 60 | { 61 | "ignore-module": "(hammerjs|core-js|zone.js)" 62 | } 63 | ], 64 | "ordered-imports": [ 65 | true, 66 | { 67 | "import-sources-order": "case-insensitive", 68 | "named-imports-order": "case-insensitive", 69 | "grouped-imports": false 70 | } 71 | ], 72 | "no-duplicate-imports": true, 73 | "import-blacklist": [ 74 | true, 75 | "rxjs/Rx" 76 | ], 77 | "no-reference": true, 78 | "typedef": [ 79 | true, 80 | "property-declaration" 81 | ], 82 | "no-inferrable-types": true, 83 | "no-object-literal-type-assertion": true, 84 | "no-angle-bracket-type-assertion": true, 85 | "callable-types": true, 86 | "no-non-null-assertion": true, 87 | "prefer-object-spread": true, 88 | "object-literal-shorthand": true, 89 | "quotemark": [ 90 | true, 91 | "single", 92 | "avoid-template", 93 | "avoid-escape" 94 | ], 95 | "prefer-template": true, 96 | "no-invalid-template-strings": true, 97 | "increment-decrement": [ 98 | true, 99 | "allow-post" 100 | ], 101 | "binary-expression-operand-order": true, 102 | "no-dynamic-delete": true, 103 | "no-bitwise": true, 104 | "use-isnan": true, 105 | "no-conditional-assignment": true, 106 | "prefer-while": true, 107 | "prefer-for-of": true, 108 | "switch-default": true, 109 | "no-switch-case-fall-through": true, 110 | "no-duplicate-switch-case": true, 111 | "no-unsafe-finally": true, 112 | "encoding": true, 113 | "cyclomatic-complexity": [ 114 | true, 115 | 20 116 | ], 117 | "indent": [ 118 | true, 119 | "spaces", 120 | 2 121 | ], 122 | "eofline": true, 123 | "curly": true, 124 | "whitespace": [ 125 | true, 126 | "check-branch", 127 | "check-decl", 128 | "check-operator", 129 | "check-module", 130 | "check-separator", 131 | "check-rest-spread", 132 | "check-type", 133 | "check-typecast", 134 | "check-type-operator", 135 | "check-preblock" 136 | ], 137 | "typedef-whitespace": [ 138 | true, 139 | { 140 | "call-signature": "nospace", 141 | "index-signature": "nospace", 142 | "parameter": "nospace", 143 | "property-declaration": "nospace", 144 | "variable-declaration": "nospace" 145 | }, 146 | { 147 | "call-signature": "onespace", 148 | "index-signature": "onespace", 149 | "parameter": "onespace", 150 | "property-declaration": "onespace", 151 | "variable-declaration": "onespace" 152 | } 153 | ], 154 | "space-before-function-paren": [ 155 | true, 156 | { 157 | "anonymous": "never", 158 | "named": "never", 159 | "asyncArrow": "always", 160 | "method": "never", 161 | "constructor": "never" 162 | } 163 | ], 164 | "space-within-parens": 0, 165 | "import-spacing": true, 166 | "no-trailing-whitespace": true, 167 | "one-line": [ 168 | true, 169 | "check-open-brace", 170 | "check-whitespace", 171 | "check-else", 172 | "check-catch", 173 | "check-finally" 174 | ], 175 | "no-consecutive-blank-lines": [ 176 | true, 177 | 1 178 | ], 179 | "semicolon": [ 180 | true, 181 | "always", 182 | "strict-bound-class-methods" 183 | ], 184 | "align": [ 185 | true, 186 | "elements", 187 | "members", 188 | "parameters", 189 | "statements" 190 | ], 191 | "trailing-comma": [ 192 | true, 193 | { 194 | "multiline": "always", 195 | "esSpecCompliant": true 196 | } 197 | ], 198 | "file-name-casing": [ 199 | true, 200 | "kebab-case" 201 | ], 202 | "class-name": true, 203 | "interface-name": [ 204 | true, 205 | "never-prefix" 206 | ], 207 | "variable-name": [ 208 | true, 209 | "check-format", 210 | "allow-leading-underscore", 211 | "ban-keywords" 212 | ], 213 | "comment-type": [ 214 | true, 215 | "singleline", 216 | "doc" 217 | ], 218 | "comment-format": [ 219 | true, 220 | "check-space" 221 | ], 222 | "jsdoc-format": [ 223 | true, 224 | "check-multiline-start" 225 | ], 226 | "no-redundant-jsdoc": true, 227 | "ban-ts-ignore": true, 228 | "no-debugger": true, 229 | "no-eval": true, 230 | "no-string-throw": true, 231 | "no-namespace": true, 232 | "no-internal-module": true, 233 | "number-literal-format": true, 234 | "no-unused-expression": [ 235 | true, 236 | "allow-fast-null-checks" 237 | ], 238 | "no-empty": true, 239 | "no-sparse-arrays": true, 240 | "ban-comma-operator": true 241 | } 242 | } 243 | --------------------------------------------------------------------------------