├── .gitignore ├── LICENSE ├── README.md ├── example └── index.js ├── giphy.gif ├── package.json └── src ├── index.js └── lib ├── Queue.js ├── Schedule.js └── errors └── QueueEmptyError.js /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Xpepermint (Kristijan Sedlak) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | Thanks to http://giphy.com/ for the logo image :). 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # qos 2 | 3 | > Safe, fast and super simple queue and schedule based on Redis. 4 | 5 | QoS (Queue or Schedule) offers a simple api for scheduling and running tasks in background. QoS is build on top of [Redis](http://redis.io). It's super fast and it uses atomic commands to ensure safe job execution in cluster environments. 6 | 7 | 8 | 9 | ## Setup 10 | 11 | ``` 12 | $ npm install --save qos 13 | ``` 14 | 15 | ## Usage 16 | 17 | Before we start make sure you have `Redis` server up and running. 18 | 19 | ### Queue 20 | 21 | Let's create a new file `./index.js` and define a simple queue. 22 | 23 | ```js 24 | import Redis from 'ioredis'; 25 | import qos from 'qos'; 26 | 27 | // initializing redis instance 28 | const redis = new Redis(); 29 | 30 | // initializing handler for processing jobs 31 | const handler = data => { 32 | console.log(`Handling job named ${data.name}`); 33 | }; 34 | 35 | // initializing queue named `myqueue` 36 | const queue = new qos.Queue(redis, 'myqueue', handler); 37 | 38 | // starting queue 39 | queue.start(); 40 | ``` 41 | 42 | First we need to pass an instance of a Redis connection to the `Queue` class. QoS should work with any Redis library that supports promises. The second argument is the name of the queue. The last argument is a function for processing jobs. 43 | 44 | We are now ready to enqueue a job using the `enqueue` command. 45 | 46 | ```js 47 | queue.enqueue({name: 'JobName'}); // returns a Promise 48 | ``` 49 | 50 | The `enqueue` command is actually a call to the handler. It accepts an argument which is passed directly to the `handler`. 51 | 52 | We can also remove a job using the `dequeue` command. 53 | 54 | ```js 55 | queue.dequeue({name: 'JobName'}); // returns a Promise 56 | ``` 57 | 58 | Jobs can also be executed without touching the queuing system using the `perform` method. 59 | 60 | ```js 61 | queue.perform({name: 'JobName'}); // returns a Promise 62 | ``` 63 | 64 | ### Schedule 65 | 66 | To schedule a job at a particular time in the future we need to use the `Schedule` class. `Schedule` is an extended `Queue` class. It has pretty much the same logic. The main difference is that we need to provide some additional information for the `enqueue` and `dequeue` commands. 67 | 68 | Let's open our `./index.js` file which we defined earlier and add a scheduler. 69 | 70 | ```js 71 | let schedule = new qos.Schedule(redis, 'myschedule'); // you can set the default `queue` through options which is optional third parameter 72 | 73 | schedule.start(); 74 | ``` 75 | 76 | Schedule a job with the delay of 10s. 77 | 78 | ```js 79 | schedule.enqueue({ 80 | queue, // you can also pass queue name (e.g. 'myqueue') 81 | at: Date.now() + 10000, 82 | data: {name: 'JobName'} // Queue job data 83 | }); // returns a Promise 84 | ``` 85 | 86 | There is one important different between `Queue` and `Schedule` classes. If we call the command above multiple times, an existing job will be replaced with a new one. This means that two identical jobs can not exist in scheduled queue. This is great and ensures that the same job will never accidentally be scheduled twice. 87 | 88 | Scheduled jobs can also be removed. 89 | 90 | ```js 91 | schedule.dequeue({ 92 | queue, 93 | data: {name: 'JobName'} 94 | }); // returns a Promise 95 | ``` 96 | 97 | We can also check if the job is schedule by using the `isEnqueued` command. 98 | 99 | ```js 100 | schedule.isEnqueued({ 101 | queue, 102 | data: {name: 'JobName'} 103 | }); // returns a Promise 104 | ``` 105 | 106 | There is also a `toggle` command which enqueues/dequeues a job based on an optional condition. 107 | 108 | ```js 109 | let condition = 1 > 0; 110 | schedule.toggle({ 111 | queue, 112 | at: Date.now() + 10000, 113 | data: {name: 'JobName'} 114 | }, condition); // returns a Promise 115 | ``` 116 | 117 | ## Example 118 | 119 | You can run the attached example with the `npm run example` command. 120 | -------------------------------------------------------------------------------- /example/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var Redis = require('ioredis'); 4 | var redis = new Redis(); 5 | var qos = require('..'); 6 | 7 | // initializing queue 8 | var queue = new qos.Queue(redis, 'qos:queue', data => { 9 | console.log('Handling job:', data); 10 | }); 11 | // starting queue 12 | queue.start(); 13 | // run job now 14 | queue.enqueue({name: 'MyJob'}); 15 | 16 | // initializing schedule 17 | var schedule = new qos.Schedule(redis, 'qos:schedule', {queue}); 18 | // starting schedule queue 19 | schedule.start(); 20 | // delay job for 5s 21 | schedule.toggle({at: Date.now()+5000, data: {name: 'MyJob'}}); 22 | -------------------------------------------------------------------------------- /giphy.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xpepermint/qos/3563b637a2c6878ef51de4e785bbd362b90d59c1/giphy.gif -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "qos", 3 | "version": "0.3.1", 4 | "description": "Safe, fast and super simple queue and schedule based on Redis.", 5 | "main": "src/index.js", 6 | "scripts": { 7 | "example": "node example/index.js", 8 | "test": "echo \"Error: no test specified\" && exit 1" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git+https://github.com/xpepermint/qos.git" 13 | }, 14 | "keywords": [ 15 | "queue", 16 | "que", 17 | "schedule", 18 | "scheduler", 19 | "task", 20 | "tasks", 21 | "job", 22 | "jobs", 23 | "background", 24 | "redis" 25 | ], 26 | "author": "Xpepermint", 27 | "license": "MIT", 28 | "bugs": { 29 | "url": "https://github.com/xpepermint/qos/issues" 30 | }, 31 | "homepage": "https://github.com/xpepermint/qos#readme", 32 | "devDependencies": { 33 | "ioredis": "^1.10.0" 34 | }, 35 | "dependencies": { 36 | "ioredis": "^1.11.0" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Queue = require('./lib/Queue'); 4 | const Schedule = require('./lib/Schedule'); 5 | 6 | module.exports = {Queue, Schedule}; 7 | -------------------------------------------------------------------------------- /src/lib/Queue.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const QueueEmptyError = require('./errors/QueueEmptyError'); 4 | 5 | module.exports = class Queue { 6 | 7 | /* 8 | * Class constructor with the required `key` parameter which represents the 9 | * name of the list used by this schedule. 10 | */ 11 | 12 | constructor(redis, key, handler) { 13 | this.redis = redis; 14 | this.key = key; 15 | this.running = false; 16 | this.timeout = null; 17 | this.handler = handler; 18 | } 19 | 20 | /* 21 | * Starts the heartbit of the schedule. 22 | */ 23 | 24 | start() { 25 | if (this.running) return; 26 | 27 | this.running = true; 28 | this.tick(); 29 | } 30 | 31 | /* 32 | * Private method which is called on every heartbit of the schedule. 33 | */ 34 | 35 | tick() { 36 | if (!this.running) return; 37 | 38 | return this.redis.rpoplpush(this.key, `${this.key}:processing`).then(value => { 39 | if (!value) throw new QueueEmptyError(); 40 | 41 | let data = this.decodeValue(value); 42 | return this.perform(data).then(res => value); 43 | }).then(value => { 44 | return this.redis.lrem(`${this.key}:processing`, '-0', value); 45 | }).then(this.tick.bind(this)).catch(this.handleError.bind(this)); 46 | } 47 | 48 | /* 49 | * Stops the heartbit of the schedule. 50 | */ 51 | 52 | stop() { 53 | clearTimeout(this.timeout); 54 | this.running = false; 55 | } 56 | 57 | /* 58 | * Private method which handles class errors. 59 | */ 60 | 61 | handleError(err) { 62 | if (!(err.name === 'QueueEmptyError')) console.log(err); 63 | 64 | clearTimeout(this.timeout); 65 | this.timeout = setTimeout(this.tick.bind(this), 1000); 66 | } 67 | 68 | /* 69 | * Returns serialized value which can be stored in redis. 70 | */ 71 | 72 | encodeValue(data) { 73 | return JSON.stringify(data); 74 | } 75 | 76 | /* 77 | * Returns unserialized value. 78 | */ 79 | 80 | decodeValue(value) { 81 | return JSON.parse(value); 82 | } 83 | 84 | /* 85 | * Places a new job on the processing list. 86 | */ 87 | 88 | enqueue(data) { 89 | let value = this.encodeValue(data); 90 | return this.redis.lpush(this.key, value); 91 | } 92 | 93 | /* 94 | * Removes a job from the processing list. Not that if a job is enqueued 95 | * multiple times then multiple values will be deleted. 96 | */ 97 | 98 | dequeue(data) { 99 | let value = this.encodeValue(data); 100 | return this.redis.lrem(this.key, '-0', value); 101 | } 102 | 103 | /* 104 | * Executes a job without touching the queuing system. 105 | */ 106 | 107 | perform(data) { 108 | return Promise.resolve().then(res => this.handler(data)); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/lib/Schedule.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Queue = require('./Queue'); 4 | const QueueEmptyError = require('./errors/QueueEmptyError'); 5 | 6 | module.exports = class Schedule extends Queue { 7 | 8 | /* 9 | * Class constructor with the required `key` parameter which represents the 10 | * name of the schedule. 11 | */ 12 | 13 | constructor(redis, key, options) { 14 | super(redis, key); 15 | this.options = options || {}; 16 | } 17 | 18 | 19 | /* 20 | * Private method which is called on every heartbit of the schedule. 21 | */ 22 | 23 | tick() { 24 | if (!this.running) return; 25 | 26 | this.redis.watch(this.key).then(status => { 27 | return this.redis.zrangebyscore(this.key, 0, Date.now(), 'LIMIT', 0, 1).then(values => values ? values[0] : null); 28 | }).then(value => { 29 | if (!value) { 30 | this.redis.unwatch(); 31 | throw new QueueEmptyError(); 32 | } 33 | 34 | let target = this.decodeValue(value); 35 | return this.redis.multi().lpush(target.key, target.value).zrem(this.key, value).exec(); 36 | }).then(this.tick.bind(this)).catch(this.handleError.bind(this)); 37 | } 38 | 39 | /* 40 | * Schedules a new job to be executed in the future. Note that two identical 41 | * jobs can not exist thus a jobs can be scheduled only once. 42 | */ 43 | 44 | enqueue(data) { 45 | if (!data.queue) { 46 | data.queue = this.options.queue; 47 | } 48 | 49 | let at = data.at || Date.now(); 50 | let key = typeof data.queue === 'string' ? data.queue : data.queue.key; 51 | let value = this.encodeValue({key, value: this.encodeValue(data.data)}); 52 | return this.redis.zadd(this.key, at, value); 53 | } 54 | 55 | /* 56 | * Removes already scheduled job. 57 | */ 58 | 59 | dequeue(data) { 60 | if (!data.queue) { 61 | data.queue = this.options.queue; 62 | } 63 | 64 | let key = typeof data.queue === 'string' ? data.queue : data.queue.key; 65 | let value = this.encodeValue({key, value: this.encodeValue(data.data)}); 66 | return this.redis.zrem(this.key, value); 67 | } 68 | 69 | /* 70 | * Tells if the job is scheduled. 71 | */ 72 | 73 | isEnqueued(data) { 74 | if (!data.queue) { 75 | data.queue = this.options.queue; 76 | } 77 | 78 | let at = data.at || Date.now(); 79 | let key = typeof data.queue === 'string' ? data.queue : data.queue.key; 80 | let value = JSON.stringify({key, value: this.encodeValue(data.data)}); 81 | return this.redis.zscore(this.key, value).then(res => !!res); 82 | } 83 | 84 | /* 85 | * Schedules or removes a job. 86 | */ 87 | 88 | toggle(data, shouldEnqueue) { 89 | let perform = (shouldEnqueue) => { 90 | if (shouldEnqueue) { 91 | return this.enqueue(data); 92 | } else { 93 | return this.dequeue(data); 94 | } 95 | }; 96 | if (typeof shouldEnqueue !== 'boolean') { 97 | return this.isEnqueued(data).then(enqueued => perform(!enqueued)); 98 | } else { 99 | return perform(shouldEnqueue); 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/lib/errors/QueueEmptyError.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | class QueueEmptyError extends Error { 4 | constructor(message) { 5 | super(message); 6 | this.name = 'QueueEmptyError'; 7 | this.message = message || 'No jobs found for processing.'; 8 | } 9 | } 10 | 11 | module.exports = QueueEmptyError; 12 | --------------------------------------------------------------------------------