├── .eslintignore ├── .eslintrc.js ├── .github ├── pull_request_template.md └── workflows │ └── test.yml ├── .gitignore ├── LICENSE ├── README.md ├── dist ├── event.d.ts ├── event.js ├── index.d.ts ├── index.js ├── job.d.ts ├── job.js ├── jobRepository.d.ts ├── jobRepository.js ├── priority.d.ts ├── priority.js ├── queue.d.ts ├── queue.js ├── state.d.ts ├── state.js ├── worker.d.ts └── worker.js ├── jest.config.js ├── package.json ├── src ├── event.ts ├── index.ts ├── job.ts ├── jobRepository.ts ├── priority.ts ├── queue.ts ├── state.ts └── worker.ts ├── test ├── JobRepository.test.ts ├── integration.test.ts ├── job.test.ts ├── queue.test.ts └── worker.test.ts ├── tsconfig.json └── yarn.lock /.eslintignore: -------------------------------------------------------------------------------- 1 | dist 2 | node_modules 3 | jest.config.js 4 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | parser: "@typescript-eslint/parser", 4 | plugins: [ 5 | "@typescript-eslint", 6 | "jest", 7 | ], 8 | extends: [ 9 | "eslint:recommended", 10 | "plugin:@typescript-eslint/eslint-recommended", 11 | "plugin:@typescript-eslint/recommended", 12 | "plugin:jest/recommended", 13 | "plugin:jest/style", 14 | ], 15 | }; 16 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | - [ ] Run `yarn prepare` 2 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | test: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - uses: actions/checkout@v2 11 | - name: Setup Node.js 12 | uses: actions/setup-node@v1 13 | with: 14 | node-version: '12.x' 15 | - name: Install dependencies 16 | run: yarn 17 | - name: Run Test 18 | run: yarn test 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | node_modules 3 | .DS_Store 4 | /.idea 5 | /build 6 | /.cache 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 Ken'ichi Saito 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 14 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 15 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 16 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 19 | OR OTHER DEALINGS IN THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # embedded-queue 2 | ![Test](https://github.com/hajipy/embedded-queue/workflows/Test/badge.svg) 3 | 4 | embedded-queue is job/message queue for Node.js and Electron. It is not required other process for persistence data, like Redis, MySQL, and so on. It persistence data by using [nedb](https://github.com/louischatriot/nedb) embedded database. 5 | 6 | ## Installation 7 | ```sh 8 | npm install --save embedded-queue 9 | ``` 10 | or 11 | ```sh 12 | yarn add embedded-queue 13 | ``` 14 | 15 | ## Basic Usage 16 | ```js 17 | const EmbeddedQueue = require("embedded-queue"); 18 | 19 | (async () => { 20 | // argument path through nedb 21 | const queue = await EmbeddedQueue.Queue.createQueue({ inMemoryOnly: true }); 22 | 23 | // set up job processor for "adder" type, concurrency is 1 24 | queue.process( 25 | "adder", 26 | async (job) => job.data.a + job.data.b, 27 | 1 28 | ); 29 | 30 | // handle job complete event 31 | queue.on( 32 | EmbeddedQueue.Event.Complete, 33 | (job, result) => { 34 | console.log("Job Completed."); 35 | console.log(` job.id: ${job.id}`); 36 | console.log(` job.type: ${job.type}`); 37 | console.log(` job.data: ${JSON.stringify(job.data)}`); 38 | console.log(` result: ${result}`); 39 | } 40 | ); 41 | 42 | // create "adder" type job 43 | await queue.createJob({ 44 | type: "adder", 45 | data: { a: 1, b: 2 }, 46 | }); 47 | 48 | // shutdown queue 49 | setTimeout(async () => { await queue.shutdown(1000); }, 1); 50 | })(); 51 | ``` 52 | 53 | ## Basic 54 | - Create Queue 55 | - Set Job Processor 56 | - Set Job Event Handler 57 | - Create Job 58 | - Shutdown Queue 59 | 60 | ### Create Queue 61 | You can create a new queue by calling `Queue.createQueue(dbOptions)`. `dbOptions` argument is pass to nedb database constructor. for more information see [nedb documents](https://github.com/louischatriot/nedb#creatingloading-a-database). `Queue.createQueue` returns a `Promise`, `await` it for initialize finish. 62 | 63 | ### Set Job Processor 64 | Job processor is a function that process single job. It is called by `Worker` and pass `Job` argument, it must return `Promise`. It runs any process(calculation, network access, etc...) and call `resolve(result)`. Required data can pass by `Job.data` object. Also you can call `Job.setProgress` for notify progress, `Job.addLog` for logging. 65 | You can set any number of job processors, each processor is associate to single job `type`, it processes only jobs of that `type`. 66 | If you want to need process many jobs that same `type` in concurrency, you can launch any number of job processor of same `type`. 67 | 68 | Finally, `queue.process` method signature is `quque.process(type, processor, concurrency)`. 69 | 70 | ### Set Job Event Handler 71 | `Queue` implements `EventEmitter`, when job is completed or job is failed, job progress updated, etc..., you can observe these events by set handlers `Queue.on(Event, Handler)`. 72 | 73 | | Event | Description | Handler Signature | 74 | |------------------|--------------------------------------------------|---------------------------| 75 | | `Event.Enqueue` | Job add to queue | `(job) => void` | 76 | | `Event.Start` | Job start processing | `(job) => void` | 77 | | `Event.Failure` | Job process fail | `(job, error) => void` | 78 | | `Event.Complete` | Job process complete | `(job, result) => void` | 79 | | `Event.Remove` | Job is removed from queue | `(job) => void` | 80 | | `Event.Error` | Error has occurred (on outside of job processor) | `(error, job?) => void` | 81 | | `Event.Progress` | Job progress update | `(job, progress) => void` | 82 | | `Event.Log` | Job log add | `(job, message) => void` | 83 | | `Event.Priority` | Job priority change | `(job, priority) => void` | 84 | 85 | `Event.Complete` event handler is most commonly used, it can receive job result from job processor. 86 | 87 | ### Create Job 88 | You can create a job by calling `Queue.createJob(data)`. `data` argument is object that contains `type`, `priority` and `data`. 89 | 90 | | Field | Type | Description | 91 | |------------|------------|-------------| 92 | | `type` | `string` | Identifier for select job processor | 93 | | `priority` | `Priority` | `Queue` picks up job that has high priority first | 94 | | `data` | `object` | Data that is used by job processor, you can set any data | 95 | 96 | `Queue.createJob(data)` returns `Promise` object, this job is associated to `Queue`. 97 | 98 | `Priority` is any of `Priority.LOW`, `Priority.NORMAL`, `Priority.MEDIUM`, `Priority.HIGH`, `Priority.CRITICAL`. 99 | 100 | ### Shutdown Queue 101 | If you want stop processing jobs, you have to call `Queue.shutdown(timeoutMilliseconds, type) => Promise`. `Queue` starts to stop running job processor, and all job processors are stopped, Promise is resolved. If stopping job processor takes long time, after `timeoutMilliseconds` `Queue` terminate job processor, and set `Job.state` to `State.FAILURE`. 102 | You can stop specified type job processor by passing second argument `type`. If `undefined` is passed, stop all type job processors. 103 | 104 | ## API 105 | 106 | ### Queue API 107 | - `createJob(data)`: Create a new `Job`, see above for usage. 108 | - `process(type, processor, concurrency)`: Set job processor, see above for usage. 109 | - `shutdown(timeoutMilliseconds, type)`: Start shutting down `Queue`, see above for usage. 110 | - `findJob(id)`: Search queue by `Job.id`. If found return `Job`, otherwise return `null`. 111 | - `listJobs(state)`: List all jobs that has specified state. If passed `undefined` return all jobs. 112 | - `removeJobById(id)`: Remove a `Job` from queue that specified id. 113 | - `removeJobsByCallback(callback)`: Remove all jobs that `callback` returns `true`. Callback signature is `(job) => boolean`. 114 | 115 | ### Job API 116 | - `setProgress(completed, total)`: Set progress, arguments are convert to percentage value(completed / total). 117 | - `addLog(message)`: Add log. 118 | - `save()`: After call it, job put into associate `Queue`, and waiting for process by job processor. 119 | - `remove()`: Remove job from `Queue`, it will not be processed anymore. 120 | - `setPriority(value)`: Set `priority` value. 121 | - `isExist()`: Return `Job` is in `Queue`. Before calling `save()` or after calling `remove()` returns `false`, otherwise `true`. 122 | - Getters 123 | - `id`: String that identifies `Job`. 124 | - `type`: String that Identifier for select job processor. 125 | - `data`: Object that is used by job processor, you can set any data. 126 | - `priority`: Number that determines processing order. 127 | - `createdAt`: Date that job is created. 128 | - `updatedAt`: Date that job is updated. 129 | - `startedAt`: Date that job processor start process. Before job start, value is `undefined`. 130 | - `completedAt`: Date that job processor complete process. Before job complete or job failed, value is `undefined`. 131 | - `failedAt`: Date that job processor occurred error. Before job complete or job complete successfully, value is `undefined`. 132 | - `state`: String that represents current `Job` state, any of `State.INACTIVE`, `State.ACTIVE`, `STATE.COMPLETE`, `State.FAILURE`. 133 | - `duration`: Number that processing time of `Job` in milliseconds. Before job complete or job failed, value is `undefined`. 134 | - `progress`: Number that `Job` progress in percentage. You can set value by calling `Job.setProgress`. When job complete, set 100 automatically. 135 | - `logs`: Array of String. You can add log by calling `Job.addLog`. 136 | 137 | ## Advanced 138 | 139 | ### Unexpectedly Termination 140 | If your program suddenly terminated without calling `Queue.shutdown` while your processor was processing jobs. These jobs remain `State.ACTIVE` in queue. When next time `Queue.createQueue` is called, these jobs are updated to `State.FAILURE` automatically. 141 | If you want reprocessing these jobs, please call `Queue.createJob` with same parameter. 142 | 143 | ## License 144 | MIT 145 | -------------------------------------------------------------------------------- /dist/event.d.ts: -------------------------------------------------------------------------------- 1 | export declare enum Event { 2 | Enqueue = "enqueue", 3 | Start = "start", 4 | Failure = "failure", 5 | Complete = "complete", 6 | Remove = "remove", 7 | Error = "error", 8 | Progress = "progress", 9 | Log = "log", 10 | Priority = "priority" 11 | } 12 | -------------------------------------------------------------------------------- /dist/event.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.Event = void 0; 4 | var Event; 5 | (function (Event) { 6 | Event["Enqueue"] = "enqueue"; 7 | Event["Start"] = "start"; 8 | Event["Failure"] = "failure"; 9 | Event["Complete"] = "complete"; 10 | Event["Remove"] = "remove"; 11 | Event["Error"] = "error"; 12 | Event["Progress"] = "progress"; 13 | Event["Log"] = "log"; 14 | Event["Priority"] = "priority"; 15 | })(Event = exports.Event || (exports.Event = {})); 16 | -------------------------------------------------------------------------------- /dist/index.d.ts: -------------------------------------------------------------------------------- 1 | export { CreateJobData, Processor, Queue } from "./queue"; 2 | export { Worker } from "./worker"; 3 | export { Job } from "./job"; 4 | export { Priority, toString as priorityToString } from "./priority"; 5 | export { State } from "./state"; 6 | export { Event } from "./event"; 7 | -------------------------------------------------------------------------------- /dist/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.Event = exports.State = exports.priorityToString = exports.Priority = exports.Job = exports.Worker = exports.Queue = void 0; 4 | var queue_1 = require("./queue"); 5 | Object.defineProperty(exports, "Queue", { enumerable: true, get: function () { return queue_1.Queue; } }); 6 | var worker_1 = require("./worker"); 7 | Object.defineProperty(exports, "Worker", { enumerable: true, get: function () { return worker_1.Worker; } }); 8 | var job_1 = require("./job"); 9 | Object.defineProperty(exports, "Job", { enumerable: true, get: function () { return job_1.Job; } }); 10 | var priority_1 = require("./priority"); 11 | Object.defineProperty(exports, "Priority", { enumerable: true, get: function () { return priority_1.Priority; } }); 12 | Object.defineProperty(exports, "priorityToString", { enumerable: true, get: function () { return priority_1.toString; } }); 13 | var state_1 = require("./state"); 14 | Object.defineProperty(exports, "State", { enumerable: true, get: function () { return state_1.State; } }); 15 | var event_1 = require("./event"); 16 | Object.defineProperty(exports, "Event", { enumerable: true, get: function () { return event_1.Event; } }); 17 | -------------------------------------------------------------------------------- /dist/job.d.ts: -------------------------------------------------------------------------------- 1 | import { Priority } from "./priority"; 2 | import { Queue } from "./queue"; 3 | import { State } from "./state"; 4 | export interface JobConstructorData { 5 | queue: Queue; 6 | id: string; 7 | type: string; 8 | priority?: Priority; 9 | data?: unknown; 10 | createdAt: Date; 11 | updatedAt: Date; 12 | startedAt?: Date; 13 | completedAt?: Date; 14 | failedAt?: Date; 15 | state?: State; 16 | duration?: number; 17 | progress?: number; 18 | logs: string[]; 19 | saved: boolean; 20 | } 21 | export declare class Job { 22 | readonly id: string; 23 | readonly type: string; 24 | readonly data: unknown | undefined; 25 | protected readonly queue: Queue; 26 | protected _priority: Priority; 27 | protected _createdAt: Date; 28 | protected _updatedAt: Date; 29 | protected _startedAt: Date | undefined; 30 | protected _completedAt: Date | undefined; 31 | protected _failedAt: Date | undefined; 32 | protected _state: State; 33 | protected _duration: number | undefined; 34 | protected _progress: number | undefined; 35 | protected _logs: string[]; 36 | protected _saved: boolean; 37 | get priority(): Priority; 38 | get createdAt(): Date; 39 | get updatedAt(): Date; 40 | get startedAt(): Date | undefined; 41 | get completedAt(): Date | undefined; 42 | get failedAt(): Date | undefined; 43 | get state(): State; 44 | get duration(): number | undefined; 45 | get progress(): number | undefined; 46 | get logs(): string[]; 47 | constructor(data: JobConstructorData); 48 | setProgress(completed: number, total: number): Promise; 49 | addLog(message: string): Promise; 50 | save(): Promise; 51 | remove(): Promise; 52 | setPriority(value: Priority): Promise; 53 | isExist(): Promise; 54 | /** @package */ 55 | setStateToActive(): Promise; 56 | /** @package */ 57 | setStateToComplete(result?: unknown): Promise; 58 | /** @package */ 59 | setStateToFailure(error: Error): Promise; 60 | protected update(): Promise; 61 | } 62 | -------------------------------------------------------------------------------- /dist/job.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.Job = void 0; 4 | const event_1 = require("./event"); 5 | const priority_1 = require("./priority"); 6 | const state_1 = require("./state"); 7 | class Job { 8 | constructor(data) { 9 | this.queue = data.queue; 10 | this.id = data.id; 11 | this.type = data.type; 12 | this._priority = data.priority || priority_1.Priority.NORMAL; 13 | this.data = data.data; 14 | this._state = data.state || state_1.State.INACTIVE; 15 | this._createdAt = data.createdAt; 16 | this._updatedAt = data.updatedAt; 17 | this._startedAt = data.startedAt; 18 | this._completedAt = data.completedAt; 19 | this._failedAt = data.failedAt; 20 | this._logs = [...data.logs]; 21 | this._duration = data.duration; 22 | this._progress = data.progress; 23 | this._saved = data.saved; 24 | } 25 | // tslint:enable:variable-name 26 | get priority() { 27 | return this._priority; 28 | } 29 | get createdAt() { 30 | return this._createdAt; 31 | } 32 | get updatedAt() { 33 | return this._updatedAt; 34 | } 35 | // noinspection JSUnusedGlobalSymbols 36 | get startedAt() { 37 | return this._startedAt; 38 | } 39 | // noinspection JSUnusedGlobalSymbols 40 | get completedAt() { 41 | return this._completedAt; 42 | } 43 | // noinspection JSUnusedGlobalSymbols 44 | get failedAt() { 45 | return this._failedAt; 46 | } 47 | get state() { 48 | return this._state; 49 | } 50 | get duration() { 51 | return this._duration; 52 | } 53 | get progress() { 54 | return this._progress; 55 | } 56 | get logs() { 57 | return [...this._logs]; 58 | } 59 | async setProgress(completed, total) { 60 | this._progress = completed * 100 / total; 61 | this._updatedAt = new Date(); 62 | await this.update(); 63 | this.queue.emit(event_1.Event.Progress, this, this._progress); 64 | } 65 | async addLog(message) { 66 | this._logs.push(message); 67 | this._updatedAt = new Date(); 68 | await this.update(); 69 | this.queue.emit(event_1.Event.Log, this, message); 70 | } 71 | async save() { 72 | if (this._saved) { 73 | await this.update(); 74 | } 75 | else { 76 | await this.queue.addJob(this); 77 | this._saved = true; 78 | this.queue.emit(event_1.Event.Enqueue, this); 79 | } 80 | return this; 81 | } 82 | async remove() { 83 | await this.queue.removeJob(this); 84 | this.queue.emit(event_1.Event.Remove, this); 85 | } 86 | async setPriority(value) { 87 | this._priority = value; 88 | this._updatedAt = new Date(); 89 | await this.update(); 90 | this.queue.emit(event_1.Event.Priority, this, value); 91 | } 92 | async isExist() { 93 | return this.queue.isExistJob(this); 94 | } 95 | /** @package */ 96 | async setStateToActive() { 97 | this._state = state_1.State.ACTIVE; 98 | const now = new Date(); 99 | this._startedAt = now; 100 | this._updatedAt = now; 101 | await this.update(); 102 | this.queue.emit(event_1.Event.Start, this); 103 | } 104 | /** @package */ 105 | async setStateToComplete(result) { 106 | this._state = state_1.State.COMPLETE; 107 | const now = new Date(); 108 | this._completedAt = now; 109 | if (this._startedAt !== undefined) { 110 | this._duration = now.getTime() - this._startedAt.getTime(); 111 | } 112 | this._updatedAt = now; 113 | await this.update(); 114 | this.queue.emit(event_1.Event.Complete, this, result); 115 | } 116 | /** @package */ 117 | async setStateToFailure(error) { 118 | this._state = state_1.State.FAILURE; 119 | const now = new Date(); 120 | this._failedAt = now; 121 | this._updatedAt = now; 122 | await this.update(); 123 | this.queue.emit(event_1.Event.Failure, this, error); 124 | } 125 | async update() { 126 | await this.queue.updateJob(this); 127 | } 128 | } 129 | exports.Job = Job; 130 | -------------------------------------------------------------------------------- /dist/jobRepository.d.ts: -------------------------------------------------------------------------------- 1 | import DataStore, { DataStoreOptions } from "nedb"; 2 | import { Job } from "./job"; 3 | import { State } from "./state"; 4 | export interface NeDbJob { 5 | _id: string; 6 | type: string; 7 | priority: number; 8 | data?: unknown; 9 | createdAt: Date; 10 | updatedAt: Date; 11 | startedAt?: Date; 12 | completedAt?: Date; 13 | failedAt?: Date; 14 | state?: State; 15 | duration?: number; 16 | progress?: number; 17 | logs: string[]; 18 | } 19 | export declare type DbOptions = DataStoreOptions; 20 | export declare class JobRepository { 21 | protected readonly db: DataStore; 22 | constructor(dbOptions?: DbOptions); 23 | init(): Promise; 24 | listJobs(state?: State): Promise; 25 | findJob(id: string): Promise; 26 | findInactiveJobByType(type: string): Promise; 27 | isExistJob(id: string): Promise; 28 | addJob(job: Job): Promise; 29 | updateJob(job: Job): Promise; 30 | removeJob(id: string): Promise; 31 | } 32 | -------------------------------------------------------------------------------- /dist/jobRepository.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __importDefault = (this && this.__importDefault) || function (mod) { 3 | return (mod && mod.__esModule) ? mod : { "default": mod }; 4 | }; 5 | Object.defineProperty(exports, "__esModule", { value: true }); 6 | exports.JobRepository = void 0; 7 | const nedb_1 = __importDefault(require("nedb")); 8 | const state_1 = require("./state"); 9 | class JobRepository { 10 | constructor(dbOptions = {}) { 11 | this.db = new nedb_1.default(dbOptions); 12 | } 13 | init() { 14 | return new Promise((resolve, reject) => { 15 | this.db.loadDatabase((error) => { 16 | if (error !== null) { 17 | reject(error); 18 | return; 19 | } 20 | resolve(); 21 | }); 22 | }); 23 | } 24 | listJobs(state) { 25 | return new Promise((resolve, reject) => { 26 | const query = (state === undefined) ? {} : { state }; 27 | this.db.find(query) 28 | .sort({ createdAt: 1 }) 29 | .exec((error, docs) => { 30 | if (error !== null) { 31 | reject(error); 32 | return; 33 | } 34 | resolve(docs); 35 | }); 36 | }); 37 | } 38 | findJob(id) { 39 | return new Promise((resolve, reject) => { 40 | this.db.findOne({ _id: id }, (error, doc) => { 41 | if (error !== null) { 42 | reject(error); 43 | return; 44 | } 45 | resolve(doc); 46 | }); 47 | }); 48 | } 49 | findInactiveJobByType(type) { 50 | return new Promise((resolve, reject) => { 51 | this.db.find({ type, state: state_1.State.INACTIVE }) 52 | .sort({ priority: -1, createdAt: 1 }) 53 | .limit(1) 54 | .exec((error, docs) => { 55 | if (error !== null) { 56 | reject(error); 57 | return; 58 | } 59 | resolve((docs.length === 0) ? null : docs[0]); 60 | }); 61 | }); 62 | } 63 | isExistJob(id) { 64 | return new Promise((resolve, reject) => { 65 | this.db.count({ _id: id }, (error, count) => { 66 | if (error !== null) { 67 | reject(error); 68 | return; 69 | } 70 | resolve(count === 1); 71 | }); 72 | }); 73 | } 74 | addJob(job) { 75 | return new Promise((resolve, reject) => { 76 | const insertDoc = { 77 | _id: job.id, 78 | type: job.type, 79 | priority: job.priority, 80 | data: job.data, 81 | createdAt: job.createdAt, 82 | updatedAt: job.updatedAt, 83 | state: job.state, 84 | logs: job.logs, 85 | }; 86 | this.db.insert(insertDoc, (error, doc) => { 87 | if (error !== null) { 88 | reject(error); 89 | return; 90 | } 91 | resolve(doc); 92 | }); 93 | }); 94 | } 95 | updateJob(job) { 96 | return new Promise((resolve, reject) => { 97 | const query = { 98 | _id: job.id, 99 | }; 100 | const updateQuery = { 101 | $set: { 102 | priority: job.priority, 103 | data: job.data, 104 | createdAt: job.createdAt, 105 | updatedAt: job.updatedAt, 106 | startedAt: job.startedAt, 107 | completedAt: job.completedAt, 108 | failedAt: job.failedAt, 109 | state: job.state, 110 | duration: job.duration, 111 | progress: job.progress, 112 | logs: job.logs, 113 | }, 114 | }; 115 | this.db.update(query, updateQuery, {}, (error, numAffected) => { 116 | if (error !== null) { 117 | reject(error); 118 | return; 119 | } 120 | if (numAffected !== 1) { 121 | reject(new Error(`update unexpected number of rows. (expected: 1, actual: ${numAffected})`)); 122 | } 123 | resolve(); 124 | }); 125 | }); 126 | } 127 | removeJob(id) { 128 | return new Promise((resolve, reject) => { 129 | this.db.remove({ _id: id }, (error) => { 130 | if (error) { 131 | reject(error); 132 | return; 133 | } 134 | resolve(); 135 | }); 136 | }); 137 | } 138 | } 139 | exports.JobRepository = JobRepository; 140 | -------------------------------------------------------------------------------- /dist/priority.d.ts: -------------------------------------------------------------------------------- 1 | export declare enum Priority { 2 | LOW = 1, 3 | NORMAL = 2, 4 | MEDIUM = 3, 5 | HIGH = 4, 6 | CRITICAL = 5 7 | } 8 | export declare function toString(priority: Priority): string; 9 | -------------------------------------------------------------------------------- /dist/priority.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.toString = exports.Priority = void 0; 4 | var Priority; 5 | (function (Priority) { 6 | Priority[Priority["LOW"] = 1] = "LOW"; 7 | Priority[Priority["NORMAL"] = 2] = "NORMAL"; 8 | Priority[Priority["MEDIUM"] = 3] = "MEDIUM"; 9 | Priority[Priority["HIGH"] = 4] = "HIGH"; 10 | Priority[Priority["CRITICAL"] = 5] = "CRITICAL"; 11 | })(Priority = exports.Priority || (exports.Priority = {})); 12 | function toString(priority) { 13 | switch (priority) { 14 | case Priority.LOW: 15 | return "LOW"; 16 | case Priority.NORMAL: 17 | return "NORMAL"; 18 | case Priority.MEDIUM: 19 | return "MEDIUM"; 20 | case Priority.HIGH: 21 | return "HIGH"; 22 | case Priority.CRITICAL: 23 | return "CRITICAL"; 24 | } 25 | } 26 | exports.toString = toString; 27 | -------------------------------------------------------------------------------- /dist/queue.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | import { EventEmitter } from "events"; 3 | import { Mutex } from "await-semaphore"; 4 | import { Job } from "./job"; 5 | import { DbOptions, JobRepository, NeDbJob } from "./jobRepository"; 6 | import { Priority } from "./priority"; 7 | import { State } from "./state"; 8 | import { Worker } from "./worker"; 9 | export interface CreateJobData { 10 | type: string; 11 | priority?: Priority; 12 | data?: unknown; 13 | } 14 | export declare type Processor = (job: Job) => Promise; 15 | interface WaitingWorkerRequest { 16 | resolve: (value: Job) => void; 17 | reject: (error: Error) => void; 18 | stillRequest: () => boolean; 19 | } 20 | export declare class Queue extends EventEmitter { 21 | static createQueue(dbOptions?: DbOptions): Promise; 22 | protected static sanitizePriority(priority: number): Priority; 23 | protected readonly repository: JobRepository; 24 | protected _workers: Worker[]; 25 | protected waitingRequests: { 26 | [type: string]: WaitingWorkerRequest[]; 27 | }; 28 | protected requestJobForProcessingMutex: Mutex; 29 | get workers(): Worker[]; 30 | protected constructor(dbOptions?: DbOptions); 31 | createJob(data: CreateJobData): Promise; 32 | process(type: string, processor: Processor, concurrency: number): void; 33 | shutdown(timeoutMilliseconds: number, type?: string | undefined): Promise; 34 | findJob(id: string): Promise; 35 | listJobs(state?: State): Promise; 36 | removeJobById(id: string): Promise; 37 | removeJobsByCallback(callback: (job: Job) => boolean): Promise; 38 | /** @package */ 39 | requestJobForProcessing(type: string, stillRequest: () => boolean): Promise; 40 | /** @package */ 41 | isExistJob(job: Job): Promise; 42 | /** @package */ 43 | addJob(job: Job): Promise; 44 | /** @package */ 45 | updateJob(job: Job): Promise; 46 | /** @package */ 47 | removeJob(job: Job): Promise; 48 | protected cleanupAfterUnexpectedlyTermination(): Promise; 49 | protected convertNeDbJobToJob(neDbJob: NeDbJob): Job; 50 | } 51 | export {}; 52 | -------------------------------------------------------------------------------- /dist/queue.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.Queue = void 0; 4 | const events_1 = require("events"); 5 | const await_semaphore_1 = require("await-semaphore"); 6 | const uuid_1 = require("uuid"); 7 | const event_1 = require("./event"); 8 | const job_1 = require("./job"); 9 | const jobRepository_1 = require("./jobRepository"); 10 | const priority_1 = require("./priority"); 11 | const state_1 = require("./state"); 12 | const worker_1 = require("./worker"); 13 | class Queue extends events_1.EventEmitter { 14 | constructor(dbOptions) { 15 | super(); 16 | this.repository = new jobRepository_1.JobRepository(dbOptions); 17 | this._workers = []; 18 | this.waitingRequests = {}; 19 | this.requestJobForProcessingMutex = new await_semaphore_1.Mutex(); 20 | } 21 | static async createQueue(dbOptions) { 22 | const queue = new Queue(dbOptions); 23 | await queue.repository.init(); 24 | await queue.cleanupAfterUnexpectedlyTermination(); 25 | return queue; 26 | } 27 | static sanitizePriority(priority) { 28 | switch (priority) { 29 | case priority_1.Priority.LOW: 30 | case priority_1.Priority.NORMAL: 31 | case priority_1.Priority.MEDIUM: 32 | case priority_1.Priority.HIGH: 33 | case priority_1.Priority.CRITICAL: 34 | return priority; 35 | } 36 | console.warn(`Invalid Priority: ${priority}`); 37 | return priority_1.Priority.NORMAL; 38 | } 39 | get workers() { 40 | return [...this._workers]; 41 | } 42 | async createJob(data) { 43 | const now = new Date(); 44 | const job = new job_1.Job(Object.assign({}, data, { 45 | queue: this, 46 | id: (0, uuid_1.v4)(), 47 | createdAt: now, 48 | updatedAt: now, 49 | logs: [], 50 | saved: false, 51 | })); 52 | return await job.save(); 53 | } 54 | process(type, processor, concurrency) { 55 | for (let i = 0; i < concurrency; i++) { 56 | const worker = new worker_1.Worker({ 57 | type, 58 | queue: this, 59 | }); 60 | worker.start(processor); 61 | this._workers.push(worker); 62 | } 63 | } 64 | async shutdown(timeoutMilliseconds, type) { 65 | const shutdownWorkers = []; 66 | for (const worker of this._workers) { 67 | if (type !== undefined && worker.type !== type) { 68 | continue; 69 | } 70 | await worker.shutdown(timeoutMilliseconds); 71 | shutdownWorkers.push(worker); 72 | } 73 | this._workers = this._workers.filter((worker) => { 74 | return shutdownWorkers.includes(worker) === false; 75 | }); 76 | } 77 | async findJob(id) { 78 | try { 79 | const neDbJob = await this.repository.findJob(id); 80 | if (neDbJob === null) { 81 | return null; 82 | } 83 | return this.convertNeDbJobToJob(neDbJob); 84 | } 85 | catch (error) { 86 | this.emit(event_1.Event.Error, error); 87 | throw error; 88 | } 89 | } 90 | async listJobs(state) { 91 | try { 92 | return await this.repository.listJobs(state).then((docs) => { 93 | return docs.map((neDbJob) => this.convertNeDbJobToJob(neDbJob)); 94 | }); 95 | } 96 | catch (error) { 97 | this.emit(event_1.Event.Error, error); 98 | throw error; 99 | } 100 | } 101 | async removeJobById(id) { 102 | let neDbJob; 103 | try { 104 | neDbJob = await this.repository.findJob(id); 105 | } 106 | catch (error) { 107 | this.emit(event_1.Event.Error, error); 108 | throw error; 109 | } 110 | if (neDbJob === null) { 111 | throw new Error(`Job(id:${id}) is not found.`); 112 | } 113 | const job = this.convertNeDbJobToJob(neDbJob); 114 | try { 115 | return await job.remove(); 116 | } 117 | catch (error) { 118 | this.emit(event_1.Event.Error, error, job); 119 | throw error; 120 | } 121 | } 122 | async removeJobsByCallback(callback) { 123 | const removedJobs = []; 124 | let job; 125 | try { 126 | const neDbJobs = await this.repository.listJobs(); 127 | for (const neDbJob of neDbJobs) { 128 | job = this.convertNeDbJobToJob(neDbJob); 129 | if (callback(job)) { 130 | removedJobs.push(job); 131 | await job.remove(); 132 | } 133 | job = undefined; 134 | } 135 | } 136 | catch (error) { 137 | this.emit(event_1.Event.Error, error, job); 138 | throw error; 139 | } 140 | return removedJobs; 141 | } 142 | /** @package */ 143 | async requestJobForProcessing(type, stillRequest) { 144 | // すでにジョブの作成を待っているリクエストがあれば、行列の末尾に足す 145 | if (this.waitingRequests[type] !== undefined && this.waitingRequests[type].length > 0) { 146 | return new Promise((resolve, reject) => { 147 | this.waitingRequests[type].push({ resolve, reject, stillRequest }); 148 | }); 149 | } 150 | // 同じジョブを多重処理しないように排他制御 151 | const releaseMutex = await this.requestJobForProcessingMutex.acquire(); 152 | try { 153 | const neDbJob = await this.repository.findInactiveJobByType(type); 154 | if (neDbJob === null) { 155 | if (this.waitingRequests[type] === undefined) { 156 | this.waitingRequests[type] = []; 157 | } 158 | return new Promise((resolve, reject) => { 159 | this.waitingRequests[type].push({ resolve, reject, stillRequest }); 160 | }); 161 | } 162 | if (stillRequest()) { 163 | const job = this.convertNeDbJobToJob(neDbJob); 164 | await job.setStateToActive(); 165 | return job; 166 | } 167 | else { 168 | return null; 169 | } 170 | } 171 | catch (error) { 172 | this.emit(event_1.Event.Error, error); 173 | throw error; 174 | } 175 | finally { 176 | releaseMutex(); 177 | } 178 | } 179 | /** @package */ 180 | async isExistJob(job) { 181 | return await this.repository.isExistJob(job.id); 182 | } 183 | /** @package */ 184 | async addJob(job) { 185 | try { 186 | const neDbJob = await this.repository.addJob(job); 187 | if (this.waitingRequests[job.type] === undefined) { 188 | return; 189 | } 190 | let processRequest = undefined; 191 | while (processRequest === undefined) { 192 | const headRequest = this.waitingRequests[job.type].shift(); 193 | if (headRequest === undefined) { 194 | break; 195 | } 196 | if (headRequest.stillRequest()) { 197 | processRequest = headRequest; 198 | } 199 | } 200 | if (processRequest === undefined) { 201 | return; 202 | } 203 | const addedJob = this.convertNeDbJobToJob(neDbJob); 204 | await addedJob.setStateToActive(); 205 | // eslint-disable-next-line @typescript-eslint/no-non-null-assertion 206 | process.nextTick(() => processRequest.resolve(addedJob)); 207 | } 208 | catch (error) { 209 | this.emit(event_1.Event.Error, error, job); 210 | throw error; 211 | } 212 | } 213 | /** @package */ 214 | async updateJob(job) { 215 | try { 216 | return await this.repository.updateJob(job); 217 | } 218 | catch (error) { 219 | this.emit(event_1.Event.Error, error, job); 220 | throw error; 221 | } 222 | } 223 | /** @package */ 224 | async removeJob(job) { 225 | try { 226 | return await this.repository.removeJob(job.id); 227 | } 228 | catch (error) { 229 | this.emit(event_1.Event.Error, error, job); 230 | throw error; 231 | } 232 | } 233 | async cleanupAfterUnexpectedlyTermination() { 234 | const jobsNeedCleanup = await this.listJobs(state_1.State.ACTIVE); 235 | for (const job of jobsNeedCleanup) { 236 | await job.setStateToFailure(new Error("unexpectedly termination")); 237 | } 238 | } 239 | convertNeDbJobToJob(neDbJob) { 240 | return new job_1.Job({ 241 | queue: this, 242 | id: neDbJob._id, 243 | type: neDbJob.type, 244 | priority: Queue.sanitizePriority(neDbJob.priority), 245 | data: neDbJob.data, 246 | createdAt: neDbJob.createdAt, 247 | updatedAt: neDbJob.updatedAt, 248 | startedAt: neDbJob.startedAt, 249 | completedAt: neDbJob.completedAt, 250 | failedAt: neDbJob.failedAt, 251 | state: neDbJob.state, 252 | duration: neDbJob.duration, 253 | progress: neDbJob.progress, 254 | logs: neDbJob.logs, 255 | saved: true, 256 | }); 257 | } 258 | } 259 | exports.Queue = Queue; 260 | -------------------------------------------------------------------------------- /dist/state.d.ts: -------------------------------------------------------------------------------- 1 | export declare enum State { 2 | INACTIVE = "INACTIVE", 3 | ACTIVE = "ACTIVE", 4 | COMPLETE = "COMPLETE", 5 | FAILURE = "FAILURE" 6 | } 7 | -------------------------------------------------------------------------------- /dist/state.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.State = void 0; 4 | var State; 5 | (function (State) { 6 | State["INACTIVE"] = "INACTIVE"; 7 | State["ACTIVE"] = "ACTIVE"; 8 | State["COMPLETE"] = "COMPLETE"; 9 | State["FAILURE"] = "FAILURE"; 10 | })(State = exports.State || (exports.State = {})); 11 | -------------------------------------------------------------------------------- /dist/worker.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | import { Job } from "./job"; 3 | import { Processor, Queue } from "./queue"; 4 | interface WorkerConstructorData { 5 | type: string; 6 | queue: Queue; 7 | } 8 | interface ShutdownInfo { 9 | timer: NodeJS.Timeout; 10 | resolve: () => void; 11 | } 12 | export declare class Worker { 13 | readonly type: string; 14 | protected readonly queue: Queue; 15 | protected shutdownInfo: ShutdownInfo | null; 16 | protected _isRunning: boolean; 17 | protected _currentJob: Job | null; 18 | get isRunning(): boolean; 19 | get currentJob(): Job | null; 20 | constructor(data: WorkerConstructorData); 21 | start(processor: Processor): void; 22 | shutdown(timeoutMilliseconds: number): Promise; 23 | protected startInternal(processor: Processor): void; 24 | protected process(processor: Processor): Promise; 25 | } 26 | export {}; 27 | -------------------------------------------------------------------------------- /dist/worker.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.Worker = void 0; 4 | class Worker { 5 | constructor(data) { 6 | this.shutdownInfo = null; 7 | // tslint:disable:variable-name 8 | this._isRunning = false; 9 | this._currentJob = null; 10 | this.type = data.type; 11 | this.queue = data.queue; 12 | } 13 | // tslint:disable:variable-name 14 | // noinspection JSUnusedGlobalSymbols 15 | get isRunning() { 16 | return this._isRunning; 17 | } 18 | // noinspection JSUnusedGlobalSymbols 19 | get currentJob() { 20 | return this._currentJob; 21 | } 22 | start(processor) { 23 | this._isRunning = true; 24 | this.startInternal(processor); 25 | } 26 | shutdown(timeoutMilliseconds) { 27 | return new Promise((resolve) => { 28 | // 実行中でなければ、何もしないで終了 29 | if (this._isRunning === false) { 30 | resolve(); 31 | return; 32 | } 33 | // 非実行状態に移行 34 | this._isRunning = false; 35 | // 処理中のジョブがなければ、シャットダウン完了 36 | if (this._currentJob === null) { 37 | resolve(); 38 | return; 39 | } 40 | // タイムアウトまでに処理中のジョブが完了しなければジョブを失敗にする 41 | this.shutdownInfo = { 42 | timer: setTimeout(async () => { 43 | // istanbul ignore if 44 | if (this._currentJob === null) { 45 | console.warn(`this._currentJob is null`); 46 | return; 47 | } 48 | await this._currentJob.setStateToFailure(new Error("shutdown timeout")); 49 | this._currentJob = null; 50 | if (this.shutdownInfo !== null) { 51 | this.shutdownInfo.resolve(); 52 | this.shutdownInfo = null; 53 | } 54 | }, timeoutMilliseconds), 55 | resolve, 56 | }; 57 | }); 58 | } 59 | startInternal(processor) { 60 | // 実行中じゃなければシャットダウンが進行中なので、処理を中断する 61 | // Note: この処理は本当は下に書きたいのだけど、TypeScriptの型認識が間違ってしまうため、ここに書いている 62 | if (this._isRunning === false) { 63 | if (this.shutdownInfo !== null) { 64 | clearTimeout(this.shutdownInfo.timer); 65 | this.shutdownInfo.resolve(); 66 | this.shutdownInfo = null; 67 | } 68 | this._currentJob = null; 69 | return; 70 | } 71 | (async () => { 72 | this._currentJob = await this.queue.requestJobForProcessing(this.type, () => this._isRunning); 73 | // 実行中じゃなければシャットダウンが進行中なので、処理を中断する 74 | if (this._isRunning === false) { 75 | // this._isRunningがfalseの場合、this.queue.requestProcessJobはnullを返すことになっている 76 | if (this._currentJob !== null) { 77 | console.warn(`this._currentJob is not null`); 78 | } 79 | this._currentJob = null; 80 | return; 81 | } 82 | await this.process(processor); 83 | // Note: 上の処理は本当はここに書きたい 84 | this.startInternal(processor); 85 | })(); 86 | } 87 | async process(processor) { 88 | // istanbul ignore if 89 | if (this._currentJob === null) { 90 | console.warn(`this._currentJob is null`); 91 | return; 92 | } 93 | let result; 94 | try { 95 | result = await processor(this._currentJob); 96 | } 97 | catch (error) { 98 | if (error instanceof Error) { 99 | await this._currentJob.setStateToFailure(error); 100 | } 101 | else { 102 | await this._currentJob.setStateToFailure(new Error("Processor is failed, and non error object is thrown.")); 103 | } 104 | this._currentJob = null; 105 | return; 106 | } 107 | if (this._currentJob === null) { 108 | return; 109 | } 110 | if (await this._currentJob.isExist() === false) { 111 | this._currentJob = null; 112 | return; 113 | } 114 | await this._currentJob.setStateToComplete(result); 115 | this._currentJob = null; 116 | } 117 | } 118 | exports.Worker = Worker; 119 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: "ts-jest", 3 | testEnvironment: "node", 4 | moduleFileExtensions: [ 5 | "js", 6 | "ts", 7 | ], 8 | collectCoverageFrom: [ 9 | "src/**/*.ts", 10 | ], 11 | }; 12 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "embedded-queue", 3 | "version": "0.0.11", 4 | "description": "Embedded job/message queue, NeDb backend.", 5 | "main": "dist/index.js", 6 | "license": "MIT", 7 | "keywords": [ 8 | "queue", 9 | "electron", 10 | "nedb" 11 | ], 12 | "author": { 13 | "name": "Ken'ichi Saito", 14 | "email": "hajime@studiohff.net" 15 | }, 16 | "dependencies": { 17 | "await-semaphore": "0.1.x", 18 | "nedb": "^1.8.0", 19 | "uuid": "8.x" 20 | }, 21 | "devDependencies": { 22 | "@types/jest": "27.x", 23 | "@types/nedb": "1.8.x", 24 | "@types/node": "*", 25 | "@types/uuid": "8.x", 26 | "@typescript-eslint/eslint-plugin": "5.x", 27 | "@typescript-eslint/parser": "5.x", 28 | "eslint": "8.x", 29 | "eslint-plugin-jest": "25.x", 30 | "jest": "27.x", 31 | "jest-mock-extended": "2.x", 32 | "ts-jest": "27.x", 33 | "typescript": "4.5.x" 34 | }, 35 | "scripts": { 36 | "prepare": "tsc", 37 | "test": "jest", 38 | "lint": "eslint . --ext .ts" 39 | }, 40 | "repository": { 41 | "type": "git", 42 | "url": "https://github.com/hajipy/embedded-queue.git" 43 | }, 44 | "bugs": { 45 | "url": "https://github.com/hajipy/embedded-queue/issues" 46 | }, 47 | "homepage": "https://github.com/hajipy/embedded-queue#readme", 48 | "resolutions": { 49 | "underscore": "^1.12.1" 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/event.ts: -------------------------------------------------------------------------------- 1 | export enum Event { 2 | Enqueue = "enqueue", 3 | Start = "start", 4 | Failure = "failure", 5 | Complete = "complete", 6 | Remove = "remove", 7 | Error = "error", 8 | Progress = "progress", 9 | Log = "log", 10 | Priority = "priority", 11 | } 12 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export { CreateJobData, Processor, Queue } from "./queue"; 2 | export { Worker } from "./worker"; 3 | export { Job } from "./job"; 4 | export { Priority, toString as priorityToString } from "./priority"; 5 | export { State } from "./state"; 6 | export { Event } from "./event"; 7 | -------------------------------------------------------------------------------- /src/job.ts: -------------------------------------------------------------------------------- 1 | import { Event } from "./event"; 2 | import { Priority } from "./priority"; 3 | import { Queue } from "./queue"; 4 | import { State } from "./state"; 5 | 6 | export interface JobConstructorData { 7 | queue: Queue; 8 | id: string; 9 | type: string; 10 | priority?: Priority; 11 | data?: unknown; 12 | createdAt: Date; 13 | updatedAt: Date; 14 | startedAt?: Date; 15 | completedAt?: Date; 16 | failedAt?: Date; 17 | state?: State; 18 | duration?: number; 19 | progress?: number; 20 | logs: string[]; 21 | saved: boolean; 22 | } 23 | 24 | export class Job { 25 | public readonly id: string; 26 | public readonly type: string; 27 | public readonly data: unknown | undefined; 28 | 29 | protected readonly queue: Queue; 30 | 31 | // tslint:disable:variable-name 32 | protected _priority: Priority; 33 | protected _createdAt: Date; 34 | protected _updatedAt: Date; 35 | protected _startedAt: Date | undefined; 36 | protected _completedAt: Date | undefined; 37 | protected _failedAt: Date | undefined; 38 | protected _state: State; 39 | protected _duration: number | undefined; 40 | protected _progress: number | undefined; 41 | protected _logs: string[]; 42 | protected _saved: boolean; 43 | // tslint:enable:variable-name 44 | 45 | public get priority(): Priority { 46 | return this._priority; 47 | } 48 | 49 | public get createdAt(): Date { 50 | return this._createdAt; 51 | } 52 | 53 | public get updatedAt(): Date { 54 | return this._updatedAt; 55 | } 56 | 57 | // noinspection JSUnusedGlobalSymbols 58 | public get startedAt(): Date | undefined { 59 | return this._startedAt; 60 | } 61 | 62 | // noinspection JSUnusedGlobalSymbols 63 | public get completedAt(): Date | undefined { 64 | return this._completedAt; 65 | } 66 | 67 | // noinspection JSUnusedGlobalSymbols 68 | public get failedAt(): Date | undefined { 69 | return this._failedAt; 70 | } 71 | 72 | public get state(): State { 73 | return this._state; 74 | } 75 | 76 | public get duration(): number | undefined { 77 | return this._duration; 78 | } 79 | 80 | public get progress(): number | undefined { 81 | return this._progress; 82 | } 83 | 84 | public get logs(): string[] { 85 | return [...this._logs]; 86 | } 87 | 88 | public constructor(data: JobConstructorData) { 89 | this.queue = data.queue; 90 | 91 | this.id = data.id; 92 | this.type = data.type; 93 | this._priority = data.priority || Priority.NORMAL; 94 | this.data = data.data; 95 | this._state = data.state || State.INACTIVE; 96 | this._createdAt = data.createdAt; 97 | this._updatedAt = data.updatedAt; 98 | this._startedAt = data.startedAt; 99 | this._completedAt = data.completedAt; 100 | this._failedAt = data.failedAt; 101 | this._logs = [...data.logs]; 102 | this._duration = data.duration; 103 | this._progress = data.progress; 104 | this._saved = data.saved; 105 | } 106 | 107 | public async setProgress(completed: number, total: number): Promise { 108 | this._progress = completed * 100 / total; 109 | this._updatedAt = new Date(); 110 | 111 | await this.update(); 112 | 113 | this.queue.emit(Event.Progress, this, this._progress); 114 | } 115 | 116 | public async addLog(message: string): Promise { 117 | this._logs.push(message); 118 | 119 | this._updatedAt = new Date(); 120 | 121 | await this.update(); 122 | 123 | this.queue.emit(Event.Log, this, message); 124 | } 125 | 126 | public async save(): Promise { 127 | if (this._saved) { 128 | await this.update(); 129 | } 130 | else { 131 | await this.queue.addJob(this); 132 | this._saved = true; 133 | 134 | this.queue.emit(Event.Enqueue, this); 135 | } 136 | 137 | return this; 138 | } 139 | 140 | public async remove(): Promise { 141 | await this.queue.removeJob(this); 142 | 143 | this.queue.emit(Event.Remove, this); 144 | } 145 | 146 | public async setPriority(value: Priority): Promise { 147 | this._priority = value; 148 | 149 | this._updatedAt = new Date(); 150 | 151 | await this.update(); 152 | 153 | this.queue.emit(Event.Priority, this, value); 154 | } 155 | 156 | public async isExist(): Promise { 157 | return this.queue.isExistJob(this); 158 | } 159 | 160 | /** @package */ 161 | public async setStateToActive(): Promise { 162 | this._state = State.ACTIVE; 163 | 164 | const now = new Date(); 165 | this._startedAt = now; 166 | this._updatedAt = now; 167 | 168 | await this.update(); 169 | 170 | this.queue.emit(Event.Start, this); 171 | } 172 | 173 | /** @package */ 174 | public async setStateToComplete(result?: unknown): Promise { 175 | this._state = State.COMPLETE; 176 | 177 | const now = new Date(); 178 | this._completedAt = now; 179 | if (this._startedAt !== undefined) { 180 | this._duration = now.getTime() - this._startedAt.getTime(); 181 | } 182 | this._updatedAt = now; 183 | 184 | await this.update(); 185 | 186 | this.queue.emit(Event.Complete, this, result); 187 | } 188 | 189 | /** @package */ 190 | public async setStateToFailure(error: Error): Promise { 191 | this._state = State.FAILURE; 192 | 193 | const now = new Date(); 194 | this._failedAt = now; 195 | this._updatedAt = now; 196 | 197 | await this.update(); 198 | 199 | this.queue.emit(Event.Failure, this, error); 200 | } 201 | 202 | protected async update(): Promise { 203 | await this.queue.updateJob(this); 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /src/jobRepository.ts: -------------------------------------------------------------------------------- 1 | import DataStore, { DataStoreOptions } from "nedb"; 2 | 3 | import { Job } from "./job"; 4 | import { State } from "./state"; 5 | 6 | export interface NeDbJob { 7 | _id: string; 8 | type: string; 9 | priority: number; 10 | data?: unknown; 11 | createdAt: Date; 12 | updatedAt: Date; 13 | startedAt?: Date; 14 | completedAt?: Date; 15 | failedAt?: Date; 16 | state?: State; 17 | duration?: number; 18 | progress?: number; 19 | logs: string[]; 20 | } 21 | 22 | export type DbOptions = DataStoreOptions; 23 | 24 | export class JobRepository { 25 | protected readonly db: DataStore; 26 | 27 | public constructor(dbOptions: DbOptions = {}) { 28 | this.db = new DataStore(dbOptions); 29 | } 30 | 31 | public init(): Promise { 32 | return new Promise((resolve, reject) => { 33 | this.db.loadDatabase((error) => { 34 | if (error !== null) { 35 | reject(error); 36 | return; 37 | } 38 | 39 | resolve(); 40 | }); 41 | }); 42 | } 43 | 44 | public listJobs(state?: State): Promise { 45 | return new Promise((resolve, reject) => { 46 | const query = (state === undefined) ? {} : { state }; 47 | 48 | this.db.find(query) 49 | .sort({ createdAt: 1 }) 50 | .exec((error, docs: NeDbJob[]) => { 51 | if (error !== null) { 52 | reject(error); 53 | return; 54 | } 55 | 56 | resolve(docs); 57 | }); 58 | }); 59 | } 60 | 61 | public findJob(id: string): Promise { 62 | return new Promise((resolve, reject) => { 63 | this.db.findOne({ _id: id }, (error, doc: NeDbJob| null) => { 64 | if (error !== null) { 65 | reject(error); 66 | return; 67 | } 68 | 69 | resolve(doc); 70 | }); 71 | }); 72 | } 73 | 74 | public findInactiveJobByType(type: string): Promise { 75 | return new Promise((resolve, reject) => { 76 | this.db.find({ type, state: State.INACTIVE }) 77 | .sort({ priority: -1, createdAt: 1 }) 78 | .limit(1) 79 | .exec((error, docs: NeDbJob[]) => { 80 | if (error !== null) { 81 | reject(error); 82 | return; 83 | } 84 | 85 | resolve((docs.length === 0) ? null : docs[0]); 86 | }); 87 | }); 88 | } 89 | 90 | public isExistJob(id: string): Promise { 91 | return new Promise((resolve, reject) => { 92 | this.db.count({ _id: id }, (error, count: number) => { 93 | if (error !== null) { 94 | reject(error); 95 | return; 96 | } 97 | 98 | resolve(count === 1); 99 | }); 100 | }); 101 | } 102 | 103 | public addJob(job: Job): Promise { 104 | return new Promise((resolve, reject) => { 105 | const insertDoc = { 106 | _id: job.id, 107 | type: job.type, 108 | priority: job.priority, 109 | data: job.data, 110 | createdAt: job.createdAt, 111 | updatedAt: job.updatedAt, 112 | state: job.state, 113 | logs: job.logs, 114 | }; 115 | 116 | this.db.insert(insertDoc, (error, doc) => { 117 | if (error !== null) { 118 | reject(error); 119 | return; 120 | } 121 | 122 | resolve(doc); 123 | }); 124 | }); 125 | } 126 | 127 | public updateJob(job: Job): Promise { 128 | return new Promise((resolve, reject) => { 129 | const query = { 130 | _id: job.id, 131 | }; 132 | const updateQuery = { 133 | $set: { 134 | priority: job.priority, 135 | data: job.data, 136 | createdAt: job.createdAt, 137 | updatedAt: job.updatedAt, 138 | startedAt: job.startedAt, 139 | completedAt: job.completedAt, 140 | failedAt: job.failedAt, 141 | state: job.state, 142 | duration: job.duration, 143 | progress: job.progress, 144 | logs: job.logs, 145 | }, 146 | }; 147 | 148 | this.db.update(query, updateQuery, {}, (error, numAffected) => { 149 | if (error !== null) { 150 | reject(error); 151 | return; 152 | } 153 | 154 | if (numAffected !== 1) { 155 | reject(new Error(`update unexpected number of rows. (expected: 1, actual: ${numAffected})`)); 156 | } 157 | 158 | resolve(); 159 | }); 160 | }); 161 | } 162 | 163 | public removeJob(id: string): Promise { 164 | return new Promise((resolve, reject) => { 165 | this.db.remove({ _id: id }, (error) => { 166 | if (error) { 167 | reject(error); 168 | return; 169 | } 170 | 171 | resolve(); 172 | }); 173 | }); 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /src/priority.ts: -------------------------------------------------------------------------------- 1 | export enum Priority { 2 | LOW = 1, 3 | NORMAL, 4 | MEDIUM, 5 | HIGH, 6 | CRITICAL, 7 | } 8 | 9 | export function toString(priority: Priority): string { 10 | switch (priority) { 11 | case Priority.LOW: 12 | return "LOW"; 13 | case Priority.NORMAL: 14 | return "NORMAL"; 15 | case Priority.MEDIUM: 16 | return "MEDIUM"; 17 | case Priority.HIGH: 18 | return "HIGH"; 19 | case Priority.CRITICAL: 20 | return "CRITICAL"; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/queue.ts: -------------------------------------------------------------------------------- 1 | import { EventEmitter } from "events"; 2 | 3 | import { Mutex } from "await-semaphore"; 4 | import { v4 as uuid } from "uuid"; 5 | 6 | import { Event } from "./event"; 7 | import { Job } from "./job"; 8 | import { DbOptions, JobRepository, NeDbJob } from "./jobRepository"; 9 | import { Priority } from "./priority"; 10 | import { State } from "./state"; 11 | import { Worker } from "./worker"; 12 | 13 | export interface CreateJobData { 14 | type: string; 15 | priority?: Priority; 16 | data?: unknown; 17 | } 18 | 19 | export type Processor = (job: Job) => Promise; 20 | 21 | interface WaitingWorkerRequest { 22 | resolve: (value: Job) => void; 23 | reject: (error: Error) => void; 24 | stillRequest: () => boolean; 25 | } 26 | 27 | export class Queue extends EventEmitter { 28 | public static async createQueue(dbOptions?: DbOptions): Promise { 29 | const queue = new Queue(dbOptions); 30 | 31 | await queue.repository.init(); 32 | 33 | await queue.cleanupAfterUnexpectedlyTermination(); 34 | 35 | return queue; 36 | } 37 | 38 | protected static sanitizePriority(priority: number): Priority { 39 | switch (priority) { 40 | case Priority.LOW: 41 | case Priority.NORMAL: 42 | case Priority.MEDIUM: 43 | case Priority.HIGH: 44 | case Priority.CRITICAL: 45 | return priority; 46 | } 47 | 48 | console.warn(`Invalid Priority: ${priority}`); 49 | return Priority.NORMAL; 50 | } 51 | 52 | protected readonly repository: JobRepository; 53 | 54 | // tslint:disable:variable-name 55 | protected _workers: Worker[]; 56 | // tslint:disable:variable-name 57 | 58 | protected waitingRequests: { [type: string]: WaitingWorkerRequest[] }; 59 | 60 | protected requestJobForProcessingMutex: Mutex; 61 | 62 | public get workers(): Worker[] { 63 | return [...this._workers]; 64 | } 65 | 66 | protected constructor(dbOptions?: DbOptions) { 67 | super(); 68 | 69 | this.repository = new JobRepository(dbOptions); 70 | this._workers = []; 71 | this.waitingRequests = {}; 72 | this.requestJobForProcessingMutex = new Mutex(); 73 | } 74 | 75 | public async createJob(data: CreateJobData): Promise { 76 | const now = new Date(); 77 | 78 | const job = new Job( 79 | Object.assign( 80 | {}, 81 | data, 82 | { 83 | queue: this, 84 | id: uuid(), 85 | createdAt: now, 86 | updatedAt: now, 87 | logs: [], 88 | saved: false, 89 | } 90 | ) 91 | ); 92 | 93 | return await job.save(); 94 | } 95 | 96 | public process(type: string, processor: Processor, concurrency: number): void { 97 | for (let i = 0; i < concurrency; i++) { 98 | const worker = new Worker({ 99 | type, 100 | queue: this, 101 | }); 102 | 103 | worker.start(processor); 104 | 105 | this._workers.push(worker); 106 | } 107 | } 108 | 109 | public async shutdown(timeoutMilliseconds: number, type?: string | undefined): Promise { 110 | const shutdownWorkers: Worker[] = []; 111 | 112 | for (const worker of this._workers) { 113 | if (type !== undefined && worker.type !== type) { 114 | continue; 115 | } 116 | 117 | await worker.shutdown(timeoutMilliseconds); 118 | 119 | shutdownWorkers.push(worker); 120 | } 121 | 122 | this._workers = this._workers.filter( 123 | (worker) => { 124 | return shutdownWorkers.includes(worker) === false; 125 | } 126 | ); 127 | } 128 | 129 | public async findJob(id: string): Promise { 130 | try { 131 | const neDbJob = await this.repository.findJob(id); 132 | 133 | if (neDbJob === null) { 134 | return null; 135 | } 136 | 137 | return this.convertNeDbJobToJob(neDbJob); 138 | } 139 | catch (error) { 140 | this.emit(Event.Error, error); 141 | throw error; 142 | } 143 | } 144 | 145 | public async listJobs(state?: State): Promise { 146 | try { 147 | return await this.repository.listJobs(state).then((docs) => { 148 | return docs.map((neDbJob) => this.convertNeDbJobToJob(neDbJob)); 149 | }); 150 | } 151 | catch (error) { 152 | this.emit(Event.Error, error); 153 | throw error; 154 | } 155 | } 156 | 157 | public async removeJobById(id: string): Promise { 158 | let neDbJob: NeDbJob | null; 159 | try { 160 | neDbJob = await this.repository.findJob(id); 161 | } 162 | catch (error) { 163 | this.emit(Event.Error, error); 164 | throw error; 165 | } 166 | 167 | if (neDbJob === null) { 168 | throw new Error(`Job(id:${id}) is not found.`); 169 | } 170 | 171 | const job = this.convertNeDbJobToJob(neDbJob); 172 | 173 | try { 174 | return await job.remove(); 175 | } 176 | catch (error) { 177 | this.emit(Event.Error, error, job); 178 | throw error; 179 | } 180 | } 181 | 182 | public async removeJobsByCallback(callback: (job: Job) => boolean): Promise { 183 | const removedJobs: Job[] = []; 184 | 185 | let job: Job | undefined; 186 | 187 | try { 188 | const neDbJobs = await this.repository.listJobs(); 189 | 190 | for (const neDbJob of neDbJobs) { 191 | job = this.convertNeDbJobToJob(neDbJob); 192 | 193 | if (callback(job)) { 194 | removedJobs.push(job); 195 | await job.remove(); 196 | } 197 | 198 | job = undefined; 199 | } 200 | } 201 | catch (error) { 202 | this.emit(Event.Error, error, job); 203 | throw error; 204 | } 205 | 206 | return removedJobs; 207 | } 208 | 209 | /** @package */ 210 | public async requestJobForProcessing(type: string, stillRequest: () => boolean): Promise { 211 | // すでにジョブの作成を待っているリクエストがあれば、行列の末尾に足す 212 | if (this.waitingRequests[type] !== undefined && this.waitingRequests[type].length > 0) { 213 | return new Promise((resolve, reject) => { 214 | this.waitingRequests[type].push({ resolve, reject, stillRequest }); 215 | }); 216 | } 217 | 218 | // 同じジョブを多重処理しないように排他制御 219 | const releaseMutex = await this.requestJobForProcessingMutex.acquire(); 220 | try { 221 | const neDbJob = await this.repository.findInactiveJobByType(type); 222 | 223 | if (neDbJob === null) { 224 | if (this.waitingRequests[type] === undefined) { 225 | this.waitingRequests[type] = []; 226 | } 227 | 228 | return new Promise((resolve, reject) => { 229 | this.waitingRequests[type].push({ resolve, reject, stillRequest }); 230 | }); 231 | } 232 | 233 | if (stillRequest()) { 234 | const job = this.convertNeDbJobToJob(neDbJob); 235 | 236 | await job.setStateToActive(); 237 | 238 | return job; 239 | } 240 | else { 241 | return null; 242 | } 243 | } 244 | catch (error) { 245 | this.emit(Event.Error, error); 246 | throw error; 247 | } 248 | finally { 249 | releaseMutex(); 250 | } 251 | } 252 | 253 | /** @package */ 254 | public async isExistJob(job: Job): Promise { 255 | return await this.repository.isExistJob(job.id); 256 | } 257 | 258 | /** @package */ 259 | public async addJob(job: Job): Promise { 260 | try { 261 | const neDbJob = await this.repository.addJob(job); 262 | 263 | if (this.waitingRequests[job.type] === undefined) { 264 | return; 265 | } 266 | 267 | let processRequest: WaitingWorkerRequest | undefined = undefined; 268 | while (processRequest === undefined) { 269 | const headRequest = this.waitingRequests[job.type].shift(); 270 | 271 | if (headRequest === undefined) { 272 | break; 273 | } 274 | 275 | if (headRequest.stillRequest()) { 276 | processRequest = headRequest; 277 | } 278 | } 279 | 280 | if (processRequest === undefined) { 281 | return; 282 | } 283 | 284 | const addedJob = this.convertNeDbJobToJob(neDbJob); 285 | 286 | await addedJob.setStateToActive(); 287 | 288 | // eslint-disable-next-line @typescript-eslint/no-non-null-assertion 289 | process.nextTick(() => processRequest!.resolve(addedJob)); 290 | } 291 | catch (error) { 292 | this.emit(Event.Error, error, job); 293 | throw error; 294 | } 295 | } 296 | 297 | /** @package */ 298 | public async updateJob(job: Job): Promise { 299 | try { 300 | return await this.repository.updateJob(job); 301 | } 302 | catch (error) { 303 | this.emit(Event.Error, error, job); 304 | throw error; 305 | } 306 | } 307 | 308 | /** @package */ 309 | public async removeJob(job: Job): Promise { 310 | try { 311 | return await this.repository.removeJob(job.id); 312 | } 313 | catch (error) { 314 | this.emit(Event.Error, error, job); 315 | throw error; 316 | } 317 | } 318 | 319 | protected async cleanupAfterUnexpectedlyTermination(): Promise { 320 | const jobsNeedCleanup = await this.listJobs(State.ACTIVE); 321 | 322 | for (const job of jobsNeedCleanup) { 323 | await job.setStateToFailure(new Error("unexpectedly termination")); 324 | } 325 | } 326 | 327 | protected convertNeDbJobToJob(neDbJob: NeDbJob): Job { 328 | return new Job({ 329 | queue: this, 330 | id: neDbJob._id, 331 | type: neDbJob.type, 332 | priority: Queue.sanitizePriority(neDbJob.priority), 333 | data: neDbJob.data, 334 | createdAt: neDbJob.createdAt, 335 | updatedAt: neDbJob.updatedAt, 336 | startedAt: neDbJob.startedAt, 337 | completedAt: neDbJob.completedAt, 338 | failedAt: neDbJob.failedAt, 339 | state: neDbJob.state, 340 | duration: neDbJob.duration, 341 | progress: neDbJob.progress, 342 | logs: neDbJob.logs, 343 | saved: true, 344 | }) 345 | } 346 | } 347 | -------------------------------------------------------------------------------- /src/state.ts: -------------------------------------------------------------------------------- 1 | export enum State { 2 | INACTIVE = "INACTIVE", 3 | ACTIVE = "ACTIVE", 4 | COMPLETE = "COMPLETE", 5 | FAILURE = "FAILURE", 6 | } 7 | -------------------------------------------------------------------------------- /src/worker.ts: -------------------------------------------------------------------------------- 1 | import { Job } from "./job"; 2 | import { Processor, Queue } from "./queue"; 3 | 4 | interface WorkerConstructorData { 5 | type: string; 6 | queue: Queue; 7 | } 8 | 9 | interface ShutdownInfo { 10 | timer: NodeJS.Timeout; 11 | resolve: () => void; 12 | } 13 | 14 | export class Worker { 15 | public readonly type: string; 16 | 17 | protected readonly queue: Queue; 18 | protected shutdownInfo: ShutdownInfo | null = null; 19 | 20 | // tslint:disable:variable-name 21 | protected _isRunning = false; 22 | protected _currentJob: Job | null = null; 23 | // tslint:disable:variable-name 24 | 25 | // noinspection JSUnusedGlobalSymbols 26 | public get isRunning(): boolean { 27 | return this._isRunning; 28 | } 29 | 30 | // noinspection JSUnusedGlobalSymbols 31 | public get currentJob(): Job | null { 32 | return this._currentJob; 33 | } 34 | 35 | public constructor(data: WorkerConstructorData) { 36 | this.type = data.type; 37 | this.queue = data.queue; 38 | } 39 | 40 | public start(processor: Processor): void { 41 | this._isRunning = true; 42 | 43 | this.startInternal(processor); 44 | } 45 | 46 | public shutdown(timeoutMilliseconds: number): Promise { 47 | return new Promise((resolve) => { 48 | // 実行中でなければ、何もしないで終了 49 | if (this._isRunning === false) { 50 | resolve(); 51 | return; 52 | } 53 | 54 | // 非実行状態に移行 55 | this._isRunning = false; 56 | 57 | // 処理中のジョブがなければ、シャットダウン完了 58 | if (this._currentJob === null) { 59 | resolve(); 60 | return; 61 | } 62 | 63 | // タイムアウトまでに処理中のジョブが完了しなければジョブを失敗にする 64 | this.shutdownInfo = { 65 | timer: setTimeout(async () => { 66 | // istanbul ignore if 67 | if (this._currentJob === null) { 68 | console.warn(`this._currentJob is null`); 69 | return; 70 | } 71 | 72 | await this._currentJob.setStateToFailure(new Error("shutdown timeout")); 73 | this._currentJob = null; 74 | 75 | if (this.shutdownInfo !== null) { 76 | this.shutdownInfo.resolve(); 77 | this.shutdownInfo = null; 78 | } 79 | }, timeoutMilliseconds), 80 | resolve, 81 | }; 82 | }); 83 | } 84 | 85 | protected startInternal(processor: Processor): void { 86 | // 実行中じゃなければシャットダウンが進行中なので、処理を中断する 87 | // Note: この処理は本当は下に書きたいのだけど、TypeScriptの型認識が間違ってしまうため、ここに書いている 88 | if (this._isRunning === false) { 89 | if (this.shutdownInfo !== null) { 90 | clearTimeout(this.shutdownInfo.timer); 91 | this.shutdownInfo.resolve(); 92 | this.shutdownInfo = null; 93 | } 94 | this._currentJob = null; 95 | return; 96 | } 97 | 98 | (async (): Promise => { 99 | this._currentJob = await this.queue.requestJobForProcessing(this.type, () => this._isRunning); 100 | 101 | // 実行中じゃなければシャットダウンが進行中なので、処理を中断する 102 | if (this._isRunning === false) { 103 | // this._isRunningがfalseの場合、this.queue.requestProcessJobはnullを返すことになっている 104 | if (this._currentJob !== null) { 105 | console.warn(`this._currentJob is not null`); 106 | } 107 | 108 | this._currentJob = null; 109 | return; 110 | } 111 | 112 | await this.process(processor); 113 | 114 | // Note: 上の処理は本当はここに書きたい 115 | 116 | this.startInternal(processor); 117 | })(); 118 | } 119 | 120 | protected async process(processor: Processor): Promise { 121 | // istanbul ignore if 122 | if (this._currentJob === null) { 123 | console.warn(`this._currentJob is null`); 124 | return; 125 | } 126 | 127 | let result: unknown; 128 | 129 | try { 130 | result = await processor(this._currentJob); 131 | } 132 | catch (error) { 133 | if (error instanceof Error) { 134 | await this._currentJob.setStateToFailure(error); 135 | } 136 | else { 137 | await this._currentJob.setStateToFailure( 138 | new Error("Processor is failed, and non error object is thrown.") 139 | ); 140 | } 141 | this._currentJob = null; 142 | return; 143 | } 144 | 145 | if (this._currentJob === null) { 146 | return; 147 | } 148 | 149 | if (await this._currentJob.isExist() === false) { 150 | this._currentJob = null; 151 | return; 152 | } 153 | 154 | await this._currentJob.setStateToComplete(result); 155 | 156 | this._currentJob = null; 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /test/JobRepository.test.ts: -------------------------------------------------------------------------------- 1 | import { mock } from "jest-mock-extended"; 2 | import DataStore from "nedb"; 3 | 4 | import { Job, Priority, Queue, State } from "../src"; 5 | import { JobRepository, NeDbJob } from "../src/jobRepository"; 6 | 7 | function dbFind(db: DataStore, _id: string): Promise { 8 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 9 | return new Promise((resolve, reject) => { 10 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 11 | db.findOne({ _id }, (error, doc: NeDbJob | null) => { 12 | if (error !== null) { 13 | reject(error); 14 | } 15 | else { 16 | resolve(doc); 17 | } 18 | }); 19 | }); 20 | } 21 | 22 | function dbInsert(db: DataStore, doc: unknown): Promise { 23 | return new Promise(((resolve, reject) => { 24 | db.insert(doc, (error) => { 25 | if (error !== null) { 26 | reject(error); 27 | } 28 | else { 29 | resolve(); 30 | } 31 | }); 32 | })); 33 | } 34 | 35 | function dbRemove(db: DataStore, _id: string): Promise { 36 | return new Promise(((resolve, reject) => { 37 | db.remove({ _id }, (error) => { 38 | if (error !== null) { 39 | reject(error); 40 | } 41 | else { 42 | resolve(); 43 | } 44 | }); 45 | })); 46 | } 47 | 48 | describe("init", () => { 49 | test("Success", async () => { 50 | const repository = new JobRepository({ 51 | inMemoryOnly: true, 52 | }); 53 | 54 | await expect( 55 | repository.init() 56 | ).resolves.toBeUndefined(); 57 | }); 58 | 59 | test("Failure", async () => { 60 | const repository = new JobRepository({ 61 | filename: "/invalid/file/path", 62 | }); 63 | 64 | await expect( 65 | repository.init() 66 | ).rejects.toThrow("no such file or directory, open '/invalid/file/path'"); 67 | }); 68 | }); 69 | 70 | describe("listJobs", () => { 71 | test("no data", async () => { 72 | const repository = new JobRepository({ 73 | inMemoryOnly: true, 74 | }); 75 | await repository.init(); 76 | 77 | const jobs = await repository.listJobs(); 78 | 79 | expect(jobs).toHaveLength(0); 80 | }); 81 | 82 | test("has data", async () => { 83 | const repository = new JobRepository({ 84 | inMemoryOnly: true, 85 | }); 86 | await repository.init(); 87 | 88 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 89 | const db: DataStore = (repository as any).db; 90 | const id = "1"; 91 | const type = "type"; 92 | const priority = Priority.NORMAL; 93 | const data = { 94 | a: "aaa", 95 | b: 123, 96 | c: { 97 | x: true, 98 | y: { 99 | z: true, 100 | }, 101 | }, 102 | }; 103 | const createdAt = new Date(2020, 4, 1, 0, 0, 0); 104 | const updatedAt = new Date(2020, 4, 2, 0, 0, 0); 105 | const startedAt = new Date(2020, 4, 3, 0, 0, 0); 106 | const completedAt = new Date(2020, 4, 4, 0, 0, 0); 107 | const failedAt = new Date(2020, 4, 5, 0, 0, 0); 108 | const state = State.INACTIVE; 109 | const duration = 123; 110 | const progress = 1 / 3; 111 | const logs = [ 112 | "First Log", 113 | "Second Log", 114 | "Third Log", 115 | ]; 116 | 117 | await dbInsert( 118 | db, 119 | { 120 | _id: id, 121 | type, 122 | priority, 123 | data, 124 | createdAt, 125 | updatedAt, 126 | startedAt, 127 | completedAt, 128 | failedAt, 129 | state, 130 | duration, 131 | progress, 132 | logs, 133 | } 134 | ); 135 | 136 | const jobs = await repository.listJobs(); 137 | 138 | expect(jobs).toHaveLength(1); 139 | expect(jobs[0]).toEqual({ 140 | _id: id, 141 | type, 142 | priority, 143 | data, 144 | createdAt, 145 | updatedAt, 146 | startedAt, 147 | completedAt, 148 | failedAt, 149 | state, 150 | duration, 151 | progress, 152 | logs, 153 | }); 154 | }); 155 | 156 | test("sort", async () => { 157 | const repository = new JobRepository({ 158 | inMemoryOnly: true, 159 | }); 160 | await repository.init(); 161 | 162 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 163 | const db: DataStore = (repository as any).db; 164 | 165 | const createdAts = [ 166 | new Date(2020, 4, 2, 0, 0, 0), 167 | new Date(2020, 4, 1, 0, 0, 0), 168 | new Date(2020, 4, 3, 0, 0, 0), 169 | ]; 170 | 171 | for (const [index, createdAt] of createdAts.entries()) { 172 | await dbInsert( 173 | db, 174 | { 175 | _id: index.toString(), 176 | type: "type", 177 | priority: Priority.NORMAL, 178 | createdAt, 179 | updatedAt: new Date(), 180 | } 181 | ); 182 | } 183 | 184 | const jobs = await repository.listJobs(); 185 | 186 | expect(jobs).toHaveLength(3); 187 | expect(jobs[0]._id).toBe("1"); 188 | expect(jobs[0].createdAt).toBe(createdAts[1]); 189 | expect(jobs[1]._id).toBe("0"); 190 | expect(jobs[1].createdAt).toBe(createdAts[0]); 191 | expect(jobs[2]._id).toBe("2"); 192 | expect(jobs[2].createdAt).toBe(createdAts[2]); 193 | }); 194 | 195 | test("state", async () => { 196 | const repository = new JobRepository({ 197 | inMemoryOnly: true, 198 | }); 199 | await repository.init(); 200 | 201 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 202 | const db: DataStore = (repository as any).db; 203 | 204 | const states = [ 205 | State.INACTIVE, 206 | State.ACTIVE, 207 | State.COMPLETE, 208 | State.FAILURE, 209 | ]; 210 | 211 | for (const [index, state] of states.entries()) { 212 | await dbInsert( 213 | db, 214 | { 215 | _id: index.toString(), 216 | type: "type", 217 | priority: Priority.NORMAL, 218 | createdAt: new Date(), 219 | updatedAt: new Date(), 220 | state, 221 | } 222 | ); 223 | } 224 | 225 | const inactiveJobs = await repository.listJobs(State.INACTIVE); 226 | expect(inactiveJobs).toHaveLength(1); 227 | expect(inactiveJobs[0]._id).toBe("0"); 228 | expect(inactiveJobs[0].state).toBe(State.INACTIVE); 229 | 230 | const activeJobs = await repository.listJobs(State.ACTIVE); 231 | expect(activeJobs).toHaveLength(1); 232 | expect(activeJobs[0]._id).toBe("1"); 233 | expect(activeJobs[0].state).toBe(State.ACTIVE); 234 | 235 | const completeJobs = await repository.listJobs(State.COMPLETE); 236 | expect(completeJobs).toHaveLength(1); 237 | expect(completeJobs[0]._id).toBe("2"); 238 | expect(completeJobs[0].state).toBe(State.COMPLETE); 239 | 240 | const failureJobs = await repository.listJobs(State.FAILURE); 241 | expect(failureJobs).toHaveLength(1); 242 | expect(failureJobs[0]._id).toBe("3"); 243 | expect(failureJobs[0].state).toBe(State.FAILURE); 244 | }); 245 | }); 246 | 247 | describe("findJob", () => { 248 | const repository = new JobRepository({ 249 | inMemoryOnly: true, 250 | }); 251 | 252 | beforeAll(async () => { 253 | await repository.init(); 254 | 255 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 256 | const db: DataStore = (repository as any).db; 257 | 258 | for (let i = 1; i <= 3; i++) { 259 | await dbInsert( 260 | db, 261 | { 262 | _id: i.toString(), 263 | type: "type", 264 | priority: Priority.NORMAL, 265 | createdAt: new Date(), 266 | updatedAt: new Date(), 267 | } 268 | ); 269 | } 270 | }); 271 | 272 | test("found", async () => { 273 | const job = await repository.findJob("1"); 274 | expect(job).not.toBeNull(); 275 | // eslint-disable-next-line @typescript-eslint/no-non-null-assertion 276 | expect(job!._id).toBe("1"); 277 | }); 278 | 279 | test("not found", async () => { 280 | const job = await repository.findJob("4"); 281 | expect(job).toBeNull(); 282 | }); 283 | }); 284 | 285 | describe("findInactiveJobByType", () => { 286 | test("found", async () => { 287 | const repository = new JobRepository({ 288 | inMemoryOnly: true, 289 | }); 290 | await repository.init(); 291 | 292 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 293 | const db: DataStore = (repository as any).db; 294 | 295 | await dbInsert( 296 | db, 297 | { 298 | _id: "1", 299 | type: "type", 300 | priority: Priority.NORMAL, 301 | createdAt: new Date(), 302 | updatedAt: new Date(), 303 | state: State.INACTIVE, 304 | } 305 | ); 306 | 307 | const job = await repository.findInactiveJobByType("type"); 308 | expect(job).not.toBeNull(); 309 | // eslint-disable-next-line @typescript-eslint/no-non-null-assertion 310 | expect(job!._id).toBe("1"); 311 | }); 312 | 313 | test("not found", async () => { 314 | const repository = new JobRepository({ 315 | inMemoryOnly: true, 316 | }); 317 | await repository.init(); 318 | 319 | const job = await repository.findInactiveJobByType("type"); 320 | expect(job).toBeNull(); 321 | }); 322 | 323 | test("sort", async () => { 324 | const repository = new JobRepository({ 325 | inMemoryOnly: true, 326 | }); 327 | await repository.init(); 328 | 329 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 330 | const db: DataStore = (repository as any).db; 331 | 332 | const priorities: Priority[] = [ 333 | Priority.LOW, 334 | Priority.NORMAL, 335 | Priority.MEDIUM, 336 | Priority.HIGH, 337 | Priority.CRITICAL, 338 | ]; 339 | 340 | const createdAts = [ 341 | new Date(2020, 4, 3, 0, 0, 0), 342 | new Date(2020, 4, 2, 0, 0, 0), 343 | new Date(2020, 4, 1, 0, 0, 0), 344 | ]; 345 | 346 | let id = 1; 347 | for (const priority of priorities) { 348 | for (const createdAt of createdAts) { 349 | await dbInsert( 350 | db, 351 | { 352 | _id: id.toString(), 353 | type: "type", 354 | priority, 355 | createdAt, 356 | updatedAt: new Date(), 357 | state: State.INACTIVE, 358 | } 359 | ); 360 | 361 | id++; 362 | } 363 | } 364 | 365 | const sortedPriorities = [...priorities].sort((lhs, rhs) => rhs - lhs); 366 | const sortedCreatedAt = [...createdAts].sort((lhs, rhs) => lhs.getTime() - rhs.getTime()); 367 | for (const priority of sortedPriorities) { 368 | for (const createdAt of sortedCreatedAt) { 369 | const job = await repository.findInactiveJobByType("type"); 370 | expect(job).not.toBeNull(); 371 | /* eslint-disable @typescript-eslint/no-non-null-assertion */ 372 | expect(job!.priority).toBe(priority); 373 | expect(job!.createdAt).toEqual(createdAt); 374 | await dbRemove(db, job!._id); 375 | /* eslint-enable @typescript-eslint/no-non-null-assertion */ 376 | } 377 | } 378 | }); 379 | }); 380 | 381 | describe("isExistJob", () => { 382 | const repository = new JobRepository({ 383 | inMemoryOnly: true, 384 | }); 385 | 386 | beforeAll(async () => { 387 | await repository.init(); 388 | 389 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 390 | const db: DataStore = (repository as any).db; 391 | 392 | for (let i = 1; i <= 3; i++) { 393 | await dbInsert( 394 | db, 395 | { 396 | _id: i.toString(), 397 | type: "type", 398 | priority: Priority.NORMAL, 399 | createdAt: new Date(), 400 | updatedAt: new Date(), 401 | } 402 | ); 403 | } 404 | }); 405 | 406 | test("exits", async () => { 407 | expect(await repository.isExistJob("1")).toBe(true); 408 | expect(await repository.isExistJob("2")).toBe(true); 409 | expect(await repository.isExistJob("3")).toBe(true); 410 | }); 411 | 412 | test("not exist", async () => { 413 | expect(await repository.isExistJob("4")).toBe(false); 414 | }); 415 | }); 416 | 417 | test("addJob", async () => { 418 | const repository = new JobRepository({ 419 | inMemoryOnly: true, 420 | }); 421 | await repository.init(); 422 | 423 | const job = new Job({ 424 | queue: mock(), 425 | id: "1", 426 | type: "type", 427 | priority: Priority.HIGH, 428 | data: { 429 | a: "aaa", 430 | b: 123, 431 | c: { 432 | x: true, 433 | y: { 434 | z: true, 435 | }, 436 | }, 437 | }, 438 | createdAt: new Date(2020, 4, 1, 0, 0, 0), 439 | updatedAt: new Date(2020, 4, 2, 0, 0, 0), 440 | state: State.INACTIVE, 441 | logs: [ 442 | "log1", 443 | "log2", 444 | "log3", 445 | ], 446 | saved: false, 447 | }); 448 | 449 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 450 | const db: DataStore = (repository as any).db; 451 | 452 | const jobDocBefore = await dbFind(db, "1"); 453 | expect(jobDocBefore).toBeNull(); 454 | 455 | await repository.addJob(job); 456 | 457 | const jobDocAfter = await dbFind(db, "1"); 458 | expect(jobDocAfter).not.toBeNull(); 459 | /* eslint-disable @typescript-eslint/no-non-null-assertion */ 460 | expect(jobDocAfter!._id).toBe(job.id); 461 | expect(jobDocAfter!.type).toBe(job.type); 462 | expect(jobDocAfter!.priority).toBe(job.priority); 463 | expect(jobDocAfter!.data).toEqual(job.data); 464 | expect(jobDocAfter!.createdAt).toEqual(job.createdAt); 465 | expect(jobDocAfter!.updatedAt).toEqual(job.updatedAt); 466 | expect(jobDocAfter!.startedAt).toBeUndefined(); 467 | expect(jobDocAfter!.completedAt).toBeUndefined(); 468 | expect(jobDocAfter!.failedAt).toBeUndefined(); 469 | expect(jobDocAfter!.state).toBe(job.state); 470 | expect(jobDocAfter!.logs).toEqual(job.logs); 471 | /* eslint-enable @typescript-eslint/no-non-null-assertion */ 472 | }); 473 | 474 | describe("updateJob", () => { 475 | const job = new Job({ 476 | queue: mock(), 477 | id: "1", 478 | type: "type", 479 | priority: Priority.HIGH, 480 | data: { 481 | a: "aaa", 482 | b: 123, 483 | c: { 484 | x: true, 485 | y: { 486 | z: true, 487 | }, 488 | }, 489 | }, 490 | createdAt: new Date(2020, 4, 1, 0, 0, 0), 491 | updatedAt: new Date(2020, 4, 2, 0, 0, 0), 492 | startedAt: new Date(2020, 4, 3, 0, 0, 0), 493 | completedAt: new Date(2020, 4, 4, 0, 0, 0), 494 | failedAt: new Date(2020, 4, 5, 0, 0, 0), 495 | state: State.ACTIVE, 496 | logs: [ 497 | "log1", 498 | "log2", 499 | "log3", 500 | ], 501 | saved: true, 502 | }); 503 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 504 | (job as any)._duration = 123; 505 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 506 | (job as any)._progress = 1 / 3; 507 | 508 | test("found", async () => { 509 | const repository = new JobRepository({ 510 | inMemoryOnly: true, 511 | }); 512 | await repository.init(); 513 | 514 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 515 | const db: DataStore = (repository as any).db; 516 | 517 | await dbInsert( 518 | db, 519 | { 520 | _id: "1", 521 | type: "type", 522 | priority: Priority.NORMAL, 523 | createdAt: new Date(2020, 4, 6, 0, 0, 0), 524 | updatedAt: new Date(2020, 4, 7, 0, 0, 0), 525 | state: State.INACTIVE, 526 | logs: [], 527 | } 528 | ); 529 | 530 | await repository.updateJob(job); 531 | 532 | const updatedDoc = await dbFind(db, "1"); 533 | expect(updatedDoc).not.toBeNull(); 534 | /* eslint-disable @typescript-eslint/no-non-null-assertion */ 535 | expect(updatedDoc!._id).toBe("1"); 536 | expect(updatedDoc!.type).toBe(job.type); 537 | expect(updatedDoc!.priority).toBe(job.priority); 538 | expect(updatedDoc!.data).toEqual(job.data); 539 | expect(updatedDoc!.createdAt).toEqual(job.createdAt); 540 | expect(updatedDoc!.updatedAt).toEqual(job.updatedAt); 541 | expect(updatedDoc!.startedAt).toEqual(job.startedAt); 542 | expect(updatedDoc!.completedAt).toEqual(job.completedAt); 543 | expect(updatedDoc!.failedAt).toEqual(job.failedAt); 544 | expect(updatedDoc!.state).toBe(job.state); 545 | expect(updatedDoc!.duration).toBe(job.duration); 546 | expect(updatedDoc!.progress).toBe(job.progress); 547 | expect(updatedDoc!.logs).toEqual(job.logs); 548 | /* eslint-enable @typescript-eslint/no-non-null-assertion */ 549 | }); 550 | 551 | test("not found", async () => { 552 | const repository = new JobRepository({ 553 | inMemoryOnly: true, 554 | }); 555 | await repository.init(); 556 | 557 | await expect( 558 | repository.updateJob(job) 559 | ).rejects.toThrow("update unexpected number of rows. (expected: 1, actual: 0)"); 560 | }); 561 | }); 562 | 563 | test("removeJob", async () => { 564 | const repository = new JobRepository({ 565 | inMemoryOnly: true, 566 | }); 567 | await repository.init(); 568 | 569 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 570 | const db: DataStore = (repository as any).db; 571 | 572 | await dbInsert( 573 | db, 574 | { 575 | _id: "1", 576 | type: "type", 577 | priority: Priority.NORMAL, 578 | createdAt: new Date(), 579 | updatedAt: new Date(), 580 | state: State.INACTIVE, 581 | logs: [], 582 | } 583 | ); 584 | 585 | await repository.removeJob("1"); 586 | 587 | const deletedDoc = await dbFind(db, "1"); 588 | expect(deletedDoc).toBeNull(); 589 | }); 590 | -------------------------------------------------------------------------------- /test/integration.test.ts: -------------------------------------------------------------------------------- 1 | import * as util from "util"; 2 | 3 | import { Event, Job, Priority, Queue, State } from "../src"; 4 | 5 | const setTimeoutPromise = util.promisify(setTimeout); 6 | 7 | test("Basic Usage", async () => { 8 | interface JobData { 9 | a: number; 10 | b: number; 11 | } 12 | 13 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 14 | function isJobData(data: any): data is JobData { 15 | if (data === null) { 16 | return false; 17 | } 18 | 19 | return typeof data.a === "number" && typeof data.b === "number"; 20 | } 21 | 22 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 23 | 24 | const processor = jest.fn().mockImplementation( 25 | async (job: Job) => { 26 | if (isJobData(job.data)) { 27 | return job.data.a + job.data.b; 28 | } 29 | } 30 | ); 31 | queue.process("adder", processor, 1); 32 | 33 | const completeHandler = jest.fn(); 34 | queue.on(Event.Complete, completeHandler); 35 | 36 | const createdJob = await queue.createJob({ 37 | type: "adder", 38 | data: { a: 1, b: 2 }, 39 | }); 40 | 41 | await setTimeoutPromise(100); 42 | 43 | expect(completeHandler).toHaveBeenCalledTimes(1); 44 | expect(completeHandler.mock.calls[0][0]).toBeInstanceOf(Job); 45 | expect(completeHandler.mock.calls[0][1]).toBe(3); 46 | 47 | const completedJob = completeHandler.mock.calls[0][0]; 48 | expect(completedJob.id).toBe(createdJob.id); 49 | const currentTimestamp = new Date(); 50 | expect(completedJob.startedAt).toBeDefined(); 51 | expect((currentTimestamp.getTime() - completedJob.startedAt.getTime())).toBeLessThan(1000); 52 | expect(completedJob.completedAt).toBeDefined(); 53 | expect((currentTimestamp.getTime() - completedJob.completedAt.getTime())).toBeLessThan(1000); 54 | expect(completedJob.duration).toBeDefined(); 55 | expect(completedJob.duration).toBe(completedJob.completedAt.getTime() - completedJob.startedAt.getTime()); 56 | expect(completedJob.failedAt).toBeUndefined(); 57 | expect(completedJob.state).toBe(State.COMPLETE); 58 | 59 | // shutdown queue 60 | setTimeout(async () => { await queue.shutdown(100); }, 1); 61 | }); 62 | 63 | describe("Event Handlers", () => { 64 | test("Enqueue", async () => { 65 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 66 | 67 | const enqueueHandler = jest.fn(); 68 | queue.on(Event.Enqueue, enqueueHandler); 69 | 70 | const createdJob = await queue.createJob({ type: "SomeType" }); 71 | 72 | expect(enqueueHandler).toHaveBeenCalledTimes(1); 73 | expect(enqueueHandler.mock.calls[0][0].id).toBe(createdJob.id); 74 | }); 75 | 76 | test("Start", async () => { 77 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 78 | 79 | const startHandler = jest.fn(); 80 | queue.on(Event.Start, startHandler); 81 | 82 | // eslint-disable-next-line @typescript-eslint/no-empty-function 83 | queue.process("SomeType", async () => {}, 1); 84 | 85 | const createdJob = await queue.createJob({ type: "SomeType" }); 86 | await setTimeoutPromise(100); 87 | 88 | expect(startHandler).toHaveBeenCalledTimes(1); 89 | expect(startHandler.mock.calls[0][0].id).toBe(createdJob.id); 90 | }); 91 | 92 | test("Failure", async () => { 93 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 94 | 95 | const failureHandler = jest.fn(); 96 | queue.on(Event.Failure, failureHandler); 97 | 98 | const error = new Error("SomeError"); 99 | queue.process("SomeType", async () => { throw error; }, 1); 100 | 101 | const createdJob = await queue.createJob({ type: "SomeType" }); 102 | await setTimeoutPromise(100); 103 | 104 | expect(failureHandler).toHaveBeenCalledTimes(1); 105 | expect(failureHandler.mock.calls[0][0].id).toBe(createdJob.id); 106 | expect(failureHandler.mock.calls[0][1]).toBe(error); 107 | }); 108 | 109 | test("Complete", async () => { 110 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 111 | 112 | const completeHandler = jest.fn(); 113 | queue.on(Event.Complete, completeHandler); 114 | 115 | const result = 123; 116 | queue.process("SomeType", async () => result, 1); 117 | 118 | const createdJob = await queue.createJob({ type: "SomeType" }); 119 | await setTimeoutPromise(100); 120 | 121 | expect(completeHandler).toHaveBeenCalledTimes(1); 122 | expect(completeHandler.mock.calls[0][0].id).toBe(createdJob.id); 123 | expect(completeHandler.mock.calls[0][1]).toBe(result); 124 | }); 125 | 126 | test("Remove", async () => { 127 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 128 | 129 | const removeHandler = jest.fn(); 130 | queue.on(Event.Remove, removeHandler); 131 | 132 | const createdJob = await queue.createJob({ type: "SomeType" }); 133 | 134 | await queue.removeJobById(createdJob.id); 135 | 136 | expect(removeHandler).toHaveBeenCalledTimes(1); 137 | expect(removeHandler.mock.calls[0][0].id).toBe(createdJob.id); 138 | }); 139 | 140 | test.skip("Error", async () => { 141 | // Skip it because I can't come up with a good test. 142 | }); 143 | 144 | test("Progress", async () => { 145 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 146 | 147 | const progressHandler = jest.fn(); 148 | queue.on(Event.Progress, progressHandler); 149 | 150 | queue.process( 151 | "SomeType", 152 | async (job: Job) => { 153 | await job.setProgress(1, 3); 154 | await job.setProgress(2, 3); 155 | await job.setProgress(3, 3); 156 | }, 157 | 1 158 | ); 159 | 160 | const createdJob = await queue.createJob({ type: "SomeType" }); 161 | await setTimeoutPromise(100); 162 | 163 | expect(progressHandler).toHaveBeenCalledTimes(3); 164 | expect(progressHandler.mock.calls[0][0].id).toBe(createdJob.id); 165 | expect(progressHandler.mock.calls[0][1]).toBeCloseTo(1 / 3 * 100); 166 | expect(progressHandler.mock.calls[1][0].id).toBe(createdJob.id); 167 | expect(progressHandler.mock.calls[1][1]).toBeCloseTo(2 / 3 * 100); 168 | expect(progressHandler.mock.calls[2][0].id).toBe(createdJob.id); 169 | expect(progressHandler.mock.calls[2][1]).toBeCloseTo(3 / 3 * 100); 170 | }); 171 | 172 | test("Log", async () => { 173 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 174 | 175 | const logHandler = jest.fn(); 176 | queue.on(Event.Log, logHandler); 177 | 178 | const createdJob = await queue.createJob({ type: "SomeType" }); 179 | 180 | queue.process( 181 | "SomeType", 182 | async (job: Job) => { 183 | await job.addLog("First Log"); 184 | await job.addLog("Second Log"); 185 | await job.addLog("Third Log"); 186 | }, 187 | 1 188 | ); 189 | 190 | await setTimeoutPromise(100); 191 | 192 | expect(logHandler).toHaveBeenCalledTimes(3); 193 | expect(logHandler.mock.calls[0][0].id).toBe(createdJob.id); 194 | expect(logHandler.mock.calls[0][1]).toBe("First Log"); 195 | expect(logHandler.mock.calls[1][0].id).toBe(createdJob.id); 196 | expect(logHandler.mock.calls[1][1]).toBe("Second Log"); 197 | expect(logHandler.mock.calls[2][0].id).toBe(createdJob.id); 198 | expect(logHandler.mock.calls[2][1]).toBe("Third Log"); 199 | }); 200 | 201 | test("Priority", async () => { 202 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 203 | 204 | const priorityHandler = jest.fn(); 205 | queue.on(Event.Priority, priorityHandler); 206 | 207 | const createdJob = await queue.createJob({ type: "SomeType", priority: Priority.NORMAL }); 208 | 209 | const newPriority = Priority.HIGH; 210 | await createdJob.setPriority(newPriority); 211 | 212 | expect(priorityHandler).toHaveBeenCalledTimes(1); 213 | expect(priorityHandler.mock.calls[0][0].id).toBe(createdJob.id); 214 | expect(priorityHandler.mock.calls[0][1]).toBe(newPriority); 215 | }); 216 | }); 217 | 218 | test("Create Job", async () => { 219 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 220 | 221 | const type = "SomeType"; 222 | const priority = Priority.NORMAL; 223 | const data = { 224 | a: "aaa", 225 | b: 123, 226 | c: { 227 | x: true, 228 | y: { 229 | z: true, 230 | }, 231 | }, 232 | }; 233 | 234 | const createdJob = await queue.createJob({ type, priority, data }); 235 | 236 | expect(createdJob.id).not.toBe(""); 237 | expect(createdJob.type).toBe(type); 238 | expect(createdJob.priority).toBe(priority); 239 | expect(createdJob.data).toEqual(data); 240 | }); 241 | 242 | test("Set Job Processor", async () => { 243 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 244 | 245 | const someTypeWorkerCount = 3 246 | // eslint-disable-next-line @typescript-eslint/no-empty-function 247 | queue.process("SomeType", async () => {}, someTypeWorkerCount); 248 | 249 | const otherTypeWorkerCount = 5; 250 | // eslint-disable-next-line @typescript-eslint/no-empty-function 251 | queue.process("OtherType", async () => {}, otherTypeWorkerCount); 252 | 253 | expect(queue.workers).toHaveLength(someTypeWorkerCount + otherTypeWorkerCount); 254 | expect(queue.workers.filter((w) => w.type === "SomeType")).toHaveLength(someTypeWorkerCount); 255 | expect(queue.workers.filter((w) => w.type === "OtherType")).toHaveLength(otherTypeWorkerCount); 256 | expect(queue.workers.every((w) => w.isRunning)).toBe(true); 257 | }); 258 | 259 | test("Shutdown Queue", async () => { 260 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 261 | 262 | const processingMilliseconds = 1000; 263 | queue.process("SomeType", () => setTimeoutPromise(processingMilliseconds), 1); 264 | 265 | expect(queue.workers).toHaveLength(1); 266 | 267 | await queue.createJob({ type: "SomeType" }); 268 | 269 | // WorkerがJobを処理し始めるまで待つ 270 | await setTimeoutPromise(100); 271 | 272 | const before = new Date(); 273 | const timeoutMilliseconds = 100; 274 | await queue.shutdown(timeoutMilliseconds); 275 | const after = new Date(); 276 | 277 | expect(queue.workers).toHaveLength(0); 278 | const shutdownMilliseconds = after.getTime() - before.getTime(); 279 | expect(shutdownMilliseconds).toBeGreaterThanOrEqual(timeoutMilliseconds); 280 | expect(shutdownMilliseconds).toBeLessThan(processingMilliseconds); 281 | }); 282 | 283 | describe("Queue API", () => { 284 | test("findJob", async () => { 285 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 286 | 287 | const createdJobs = await Promise.all([ 288 | queue.createJob({ type: "SomeType" }), 289 | queue.createJob({ type: "SomeType" }), 290 | queue.createJob({ type: "SomeType" }), 291 | ]); 292 | 293 | for (const createdJob of createdJobs) { 294 | const foundJob = await queue.findJob(createdJob.id); 295 | expect(foundJob).not.toBeNull(); 296 | // eslint-disable-next-line @typescript-eslint/no-non-null-assertion 297 | expect(foundJob!.id).toBe(createdJob.id); 298 | } 299 | 300 | const notFoundJob = await queue.findJob("invalid-id"); 301 | expect(notFoundJob).toBeNull(); 302 | }); 303 | 304 | test("listJobs", async () => { 305 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 306 | 307 | const createdJobs: Job[] = []; 308 | createdJobs.push(await queue.createJob({ type: "SomeType" })); 309 | await setTimeoutPromise(100); 310 | createdJobs.push(await queue.createJob({ type: "SomeType" })); 311 | await setTimeoutPromise(100); 312 | createdJobs.push(await queue.createJob({ type: "SomeType" })); 313 | 314 | const jobs = await queue.listJobs(); 315 | 316 | expect(jobs).toHaveLength(createdJobs.length); 317 | for (let i = 0; i < jobs.length; i++) { 318 | expect(jobs[i].id).toBe(createdJobs[i].id); 319 | } 320 | }); 321 | 322 | test("removeJobById", async () => { 323 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 324 | 325 | const createdJobs = await Promise.all([ 326 | queue.createJob({ type: "SomeType" }), 327 | queue.createJob({ type: "SomeType" }), 328 | queue.createJob({ type: "SomeType" }), 329 | ]); 330 | 331 | for (let i = 0; i < createdJobs.length; i++) { 332 | expect(await queue.listJobs()).toHaveLength(createdJobs.length - i); 333 | 334 | await queue.removeJobById(createdJobs[i].id); 335 | 336 | const job = await queue.findJob(createdJobs[i].id); 337 | expect(job).toBeNull(); 338 | 339 | expect(await queue.listJobs()).toHaveLength(createdJobs.length - i - 1); 340 | } 341 | }); 342 | 343 | test("removeJobsByCallback", async () => { 344 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 345 | 346 | const createdJobs: Job[] = []; 347 | createdJobs.push(await queue.createJob({ type: "SomeType" })); 348 | await setTimeoutPromise(100); 349 | createdJobs.push(await queue.createJob({ type: "SomeType" })); 350 | await setTimeoutPromise(100); 351 | createdJobs.push(await queue.createJob({ type: "SomeType" })); 352 | 353 | await queue.removeJobsByCallback((job: Job) => job.id !== createdJobs[1].id); 354 | 355 | const jobs = await queue.listJobs(); 356 | 357 | expect(jobs).toHaveLength(1); 358 | expect(jobs[0].id).toBe(createdJobs[1].id); 359 | }); 360 | }); 361 | 362 | describe("Job API", () => { 363 | test("setProgress", async () => { 364 | expect.assertions(3); 365 | 366 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 367 | 368 | queue.process( 369 | "SomeType", 370 | async (job: Job) => { 371 | await job.setProgress(1, 3); 372 | expect(job.progress).toBeCloseTo(1 / 3 * 100); 373 | 374 | await job.setProgress(2, 3); 375 | expect(job.progress).toBeCloseTo(2 / 3 * 100); 376 | 377 | await job.setProgress(3, 3); 378 | expect(job.progress).toBeCloseTo(3 / 3 * 100); 379 | }, 380 | 1 381 | ); 382 | 383 | await queue.createJob({ type: "SomeType" }); 384 | await setTimeoutPromise(100); 385 | }); 386 | 387 | test("remove", async () => { 388 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 389 | 390 | const createdJob = await queue.createJob({ type: "SomeType" }); 391 | 392 | expect(await createdJob.isExist()).toBe(true); 393 | 394 | await createdJob.remove(); 395 | 396 | expect(await createdJob.isExist()).toBe(false); 397 | }); 398 | }); 399 | 400 | test("Multiple Workers", async () => { 401 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 402 | 403 | let completedJobCount = 0; 404 | queue.on(Event.Complete, () => { 405 | completedJobCount++; 406 | }); 407 | 408 | let createdJobCount = 0; 409 | for (let i = 0; i < 10; i++) { 410 | await queue.createJob({ type: "SomeType" }); 411 | createdJobCount++; 412 | } 413 | 414 | expect(completedJobCount).toBe(0); 415 | 416 | // eslint-disable-next-line @typescript-eslint/no-empty-function 417 | queue.process("SomeType", async () => {}, 5); 418 | 419 | // WorkerがJobを処理し終えるまで待つ 420 | await setTimeoutPromise(100); 421 | 422 | expect(completedJobCount).toBe(createdJobCount); 423 | 424 | for (let i = 0; i < 10; i++) { 425 | await queue.createJob({ type: "SomeType" }); 426 | createdJobCount++; 427 | } 428 | 429 | // WorkerがJobを処理し終えるまで待つ 430 | await setTimeoutPromise(100); 431 | 432 | expect(completedJobCount).toBe(createdJobCount); 433 | }); 434 | -------------------------------------------------------------------------------- /test/job.test.ts: -------------------------------------------------------------------------------- 1 | import { mock, mockReset } from "jest-mock-extended"; 2 | 3 | import { Event, Job, Priority, Queue, State } from "../src"; 4 | 5 | test("constructor", () => { 6 | const id = "id"; 7 | const type = "type"; 8 | const priority = Priority.HIGH; 9 | const data = { 10 | a: "aaa", 11 | b: 123, 12 | c: { 13 | x: true, 14 | y: { 15 | z: true, 16 | }, 17 | }, 18 | }; 19 | const createdAt = new Date(2020, 4, 1, 0, 0, 0); 20 | const updatedAt = new Date(2020, 4, 2, 0, 0, 0); 21 | const startedAt = new Date(2020, 4, 3, 0, 0, 0); 22 | const completedAt = new Date(2020, 4, 4, 0, 0, 0); 23 | const failedAt = new Date(2020, 4, 5, 0, 0, 0); 24 | const state = State.INACTIVE; 25 | const duration = 123; 26 | const progress = 1 / 3; 27 | const logs = [ 28 | "log1", 29 | "log2", 30 | "log3", 31 | ]; 32 | 33 | const job = new Job({ 34 | queue: mock(), 35 | id, 36 | type, 37 | priority, 38 | data, 39 | createdAt, 40 | updatedAt, 41 | startedAt, 42 | completedAt, 43 | failedAt, 44 | state, 45 | duration, 46 | progress, 47 | logs, 48 | saved: true, 49 | }); 50 | 51 | expect(job.id).toBe(id); 52 | expect(job.type).toBe(type); 53 | expect(job.priority).toBe(priority); 54 | expect(job.data).toEqual(data); 55 | expect(job.createdAt).toBe(createdAt); 56 | expect(job.updatedAt).toBe(updatedAt); 57 | expect(job.completedAt).toBe(completedAt); 58 | expect(job.failedAt).toBe(failedAt); 59 | expect(job.state).toBe(state); 60 | expect(job.duration).toBe(duration); 61 | expect(job.progress).toBe(progress); 62 | expect(job.logs).toEqual(logs); 63 | }); 64 | 65 | test("setProgress", async () => { 66 | const mockedQueue = mock(); 67 | const updatedAt = new Date(2020, 4, 1, 0, 0, 0); 68 | 69 | const job = new Job({ 70 | queue: mockedQueue, 71 | id: "id", 72 | type: "type", 73 | createdAt: new Date(), 74 | updatedAt, 75 | logs: [], 76 | saved: true, 77 | }); 78 | 79 | expect(job.progress).toBeUndefined(); 80 | 81 | await job.setProgress(1, 6); 82 | expect(job.progress).toBeCloseTo(16.67, 2); 83 | expect(job.updatedAt).not.toBe(updatedAt); 84 | expect(mockedQueue.updateJob).toHaveBeenCalledTimes(1); 85 | expect(mockedQueue.updateJob.mock.calls[0][0]).toBe(job); 86 | expect(mockedQueue.emit).toHaveBeenCalledTimes(1); 87 | expect(mockedQueue.emit.mock.calls[0][0]).toBe(Event.Progress); 88 | expect(mockedQueue.emit.mock.calls[0][1]).toBe(job); 89 | expect(mockedQueue.emit.mock.calls[0][2]).toBe(job.progress); 90 | 91 | await job.setProgress(2, 6); 92 | expect(job.progress).toBeCloseTo(33.33, 2); 93 | 94 | await job.setProgress(3, 6); 95 | expect(job.progress).toBeCloseTo(50.00, 2); 96 | 97 | await job.setProgress(4, 6); 98 | expect(job.progress).toBeCloseTo(66.67, 2); 99 | 100 | await job.setProgress(5, 6); 101 | expect(job.progress).toBeCloseTo(83.33, 2); 102 | 103 | await job.setProgress(6, 6); 104 | expect(job.progress).toBeCloseTo(100.00, 2); 105 | }); 106 | 107 | test("addLog", async () => { 108 | const mockedQueue = mock(); 109 | const updatedAt = new Date(2020, 4, 1, 0, 0, 0); 110 | 111 | const job = new Job({ 112 | queue: mockedQueue, 113 | id: "id", 114 | type: "type", 115 | createdAt: new Date(), 116 | updatedAt, 117 | logs: [], 118 | saved: true, 119 | }); 120 | 121 | expect(job.logs).toHaveLength(0); 122 | 123 | await job.addLog("First Log"); 124 | expect(job.logs).toEqual([ 125 | "First Log", 126 | ]); 127 | expect(job.updatedAt).not.toBe(updatedAt); 128 | expect(mockedQueue.updateJob).toHaveBeenCalledTimes(1); 129 | expect(mockedQueue.updateJob.mock.calls[0][0]).toBe(job); 130 | expect(mockedQueue.emit).toHaveBeenCalledTimes(1); 131 | expect(mockedQueue.emit.mock.calls[0][0]).toBe(Event.Log); 132 | expect(mockedQueue.emit.mock.calls[0][1]).toBe(job); 133 | 134 | await job.addLog("Second Log"); 135 | expect(job.logs).toEqual([ 136 | "First Log", 137 | "Second Log", 138 | ]); 139 | 140 | await job.addLog("Third Log"); 141 | expect(job.logs).toEqual([ 142 | "First Log", 143 | "Second Log", 144 | "Third Log", 145 | ]); 146 | }); 147 | 148 | describe("save", () => { 149 | const mockedQueue = mock(); 150 | const id = "id"; 151 | const type = "type"; 152 | const createdAt = new Date(); 153 | const updatedAt = new Date(); 154 | const logs: string[] = []; 155 | 156 | beforeEach(() => { 157 | mockReset(mockedQueue); 158 | }); 159 | 160 | test("new job", async () => { 161 | const job = new Job({ 162 | queue: mockedQueue, 163 | id, 164 | type, 165 | createdAt, 166 | updatedAt, 167 | logs, 168 | saved: false, 169 | }); 170 | 171 | await job.save(); 172 | 173 | expect(mockedQueue.addJob).toHaveBeenCalledTimes(1); 174 | expect(mockedQueue.addJob.mock.calls[0][0]).toBe(job); 175 | expect(mockedQueue.emit).toHaveBeenCalledTimes(1); 176 | expect(mockedQueue.emit.mock.calls[0][0]).toBe(Event.Enqueue); 177 | expect(mockedQueue.emit.mock.calls[0][1]).toBe(job); 178 | }); 179 | 180 | test("exist job", async () => { 181 | const job = new Job({ 182 | queue: mockedQueue, 183 | id, 184 | type, 185 | createdAt, 186 | updatedAt, 187 | logs, 188 | saved: true, 189 | }); 190 | 191 | await job.save(); 192 | 193 | expect(mockedQueue.updateJob).toHaveBeenCalledTimes(1); 194 | expect(mockedQueue.updateJob.mock.calls[0][0]).toBe(job); 195 | }); 196 | }); 197 | 198 | test("remove", async () => { 199 | const mockedQueue = mock(); 200 | 201 | const job = new Job({ 202 | queue: mockedQueue, 203 | id: "id", 204 | type: "type", 205 | createdAt: new Date(), 206 | updatedAt: new Date(), 207 | logs: [], 208 | saved: true, 209 | }); 210 | 211 | await job.remove(); 212 | 213 | expect(mockedQueue.removeJob).toHaveBeenCalledTimes(1); 214 | expect(mockedQueue.removeJob.mock.calls[0][0]).toBe(job); 215 | expect(mockedQueue.emit).toHaveBeenCalledTimes(1); 216 | expect(mockedQueue.emit.mock.calls[0][0]).toBe(Event.Remove); 217 | expect(mockedQueue.emit.mock.calls[0][1]).toBe(job); 218 | }); 219 | 220 | test("setPriority", async () => { 221 | const mockedQueue = mock(); 222 | const updatedAt = new Date(2020, 4, 1, 0, 0, 0); 223 | 224 | const job = new Job({ 225 | queue: mockedQueue, 226 | id: "id", 227 | type: "type", 228 | priority: Priority.LOW, 229 | createdAt: new Date(), 230 | updatedAt, 231 | logs: [], 232 | saved: true, 233 | }); 234 | 235 | expect(job.priority).toBe(Priority.LOW); 236 | 237 | await job.setPriority(Priority.NORMAL); 238 | 239 | expect(job.priority).toBe(Priority.NORMAL); 240 | expect(job.updatedAt).not.toBe(updatedAt); 241 | expect(mockedQueue.updateJob).toHaveBeenCalledTimes(1); 242 | expect(mockedQueue.updateJob.mock.calls[0][0]).toBe(job); 243 | expect(mockedQueue.emit).toHaveBeenCalledTimes(1); 244 | expect(mockedQueue.emit.mock.calls[0][0]).toBe(Event.Priority); 245 | expect(mockedQueue.emit.mock.calls[0][1]).toBe(job); 246 | }); 247 | 248 | describe("isExist", () => { 249 | const mockedQueue = mock(); 250 | const id = "id"; 251 | const type = "type"; 252 | const createdAt = new Date(); 253 | const updatedAt = new Date(); 254 | const logs: string[] = []; 255 | 256 | const job = new Job({ 257 | queue: mockedQueue, 258 | id, 259 | type, 260 | createdAt, 261 | updatedAt, 262 | logs, 263 | saved: false, 264 | }); 265 | 266 | beforeEach(() => { 267 | mockReset(mockedQueue); 268 | }); 269 | 270 | test("is exist", async () => { 271 | mockedQueue.isExistJob.mockResolvedValue(true); 272 | 273 | expect(await job.isExist()).toBe(true); 274 | }); 275 | 276 | test("is not exist", async () => { 277 | mockedQueue.isExistJob.mockResolvedValue(false); 278 | 279 | expect(await job.isExist()).toBe(false); 280 | }); 281 | }); 282 | 283 | test("setStateToActive", async () => { 284 | const mockedQueue = mock(); 285 | const updatedAt = new Date(2020, 4, 1, 0, 0, 0); 286 | 287 | const job = new Job({ 288 | queue: mockedQueue, 289 | id: "id", 290 | type: "type", 291 | createdAt: new Date(), 292 | updatedAt, 293 | startedAt: undefined, 294 | logs: [], 295 | saved: true, 296 | }); 297 | 298 | await job.setStateToActive(); 299 | 300 | expect(job.startedAt).toBeDefined(); 301 | expect(job.updatedAt).not.toBe(updatedAt); 302 | expect(mockedQueue.updateJob).toHaveBeenCalledTimes(1); 303 | expect(mockedQueue.updateJob.mock.calls[0][0]).toBe(job); 304 | expect(mockedQueue.emit).toHaveBeenCalledTimes(1); 305 | expect(mockedQueue.emit.mock.calls[0][0]).toBe(Event.Start); 306 | }); 307 | 308 | test("setStateToComplete", async () => { 309 | const mockedQueue = mock(); 310 | const updatedAt = new Date(2020, 4, 1, 0, 0, 0); 311 | const startedAt = new Date(2020, 4, 2, 0, 0, 0); 312 | 313 | const job = new Job({ 314 | queue: mockedQueue, 315 | id: "id", 316 | type: "type", 317 | createdAt: new Date(), 318 | updatedAt, 319 | startedAt, 320 | logs: [], 321 | saved: true, 322 | }); 323 | 324 | expect(job.completedAt).toBeUndefined(); 325 | expect(job.duration).toBeUndefined(); 326 | 327 | const result = { 328 | a: "aaa", 329 | b: 123, 330 | c: { 331 | x: true, 332 | y: { 333 | z: true, 334 | }, 335 | }, 336 | }; 337 | 338 | await job.setStateToComplete(result); 339 | 340 | expect(job.completedAt).toBeDefined(); 341 | expect(job.duration).toBeCloseTo((new Date()).getTime() - startedAt.getTime(), -3); // 1 seconds or less 342 | expect(job.updatedAt).not.toBe(updatedAt); 343 | expect(mockedQueue.updateJob).toHaveBeenCalledTimes(1); 344 | expect(mockedQueue.updateJob.mock.calls[0][0]).toBe(job); 345 | expect(mockedQueue.emit).toHaveBeenCalledTimes(1); 346 | expect(mockedQueue.emit.mock.calls[0][0]).toBe(Event.Complete); 347 | expect(mockedQueue.emit.mock.calls[0][1]).toBe(job); 348 | expect(mockedQueue.emit.mock.calls[0][2]).toEqual(result); 349 | }); 350 | 351 | test("setStateToFailure", async () => { 352 | const mockedQueue = mock(); 353 | const updatedAt = new Date(2020, 4, 1, 0, 0, 0); 354 | const startedAt = new Date(2020, 4, 2, 0, 0, 0); 355 | 356 | const job = new Job({ 357 | queue: mockedQueue, 358 | id: "id", 359 | type: "type", 360 | createdAt: new Date(), 361 | updatedAt, 362 | startedAt, 363 | logs: [], 364 | saved: true, 365 | }); 366 | 367 | expect(job.failedAt).toBeUndefined(); 368 | 369 | const error = new Error("my error"); 370 | await job.setStateToFailure(error); 371 | 372 | expect(job.failedAt).toBeDefined(); 373 | expect(job.updatedAt).not.toBe(updatedAt); 374 | expect(mockedQueue.updateJob).toHaveBeenCalledTimes(1); 375 | expect(mockedQueue.updateJob.mock.calls[0][0]).toBe(job); 376 | expect(mockedQueue.emit).toHaveBeenCalledTimes(1); 377 | expect(mockedQueue.emit.mock.calls[0][0]).toBe(Event.Failure); 378 | expect(mockedQueue.emit.mock.calls[0][1]).toBe(job); 379 | expect(mockedQueue.emit.mock.calls[0][2]).toEqual(error); 380 | }); 381 | -------------------------------------------------------------------------------- /test/queue.test.ts: -------------------------------------------------------------------------------- 1 | import { v4 as uuid } from "uuid"; 2 | 3 | import { Job, Priority, Queue, State } from "../src"; 4 | import { JobRepository, NeDbJob } from "../src/jobRepository"; 5 | 6 | // Note: Same as src/queue.ts 7 | interface WaitingWorkerRequest { 8 | resolve: (value: Job) => void; 9 | reject: (error: Error) => void; 10 | } 11 | 12 | jest.mock("uuid"); 13 | 14 | test("process & shutdown", async () => { 15 | const queue = await Queue.createQueue({ 16 | inMemoryOnly: true, 17 | }); 18 | 19 | const type1 = "type1"; 20 | const processor1 = jest.fn(); 21 | const concurrency1 = 2; 22 | queue.process(type1, processor1, concurrency1); 23 | 24 | expect(queue.workers).toHaveLength(concurrency1); 25 | for (let i = 0; i < concurrency1; i++) { 26 | expect(queue.workers[i].type).toBe(type1); 27 | expect(queue.workers[i].isRunning).toBe(true); 28 | } 29 | 30 | const type2 = "type2"; 31 | const processor2 = jest.fn(); 32 | const concurrency2 = 3; 33 | queue.process(type2, processor2, concurrency2); 34 | 35 | expect(queue.workers).toHaveLength(concurrency1 + concurrency2); 36 | for (let i = concurrency1; i < concurrency2; i++) { 37 | expect(queue.workers[i].type).toBe(type2); 38 | expect(queue.workers[i].isRunning).toBe(true); 39 | } 40 | 41 | const spiedWorkersShutdown = queue.workers.map((worker) => jest.spyOn(worker, "shutdown")); 42 | const timeoutMilliseconds = 100; 43 | await queue.shutdown(timeoutMilliseconds, type1); 44 | 45 | expect(queue.workers).toHaveLength(concurrency2); 46 | for (let i = 0; i < concurrency1; i++) { 47 | expect(spiedWorkersShutdown[i]).toHaveBeenCalledTimes(1); 48 | expect(spiedWorkersShutdown[i].mock.calls[0][0]).toBe(timeoutMilliseconds); 49 | } 50 | for (let i = concurrency1; i < concurrency2; i++) { 51 | expect(spiedWorkersShutdown[i]).not.toHaveBeenCalled(); 52 | expect(queue.workers[i].type).toBe(type2); 53 | expect(queue.workers[i].isRunning).toBe(true); 54 | } 55 | 56 | await queue.shutdown(timeoutMilliseconds, type2); 57 | 58 | expect(queue.workers).toHaveLength(0); 59 | for (let i = concurrency1; i < concurrency2; i++) { 60 | expect(spiedWorkersShutdown[i]).toHaveBeenCalledTimes(1); 61 | expect(spiedWorkersShutdown[i].mock.calls[0][0]).toBe(timeoutMilliseconds); 62 | } 63 | }); 64 | 65 | test("createJob", async () => { 66 | const uuidValue = "12345678-90ab-cdef-1234-567890abcdef"; 67 | // eslint-disable-next-line 68 | (uuid as any).mockReturnValue(uuidValue); 69 | 70 | const queue = await Queue.createQueue({ 71 | inMemoryOnly: true, 72 | }); 73 | 74 | const type = "type"; 75 | const priority = Priority.HIGH; 76 | const data = { 77 | a: "aaa", 78 | b: 123, 79 | c: { 80 | x: true, 81 | y: { 82 | z: true, 83 | }, 84 | }, 85 | }; 86 | 87 | const job = await queue.createJob({ 88 | type, 89 | priority, 90 | data, 91 | }); 92 | 93 | const currentTimestamp = new Date(); 94 | expect(job.id).toBe(uuidValue); 95 | expect(job.type).toBe(type); 96 | expect(job.data).toBe(data); 97 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 98 | expect((job as any).queue).toBe(queue); 99 | expect(job.priority).toBe(priority); 100 | expect(currentTimestamp.getTime() - job.createdAt.getTime()).toBeLessThan(100); 101 | expect(currentTimestamp.getTime() - job.updatedAt.getTime()).toBeLessThan(100); 102 | expect(job.logs).toHaveLength(0); 103 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 104 | expect((job as any)._saved).toBe(true); 105 | }); 106 | 107 | describe("findJob", () => { 108 | test("found", async () => { 109 | const queue = await Queue.createQueue({ 110 | inMemoryOnly: true, 111 | }); 112 | 113 | const nedbJob: NeDbJob = { 114 | _id: "1", 115 | type: "type", 116 | priority: Priority.NORMAL, 117 | data: { 118 | a: "aaa", 119 | b: 123, 120 | c: { 121 | x: true, 122 | y: { 123 | z: true, 124 | }, 125 | }, 126 | }, 127 | createdAt: new Date(2020, 4, 1, 0, 0, 0), 128 | updatedAt: new Date(2020, 4, 2, 0, 0, 0), 129 | startedAt: new Date(2020, 4, 3, 0, 0, 0), 130 | completedAt: new Date(2020, 4, 4, 0, 0, 0), 131 | failedAt: new Date(2020, 4, 5, 0, 0, 0), 132 | state: State.INACTIVE, 133 | duration: 123, 134 | progress: 1 / 3, 135 | logs: [ 136 | "First Log", 137 | "Second Log", 138 | "Third Log", 139 | ], 140 | }; 141 | 142 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 143 | const repository = (queue as any).repository as JobRepository; 144 | const mockedRepositoryFindJob = jest.fn().mockResolvedValue(nedbJob); 145 | repository.findJob = mockedRepositoryFindJob; 146 | 147 | const id = "1"; 148 | const job = await queue.findJob(id); 149 | 150 | expect(job).not.toBeNull(); 151 | /* eslint-disable @typescript-eslint/no-non-null-assertion, @typescript-eslint/no-explicit-any */ 152 | expect((job as any).queue).toBe(queue); 153 | expect(job!.id).toBe(nedbJob._id); 154 | expect(job!.type).toBe(nedbJob.type); 155 | expect(job!.priority).toBe(nedbJob.priority); 156 | expect(job!.data).toEqual(nedbJob.data); 157 | expect(job!.createdAt).toEqual(nedbJob.createdAt); 158 | expect(job!.updatedAt).toEqual(nedbJob.updatedAt); 159 | expect(job!.startedAt).toEqual(nedbJob.startedAt); 160 | expect(job!.completedAt).toEqual(nedbJob.completedAt); 161 | expect(job!.failedAt).toEqual(nedbJob.failedAt); 162 | expect(job!.state).toBe(nedbJob.state); 163 | expect(job!.duration).toBe(nedbJob.duration); 164 | expect(job!.progress).toBe(nedbJob.progress); 165 | expect(job!.logs).toEqual(nedbJob.logs); 166 | expect((job as any)._saved).toBe(true); 167 | /* eslint-enable @typescript-eslint/no-non-null-assertion, @typescript-eslint/no-explicit-any */ 168 | 169 | expect(mockedRepositoryFindJob).toHaveBeenCalledTimes(1); 170 | expect(mockedRepositoryFindJob.mock.calls[0][0]).toBe(id); 171 | }); 172 | 173 | test("not found", async () => { 174 | const queue = await Queue.createQueue({ 175 | inMemoryOnly: true, 176 | }); 177 | 178 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 179 | const repository = (queue as any).repository as JobRepository; 180 | const mockedRepositoryFindJob = jest.fn().mockResolvedValue(null); 181 | repository.findJob = mockedRepositoryFindJob; 182 | 183 | const id = "1"; 184 | const job = await queue.findJob(id); 185 | 186 | expect(job).toBeNull(); 187 | expect(mockedRepositoryFindJob).toHaveBeenCalledTimes(1); 188 | expect(mockedRepositoryFindJob.mock.calls[0][0]).toBe(id); 189 | }); 190 | }); 191 | 192 | test("listJobs", async () => { 193 | const queue = await Queue.createQueue({ 194 | inMemoryOnly: true, 195 | }); 196 | 197 | const nedbJobs: NeDbJob[] = [ 198 | { 199 | _id: "1", 200 | type: "type", 201 | priority: Priority.NORMAL, 202 | data: { 203 | a: "aaa", 204 | b: 123, 205 | c: { 206 | x: true, 207 | y: { 208 | z: true, 209 | }, 210 | }, 211 | }, 212 | createdAt: new Date(2020, 4, 1, 0, 0, 0), 213 | updatedAt: new Date(2020, 4, 2, 0, 0, 0), 214 | startedAt: new Date(2020, 4, 3, 0, 0, 0), 215 | completedAt: new Date(2020, 4, 4, 0, 0, 0), 216 | failedAt: new Date(2020, 4, 5, 0, 0, 0), 217 | state: State.INACTIVE, 218 | duration: 123, 219 | progress: 1 / 3, 220 | logs: [ 221 | "First Log", 222 | "Second Log", 223 | "Third Log", 224 | ], 225 | }, 226 | { 227 | _id: "2", 228 | type: "type", 229 | priority: Priority.HIGH, 230 | createdAt: new Date(2020, 4, 6, 0, 0, 0), 231 | updatedAt: new Date(2020, 4, 7, 0, 0, 0), 232 | startedAt: new Date(2020, 4, 8, 0, 0, 0), 233 | completedAt: new Date(2020, 4, 9, 0, 0, 0), 234 | failedAt: new Date(2020, 4, 10, 0, 0, 0), 235 | state: State.ACTIVE, 236 | duration: 234, 237 | progress: 2 / 3, 238 | logs: [ 239 | ], 240 | }, 241 | { 242 | _id: "3", 243 | type: "type", 244 | priority: Priority.LOW, 245 | createdAt: new Date(2020, 4, 11, 0, 0, 0), 246 | updatedAt: new Date(2020, 4, 12, 0, 0, 0), 247 | startedAt: new Date(2020, 4, 13, 0, 0, 0), 248 | completedAt: new Date(2020, 14, 9, 0, 0, 0), 249 | failedAt: new Date(2020, 4, 15, 0, 0, 0), 250 | state: State.COMPLETE, 251 | duration: 345, 252 | progress: 3 / 4, 253 | logs: [ 254 | ], 255 | }, 256 | ]; 257 | 258 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 259 | const repository = (queue as any).repository as JobRepository; 260 | const mockedRepositoryListJobs = jest.fn().mockResolvedValue(nedbJobs); 261 | repository.listJobs = mockedRepositoryListJobs; 262 | 263 | const state = State.ACTIVE; 264 | const jobs = await queue.listJobs(state); 265 | 266 | expect(jobs).toHaveLength(3); 267 | 268 | for (const [index, job] of jobs.entries()) { 269 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 270 | expect((job as any).queue).toBe(queue); 271 | expect(job.id).toBe(nedbJobs[index]._id); 272 | expect(job.type).toBe(nedbJobs[index].type); 273 | expect(job.priority).toBe(nedbJobs[index].priority); 274 | expect(job.data).toEqual(nedbJobs[index].data); 275 | expect(job.createdAt).toEqual(nedbJobs[index].createdAt); 276 | expect(job.updatedAt).toEqual(nedbJobs[index].updatedAt); 277 | expect(job.startedAt).toEqual(nedbJobs[index].startedAt); 278 | expect(job.completedAt).toEqual(nedbJobs[index].completedAt); 279 | expect(job.failedAt).toEqual(nedbJobs[index].failedAt); 280 | expect(job.state).toBe(nedbJobs[index].state); 281 | expect(job.duration).toBe(nedbJobs[index].duration); 282 | expect(job.progress).toBe(nedbJobs[index].progress); 283 | expect(job.logs).toEqual(nedbJobs[index].logs); 284 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 285 | expect((job as any)._saved).toBe(true); 286 | } 287 | expect(mockedRepositoryListJobs).toHaveBeenCalledTimes(1); 288 | expect(mockedRepositoryListJobs.mock.calls[0][0]).toBe(state); 289 | }); 290 | 291 | describe("removeJobById", () => { 292 | test("found", async () => { 293 | const queue = await Queue.createQueue({ 294 | inMemoryOnly: true, 295 | }); 296 | 297 | const nedbJob: NeDbJob = { 298 | _id: "1", 299 | type: "type", 300 | priority: Priority.NORMAL, 301 | data: { 302 | a: "aaa", 303 | b: 123, 304 | c: { 305 | x: true, 306 | y: { 307 | z: true, 308 | }, 309 | }, 310 | }, 311 | createdAt: new Date(2020, 4, 1, 0, 0, 0), 312 | updatedAt: new Date(2020, 4, 2, 0, 0, 0), 313 | startedAt: new Date(2020, 4, 3, 0, 0, 0), 314 | completedAt: new Date(2020, 4, 4, 0, 0, 0), 315 | failedAt: new Date(2020, 4, 5, 0, 0, 0), 316 | state: State.INACTIVE, 317 | duration: 123, 318 | progress: 1 / 3, 319 | logs: [ 320 | "First Log", 321 | "Second Log", 322 | "Third Log", 323 | ], 324 | }; 325 | 326 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 327 | const repository = (queue as any).repository as JobRepository; 328 | const mockedRepositoryFindJob = jest.fn().mockResolvedValue(nedbJob); 329 | repository.findJob = mockedRepositoryFindJob; 330 | 331 | const id = "1"; 332 | await queue.removeJobById(id); 333 | 334 | expect(mockedRepositoryFindJob).toHaveBeenCalledTimes(1); 335 | expect(mockedRepositoryFindJob.mock.calls[0][0]).toBe(id); 336 | }); 337 | 338 | test("not found", async () => { 339 | const queue = await Queue.createQueue({ 340 | inMemoryOnly: true, 341 | }); 342 | 343 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 344 | const repository = (queue as any).repository as JobRepository; 345 | const mockedRepositoryFindJob = jest.fn().mockResolvedValue(null); 346 | repository.findJob = mockedRepositoryFindJob; 347 | 348 | const id = "1"; 349 | await expect( 350 | queue.removeJobById(id) 351 | ).rejects.toThrow(); 352 | 353 | expect(mockedRepositoryFindJob).toHaveBeenCalledTimes(1); 354 | expect(mockedRepositoryFindJob.mock.calls[0][0]).toBe(id); 355 | }); 356 | }); 357 | 358 | test("removeJobsByCallback", async () => { 359 | const queue = await Queue.createQueue({ 360 | inMemoryOnly: true, 361 | }); 362 | 363 | const nedbJobs: NeDbJob[] = [ 364 | { 365 | _id: "1", 366 | type: "type", 367 | priority: Priority.NORMAL, 368 | data: { 369 | a: "aaa", 370 | b: 123, 371 | c: { 372 | x: true, 373 | y: { 374 | z: true, 375 | }, 376 | }, 377 | }, 378 | createdAt: new Date(2020, 4, 1, 0, 0, 0), 379 | updatedAt: new Date(2020, 4, 2, 0, 0, 0), 380 | startedAt: new Date(2020, 4, 3, 0, 0, 0), 381 | completedAt: new Date(2020, 4, 4, 0, 0, 0), 382 | failedAt: new Date(2020, 4, 5, 0, 0, 0), 383 | state: State.INACTIVE, 384 | duration: 123, 385 | progress: 1 / 3, 386 | logs: [ 387 | "First Log", 388 | "Second Log", 389 | "Third Log", 390 | ], 391 | }, 392 | { 393 | _id: "2", 394 | type: "type", 395 | priority: Priority.HIGH, 396 | createdAt: new Date(2020, 4, 6, 0, 0, 0), 397 | updatedAt: new Date(2020, 4, 7, 0, 0, 0), 398 | startedAt: new Date(2020, 4, 8, 0, 0, 0), 399 | completedAt: new Date(2020, 4, 9, 0, 0, 0), 400 | failedAt: new Date(2020, 4, 10, 0, 0, 0), 401 | state: State.ACTIVE, 402 | duration: 234, 403 | progress: 2 / 3, 404 | logs: [ 405 | ], 406 | }, 407 | { 408 | _id: "3", 409 | type: "type", 410 | priority: Priority.LOW, 411 | createdAt: new Date(2020, 4, 11, 0, 0, 0), 412 | updatedAt: new Date(2020, 4, 12, 0, 0, 0), 413 | startedAt: new Date(2020, 4, 13, 0, 0, 0), 414 | completedAt: new Date(2020, 14, 9, 0, 0, 0), 415 | failedAt: new Date(2020, 4, 15, 0, 0, 0), 416 | state: State.COMPLETE, 417 | duration: 345, 418 | progress: 3 / 4, 419 | logs: [ 420 | ], 421 | }, 422 | ]; 423 | 424 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 425 | const repository = (queue as any).repository as JobRepository; 426 | const mockedRepositoryListJobs = jest.fn().mockResolvedValue(nedbJobs); 427 | repository.listJobs = mockedRepositoryListJobs; 428 | 429 | const removeCallback = jest.fn().mockImplementation((job: Job) => job.id !== "2"); 430 | 431 | const removedJobs = await queue.removeJobsByCallback(removeCallback); 432 | 433 | expect(removedJobs).toHaveLength(2); 434 | 435 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 436 | expect((removedJobs[0] as any).queue).toBe(queue); 437 | expect(removedJobs[0].id).toBe(nedbJobs[0]._id); 438 | expect(removedJobs[0].type).toBe(nedbJobs[0].type); 439 | expect(removedJobs[0].priority).toBe(nedbJobs[0].priority); 440 | expect(removedJobs[0].data).toEqual(nedbJobs[0].data); 441 | expect(removedJobs[0].createdAt).toEqual(nedbJobs[0].createdAt); 442 | expect(removedJobs[0].updatedAt).toEqual(nedbJobs[0].updatedAt); 443 | expect(removedJobs[0].startedAt).toEqual(nedbJobs[0].startedAt); 444 | expect(removedJobs[0].completedAt).toEqual(nedbJobs[0].completedAt); 445 | expect(removedJobs[0].failedAt).toEqual(nedbJobs[0].failedAt); 446 | expect(removedJobs[0].state).toBe(nedbJobs[0].state); 447 | expect(removedJobs[0].duration).toBe(nedbJobs[0].duration); 448 | expect(removedJobs[0].progress).toBe(nedbJobs[0].progress); 449 | expect(removedJobs[0].logs).toEqual(nedbJobs[0].logs); 450 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 451 | expect((removedJobs[0] as any)._saved).toBe(true); 452 | 453 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 454 | expect((removedJobs[1] as any).queue).toBe(queue); 455 | expect(removedJobs[1].id).toBe(nedbJobs[2]._id); 456 | expect(removedJobs[1].type).toBe(nedbJobs[2].type); 457 | expect(removedJobs[1].priority).toBe(nedbJobs[2].priority); 458 | expect(removedJobs[1].data).toEqual(nedbJobs[2].data); 459 | expect(removedJobs[1].createdAt).toEqual(nedbJobs[2].createdAt); 460 | expect(removedJobs[1].updatedAt).toEqual(nedbJobs[2].updatedAt); 461 | expect(removedJobs[1].startedAt).toEqual(nedbJobs[2].startedAt); 462 | expect(removedJobs[1].completedAt).toEqual(nedbJobs[2].completedAt); 463 | expect(removedJobs[1].failedAt).toEqual(nedbJobs[2].failedAt); 464 | expect(removedJobs[1].state).toBe(nedbJobs[2].state); 465 | expect(removedJobs[1].duration).toBe(nedbJobs[2].duration); 466 | expect(removedJobs[1].progress).toBe(nedbJobs[2].progress); 467 | expect(removedJobs[1].logs).toEqual(nedbJobs[2].logs); 468 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 469 | expect((removedJobs[1] as any)._saved).toBe(true); 470 | 471 | expect(mockedRepositoryListJobs).toHaveBeenCalledTimes(1); 472 | 473 | expect(removeCallback).toHaveBeenCalledTimes(3); 474 | expect((removeCallback.mock.calls[0][0] as Job).id).toBe(nedbJobs[0]._id); 475 | expect((removeCallback.mock.calls[1][0] as Job).id).toBe(nedbJobs[1]._id); 476 | expect((removeCallback.mock.calls[2][0] as Job).id).toBe(nedbJobs[2]._id); 477 | }); 478 | 479 | describe("requestJobForProcessing", () => { 480 | test("immediately found", async () => { 481 | const uuidValue = "12345678-90ab-cdef-1234-567890abcdef"; 482 | // eslint-disable-next-line 483 | (uuid as any).mockReturnValue(uuidValue); 484 | 485 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 486 | 487 | await queue.createJob({ 488 | type: "type", 489 | priority: Priority.NORMAL, 490 | data: {}, 491 | }); 492 | 493 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 494 | const waitingRequests: { [type: string]: WaitingWorkerRequest[] } = (queue as any).waitingRequests; 495 | 496 | expect(waitingRequests["type"]).toBeUndefined(); 497 | const job = await queue.requestJobForProcessing("type", () => true); 498 | expect(job).not.toBeNull(); 499 | expect(job?.id).toBe(uuidValue); 500 | expect(job?.state).toBe(State.ACTIVE); 501 | expect(waitingRequests["type"]).toBeUndefined(); 502 | }); 503 | 504 | test("queue waiting", async () => { 505 | const uuidValues = [ 506 | "12345678-90ab-cdef-1234-567890abcdef", 507 | "23456789-0abc-def1-2345-67890abcdef1", 508 | "34567890-abcd-ef12-3456-7890abcdef12", 509 | ]; 510 | // eslint-disable-next-line 511 | (uuid as any) 512 | .mockReturnValueOnce(uuidValues[0]) 513 | .mockReturnValueOnce(uuidValues[1]) 514 | .mockReturnValueOnce(uuidValues[2]); 515 | 516 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 517 | 518 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 519 | const waitingRequests: { [type: string]: WaitingWorkerRequest[] } = (queue as any).waitingRequests; 520 | expect(waitingRequests["type1"]).toBeUndefined(); 521 | expect(waitingRequests["type2"]).toBeUndefined(); 522 | 523 | const findJobPromise1 = queue.requestJobForProcessing("type1", () => true); 524 | await new Promise((resolve) => setTimeout(resolve, 100)); 525 | expect(waitingRequests["type1"]).toBeDefined(); 526 | expect(waitingRequests["type1"]).toHaveLength(1); 527 | expect(waitingRequests["type2"]).toBeUndefined(); 528 | 529 | const findJobPromise2 = queue.requestJobForProcessing("type1", () => true); 530 | await new Promise((resolve) => setTimeout(resolve, 100)); 531 | expect(waitingRequests["type1"]).toHaveLength(2); 532 | expect(waitingRequests["type2"]).toBeUndefined(); 533 | 534 | const findJobPromise3 = queue.requestJobForProcessing("type2", () => true); 535 | await new Promise((resolve) => setTimeout(resolve, 100)); 536 | expect(waitingRequests["type1"]).toHaveLength(2); 537 | expect(waitingRequests["type2"]).toBeDefined(); 538 | expect(waitingRequests["type2"]).toHaveLength(1); 539 | 540 | await queue.createJob({ 541 | type: "type1", 542 | priority: Priority.NORMAL, 543 | data: {}, 544 | }); 545 | const job1 = await findJobPromise1; 546 | expect(job1).not.toBeNull(); 547 | expect(job1?.id).toBe(uuidValues[0]); 548 | expect(job1?.state).toBe(State.ACTIVE); 549 | expect(waitingRequests["type1"]).toHaveLength(1); 550 | expect(waitingRequests["type2"]).toHaveLength(1); 551 | 552 | await queue.createJob({ 553 | type: "type1", 554 | priority: Priority.NORMAL, 555 | data: {}, 556 | }); 557 | const job2 = await findJobPromise2; 558 | expect(job2).not.toBeNull(); 559 | expect(job2?.id).toBe(uuidValues[1]); 560 | expect(job2?.state).toBe(State.ACTIVE); 561 | expect(waitingRequests["type1"]).toHaveLength(0); 562 | expect(waitingRequests["type2"]).toHaveLength(1); 563 | 564 | await queue.createJob({ 565 | type: "type2", 566 | priority: Priority.NORMAL, 567 | data: {}, 568 | }); 569 | const job3 = await findJobPromise3; 570 | expect(job3).not.toBeNull(); 571 | expect(job3?.id).toBe(uuidValues[2]); 572 | expect(job3?.state).toBe(State.ACTIVE); 573 | expect(waitingRequests["type1"]).toHaveLength(0); 574 | expect(waitingRequests["type2"]).toHaveLength(0); 575 | }); 576 | 577 | describe("job is already unnecessary", () => { 578 | test("immediately found", async () => { 579 | const uuidValue = "12345678-90ab-cdef-1234-567890abcdef"; 580 | // eslint-disable-next-line 581 | (uuid as any).mockReturnValue(uuidValue); 582 | 583 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 584 | 585 | await queue.createJob({ 586 | type: "type", 587 | priority: Priority.NORMAL, 588 | data: {}, 589 | }); 590 | 591 | const job1 = await queue.requestJobForProcessing("type", () => false); 592 | expect(job1).toBeNull(); 593 | 594 | const job2 = await queue.requestJobForProcessing("type", () => false); 595 | expect(job2).toBeNull(); 596 | 597 | const job3 = await queue.requestJobForProcessing("type", () => true); 598 | expect(job3).not.toBeNull(); 599 | expect(job3?.id).toBe(uuidValue); 600 | }); 601 | 602 | test("queue waiting", async () => { 603 | const uuidValue = "12345678-90ab-cdef-1234-567890abcdef"; 604 | // eslint-disable-next-line 605 | (uuid as any).mockReturnValue(uuidValue); 606 | 607 | const queue = await Queue.createQueue({ inMemoryOnly: true }); 608 | 609 | const findJobPromise1 = queue.requestJobForProcessing("type", () => false); 610 | const findJobPromise2 = queue.requestJobForProcessing("type", () => false); 611 | const findJobPromise3 = queue.requestJobForProcessing("type", () => true); 612 | 613 | await queue.createJob({ 614 | type: "type", 615 | priority: Priority.NORMAL, 616 | data: {}, 617 | }); 618 | 619 | const job1 = await findJobPromise1; 620 | expect(job1).toBeNull(); 621 | 622 | const job2 = await findJobPromise2; 623 | expect(job2).toBeNull(); 624 | 625 | const job3 = await findJobPromise3; 626 | expect(job3).not.toBeNull(); 627 | expect(job3?.id).toBe(uuidValue); 628 | }); 629 | }); 630 | }); 631 | -------------------------------------------------------------------------------- /test/worker.test.ts: -------------------------------------------------------------------------------- 1 | import * as util from "util"; 2 | 3 | import { mock } from "jest-mock-extended"; 4 | 5 | import { Job, Queue, State, Worker } from "../src"; 6 | 7 | const setTimeoutPromise = util.promisify(setTimeout); 8 | 9 | function createJob(queue: Queue, id: string): Job { 10 | return new Job({ 11 | queue, 12 | id, 13 | type: "type", 14 | createdAt: new Date(), 15 | updatedAt: new Date(), 16 | state: State.INACTIVE, 17 | logs: [], 18 | saved: true, 19 | }); 20 | } 21 | 22 | test("basic", async () => { 23 | const mockedQueue = mock(); 24 | 25 | const job = createJob(mockedQueue, "1"); 26 | const spiedJobSetStateToComplete = jest.spyOn(job, "setStateToComplete"); 27 | const spiedJobSetStateToFailure = jest.spyOn(job, "setStateToFailure"); 28 | 29 | mockedQueue.requestJobForProcessing 30 | .mockResolvedValueOnce(job) 31 | .mockResolvedValueOnce(createJob(mockedQueue, "2")); 32 | 33 | const worker = new Worker({ 34 | type: "type", 35 | queue: mockedQueue 36 | }); 37 | 38 | // eslint-disable-next-line @typescript-eslint/no-empty-function 39 | const processor = jest.fn().mockImplementation(async (job: Job) => { 40 | switch (job.id) { 41 | case "1": 42 | return 123; 43 | case "2": 44 | await worker.shutdown(100); 45 | break; 46 | } 47 | }); 48 | 49 | worker.start(processor); 50 | 51 | expect(worker.isRunning).toBe(true); 52 | 53 | await setTimeoutPromise(500); // wait for shutting down queue 54 | 55 | expect(worker.isRunning).toBe(false); 56 | expect(processor).toHaveBeenCalledTimes(2); 57 | expect(spiedJobSetStateToComplete).toHaveBeenCalledTimes(1); 58 | expect(spiedJobSetStateToComplete.mock.calls[0][0]).toBe(123); 59 | expect(spiedJobSetStateToFailure).not.toHaveBeenCalled(); 60 | }); 61 | 62 | describe("shutdown", () => { 63 | test("idle", async () => { 64 | const mockedQueue = mock(); 65 | 66 | // eslint-disable-next-line @typescript-eslint/no-empty-function 67 | mockedQueue.requestJobForProcessing.mockReturnValue(new Promise(() => {})); 68 | 69 | const worker = new Worker({ 70 | type: "type", 71 | queue: mockedQueue 72 | }); 73 | 74 | const processor = jest.fn(); 75 | 76 | worker.start(processor); 77 | 78 | expect(worker.isRunning).toBe(true); 79 | 80 | await worker.shutdown(100); 81 | 82 | expect(worker.isRunning).toBe(false); 83 | expect(worker.currentJob).toBeNull(); 84 | expect(processor).not.toHaveBeenCalled(); 85 | }); 86 | 87 | test("is not running", async () => { 88 | const mockedQueue = mock(); 89 | 90 | const worker = new Worker({ 91 | type: "type", 92 | queue: mockedQueue 93 | }); 94 | 95 | await worker.shutdown(100); 96 | 97 | expect(worker.isRunning).toBe(false); 98 | expect(worker.currentJob).toBeNull(); 99 | }); 100 | 101 | test("processing job is finished before timed out", async () => { 102 | const mockedQueue = mock(); 103 | 104 | const job = createJob(mockedQueue, "1"); 105 | const spiedJobSetStateToComplete = jest.spyOn(job, "setStateToComplete"); 106 | const spiedJobSetStateToFailure = jest.spyOn(job, "setStateToFailure"); 107 | 108 | mockedQueue.requestJobForProcessing.mockResolvedValueOnce(job); 109 | 110 | const worker = new Worker({ 111 | type: "type", 112 | queue: mockedQueue 113 | }); 114 | 115 | const processor = jest.fn().mockImplementation(async () => { 116 | return setTimeoutPromise(500); 117 | }); 118 | 119 | worker.start(processor); 120 | await setTimeoutPromise(100); 121 | 122 | expect(worker.isRunning).toBe(true); 123 | 124 | await worker.shutdown(1000); 125 | 126 | expect(worker.isRunning).toBe(false); 127 | expect(worker.currentJob).toBeNull(); 128 | expect(processor).toHaveBeenCalledTimes(1); 129 | expect(spiedJobSetStateToComplete).toHaveBeenCalledTimes(1); 130 | expect(spiedJobSetStateToFailure).not.toHaveBeenCalled(); 131 | }); 132 | 133 | test("processing job is timed out", async () => { 134 | const mockedQueue = mock(); 135 | 136 | const job = createJob(mockedQueue, "1"); 137 | const spiedJobSetStateToComplete = jest.spyOn(job, "setStateToComplete"); 138 | const spiedJobSetStateToFailure = jest.spyOn(job, "setStateToFailure"); 139 | 140 | mockedQueue.requestJobForProcessing.mockResolvedValueOnce(job); 141 | 142 | const worker = new Worker({ 143 | type: "type", 144 | queue: mockedQueue 145 | }); 146 | 147 | const processor = jest.fn().mockImplementation(async () => { 148 | return setTimeoutPromise(1000); 149 | }); 150 | 151 | worker.start(processor); 152 | await setTimeoutPromise(100); 153 | 154 | expect(worker.isRunning).toBe(true); 155 | 156 | await worker.shutdown(500); 157 | 158 | expect(worker.isRunning).toBe(false); 159 | expect(worker.currentJob).toBeNull(); 160 | expect(processor).toHaveBeenCalledTimes(1); 161 | expect(spiedJobSetStateToComplete).not.toHaveBeenCalled(); 162 | expect(spiedJobSetStateToFailure).toHaveBeenCalledTimes(1); 163 | expect(spiedJobSetStateToFailure.mock.calls[0][0].message).toBe("shutdown timeout") 164 | }); 165 | }); 166 | 167 | test("processor failed", async () => { 168 | const mockedQueue = mock(); 169 | 170 | const job = createJob(mockedQueue, "1"); 171 | const spiedJobSetStateToComplete = jest.spyOn(job, "setStateToComplete"); 172 | const spiedJobSetStateToFailure = jest.spyOn(job, "setStateToFailure"); 173 | 174 | mockedQueue.requestJobForProcessing 175 | .mockResolvedValueOnce(job) 176 | .mockResolvedValueOnce(createJob(mockedQueue, "2")); 177 | 178 | const worker = new Worker({ 179 | type: "type", 180 | queue: mockedQueue 181 | }); 182 | 183 | const processor = jest.fn().mockImplementation(async (job: Job) => { 184 | switch (job.id) { 185 | case "1": 186 | throw new Error("Some Error"); 187 | case "2": 188 | await worker.shutdown(100); 189 | break; 190 | } 191 | }); 192 | 193 | worker.start(processor); 194 | await setTimeoutPromise(100); 195 | 196 | expect(spiedJobSetStateToComplete).not.toHaveBeenCalled(); 197 | expect(spiedJobSetStateToFailure).toHaveBeenCalledTimes(1); 198 | expect(spiedJobSetStateToFailure.mock.calls[0][0].message).toBe("Some Error"); 199 | }); 200 | 201 | test("processing job is deleted", async () => { 202 | const mockedQueue = mock(); 203 | 204 | const job1 = createJob(mockedQueue, "1"); 205 | const spiedJobSetStateToComplete = jest.spyOn(job1, "setStateToComplete"); 206 | const spiedJobSetStateToFailure = jest.spyOn(job1, "setStateToFailure"); 207 | 208 | mockedQueue.requestJobForProcessing 209 | .mockResolvedValueOnce(job1) 210 | .mockResolvedValueOnce(createJob(mockedQueue, "2")); 211 | 212 | const worker = new Worker({ 213 | type: "type", 214 | queue: mockedQueue 215 | }); 216 | 217 | const processor = jest.fn().mockImplementation(async (job: Job) => { 218 | switch (job.id) { 219 | case "1": 220 | job1.isExist = (): Promise => Promise.resolve(false); 221 | break; 222 | case "2": 223 | await worker.shutdown(100); 224 | break; 225 | } 226 | }); 227 | 228 | worker.start(processor); 229 | await setTimeoutPromise(100); 230 | 231 | expect(spiedJobSetStateToComplete).not.toHaveBeenCalled(); 232 | expect(spiedJobSetStateToFailure).not.toHaveBeenCalled(); 233 | }); 234 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | "target": "es2017", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017','ES2018' or 'ESNEXT'. */ 5 | "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 6 | // "lib": [], /* Specify library files to be included in the compilation. */ 7 | // "allowJs": true, /* Allow javascript files to be compiled. */ 8 | // "checkJs": true, /* Report errors in .js files. */ 9 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 10 | "declaration": true, /* Generates corresponding '.d.ts' file. */ 11 | // "sourceMap": true, /* Generates corresponding '.map' file. */ 12 | // "outFile": "./", /* Concatenate and emit output to single file. */ 13 | "outDir": "./dist", /* Redirect output structure to the directory. */ 14 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 15 | // "removeComments": true, /* Do not emit comments to output. */ 16 | // "noEmit": true, /* Do not emit outputs. */ 17 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 18 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 19 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 20 | 21 | /* Strict Type-Checking Options */ 22 | "strict": true, /* Enable all strict type-checking options. */ 23 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 24 | // "strictNullChecks": true, /* Enable strict null checks. */ 25 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 26 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 27 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 28 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 29 | 30 | /* Additional Checks */ 31 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 32 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 33 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 34 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 35 | 36 | /* Module Resolution Options */ 37 | // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 38 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 39 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 40 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 41 | // "typeRoots": [], /* List of folders to include type definitions from. */ 42 | // "types": [], /* Type declaration files to be included in compilation. */ 43 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 44 | "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 45 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 46 | 47 | /* Source Map Options */ 48 | // "sourceRoot": "./", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 49 | // "mapRoot": "./", /* Specify the location where debugger should locate map files instead of generated locations. */ 50 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 51 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 52 | 53 | /* Experimental Options */ 54 | "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 55 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 56 | }, 57 | "include": [ 58 | "src/**/*.ts" 59 | ], 60 | "exclude": [ 61 | "**/*.test.ts" 62 | ] 63 | } 64 | --------------------------------------------------------------------------------