├── .babelrc
├── .eslintignore
├── .eslintrc.js
├── .gitignore
├── .travis.yml
├── LICENSE
├── Models
├── Queue.js
└── Worker.js
├── README.md
├── config
├── Database.js
└── config.js
├── docs
├── easy-os-background-tasks-in-react-native.md
├── easy-os-background-tasks-in-react-native.png
└── logo.png
├── index.js
├── package.json
├── tests
├── Queue.test.js
└── Worker.test.js
└── yarn.lock
/.babelrc:
--------------------------------------------------------------------------------
1 | {
2 | "presets": ["react-native"]
3 | }
4 |
--------------------------------------------------------------------------------
/.eslintignore:
--------------------------------------------------------------------------------
1 | /coverage
--------------------------------------------------------------------------------
/.eslintrc.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | "env": {
3 | "browser": true,
4 | "es6": true
5 | },
6 | "parser": "babel-eslint",
7 | "extends": [
8 | "eslint:recommended"
9 | ],
10 | "parserOptions": {
11 | "sourceType": "module"
12 | },
13 | "rules": {
14 | "indent": [
15 | "error",
16 | 2,
17 | { "SwitchCase": 1 }
18 | ],
19 | "brace-style": [
20 | "error",
21 | "1tbs",
22 | { "allowSingleLine": true }
23 | ],
24 | "linebreak-style": [
25 | "error",
26 | "unix"
27 | ],
28 | "quotes": [
29 | "error",
30 | "single"
31 | ],
32 | "semi": [
33 | "error",
34 | "always"
35 | ],
36 | "no-var": "error"
37 | }
38 | };
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /node_modules
2 | /coverage
3 | /.idea
4 | /reactNativeQueue.realm*
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 |
2 | #
3 | # Travis CI config
4 | # https://docs.travis-ci.com/user/customizing-the-build/
5 | #
6 |
7 | env:
8 | - COVERALLS_ENV=production
9 |
10 | branches:
11 | only:
12 | - master
13 |
14 | language: node_js
15 |
16 | node_js:
17 | - "7"
18 |
19 | cache:
20 | directories:
21 | - "node_modules"
22 |
23 | script:
24 | - npm run lint
25 | - npm test
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2017 Reid Mayo
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/Models/Queue.js:
--------------------------------------------------------------------------------
1 | /**
2 | *
3 | * Queue Model
4 | *
5 | * Queue Job Realm Schema defined in ../config/Database
6 | *
7 | */
8 |
9 | import Database from '../config/Database';
10 | import uuid from 'react-native-uuid';
11 | import Worker from './Worker';
12 | import promiseReflect from 'promise-reflect';
13 |
14 |
15 | export class Queue {
16 |
17 | /**
18 | *
19 | * Set initial class properties.
20 | *
21 | * @constructor
22 | */
23 | constructor() {
24 | this.realm = null;
25 | this.worker = new Worker();
26 | this.status = 'inactive';
27 | }
28 |
29 | /**
30 | *
31 | * Initializes the queue by connecting to Realm database.
32 | *
33 | */
34 | async init() {
35 | if (this.realm === null) {
36 | this.realm = await Database.getRealmInstance();
37 | }
38 | }
39 |
40 | /**
41 | *
42 | * Add a worker function to the queue.
43 | *
44 | * Worker will be called to execute jobs associated with jobName.
45 | *
46 | * Worker function will receive job id and job payload as parameters.
47 | *
48 | * Example:
49 | *
50 | * function exampleJobWorker(id, payload) {
51 | * console.log(id); // UUID of job.
52 | * console.log(payload); // Payload of data related to job.
53 | * }
54 | *
55 | * @param jobName {string} - Name associated with jobs assigned to this worker.
56 | * @param worker {function} - The worker function that will execute jobs.
57 | * @param options {object} - Worker options. See README.md for worker options info.
58 | */
59 | addWorker(jobName, worker, options = {}) {
60 | this.worker.addWorker(jobName, worker, options);
61 | }
62 |
63 | /**
64 | *
65 | * Delete worker function from queue.
66 | *
67 | * @param jobName {string} - Name associated with jobs assigned to this worker.
68 | */
69 | removeWorker(jobName) {
70 | this.worker.removeWorker(jobName);
71 | }
72 |
73 | /**
74 | *
75 | * Creates a new job and adds it to queue.
76 | *
77 | * Queue will automatically start processing unless startQueue param is set to false.
78 | *
79 | * @param name {string} - Name associated with job. The worker function assigned to this name will be used to execute this job.
80 | * @param payload {object} - Object of arbitrary data to be passed into worker function when job executes.
81 | * @param options {object} - Job related options like timeout etc. See README.md for job options info.
82 | * @param startQueue - {boolean} - Whether or not to immediately begin prcessing queue. If false queue.start() must be manually called.
83 | */
84 | createJob(name, payload = {}, options = {}, startQueue = true) {
85 |
86 | if (!name) {
87 | throw new Error('Job name must be supplied.');
88 | }
89 |
90 | // Validate options
91 | if (options.timeout < 0 || options.attempts < 0) {
92 | throw new Error('Invalid job option.');
93 | }
94 |
95 | this.realm.write(() => {
96 |
97 | this.realm.create('Job', {
98 | id: uuid.v4(),
99 | name,
100 | payload: JSON.stringify(payload),
101 | data: JSON.stringify({
102 | attempts: options.attempts || 1
103 | }),
104 | priority: options.priority || 0,
105 | active: false,
106 | timeout: (options.timeout >= 0) ? options.timeout : 25000,
107 | created: new Date(),
108 | failed: null
109 | });
110 |
111 | });
112 |
113 | // Start queue on job creation if it isn't running by default.
114 | if (startQueue && this.status == 'inactive') {
115 | this.start();
116 | }
117 |
118 | }
119 |
120 | /**
121 | *
122 | * Start processing the queue.
123 | *
124 | * If queue was not started automatically during queue.createJob(), this
125 | * method should be used to manually start the queue.
126 | *
127 | * If queue.start() is called again when queue is already running,
128 | * queue.start() will return early with a false boolean value instead
129 | * of running multiple queue processing loops concurrently.
130 | *
131 | * Lifespan can be passed to start() in order to run the queue for a specific amount of time before stopping.
132 | * This is useful, as an example, for OS background tasks which typically are time limited.
133 | *
134 | * NOTE: If lifespan is set, only jobs with a timeout property at least 500ms less than remaining lifespan will be processed
135 | * during queue processing lifespan. This is to buffer for the small amount of time required to query Realm for suitable
136 | * jobs, and to mark such jobs as complete or failed when job finishes processing.
137 | *
138 | * IMPORTANT: Jobs with timeout set to 0 that run indefinitely will not be processed if the queue is running with a lifespan.
139 | *
140 | * @param lifespan {number} - If lifespan is passed, the queue will start up and run for lifespan ms, then queue will be stopped.
141 | * @return {boolean|undefined} - False if queue is already started. Otherwise nothing is returned when queue finishes processing.
142 | */
143 | async start(lifespan = 0) {
144 |
145 | // If queue is already running, don't fire up concurrent loop.
146 | if (this.status == 'active') {
147 | return false;
148 | }
149 |
150 | this.status = 'active';
151 |
152 | // Get jobs to process
153 | const startTime = Date.now();
154 | let lifespanRemaining = null;
155 | let concurrentJobs = [];
156 |
157 | if (lifespan !== 0) {
158 | lifespanRemaining = lifespan - (Date.now() - startTime);
159 | lifespanRemaining = (lifespanRemaining === 0) ? -1 : lifespanRemaining; // Handle exactly zero lifespan remaining edge case.
160 | concurrentJobs = await this.getConcurrentJobs(lifespanRemaining);
161 | } else {
162 | concurrentJobs = await this.getConcurrentJobs();
163 | }
164 |
165 | while (this.status == 'active' && concurrentJobs.length) {
166 |
167 | // Loop over jobs and process them concurrently.
168 | const processingJobs = concurrentJobs.map( job => {
169 | return this.processJob(job);
170 | });
171 |
172 | // Promise Reflect ensures all processingJobs resolve so
173 | // we don't break await early if one of the jobs fails.
174 | await Promise.all(processingJobs.map(promiseReflect));
175 |
176 | // Get next batch of jobs.
177 | if (lifespan !== 0) {
178 | lifespanRemaining = lifespan - (Date.now() - startTime);
179 | lifespanRemaining = (lifespanRemaining === 0) ? -1 : lifespanRemaining; // Handle exactly zero lifespan remaining edge case.
180 | concurrentJobs = await this.getConcurrentJobs(lifespanRemaining);
181 | } else {
182 | concurrentJobs = await this.getConcurrentJobs();
183 | }
184 |
185 | }
186 |
187 | this.status = 'inactive';
188 |
189 | }
190 |
191 | /**
192 | *
193 | * Stop processing queue.
194 | *
195 | * If queue.stop() is called, queue will stop processing until
196 | * queue is restarted by either queue.createJob() or queue.start().
197 | *
198 | */
199 | stop() {
200 | this.status = 'inactive';
201 | }
202 |
203 | /**
204 | *
205 | * Get a collection of all the jobs in the queue.
206 | *
207 | * @param sync {boolean} - This should be true if you want to guarantee job data is fresh. Otherwise you could receive job data that is not up to date if a write transaction is occuring concurrently.
208 | * @return {promise} - Promise that resolves to a collection of all the jobs in the queue.
209 | */
210 | async getJobs(sync = false) {
211 |
212 | if (sync) {
213 |
214 | let jobs = null;
215 | this.realm.write(() => {
216 |
217 | jobs = this.realm.objects('Job');
218 |
219 | });
220 |
221 | return jobs;
222 |
223 | } else {
224 | return await this.realm.objects('Job');
225 | }
226 |
227 | }
228 |
229 | /**
230 | *
231 | * Get the next job(s) that should be processed by the queue.
232 | *
233 | * If the next job to be processed by the queue is associated with a
234 | * worker function that has concurrency X > 1, then X related (jobs with same name)
235 | * jobs will be returned.
236 | *
237 | * If queue is running with a lifespan, only jobs with timeouts at least 500ms < than REMAINING lifespan
238 | * AND a set timeout (ie timeout > 0) will be returned. See Queue.start() for more info.
239 | *
240 | * @param queueLifespanRemaining {number} - The remaining lifespan of the current queue process (defaults to indefinite).
241 | * @return {promise} - Promise resolves to an array of job(s) to be processed next by the queue.
242 | */
243 | async getConcurrentJobs(queueLifespanRemaining = 0) {
244 |
245 | let concurrentJobs = [];
246 |
247 | this.realm.write(() => {
248 |
249 | // Get next job from queue.
250 | let nextJob = null;
251 |
252 | // Build query string
253 | // If queueLife
254 | const timeoutUpperBound = (queueLifespanRemaining - 500 > 0) ? queueLifespanRemaining - 499 : 0; // Only get jobs with timeout at least 500ms < queueLifespanRemaining.
255 |
256 | const initialQuery = (queueLifespanRemaining)
257 | ? 'active == FALSE AND failed == null AND timeout > 0 AND timeout < ' + timeoutUpperBound
258 | : 'active == FALSE AND failed == null';
259 |
260 | let jobs = this.realm.objects('Job')
261 | .filtered(initialQuery)
262 | .sorted([['priority', true], ['created', false]]);
263 |
264 | if (jobs.length) {
265 | nextJob = jobs[0];
266 | }
267 |
268 | // If next job exists, get concurrent related jobs appropriately.
269 | if (nextJob) {
270 |
271 | const concurrency = this.worker.getConcurrency(nextJob.name);
272 |
273 | const allRelatedJobsQuery = (queueLifespanRemaining)
274 | ? 'name == "'+ nextJob.name +'" AND active == FALSE AND failed == null AND timeout > 0 AND timeout < ' + timeoutUpperBound
275 | : 'name == "'+ nextJob.name +'" AND active == FALSE AND failed == null';
276 |
277 | const allRelatedJobs = this.realm.objects('Job')
278 | .filtered(allRelatedJobsQuery)
279 | .sorted([['priority', true], ['created', false]]);
280 |
281 | let jobsToMarkActive = allRelatedJobs.slice(0, concurrency);
282 |
283 | // Grab concurrent job ids to reselect jobs as marking these jobs as active will remove
284 | // them from initial selection when write transaction exits.
285 | // See: https://stackoverflow.com/questions/47359368/does-realm-support-select-for-update-style-read-locking/47363356#comment81772710_47363356
286 | const concurrentJobIds = jobsToMarkActive.map( job => job.id);
287 |
288 | // Mark concurrent jobs as active
289 | jobsToMarkActive = jobsToMarkActive.map( job => {
290 | job.active = true;
291 | });
292 |
293 | // Reselect now-active concurrent jobs by id.
294 | const reselectQuery = concurrentJobIds.map( jobId => 'id == "' + jobId + '"').join(' OR ');
295 | const reselectedJobs = this.realm.objects('Job')
296 | .filtered(reselectQuery)
297 | .sorted([['priority', true], ['created', false]]);
298 |
299 | concurrentJobs = reselectedJobs.slice(0, concurrency);
300 |
301 | }
302 |
303 | });
304 |
305 | return concurrentJobs;
306 |
307 | }
308 |
309 | /**
310 | *
311 | * Process a job.
312 | *
313 | * Job lifecycle callbacks are called as appropriate throughout the job processing lifecycle.
314 | *
315 | * Job is deleted upon successful completion.
316 | *
317 | * If job fails execution via timeout or other exception, error will be
318 | * logged to job.data.errors array and job will be reset to inactive status.
319 | * Job will be re-attempted up to the specified "attempts" setting (defaults to 1),
320 | * after which it will be marked as failed and not re-attempted further.
321 | *
322 | * @param job {object} - Job realm model object
323 | */
324 | async processJob(job) {
325 |
326 | // Data must be cloned off the realm job object for several lifecycle callbacks to work correctly.
327 | // This is because realm job is deleted before some callbacks are called if job processed successfully.
328 | // More info: https://github.com/billmalarky/react-native-queue/issues/2#issuecomment-361418965
329 | const jobName = job.name;
330 | const jobId = job.id;
331 | const jobPayload = JSON.parse(job.payload);
332 |
333 | // Fire onStart job lifecycle callback
334 | this.worker.executeJobLifecycleCallback('onStart', jobName, jobId, jobPayload);
335 |
336 | try {
337 |
338 | await this.worker.executeJob(job);
339 |
340 | // On successful job completion, remove job
341 | this.realm.write(() => {
342 |
343 | this.realm.delete(job);
344 |
345 | });
346 |
347 | // Job has processed successfully, fire onSuccess and onComplete job lifecycle callbacks.
348 | this.worker.executeJobLifecycleCallback('onSuccess', jobName, jobId, jobPayload);
349 | this.worker.executeJobLifecycleCallback('onComplete', jobName, jobId, jobPayload);
350 |
351 | } catch (error) {
352 |
353 | // Handle job failure logic, including retries.
354 | let jobData = JSON.parse(job.data);
355 |
356 | this.realm.write(() => {
357 |
358 | // Increment failed attempts number
359 | if (!jobData.failedAttempts) {
360 | jobData.failedAttempts = 1;
361 | } else {
362 | jobData.failedAttempts++;
363 | }
364 |
365 | // Log error
366 | if (!jobData.errors) {
367 | jobData.errors = [ error.message ];
368 | } else {
369 | jobData.errors.push(error.message);
370 | }
371 |
372 | job.data = JSON.stringify(jobData);
373 |
374 | // Reset active status
375 | job.active = false;
376 |
377 | // Mark job as failed if too many attempts
378 | if (jobData.failedAttempts >= jobData.attempts) {
379 | job.failed = new Date();
380 | }
381 |
382 | });
383 |
384 | // Execute job onFailure lifecycle callback.
385 | this.worker.executeJobLifecycleCallback('onFailure', jobName, jobId, jobPayload);
386 |
387 | // If job has failed all attempts execute job onFailed and onComplete lifecycle callbacks.
388 | if (jobData.failedAttempts >= jobData.attempts) {
389 | this.worker.executeJobLifecycleCallback('onFailed', jobName, jobId, jobPayload);
390 | this.worker.executeJobLifecycleCallback('onComplete', jobName, jobId, jobPayload);
391 | }
392 |
393 | }
394 |
395 | }
396 |
397 | /**
398 | *
399 | * Delete jobs in the queue.
400 | *
401 | * If jobName is supplied, only jobs associated with that name
402 | * will be deleted. Otherwise all jobs in queue will be deleted.
403 | *
404 | * @param jobName {string} - Name associated with job (and related job worker).
405 | */
406 | flushQueue(jobName = null) {
407 |
408 | if (jobName) {
409 |
410 | this.realm.write(() => {
411 |
412 | let jobs = this.realm.objects('Job')
413 | .filtered('name == "' + jobName + '"');
414 |
415 | if (jobs.length) {
416 | this.realm.delete(jobs);
417 | }
418 |
419 | });
420 |
421 | } else {
422 | this.realm.write(() => {
423 |
424 | this.realm.deleteAll();
425 |
426 | });
427 | }
428 |
429 | }
430 |
431 |
432 | }
433 |
434 | /**
435 | *
436 | * Factory should be used to create a new queue instance.
437 | *
438 | * @return {Queue} - A queue instance.
439 | */
440 | export default async function queueFactory() {
441 |
442 | const queue = new Queue();
443 | await queue.init();
444 |
445 | return queue;
446 |
447 | }
448 |
--------------------------------------------------------------------------------
/Models/Worker.js:
--------------------------------------------------------------------------------
1 | /**
2 | *
3 | * Worker Model
4 | *
5 | */
6 |
7 | export default class Worker {
8 |
9 | /**
10 | *
11 | * Singleton map of all worker functions assigned to queue.
12 | *
13 | */
14 | static workers = {};
15 |
16 | /**
17 | *
18 | * Assign a worker function to the queue.
19 | *
20 | * Worker will be called to execute jobs associated with jobName.
21 | *
22 | * Worker function will receive job id and job payload as parameters.
23 | *
24 | * Example:
25 | *
26 | * function exampleJobWorker(id, payload) {
27 | * console.log(id); // UUID of job.
28 | * console.log(payload); // Payload of data related to job.
29 | * }
30 | *
31 | * @param jobName {string} - Name associated with jobs assigned to this worker.
32 | * @param worker {function} - The worker function that will execute jobs.
33 | * @param options {object} - Worker options. See README.md for worker options info.
34 | */
35 | addWorker(jobName, worker, options = {}) {
36 |
37 | // Validate input.
38 | if (!jobName || !worker) {
39 | throw new Error('Job name and associated worker function must be supplied.');
40 | }
41 |
42 | // Attach options to worker
43 | worker.options = {
44 | concurrency: options.concurrency || 1,
45 | onStart: options.onStart || null,
46 | onSuccess: options.onSuccess || null,
47 | onFailure: options.onFailure || null,
48 | onFailed: options.onFailed || null,
49 | onComplete: options.onComplete || null
50 | };
51 |
52 | Worker.workers[jobName] = worker;
53 | }
54 |
55 | /**
56 | *
57 | * Un-assign worker function from queue.
58 | *
59 | * @param jobName {string} - Name associated with jobs assigned to this worker.
60 | */
61 | removeWorker(jobName) {
62 | delete Worker.workers[jobName];
63 | }
64 |
65 | /**
66 | *
67 | * Get the concurrency setting for a worker.
68 | *
69 | * Worker concurrency defaults to 1.
70 | *
71 | * @param jobName {string} - Name associated with jobs assigned to this worker.
72 | * @throws Throws error if no worker is currently assigned to passed in job name.
73 | * @return {number}
74 | */
75 | getConcurrency(jobName) {
76 |
77 | // If no worker assigned to job name, throw error.
78 | if (!Worker.workers[jobName]) {
79 | throw new Error('Job ' + jobName + ' does not have a worker assigned to it.');
80 | }
81 |
82 | return Worker.workers[jobName].options.concurrency;
83 |
84 | }
85 |
86 | /**
87 | *
88 | * Execute the worker function assigned to the passed in job name.
89 | *
90 | * If job has a timeout setting, job will fail with a timeout exception upon reaching timeout.
91 | *
92 | * @throws Throws error if no worker is currently assigned to passed in job name.
93 | * @param job {object} - Job realm model object
94 | */
95 | async executeJob(job) {
96 |
97 | // If no worker assigned to job name, throw error.
98 | if (!Worker.workers[job.name]) {
99 | throw new Error('Job ' + job.name + ' does not have a worker assigned to it.');
100 | }
101 |
102 | // Data must be cloned off the realm job object for the timeout logic promise race.
103 | // More info: https://github.com/billmalarky/react-native-queue/issues/2#issuecomment-361418965
104 | const jobId = job.id;
105 | const jobName = job.name;
106 | const jobTimeout = job.timeout;
107 | const jobPayload = JSON.parse(job.payload);
108 |
109 | if (jobTimeout > 0) {
110 |
111 | let timeoutPromise = new Promise((resolve, reject) => {
112 |
113 | setTimeout(() => {
114 | reject(new Error('TIMEOUT: Job id: ' + jobId + ' timed out in ' + jobTimeout + 'ms.'));
115 | }, jobTimeout);
116 |
117 | });
118 |
119 | await Promise.race([timeoutPromise, Worker.workers[jobName](jobId, jobPayload)]);
120 |
121 | } else {
122 | await Worker.workers[jobName](jobId, jobPayload);
123 | }
124 |
125 | }
126 |
127 | /**
128 | *
129 | * Execute an asynchronous job lifecycle callback associated with related worker.
130 | *
131 | * @param callbackName {string} - Job lifecycle callback name.
132 | * @param jobName {string} - Name associated with jobs assigned to related worker.
133 | * @param jobId {string} - Unique id associated with job.
134 | * @param jobPayload {object} - Data payload associated with job.
135 | */
136 | async executeJobLifecycleCallback(callbackName, jobName, jobId, jobPayload) {
137 |
138 | // Validate callback name
139 | const validCallbacks = ['onStart', 'onSuccess', 'onFailure', 'onFailed', 'onComplete'];
140 | if (!validCallbacks.includes(callbackName)) {
141 | throw new Error('Invalid job lifecycle callback name.');
142 | }
143 |
144 | // Fire job lifecycle callback if set.
145 | // Uses a try catch statement to gracefully degrade errors in production.
146 | if (Worker.workers[jobName].options[callbackName]) {
147 |
148 | try {
149 | await Worker.workers[jobName].options[callbackName](jobId, jobPayload);
150 | } catch (error) {
151 | console.error(error); // eslint-disable-line no-console
152 | }
153 |
154 | }
155 |
156 | }
157 |
158 | }
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | 
2 |
3 | # React Native Queue
4 | #### Simple. Powerful. Persistent.
5 |
6 | [](https://travis-ci.org/billmalarky/react-native-queue)
7 | [](https://github.com/billmalarky/react-native-queue/blob/master/LICENSE)
8 | [](https://github.com/billmalarky/react-native-queue/blob/master/.eslintrc.js)
9 | [](http://usejsdoc.org/)
10 | [](https://coveralls.io/github/billmalarky/react-native-queue?branch=master)
11 |
12 | A React Native at-least-once priority job queue / task queue backed by persistent Realm storage. Jobs will persist until completed, even if user closes and re-opens app. React Native Queue is easily integrated into OS background processes (services) so you can ensure the queue will continue to process until all jobs are completed even if app isn't in focus. It also plays well with Workers so your jobs can be thrown on the queue, then processed in dedicated worker threads for greatly improved processing performance.
13 |
14 | ## Table of Contents
15 |
16 | * [Features](#features)
17 | * [Example Use Cases](#example-use-cases)
18 | * [Installation](#installation)
19 | * [Basic Usage](#basic-usage)
20 | * [Options and Job Lifecycle Callbacks](#options-and-job-lifecycle-callbacks)
21 | * [Testing with Jest](#testing-with-jest)
22 | * [Caveats](#caveats)
23 | * [Advanced Usage Examples](#advanced-usage-examples)
24 | * [Advanced Job Full Example](#advanced-job-full-example)
25 | * [OS Background Task Full Example](#os-background-task-full-example)
26 | * [Tutorials](#tutorials)
27 |
28 | ## Features
29 |
30 | * **Simple API:** Set up job workers and begin creating your jobs in minutes with just two basic API calls
31 | * queue.addWorker(name, workerFunction, options = {})
32 | * queue.createJob(name, payload = {}, options = {}, startQueue = true)
33 | * **Powerful options:** Easily modify default functionality. Set job timeouts, number of retry attempts, priority, job lifecycle callbacks, and worker concurrency with an options object. Start queue processing with a lifespan to easily meet OS background task time limits.
34 | * **Persistent Jobs:** Jobs are persisted with Realm. Because jobs persist, you can easily continue to process jobs across app restarts or in OS background tasks until completed or failed (or app is uninstalled).
35 | * **Powerful Integrations:** React Native Queue was designed to play well with others. The queue quickly integrates with a variety of OS background task and Worker packages so processing your jobs in a background service or dedicated thread have never been easier.
36 |
37 | ## Example Use Cases
38 |
39 | **React Native Queue is designed to be a swiss army knife for task management in React Native**. It abstracts away the many annoyances related to processing complex tasks, like durability, retry-on-failure, timeouts, chaining processes, and more. **Just throw your jobs onto the queue and relax - they're covered**.
40 |
41 | Need advanced task functionality like dedicated worker threads or OS services? Easy:
42 |
43 | * **React Native Queue + React Native Background Task:** [Simple and Powerful OS services](#os-background-task-full-example) that fire when your app is closed.
44 | * **React Native Queue + React Native Workers:** Spinning up [dedicated worker threads](https://gist.github.com/billmalarky/1d1f72a267a1608606410602aa63cabf) for CPU intensive tasks has never been easier.
45 |
46 | **Example Queue Tasks:**
47 |
48 | * Downloading content for offline access.
49 | * Media processing.
50 | * Cache Warming.
51 | * _Durable_ API calls to external services, such as publishing content to a variety of 3rd party distribution channel APIs.
52 | * Complex and time-consuming jobs that you want consistently processed regardless if app is open, closed, or repeatedly opened and closed.
53 | * Complex tasks with multiple linked dependant steps (job chaining).
54 |
55 | ## Installation
56 |
57 | ```bash
58 | $ npm install --save react-native-queue
59 | ```
60 |
61 | Or
62 |
63 | ```bash
64 | $ yarn add react-native-queue
65 | ```
66 |
67 | Then, because this package has a depedency on [Realm](https://github.com/realm/realm-js) you will need to link this native package by running:
68 |
69 | ```bash
70 | $ react-native link realm
71 | ```
72 |
73 | Linking realm **should only be done once**, reinstalling node_modules with npm or yarn does not require running the above command again.
74 |
75 | To troubleshoot linking, refer to [the realm installation instructions](https://realm.io/docs/javascript/latest/#getting-started).
76 |
77 | ## Basic Usage
78 |
79 | React Native Queue is a standard job/task queue built specifically for react native applications. If you have a long-running task, or a large number of tasks, consider turning that task into a job(s) and throwing it/them onto the queue to be processed in the background instead of blocking your UI until task(s) complete.
80 |
81 | Creating and processing jobs consists of:
82 |
83 | 1. Importing and initializing React Native Queue
84 | 2. Registering worker functions (the functions that execute your jobs).
85 | 3. Creating jobs.
86 | 4. Starting the queue (note this happens automatically on job creation, but sometimes the queue must be explicitly started such as in a OS background task or on app restart). Queue can be started with a lifespan in order to limit queue processing time.
87 |
88 | ```js
89 |
90 | import queueFactory from 'react-native-queue';
91 |
92 | // Of course this line needs to be in the context of an async function,
93 | // otherwise use queueFactory.then((queue) => { console.log('add workers and jobs here'); });
94 | const queue = await queueFactory();
95 |
96 | // Register the worker function for "example-job" jobs.
97 | queue.addWorker('example-job', async (id, payload) => {
98 | console.log('EXECUTING "example-job" with id: ' + id);
99 | console.log(payload, 'payload');
100 |
101 | await new Promise((resolve) => {
102 | setTimeout(() => {
103 | console.log('"example-job" has completed!');
104 | resolve();
105 | }, 5000);
106 | });
107 |
108 | });
109 |
110 | // Create a couple "example-job" jobs.
111 |
112 | // Example job passes a payload of data to 'example-job' worker.
113 | // Default settings are used (note the empty options object).
114 | // Because false is passed, the queue won't automatically start when this job is created, so usually queue.start()
115 | // would have to be manually called. However in the final createJob() below we don't pass false so it will start the queue.
116 | // NOTE: We pass false for example purposes. In most scenarios starting queue on createJob() is perfectly fine.
117 | queue.createJob('example-job', {
118 | emailAddress: 'foo@bar.com',
119 | randomData: {
120 | random: 'object',
121 | of: 'arbitrary data'
122 | }
123 | }, {}, false);
124 |
125 | // Create another job with an example timeout option set.
126 | // false is passed so queue still hasn't started up.
127 | queue.createJob('example-job', {
128 | emailAddress: 'example@gmail.com',
129 | randomData: {
130 | random: 'object',
131 | of: 'arbitrary data'
132 | }
133 | }, {
134 | timeout: 1000 // This job will timeout in 1000 ms and be marked failed (since worker takes 5000 ms to complete).
135 | }, false);
136 |
137 | // This will automatically start the queue after adding the new job so we don't have to manually call queue.start().
138 | queue.createJob('example-job', {
139 | emailAddress: 'another@gmail.com',
140 | randomData: {
141 | random: 'object',
142 | of: 'arbitrary data'
143 | }
144 | });
145 |
146 | console.log('The above jobs are processing in the background of app now.');
147 |
148 | ```
149 |
150 | ## Options and Job Lifecycle Callbacks
151 |
152 | #### Worker Options (includes async job lifecycle callbacks)
153 |
154 | queue.addWorker() accepts an options object in order to tweak standard functionality and allow you to hook into asynchronous job lifecycle callbacks.
155 |
156 | **IMPORTANT: Job Lifecycle callbacks are called asynchronously.** They do not block job processing or each other. Don't put logic in onStart that you expect to be completed before the actual job process begins executing. Don't put logic in onFailure you expect to be completed before onFailed is called. You can, of course, assume that the job process has completed (or failed) before onSuccess, onFailure, onFailed, or onComplete are asynchonrously called.
157 |
158 | ```js
159 |
160 | queue.addWorker('job-name-here', async (id, payload) => { console.log(id); }, {
161 |
162 | // Set max number of jobs for this worker to process concurrently.
163 | // Defaults to 1.
164 | concurrency: 5,
165 |
166 | // JOB LIFECYCLE CALLBACKS
167 |
168 | // onStart job callback handler is fired when a job begins processing.
169 | //
170 | // IMPORTANT: Job lifecycle callbacks are executed asynchronously and do not block job processing
171 | // (even if the callback returns a promise it will not be "awaited" on).
172 | // As such, do not place any logic in onStart that your actual job worker function will depend on,
173 | // this type of logic should of course go inside the job worker function itself.
174 | onStart: async (id, payload) => {
175 |
176 | console.log('Job "job-name-here" with id ' + id + ' has started processing.');
177 |
178 | },
179 |
180 | // onSuccess job callback handler is fired after a job successfully completes processing.
181 | onSuccess: async (id, payload) => {
182 |
183 | console.log('Job "job-name-here" with id ' + id + ' was successful.');
184 |
185 | },
186 |
187 | // onFailure job callback handler is fired after each time a job fails (onFailed also fires if job has reached max number of attempts).
188 | onFailure: async (id, payload) => {
189 |
190 | console.log('Job "job-name-here" with id ' + id + ' had an attempt end in failure.');
191 |
192 | },
193 |
194 | // onFailed job callback handler is fired if job fails enough times to reach max number of attempts.
195 | onFailed: async (id, payload) => {
196 |
197 | console.log('Job "job-name-here" with id ' + id + ' has failed.');
198 |
199 | },
200 |
201 | // onComplete job callback handler fires after job has completed processing successfully or failed entirely.
202 | onComplete: async (id, payload) => {
203 |
204 | console.log('Job "job-name-here" with id ' + id + ' has completed processing.');
205 |
206 | }
207 |
208 | });
209 |
210 | ```
211 |
212 | #### Job Options
213 |
214 | queue.createJob() accepts an options object in order to tweak standard functionality.
215 |
216 | ```js
217 |
218 | queue.createJob('job-name-here', {foo: 'bar'}, {
219 |
220 | // Higher priority jobs (10) get processed before lower priority jobs (-10).
221 | // Any int will work, priority 1000 will be processed before priority 10, though this is probably overkill.
222 | // Defaults to 0.
223 | priority: 10, // High priority
224 |
225 | // Timeout in ms before job is considered failed.
226 | // Use this setting to kill off hanging jobs that are clogging up
227 | // your queue, or ensure your jobs finish in a timely manner if you want
228 | // to execute jobs in OS background tasks.
229 | //
230 | // IMPORTANT: Jobs are required to have a timeout > 0 set in order to be processed
231 | // by a queue that has been started with a lifespan. As such, if you want to process
232 | // jobs in an OS background task, you MUST give the jobs a timeout setting.
233 | //
234 | // Setting this option to 0 means never timeout.
235 | //
236 | // Defaults to 25000.
237 | timeout: 30000, // Timeout in 30 seconds
238 |
239 | // Number of times to attempt a failing job before marking job as failed and moving on.
240 | // Defaults to 1.
241 | attempts: 4, // If this job fails to process 4 times in a row, it will be marked as failed.
242 |
243 | });
244 |
245 |
246 | ```
247 |
248 | ## Testing with Jest
249 |
250 | Because realm will write database files to the root test directory when running jest tests, you will need to add the following to your gitignore file if you use tests.
251 |
252 | ```text
253 | /reactNativeQueue.realm*
254 | ```
255 |
256 | ## Caveats
257 |
258 | **Jobs must be idempotent.** As with most queues, there are certain scenarios that could lead to React Native Queue processing a job more than once. For example, a job could timeout locally but remote server actions kicked off by the job could continue to execute. If the job is retried then effectively the remote code will be run twice. Furthermore, a job could fail due to some sort of exception halfway through then the next time it runs the first half of the job has already been executed once. Always design your React Native Queue jobs to be idempotent. If this is not possible, set job "attempts" option to be 1 (the default setting), and then you will have to write custom logic to handle the event of a job failing (perhaps via a job chain).
259 |
260 | ## Advanced Usage Examples
261 |
262 | #### Advanced Job Full Example
263 |
264 | ```js
265 |
266 | import React, { Component } from 'react';
267 | import {
268 | Platform,
269 | StyleSheet,
270 | Text,
271 | View,
272 | Button
273 | } from 'react-native';
274 |
275 | import queueFactory from 'react-native-queue';
276 |
277 | export default class App extends Component<{}> {
278 |
279 | constructor(props) {
280 | super(props);
281 |
282 | this.state = {
283 | queue: null
284 | };
285 |
286 | this.init();
287 |
288 | }
289 |
290 | async init() {
291 |
292 | const queue = await queueFactory();
293 |
294 | //
295 | // Standard Job Example
296 | // Nothing fancy about this job.
297 | //
298 | queue.addWorker('standard-example', async (id, payload) => {
299 | console.log('standard-example job '+id+' executed.');
300 | });
301 |
302 | //
303 | // Recursive Job Example
304 | // This job creates itself over and over.
305 | //
306 | let recursionCounter = 1;
307 | queue.addWorker('recursive-example', async (id, payload) => {
308 | console.log('recursive-example job '+ id +' started');
309 | console.log(recursionCounter, 'recursionCounter');
310 |
311 | recursionCounter++;
312 |
313 | await new Promise((resolve) => {
314 | setTimeout(() => {
315 | console.log('recursive-example '+ id +' has completed!');
316 |
317 | // Keep creating these jobs until counter reaches 3.
318 | if (recursionCounter <= 3) {
319 | queue.createJob('recursive-example');
320 | }
321 |
322 | resolve();
323 | }, 1000);
324 | });
325 |
326 | });
327 |
328 | //
329 | // Job Chaining Example
330 | // When job completes, it creates a new job to handle the next step
331 | // of your process. Breaking large jobs up into smaller jobs and then
332 | // chaining them together will allow you to handle large tasks in
333 | // OS background tasks, that are limited to 30 seconds of
334 | // execution every 15 minutes on iOS and Android.
335 | //
336 | queue.addWorker('start-job-chain', async (id, payload) => {
337 | console.log('start-job-chain job '+ id +' started');
338 | console.log('step: ' + payload.step);
339 |
340 | await new Promise((resolve) => {
341 | setTimeout(() => {
342 | console.log('start-job-chain '+ id +' has completed!');
343 |
344 | // Create job for next step in chain
345 | queue.createJob('job-chain-2nd-step', {
346 | callerJobName: 'start-job-chain',
347 | step: payload.step + 1
348 | });
349 |
350 | resolve();
351 | }, 1000);
352 | });
353 |
354 | });
355 |
356 | queue.addWorker('job-chain-2nd-step', async (id, payload) => {
357 | console.log('job-chain-2nd-step job '+ id +' started');
358 | console.log('step: ' + payload.step);
359 |
360 | await new Promise((resolve) => {
361 | setTimeout(() => {
362 | console.log('job-chain-2nd-step '+ id +' has completed!');
363 |
364 | // Create job for last step in chain
365 | queue.createJob('job-chain-final-step', {
366 | callerJobName: 'job-chain-2nd-step',
367 | step: payload.step + 1
368 | });
369 |
370 | resolve();
371 | }, 1000);
372 | });
373 |
374 | });
375 |
376 | queue.addWorker('job-chain-final-step', async (id, payload) => {
377 | console.log('job-chain-final-step job '+ id +' started');
378 | console.log('step: ' + payload.step);
379 |
380 | await new Promise((resolve) => {
381 | setTimeout(() => {
382 | console.log('job-chain-final-step '+ id +' has completed!');
383 | console.log('Job chain is now completed!');
384 |
385 | resolve();
386 | }, 1000);
387 | });
388 |
389 | });
390 |
391 | // Start queue to process any jobs that hadn't finished when app was last closed.
392 | queue.start();
393 |
394 | // Attach initialized queue to state.
395 | this.setState({
396 | queue
397 | });
398 |
399 | }
400 |
401 | makeJob(jobName, payload = {}) {
402 | this.state.queue.createJob(jobName, payload);
403 | }
404 |
405 | render() {
406 |
407 | return (
408 |
409 |
410 | Welcome to React Native!
411 |
412 | {this.state.queue &&
416 | );
417 |
418 | }
419 | }
420 |
421 | const styles = StyleSheet.create({
422 | container: {
423 | flex: 1,
424 | justifyContent: 'center',
425 | alignItems: 'center',
426 | backgroundColor: '#F5FCFF',
427 | },
428 | welcome: {
429 | fontSize: 20,
430 | textAlign: 'center',
431 | margin: 10,
432 | },
433 | });
434 |
435 |
436 | ```
437 |
438 | #### OS Background Task Full Example
439 |
440 | For the purpose of this example we will use the [React Native Background Task](https://github.com/jamesisaac/react-native-background-task) module, but you could integrate React Native Queue with any acceptable OS background task module.
441 |
442 | Follow the [installation steps](https://github.com/jamesisaac/react-native-background-task#installation) for React Native Background Task.
443 |
444 | ```js
445 |
446 | import React, { Component } from 'react';
447 | import {
448 | Platform,
449 | StyleSheet,
450 | Text,
451 | View,
452 | Button,
453 | AsyncStorage
454 | } from 'react-native';
455 |
456 | import BackgroundTask from 'react-native-background-task'
457 | import queueFactory from 'react-native-queue';
458 |
459 | BackgroundTask.define(async () => {
460 |
461 | // Init queue
462 | queue = await queueFactory();
463 |
464 | // Register worker
465 | queue.addWorker('background-example', async (id, payload) => {
466 |
467 | // Load some arbitrary data while the app is in the background
468 | if (payload.name == 'luke') {
469 | await AsyncStorage.setItem('lukeData', 'Luke Skywalker arbitrary data loaded!');
470 | } else {
471 | await AsyncStorage.setItem('c3poData', 'C-3PO arbitrary data loaded!');
472 | }
473 |
474 | });
475 |
476 | // Start the queue with a lifespan
477 | // IMPORTANT: OS background tasks are limited to 30 seconds or less.
478 | // NOTE: Queue lifespan logic will attempt to stop queue processing 500ms less than passed lifespan for a healthy shutdown buffer.
479 | // IMPORTANT: Queue processing started with a lifespan will ONLY process jobs that have a defined timeout set.
480 | // Additionally, lifespan processing will only process next job if job.timeout < (remainingLifespan - 500).
481 | await queue.start(20000); // Run queue for at most 20 seconds.
482 |
483 | // finish() must be called before OS hits timeout.
484 | BackgroundTask.finish();
485 |
486 | });
487 |
488 | export default class App extends Component<{}> {
489 |
490 | constructor(props) {
491 | super(props);
492 |
493 | this.state = {
494 | queue: null,
495 | data: null
496 | };
497 |
498 | this.init();
499 |
500 | }
501 |
502 | async init() {
503 |
504 | const queue = await queueFactory();
505 |
506 | // Add the worker.
507 | queue.addWorker('background-example', async (id, payload) => {
508 | // Worker has to be defined before related jobs can be added to queue.
509 | // Since this example is only concerned with OS background task worker execution,
510 | // We will make this a dummy function in this context.
511 | console.log(id);
512 | });
513 |
514 | // Attach initialized queue to state.
515 | this.setState({
516 | queue
517 | });
518 |
519 | }
520 |
521 | componentDidMount() {
522 | BackgroundTask.schedule(); // Schedule the task to run every ~15 min if app is closed.
523 | }
524 |
525 | makeJob(jobName, payload = {}) {
526 | console.log('job is created but will not execute until the above OS background task runs in ~15 min');
527 | this.state.queue.createJob(jobName, payload, {
528 |
529 | timeout: 5000 // IMPORTANT: If queue processing is started with a lifespan ie queue.start(lifespan) it will ONLY process jobs with a defined timeout.
530 |
531 | }, false); // Pass false so queue doesn't get started here (we want the queue to start only in OS background task in this example).
532 | }
533 |
534 | async checkData() {
535 |
536 | const lukeData = await AsyncStorage.getItem('lukeData');
537 | const c3poData = await AsyncStorage.getItem('c3poData');
538 |
539 | this.setState({
540 | data: {
541 | lukeData: (lukeData) ? lukeData : 'No data loaded from OS background task yet for Luke Skywalker.',
542 | c3poData: (c3poData) ? c3poData : 'No data loaded from OS background task yet for C-3PO.'
543 | }
544 | });
545 |
546 | }
547 |
548 | render() {
549 |
550 | let output = 'No data loaded from OS background task yet.';
551 | if (this.state.data) {
552 | output = JSON.stringify(this.state.data);
553 | }
554 |
555 | return (
556 |
557 |
558 | Welcome to React Native!
559 |
560 | Click buttons below to add OS background task jobs.
561 | Then Close App (task will not fire if app is in focus).
562 | Job will exec in ~15 min in OS background.
563 | {this.state.queue &&
568 | );
569 |
570 | }
571 | }
572 |
573 | const styles = StyleSheet.create({
574 | container: {
575 | flex: 1,
576 | justifyContent: 'center',
577 | alignItems: 'center',
578 | backgroundColor: '#F5FCFF',
579 | },
580 | welcome: {
581 | fontSize: 20,
582 | textAlign: 'center',
583 | margin: 10,
584 | },
585 | });
586 |
587 | ```
588 |
589 | ## Tutorials
590 |
591 | #### Easy OS Background Tasks in React Native
592 |
593 | An in-depth guide to setting up background tasks / services that run periodically when the app is closed.
594 |
595 | * [Hosted on Medium](https://hackernoon.com/easy-os-background-tasks-in-react-native-bc4476c48b8a)
596 | * [Raw Markdown](/docs/easy-os-background-tasks-in-react-native.md)
597 |
598 |
--------------------------------------------------------------------------------
/config/Database.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Realm database bootstrap
3 | */
4 |
5 | import { Config } from './config';
6 | import Realm from 'realm';
7 |
8 | const JobSchema = {
9 | name: 'Job',
10 | primaryKey: 'id',
11 | properties: {
12 | id: 'string', // UUID.
13 | name: 'string', // Job name to be matched with worker function.
14 | payload: 'string', // Job payload stored as JSON.
15 | data: 'string', // Store arbitrary data like "failed attempts" as JSON.
16 | priority: 'int', // -5 to 5 to indicate low to high priority.
17 | active: { type: 'bool', default: false}, // Whether or not job is currently being processed.
18 | timeout: 'int', // Job timeout in ms. 0 means no timeout.
19 | created: 'date', // Job creation timestamp.
20 | failed: 'date?' // Job failure timestamp (null until failure).
21 | }
22 | };
23 |
24 | export default class Database {
25 |
26 | static realmInstance = null; // Use a singleton connection to realm for performance.
27 |
28 | static async getRealmInstance(options = {}) {
29 |
30 | // Connect to realm if database singleton instance has not already been created.
31 | if (Database.realmInstance === null) {
32 |
33 | Database.realmInstance = await Realm.open({
34 | path: options.realmPath || Config.REALM_PATH,
35 | schemaVersion: Config.REALM_SCHEMA_VERSION,
36 | schema: [JobSchema]
37 |
38 | // Look up shouldCompactOnLaunch to auto-vacuum https://github.com/realm/realm-js/pull/1209/files
39 |
40 | });
41 |
42 | }
43 |
44 | return Database.realmInstance;
45 |
46 | }
47 |
48 | }
--------------------------------------------------------------------------------
/config/config.js:
--------------------------------------------------------------------------------
1 | /**
2 | *
3 | * Config constants
4 | *
5 | */
6 |
7 | export const Config = {
8 | REALM_PATH: 'reactNativeQueue.realm', // Name of realm database.
9 | REALM_SCHEMA_VERSION: 0 // Must be incremented if data model updates.
10 | };
--------------------------------------------------------------------------------
/docs/easy-os-background-tasks-in-react-native.md:
--------------------------------------------------------------------------------
1 |
2 | # Easy OS Background Tasks in React Native
3 |
4 | 
5 |
6 | **Thanks to a couple of relatively new libraries, running tasks in a background thread, also known as a service, when your react native app is closed has never been easier.**
7 |
8 | Today I’ll walk you through setting up tasks that will run periodically even when your app is closed. If you already have React Native setup, and you’re staring at your IDE right now, it will only take you ~15 minutes to be fully up and running with this complete example.
9 |
10 | **We will use two libraries to accomplish this:**
11 |
12 | * [React Native Queue](https://github.com/billmalarky/react-native-queue): Control flow and job management.
13 |
14 | * [React Native Background Task](https://github.com/jamesisaac/react-native-background-task): Register js handler function that will be executed when app is closed.
15 |
16 | **In our example, we will do basic image pre-fetching (yeah yeah it’s a bit pointless but it’s easy to understand for illustrative purposes).**
17 |
18 | **Examples of more realistic use cases for this functionality:**
19 |
20 | * Downloading content for offline access.
21 |
22 | * Media processing.
23 |
24 | * Cache Warming.
25 |
26 | * *Durable* API calls to external services, such as publishing content to a variety of 3rd party distribution channel APIs.
27 |
28 | * Complex and time-consuming jobs that you want consistently processed regardless if app is open, closed, or repeatedly opened and closed.
29 |
30 | ## Installation
31 |
32 | First create the skeleton React Native app in your working directory
33 |
34 | $ react-native init backgroundexample
35 |
36 | Quickly install [react-native-queue](https://github.com/billmalarky/react-native-queue#installation) and [react-native-background-task](https://github.com/jamesisaac/react-native-background-task#installation) packages and link them (note react-native-background-fetch is an optional dependency of react-native-background-task required for iOS support).
37 |
38 | $ yarn add react-native-queue
39 | $ react-native link realm
40 | $ yarn add react-native-background-task
41 | $ react-native link react-native-background-task
42 | $ yarn add react-native-background-fetch@2.0.x
43 | $ react-native link react-native-background-fetch
44 |
45 | Manually update the onCreate() method in MainApplication.java like so
46 |
47 | // Update the following file as seen below
48 | // android/app/src/main/java/com/backgroundexample/MainApplication.java
49 |
50 | @Override
51 | public void onCreate() {
52 | super.onCreate();
53 | SoLoader.init(this, /* native exopackage */ false);
54 | BackgroundTaskPackage.useContext(this); // ADD ME HERE!
55 | }
56 |
57 | ## Building The Feature
58 |
59 | First lets update the react native skeleton app to include a screen of images that is toggled with a button press. Nothing fancy.
60 |
61 | ### Toggle Screen App.js Changes
62 |
63 | Nothing fancy here. Just add ScrollView, Button, Image imports, modify the container style, add the image style, and make some small updates to the skeleton App class.
64 |
65 | import React, { Component } from 'react';
66 | import {
67 | Platform,
68 | StyleSheet,
69 | Text,
70 | ScrollView,
71 | Button,
72 | Image
73 | } from 'react-native';
74 |
75 | export default class App extends Component<{}> {
76 |
77 | constructor(props) {
78 | super(props);
79 |
80 | this.state = {
81 | showImages: false
82 | };
83 |
84 | }
85 |
86 | render() {
87 | return (
88 |
89 |
96 | );
97 | }
98 | }
99 |
100 | const styles = StyleSheet.create({
101 | container: {
102 | padding: 10
103 | },
104 | welcome: {
105 | fontSize: 20,
106 | textAlign: 'center',
107 | margin: 10,
108 | },
109 | image: {
110 | width:150,
111 | height: 204
112 | }
113 | });
114 |
115 | ### Integrating the background task
116 |
117 | **At the top of App.js we will define the js function we want the OS to call periodically in the background when the app is closed (the “background task”).**
118 |
119 | What we want to happen in this background task function, is to initialize the queue, and immediately start pulling jobs off of the queue and processing as many as we can before hitting the 30 second timeout imposed by iOS (Android does not have this timeout limitation but we need to adhere to the strictest constraints for cross-platform support) for background service functions. **Because of this hard timeout limit, we will call queue.start(lifespan) with a lifespan of 25 seconds**. This way, the queue will start processing jobs for at most 25 seconds (or until queue is cleared), then stop processing guaranteeing us time to call the required Background.finish() before the OS times out the function.
120 |
121 | In our example, 25 seconds will be more than enough to churn through the entire queue, seeing as we’re only gonna pre-fetch 3 images. However, imagine if we were pre-fetching 10,000 images. **The queue keeps the jobs durable** (they won’t be deleted until completed, and can auto-retry on failure), so every ~15 min when the OS fires this function in the background again, another batch of images would be pre-fetched, and sooner or later all of the images would be pre-fetched all behind the scenes.
122 |
123 | import BackgroundTask from 'react-native-background-task'
124 | import queueFactory from 'react-native-queue';
125 |
126 | BackgroundTask.define(async () => {
127 |
128 | // Init queue
129 | queue = await queueFactory();
130 |
131 | // Register job worker
132 | queue.addWorker('pre-fetch-image', async (id, payload) => {
133 |
134 | Image.prefetch(payload.imageUrl);
135 |
136 | });
137 |
138 | // Start the queue with a lifespan
139 | // IMPORTANT: OS background tasks are limited to 30 seconds or less.
140 | // NOTE: Queue lifespan logic will attempt to stop queue processing 500ms less than passed lifespan for a healthy shutdown buffer.
141 | // IMPORTANT: Queue processing started with a lifespan will ONLY process jobs that have a defined timeout set.
142 | // Additionally, lifespan processing will only process next job if job.timeout < (remainingLifespan - 500).
143 | await queue.start(25000); // Run queue for at most 25 seconds.
144 |
145 | // finish() must be called before OS hits timeout.
146 | BackgroundTask.finish();
147 |
148 | });
149 |
150 | Then add a componentDidMount() lifecycle method to the App component to schedule the background task when the app mounts.
151 |
152 | componentDidMount() {
153 | BackgroundTask.schedule(); // Schedule the task to run every ~15 min if app is closed.
154 | }
155 |
156 | Your App.js file should now look something like this:
157 |
158 | import React, { Component } from 'react';
159 | import {
160 | Platform,
161 | StyleSheet,
162 | Text,
163 | ScrollView,
164 | Button,
165 | Image
166 | } from 'react-native';
167 |
168 | import BackgroundTask from 'react-native-background-task'
169 | import queueFactory from 'react-native-queue';
170 |
171 | BackgroundTask.define(async () => {
172 |
173 | // Init queue
174 | queue = await queueFactory();
175 |
176 | // Register job worker
177 | queue.addWorker('pre-fetch-image', async (id, payload) => {
178 |
179 | Image.prefetch(payload.imageUrl);
180 |
181 | });
182 |
183 | // Start the queue with a lifespan
184 | // IMPORTANT: OS background tasks are limited to 30 seconds or less.
185 | // NOTE: Queue lifespan logic will attempt to stop queue processing 500ms less than passed lifespan for a healthy shutdown buffer.
186 | // IMPORTANT: Queue processing started with a lifespan will ONLY process jobs that have a defined timeout set.
187 | // Additionally, lifespan processing will only process next job if job.timeout < (remainingLifespan - 500).
188 | await queue.start(25000); // Run queue for at most 25 seconds.
189 |
190 | // finish() must be called before OS hits timeout.
191 | BackgroundTask.finish();
192 |
193 | });
194 |
195 | export default class App extends Component<{}> {
196 |
197 | constructor(props) {
198 | super(props);
199 |
200 | this.state = {
201 | showImages: false
202 | };
203 |
204 | }
205 |
206 | componentDidMount() {
207 | BackgroundTask.schedule(); // Schedule the task to run every ~15 min if app is closed.
208 | }
209 |
210 | render() {
211 | return (
212 |
213 |
220 | );
221 | }
222 | }
223 |
224 | const styles = StyleSheet.create({
225 | container: {
226 | padding: 10
227 | },
228 | welcome: {
229 | fontSize: 20,
230 | textAlign: 'center',
231 | margin: 10,
232 | },
233 | image: {
234 | width:150,
235 | height: 204
236 | }
237 | });
238 |
239 | ### Adding Queue Jobs
240 |
241 | Now that we’ve got our background task setup to initialize the queue and process jobs when our app is closed, we need a way to actually add jobs to the queue!
242 |
243 | First we’ll initialize the queue in the app so we can use it to create jobs.
244 |
245 | Reference the final App.js file below in lines 41–54 to make the necessary updates to the constructor() in order to initialize the queue inside the app.
246 |
247 | After the queue is initialized, create the createPrefetchJobs() class method seen below in lines 60–86. Inside this method we will reference the queue instance stored in the app component state to create the 3 jobs that prefetch images to throw on the queue. Notice that we pass false as the last parameter to createJob(), this stops the queue from starting up processing immediately (which is the default behavior). In this example we don’t want the queue to process in the main app thread, so we’ll only call queue.start() in the background task.
248 |
249 | Last but not least, update render() in line 92 to add the “Pre-fetch Images” button and wire it to the createPrefetchJobs() method we created earlier.
250 |
251 | import React, { Component } from 'react';
252 | import {
253 | Platform,
254 | StyleSheet,
255 | Text,
256 | ScrollView,
257 | Button,
258 | Image
259 | } from 'react-native';
260 |
261 | import BackgroundTask from 'react-native-background-task'
262 | import queueFactory from 'react-native-queue';
263 |
264 | BackgroundTask.define(async () => {
265 |
266 | // Init queue
267 | queue = await queueFactory();
268 |
269 | // Register job worker
270 | queue.addWorker('pre-fetch-image', async (id, payload) => {
271 |
272 | Image.prefetch(payload.imageUrl);
273 |
274 | });
275 |
276 | // Start the queue with a lifespan
277 | // IMPORTANT: OS background tasks are limited to 30 seconds or less.
278 | // NOTE: Queue lifespan logic will attempt to stop queue processing 500ms less than passed lifespan for a healthy shutdown buffer.
279 | // IMPORTANT: Queue processing started with a lifespan will ONLY process jobs that have a defined timeout set.
280 | // Additionally, lifespan processing will only process next job if job.timeout < (remainingLifespan - 500).
281 | await queue.start(25000); // Run queue for at most 25 seconds.
282 |
283 | // finish() must be called before OS hits timeout.
284 | BackgroundTask.finish();
285 |
286 | });
287 |
288 | export default class App extends Component<{}> {
289 |
290 | constructor(props) {
291 | super(props);
292 |
293 | this.state = {
294 | queue: null,
295 | showImages: false
296 | };
297 |
298 | queueFactory()
299 | .then(queue => {
300 | this.setState({queue});
301 | });
302 |
303 | }
304 |
305 | componentDidMount() {
306 | BackgroundTask.schedule(); // Schedule the task to run every ~15 min if app is closed.
307 | }
308 |
309 | createPrefetchJobs() {
310 |
311 | // Create the prefetch job for the first component.
312 | this.state.queue.createJob(
313 | 'pre-fetch-image',
314 | { imageUrl: 'https://i.imgur.com/kPkQTic.jpg' }, // Supply the image url we want prefetched in this job to the payload.
315 | { attempts: 5, timeout: 15000 }, // Retry job on failure up to 5 times. Timeout job in 15 sec (prefetch is probably hanging if it takes that long).
316 | false // Must pass false as the last param so the queue starts up in the background task instead of immediately.
317 | );
318 |
319 | // Create the prefetch job for the second component.
320 | this.state.queue.createJob(
321 | 'pre-fetch-image',
322 | { imageUrl: 'https://i.redd.it/uwvjph19mltz.jpg' }, // Supply the image url we want prefetched in this job to the payload.
323 | { attempts: 5, timeout: 15000 }, // Retry job on failure up to 5 times. Timeout job in 15 sec (prefetch is probably hanging if it takes that long).
324 | false // Must pass false as the last param so the queue starts up in the background task instead of immediately.
325 | );
326 |
327 | // Create the prefetch job for the third component.
328 | this.state.queue.createJob(
329 | 'pre-fetch-image',
330 | { imageUrl: 'https://i.redd.it/39w0xd9ersxz.jpg' }, // Supply the image url we want prefetched in this job to the payload.
331 | { attempts: 5, timeout: 15000 }, // Retry job on failure up to 5 times. Timeout job in 15 sec (prefetch is probably hanging if it takes that long).
332 | false // Must pass false as the last param so the queue starts up in the background task instead of immediately.
333 | );
334 |
335 | }
336 |
337 | render() {
338 | return (
339 |
340 |
348 | );
349 | }
350 | }
351 |
352 | const styles = StyleSheet.create({
353 | container: {
354 | padding: 10
355 | },
356 | welcome: {
357 | fontSize: 20,
358 | textAlign: 'center',
359 | margin: 10,
360 | },
361 | image: {
362 | width:150,
363 | height: 204
364 | }
365 | });
366 |
367 | ### You’re Done!
368 |
369 | Now boot up your react native app on an actual device **(background tasks WILL NOT FIRE in simulators)**. Once the app is booted, click the prefetch button to queue the jobs.
370 |
371 | Now all that’s left is to unfocus the app, and wait. **OS Background tasks WILL NOT fire if the app is in focus** (that would sort of be against the entire point). After ~15 minutes, the OS will fire up the background task, initialize the queue, and start the queue up churning through your 3 prefetch jobs.
372 |
373 | At this point your remote images have been prefetched to the phone’s local disk, and when you click “toggle screen” to view the images, they will be pulled from your local disk instead of the network.
374 |
375 | ### Questions? Troubleshooting.
376 |
377 | If you’re having any issues, or have any questions, feel free to contact me directly and I can help you.
378 |
--------------------------------------------------------------------------------
/docs/easy-os-background-tasks-in-react-native.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/billmalarky/react-native-queue/5c45bf92a6c24275feb753cc95366e74c754a956/docs/easy-os-background-tasks-in-react-native.png
--------------------------------------------------------------------------------
/docs/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/billmalarky/react-native-queue/5c45bf92a6c24275feb753cc95366e74c754a956/docs/logo.png
--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
1 | /**
2 | *
3 | * Bootstrap.
4 | *
5 | * @module ReactNativeQueue
6 | */
7 |
8 | import QueueFactory from './Models/Queue';
9 |
10 | export default QueueFactory;
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "react-native-queue",
3 | "version": "1.2.1",
4 | "description": "A React Native Job Queue",
5 | "main": "index.js",
6 | "scripts": {
7 | "test": "./node_modules/.bin/jest --coverage && if [ \"$COVERALLS_ENV\" = \"production\" ]; then cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js; fi",
8 | "lint": "eslint ."
9 | },
10 | "repository": {
11 | "type": "git",
12 | "url": "git+https://github.com/billmalarky/react-native-queue.git"
13 | },
14 | "keywords": [
15 | "react",
16 | "react-native",
17 | "queue",
18 | "job-queue",
19 | "task-queue"
20 | ],
21 | "author": "Reid Mayo",
22 | "license": "MIT",
23 | "bugs": {
24 | "url": "https://github.com/billmalarky/react-native-queue/issues"
25 | },
26 | "homepage": "https://github.com/billmalarky/react-native-queue#readme",
27 | "dependencies": {
28 | "promise-reflect": "^1.1.0",
29 | "react-native-uuid": "^1.4.9",
30 | "realm": "^2.0.12"
31 | },
32 | "devDependencies": {
33 | "babel-eslint": "^8.0.3",
34 | "babel-jest": "^21.2.0",
35 | "babel-preset-react-native": "^4.0.0",
36 | "coveralls": "^3.0.0",
37 | "eslint": "^4.12.1",
38 | "jest": "^21.2.1",
39 | "should": "^13.1.3"
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/tests/Queue.test.js:
--------------------------------------------------------------------------------
1 |
2 | // Define globals for eslint.
3 | /* global describe it beforeEach process jest */
4 |
5 | // Load dependencies
6 | import should from 'should'; // eslint-disable-line no-unused-vars
7 | import QueueFactory, { Queue } from '../Models/Queue';
8 | import Worker from '../Models/Worker';
9 |
10 | describe('Models/Queue', function() {
11 |
12 | beforeEach(async () => {
13 |
14 | // Make sure each test starts with a fresh database.
15 | const queue = await QueueFactory();
16 | queue.flushQueue();
17 |
18 | });
19 |
20 | //
21 | // QUEUE LIFESPAN TESTING
22 | //
23 |
24 | it('#start(lifespan) queue with lifespan does not process jobs that have no timeout set.', async () => {
25 |
26 | const queue = await QueueFactory();
27 | const jobName = 'job-name';
28 |
29 | queue.addWorker(jobName, () => {});
30 |
31 | // Create a couple jobs
32 | queue.createJob(jobName, {}, {}, false);
33 | queue.createJob(jobName, {}, {}, false);
34 | queue.createJob(jobName, {}, {}, false);
35 |
36 | // startQueue is false so queue should not have started.
37 | queue.status.should.equal('inactive');
38 |
39 | // Start queue, don't await so this test can continue while queue processes.
40 | queue.start(10000);
41 |
42 | queue.status.should.equal('active');
43 |
44 | // Because no jobs should be processed due to not having defined timeouts
45 | // queue should become inactive well before 10 seconds passes.
46 |
47 | // wait a tick
48 | await new Promise((resolve) => {
49 | setTimeout(resolve, 0);
50 | });
51 |
52 | queue.status.should.equal('inactive');
53 |
54 | });
55 |
56 | it('#start(lifespan) FUNDAMENTAL TEST (queue started with lifespan will process a job with job timeout set).', async () => {
57 |
58 | const queue = await QueueFactory();
59 | queue.flushQueue();
60 | const jobName = 'job-name';
61 |
62 | // Track the jobs that have executed to test against.
63 | let executedJobs = [];
64 |
65 | queue.addWorker(jobName, async (id, payload) => {
66 |
67 | // Track jobs that exec
68 | executedJobs.push(payload.trackingName);
69 |
70 | });
71 |
72 | // Create a job but don't auto-start queue
73 | queue.createJob(jobName, {
74 | trackingName: jobName
75 | }, {
76 | timeout: 100
77 | }, false);
78 |
79 | // startQueue is false so queue should not have started.
80 | queue.status.should.equal('inactive');
81 |
82 | // Start queue with lifespan, don't await so this test can continue while queue processes.
83 | queue.start(750);
84 |
85 | queue.status.should.equal('active');
86 |
87 | // wait a bit for queue to process job
88 | await new Promise((resolve) => {
89 | setTimeout(resolve, 750);
90 | });
91 |
92 | //Check that the correct jobs executed.
93 | executedJobs.should.deepEqual(['job-name']);
94 |
95 | // Queue should have stopped.
96 | queue.status.should.equal('inactive');
97 |
98 | });
99 |
100 | it('#start(lifespan) BASIC TEST (One job type, default job/worker options): queue will process jobs with timeout set as expected until lifespan ends.', async () => {
101 |
102 | // This test will intermittently fail in CI environments like travis-ci.
103 | // Intermittent failure is a result of the poor performance of CI environments
104 | // causing the timeouts in this test to become really flakey (setTimeout can't
105 | // guarantee exact time of function execution, and in a high load env execution can
106 | // be significantly delayed.
107 | if (process.env.COVERALLS_ENV == 'production') {
108 | return true;
109 | }
110 |
111 | const queue = await QueueFactory();
112 | queue.flushQueue();
113 | const jobName = 'job-name';
114 | const queueLifespan = 2000;
115 | let remainingLifespan = queueLifespan;
116 |
117 | // Track the jobs that have executed to test against.
118 | let executedJobs = [];
119 |
120 | // We need to be able to throw an error outside of
121 | // job workers, because errors thrown inside a job
122 | // worker function are caught and logged by job processing
123 | // logic. They will not fail the test. So track bad jobs
124 | // and throw error after jobs finish processing.
125 | let badJobs = [];
126 |
127 | queue.addWorker(jobName, async (id, payload) => {
128 |
129 | // Track jobs that exec
130 | executedJobs.push(payload.trackingName);
131 |
132 | // Detect jobs that should't be picked up by lifespan queue.
133 | if (remainingLifespan - 500 < payload.payloadOptionsTimeout) {
134 | badJobs.push({id, payload});
135 | }
136 |
137 | remainingLifespan = remainingLifespan - payload.payloadTimeout;
138 |
139 | await new Promise((resolve) => {
140 | setTimeout(resolve, payload.payloadTimeout);
141 | });
142 |
143 | }, { concurrency: 1});
144 |
145 | // 2000 (lifespan) - 200 (job1) - 200 (job2) - 1000 (job3) - 50 (job 4) - 100 (timeout value for job 5 overflows remaining lifespan + 500ms for buffer so job5 will not exec) < 500
146 |
147 | // Create a couple jobs
148 | queue.createJob(jobName, {
149 | trackingName: 'job1',
150 | payloadTimeout: 200,
151 | payloadOptionsTimeout: 300 // Mirror the actual job options timeout in payload so we can use it for testing.
152 | }, {
153 | timeout: 300
154 | }, false);
155 |
156 | // Since more than one job can be written in 1 ms, we need to add a slight delay
157 | // in order to control the order jobs come off the queue (since they are time sorted)
158 | // If multiple jobs are written in the same ms, Realm can't be deterministic about job
159 | // ordering when we pop jobs off the top of the queue.
160 | await new Promise((resolve) => { setTimeout(resolve, 25); });
161 |
162 | queue.createJob(jobName, {
163 | trackingName: 'job2',
164 | payloadTimeout: 200,
165 | payloadOptionsTimeout: 300 // Mirror the actual job options timeout in payload so we can use it for testing.
166 | }, {
167 | timeout: 300
168 | }, false);
169 | await new Promise((resolve) => { setTimeout(resolve, 25); });
170 |
171 | queue.createJob(jobName, {
172 | trackingName: 'job3',
173 | payloadTimeout: 1000,
174 | payloadOptionsTimeout: 1100 // Mirror the actual job options timeout in payload so we can use it for testing.
175 | }, {
176 | timeout: 1100
177 | }, false);
178 | await new Promise((resolve) => { setTimeout(resolve, 25); });
179 |
180 | queue.createJob(jobName, {
181 | trackingName: 'job4',
182 | payloadTimeout: 500,
183 | payloadOptionsTimeout: 600 // Mirror the actual job options timeout in payload so we can use it for testing.
184 | }, {
185 | timeout: 600
186 | }, false);
187 | await new Promise((resolve) => { setTimeout(resolve, 25); });
188 |
189 | queue.createJob(jobName, {
190 | trackingName: 'job5',
191 | payloadTimeout: 50,
192 | payloadOptionsTimeout: 75 // Mirror the actual job options timeout in payload so we can use it for testing.
193 | }, {
194 | timeout: 75
195 | }, false);
196 | await new Promise((resolve) => { setTimeout(resolve, 25); });
197 |
198 | queue.createJob(jobName, {
199 | trackingName: 'job6',
200 | payloadTimeout: 25,
201 | payloadOptionsTimeout: 100 // Mirror the actual job options timeout in payload so we can use it for testing.
202 | }, {
203 | timeout: 100
204 | }, false);
205 | await new Promise((resolve) => { setTimeout(resolve, 25); });
206 |
207 | queue.createJob(jobName, {
208 | trackingName: 'job7',
209 | payloadTimeout: 1100,
210 | payloadOptionsTimeout: 1200 // Mirror the actual job options timeout in payload so we can use it for testing.
211 | }, {
212 | timeout: 1200
213 | }, false);
214 | await new Promise((resolve) => { setTimeout(resolve, 25); });
215 |
216 | // startQueue is false so queue should not have started.
217 | queue.status.should.equal('inactive');
218 |
219 | const queueStartTime = Date.now();
220 |
221 | // Start queue, don't await so this test can continue while queue processes.
222 | await queue.start(queueLifespan);
223 |
224 | const queueEndTime = Date.now();
225 | const queueProcessTime = queueStartTime - queueEndTime;
226 |
227 | if (queueProcessTime > queueLifespan) {
228 | throw new Error('ERROR: Queue did not complete before lifespan ended.');
229 | }
230 |
231 | if (badJobs.length) {
232 | throw new Error('ERROR: Queue with lifespan picked up bad jobs it did not have enough remaining lifespan to execute: ' + JSON.stringify(badJobs));
233 | }
234 |
235 | // Queue should have stopped.
236 | queue.status.should.equal('inactive');
237 |
238 | //Check that the correct jobs executed.
239 | executedJobs.should.deepEqual(['job1', 'job2', 'job4', 'job5', 'job6']);
240 |
241 | // Check jobs that couldn't be picked up are still in the queue.
242 | const remainingJobs = await queue.getJobs(true);
243 |
244 | const remainingJobNames = remainingJobs.map( job => {
245 | const payload = JSON.parse(job.payload);
246 | return payload.trackingName;
247 | });
248 |
249 | // queue.getJobs() doesn't order jobs in any particular way so just
250 | // check that the jobs still exist on the queue.
251 | remainingJobNames.should.containDeep(['job3', 'job7']);
252 |
253 | });
254 |
255 | it('#start(lifespan) ADVANCED TEST FULL (Multiple job names, job timeouts, concurrency, priority) - ONLY RUN IN NON-CI ENV: queue will process jobs with timeout set as expected until lifespan ends.', async () => {
256 |
257 | // This test will intermittently fail in CI environments like travis-ci.
258 | // Intermittent failure is a result of the poor performance of CI environments
259 | // causing the timeouts in this test to become really flakey (setTimeout can't
260 | // guarantee exact time of function execution, and in a high load env execution can
261 | // be significantly delayed.
262 | if (process.env.COVERALLS_ENV == 'production') {
263 | return true;
264 | }
265 |
266 | const queue = await QueueFactory();
267 | queue.flushQueue();
268 | const jobName = 'job-name';
269 | const anotherJobName = 'another-job-name';
270 | const timeoutJobName = 'timeout-job-name';
271 | const concurrentJobName = 'concurrent-job-name';
272 | const queueLifespan = 5300;
273 | let remainingLifespan = queueLifespan;
274 |
275 | // Track the jobs that have executed to test against.
276 | let executedJobs = [];
277 |
278 | // We need to be able to throw an error outside of
279 | // job workers, because errors thrown inside a job
280 | // worker function are caught and logged by job processing
281 | // logic. They will not fail the test. So track bad jobs
282 | // and throw error after jobs finish processing.
283 | let badJobs = [];
284 |
285 | queue.addWorker(jobName, async (id, payload) => {
286 |
287 | // Track jobs that exec
288 | executedJobs.push(payload.trackingName);
289 |
290 | // Detect jobs that should't be picked up by lifespan queue.
291 | if (remainingLifespan - 500 < payload.payloadOptionsTimeout) {
292 | badJobs.push({id, payload});
293 | }
294 |
295 | remainingLifespan = remainingLifespan - payload.payloadTimeout;
296 |
297 | await new Promise((resolve) => {
298 | setTimeout(resolve, payload.payloadTimeout);
299 | });
300 |
301 | }, { concurrency: 1});
302 |
303 | queue.addWorker(anotherJobName, async (id, payload) => {
304 |
305 | // Track jobs that exec
306 | executedJobs.push(payload.trackingName);
307 |
308 | // Detect jobs that should't be picked up by lifespan queue.
309 | if (remainingLifespan - 500 < payload.payloadOptionsTimeout) {
310 | badJobs.push({id, payload});
311 | }
312 |
313 | remainingLifespan = remainingLifespan - payload.payloadTimeout;
314 |
315 | await new Promise((resolve) => {
316 | setTimeout(resolve, payload.payloadTimeout);
317 | });
318 |
319 | }, { concurrency: 1});
320 |
321 | queue.addWorker(timeoutJobName, async (id, payload) => {
322 |
323 | // Track jobs that exec
324 | executedJobs.push(payload.trackingName);
325 |
326 | // Detect jobs that should't be picked up by lifespan queue.
327 | if (remainingLifespan - 500 < payload.payloadOptionsTimeout) {
328 | badJobs.push({id, payload});
329 | }
330 |
331 | remainingLifespan = remainingLifespan - payload.payloadOptionsTimeout;
332 |
333 | await new Promise((resolve) => {
334 | setTimeout(resolve, payload.payloadTimeout);
335 | });
336 |
337 | }, { concurrency: 1});
338 |
339 | queue.addWorker(concurrentJobName, async (id, payload) => {
340 |
341 | // Track jobs that exec
342 | executedJobs.push(payload.trackingName);
343 |
344 | // Detect jobs that should't be picked up by lifespan queue.
345 | if (remainingLifespan - 500 < payload.payloadOptionsTimeout) {
346 | badJobs.push({id, payload});
347 | }
348 |
349 |
350 | // Since these all run concurrently, only subtract the job with the longest
351 | // timeout that will presumabely finish last.
352 | if (payload.payloadTimeout == 600) {
353 | remainingLifespan = remainingLifespan - payload.payloadTimeout;
354 | }
355 |
356 |
357 | await new Promise((resolve) => {
358 | setTimeout(resolve, payload.payloadTimeout);
359 | });
360 |
361 | }, { concurrency: 4});
362 |
363 | // Create a couple jobs
364 | queue.createJob(jobName, {
365 | trackingName: 'job1-job-name-payloadTimeout(100)-timeout(200)-priority(-1)',
366 | payloadTimeout: 100,
367 | payloadOptionsTimeout: 200 // Mirror the actual job options timeout in payload so we can use it for testing.
368 | }, {
369 | timeout: 200,
370 | priority: -1
371 | }, false);
372 |
373 | // Since more than one job can be written in 1 ms, we need to add a slight delay
374 | // in order to control the order jobs come off the queue (since they are time sorted)
375 | // If multiple jobs are written in the same ms, Realm can't be deterministic about job
376 | // ordering when we pop jobs off the top of the queue.
377 | await new Promise((resolve) => { setTimeout(resolve, 25); });
378 |
379 | queue.createJob(anotherJobName, {
380 | trackingName: 'job2-another-job-name-payloadTimeout(1000)-timeout(1100)-priority(0)',
381 | payloadTimeout: 1000,
382 | payloadOptionsTimeout: 1100 // Mirror the actual job options timeout in payload so we can use it for testing.
383 | }, {
384 | timeout: 1100
385 | }, false);
386 | await new Promise((resolve) => { setTimeout(resolve, 25); });
387 |
388 | queue.createJob(anotherJobName, {
389 | trackingName: 'job3-another-job-name-payloadTimeout(750)-timeout(800)-priority(10)',
390 | payloadTimeout: 750,
391 | payloadOptionsTimeout: 800 // Mirror the actual job options timeout in payload so we can use it for testing.
392 | }, {
393 | timeout: 800,
394 | priority: 10
395 | }, false);
396 | await new Promise((resolve) => { setTimeout(resolve, 25); });
397 |
398 | queue.createJob(jobName, {
399 | trackingName: 'job4-job-name-payloadTimeout(10000)-timeout(10100)-priority(0)',
400 | payloadTimeout: 10000,
401 | payloadOptionsTimeout: 10100 // Mirror the actual job options timeout in payload so we can use it for testing.
402 | }, {
403 | timeout: 10100
404 | }, false);
405 | await new Promise((resolve) => { setTimeout(resolve, 25); });
406 |
407 | queue.createJob(jobName, {
408 | trackingName: 'job5-job-name-payloadTimeout(400)-timeout(500)-priority(0)',
409 | payloadTimeout: 400,
410 | payloadOptionsTimeout: 500 // Mirror the actual job options timeout in payload so we can use it for testing.
411 | }, {
412 | timeout: 500
413 | }, false);
414 | await new Promise((resolve) => { setTimeout(resolve, 25); });
415 |
416 | queue.createJob(timeoutJobName, {
417 | trackingName: 'job6-timeout-job-name-payloadTimeout(10000)-timeout(500)-priority(0)',
418 | payloadTimeout: 10000,
419 | payloadOptionsTimeout: 500 // Mirror the actual job options timeout in payload so we can use it for testing.
420 | }, {
421 | timeout: 500
422 | }, false);
423 | await new Promise((resolve) => { setTimeout(resolve, 25); });
424 |
425 | queue.createJob(jobName, {
426 | trackingName: 'job7-job-name-payloadTimeout(1000)-timeout(1100)-priority(1)',
427 | payloadTimeout: 1000,
428 | payloadOptionsTimeout: 1100 // Mirror the actual job options timeout in payload so we can use it for testing.
429 | }, {
430 | timeout: 1100,
431 | priority: 1
432 | }, false);
433 | await new Promise((resolve) => { setTimeout(resolve, 25); });
434 |
435 |
436 | // Create concurrent jobs
437 | queue.createJob(concurrentJobName, {
438 | trackingName: 'job8-concurrent-job-name-payloadTimeout(500)-timeout(600)-priority(0)',
439 | payloadTimeout: 500,
440 | payloadOptionsTimeout: 600 // Mirror the actual job options timeout in payload so we can use it for testing.
441 | }, {
442 | timeout: 600,
443 | priority: 0
444 | }, false);
445 | await new Promise((resolve) => { setTimeout(resolve, 25); });
446 |
447 | queue.createJob(concurrentJobName, {
448 | trackingName: 'job9-concurrent-job-name-payloadTimeout(510)-timeout(600)-priority(0)',
449 | payloadTimeout: 510,
450 | payloadOptionsTimeout: 600 // Mirror the actual job options timeout in payload so we can use it for testing.
451 | }, {
452 | timeout: 600,
453 | priority: 0
454 | }, false);
455 | await new Promise((resolve) => { setTimeout(resolve, 25); });
456 |
457 | queue.createJob(concurrentJobName, {
458 | trackingName: 'job10-concurrent-job-name-payloadTimeout(10000)-timeout(10100)-priority(0)', // THIS JOB WILL BE SKIPPED BY getConcurrentJobs() due to timeout too long.
459 | payloadTimeout: 10000,
460 | payloadOptionsTimeout: 10100 // Mirror the actual job options timeout in payload so we can use it for testing.
461 | }, {
462 | timeout: 10100,
463 | priority: 0
464 | }, false);
465 | await new Promise((resolve) => { setTimeout(resolve, 25); });
466 |
467 | queue.createJob(concurrentJobName, {
468 | trackingName: 'job11-concurrent-job-name-payloadTimeout(600)-timeout(700)-priority(0)',
469 | payloadTimeout: 600,
470 | payloadOptionsTimeout: 700 // Mirror the actual job options timeout in payload so we can use it for testing.
471 | }, {
472 | timeout: 700,
473 | priority: 0
474 | }, false);
475 | await new Promise((resolve) => { setTimeout(resolve, 25); });
476 |
477 | queue.createJob(jobName, {
478 | trackingName: 'job12-job-name-payloadTimeout(100)-timeout(200)-priority(0)',
479 | payloadTimeout: 100,
480 | payloadOptionsTimeout: 200 // Mirror the actual job options timeout in payload so we can use it for testing.
481 | }, {
482 | timeout: 200
483 | }, false);
484 | await new Promise((resolve) => { setTimeout(resolve, 25); });
485 |
486 | queue.createJob(jobName, {
487 | trackingName: 'job13-job-name-payloadTimeout(400)-timeout(500)-priority(0)', // THIS JOB WON'T BE RUN BECAUSE THE TIMEOUT IS 500 AND ONLY 950ms left by this pount. 950 - 500 = 450 and 500 remaining is min for job to be pulled.
488 | payloadTimeout: 400,
489 | payloadOptionsTimeout: 500 // Mirror the actual job options timeout in payload so we can use it for testing.
490 | }, {
491 | timeout: 500
492 | }, false);
493 | await new Promise((resolve) => { setTimeout(resolve, 25); });
494 |
495 | queue.createJob(jobName, {
496 | trackingName: 'job14-job-name-payloadTimeout(100)-timeout(200)-priority(0)',
497 | payloadTimeout: 100,
498 | payloadOptionsTimeout: 200 // Mirror the actual job options timeout in payload so we can use it for testing.
499 | }, {
500 | timeout: 200
501 | }, false);
502 | await new Promise((resolve) => { setTimeout(resolve, 25); });
503 |
504 | queue.createJob(jobName, {
505 | trackingName: 'job15-job-name-payloadTimeout(500)-timeout(600)-priority(0)', // THIS JOB WON'T BE RUN BECAUSE out of time!
506 | payloadTimeout: 500,
507 | payloadOptionsTimeout: 600 // Mirror the actual job options timeout in payload so we can use it for testing.
508 | }, {
509 | timeout: 600
510 | }, false);
511 | await new Promise((resolve) => { setTimeout(resolve, 25); });
512 |
513 | // startQueue is false so queue should not have started.
514 | queue.status.should.equal('inactive');
515 |
516 | const queueStartTime = Date.now();
517 |
518 | // Start queue, don't await so this test can continue while queue processes.
519 | await queue.start(queueLifespan);
520 |
521 | const queueEndTime = Date.now();
522 | const queueProcessTime = queueStartTime - queueEndTime;
523 |
524 | if (queueProcessTime > queueLifespan) {
525 | throw new Error('ERROR: Queue did not complete before lifespan ended.');
526 | }
527 |
528 | if (badJobs.length) {
529 | throw new Error('ERROR: Queue with lifespan picked up bad jobs it did not have enough remaining lifespan to execute: ' + JSON.stringify(badJobs));
530 | }
531 |
532 | // Queue should have stopped.
533 | queue.status.should.equal('inactive');
534 |
535 | //Check that the correct jobs executed.
536 | executedJobs.should.deepEqual([
537 | 'job3-another-job-name-payloadTimeout(750)-timeout(800)-priority(10)',
538 | 'job7-job-name-payloadTimeout(1000)-timeout(1100)-priority(1)',
539 | 'job2-another-job-name-payloadTimeout(1000)-timeout(1100)-priority(0)',
540 | 'job5-job-name-payloadTimeout(400)-timeout(500)-priority(0)',
541 | 'job6-timeout-job-name-payloadTimeout(10000)-timeout(500)-priority(0)', // This job executes but isn't deleted because it fails due to timeout.
542 | 'job8-concurrent-job-name-payloadTimeout(500)-timeout(600)-priority(0)',
543 | 'job9-concurrent-job-name-payloadTimeout(510)-timeout(600)-priority(0)',
544 | 'job11-concurrent-job-name-payloadTimeout(600)-timeout(700)-priority(0)',
545 | 'job12-job-name-payloadTimeout(100)-timeout(200)-priority(0)',
546 | 'job14-job-name-payloadTimeout(100)-timeout(200)-priority(0)',
547 | 'job1-job-name-payloadTimeout(100)-timeout(200)-priority(-1)'
548 | ]);
549 |
550 | // Check jobs that couldn't be picked up are still in the queue.
551 | const remainingJobs = await queue.getJobs(true);
552 |
553 | const remainingJobNames = remainingJobs.map( job => {
554 | const payload = JSON.parse(job.payload);
555 | return payload.trackingName;
556 | });
557 |
558 | // queue.getJobs() doesn't order jobs in any particular way so just
559 | // check that the jobs still exist on the queue.
560 | remainingJobNames.should.containDeep([
561 | 'job4-job-name-payloadTimeout(10000)-timeout(10100)-priority(0)',
562 | 'job6-timeout-job-name-payloadTimeout(10000)-timeout(500)-priority(0)',
563 | 'job10-concurrent-job-name-payloadTimeout(10000)-timeout(10100)-priority(0)',
564 | 'job13-job-name-payloadTimeout(400)-timeout(500)-priority(0)',
565 | 'job15-job-name-payloadTimeout(500)-timeout(600)-priority(0)'
566 | ]);
567 |
568 | }, 10000); // Increase timeout of this advanced test to 10 seconds.
569 |
570 | it('#start(lifespan) "Zero lifespanRemaining" edge case #1 is properly handled.', async () => {
571 |
572 | // Mock Date.now()
573 | Date.now = jest.fn();
574 | Date.now.mockReturnValueOnce(0);
575 | Date.now.mockReturnValueOnce(1000);
576 |
577 | const queue = await QueueFactory();
578 | const jobName = 'job-name';
579 | let counter = 0;
580 |
581 | queue.addWorker(jobName, () => {
582 | counter++;
583 | });
584 |
585 | // Create a job
586 | queue.createJob(jobName, {}, {
587 | timeout: 100 // Timeout must be set to test that job still isn't grabbed during "zero lifespanRemaining" edge case.
588 | }, false);
589 |
590 | // startQueue is false so queue should not have started.
591 | queue.status.should.equal('inactive');
592 |
593 | // Start queue, don't await so this test can continue while queue processes.
594 | await queue.start(1000);
595 |
596 | // Queue should be inactive again.
597 | queue.status.should.equal('inactive');
598 |
599 | // Since we hit "zero lifespanRemaining" edge case, the job should never have been pulled
600 | // off the queue and processed. So counter should remain 0 and job should still exist.
601 | counter.should.equal(0);
602 |
603 | const jobs = await queue.getJobs(true);
604 |
605 | jobs.length.should.equal(1);
606 |
607 | });
608 |
609 | it('#start(lifespan) "Zero lifespanRemaining" edge case #2 is properly handled.', async () => {
610 |
611 | // Mock Date.now()
612 | Date.now = jest.fn();
613 | Date.now.mockReturnValueOnce(0);
614 | Date.now.mockReturnValueOnce(500);
615 | Date.now.mockReturnValueOnce(2000);
616 |
617 | const queue = await QueueFactory();
618 | const jobName = 'job-name';
619 | let counter = 0;
620 |
621 | queue.addWorker(jobName, () => {
622 | counter++;
623 | });
624 |
625 | // Create jobs
626 | queue.createJob(jobName, {}, {
627 | timeout: 100 // Timeout must be set to test that job still isn't grabbed during "zero lifespanRemaining" edge case.
628 | }, false);
629 |
630 | await new Promise((resolve) => { setTimeout(resolve, 25); }); // Space out inserts so time sorting is deterministic.
631 |
632 | queue.createJob(jobName, {
633 | testIdentifier: 'this is 2nd job'
634 | }, {
635 | timeout: 100 // Timeout must be set to test that job still isn't grabbed during "zero lifespanRemaining" edge case.
636 | }, false);
637 |
638 | // startQueue is false so queue should not have started.
639 | queue.status.should.equal('inactive');
640 |
641 | // Start queue, don't await so this test can continue while queue processes.
642 | await queue.start(2000);
643 |
644 | // Queue should be inactive again.
645 | queue.status.should.equal('inactive');
646 |
647 | // Since we skipped first "zero lifespanRemaining" edge case, one job should
648 | // be processed. However since we hit 2nd "zero lifespanRemaining" edge case,
649 | // second job should never be pulled off queue and so second job should still exist.
650 | counter.should.equal(1);
651 |
652 | const jobs = await queue.getJobs(true);
653 |
654 | const jobPayload = JSON.parse(jobs[0].payload);
655 |
656 | jobPayload.testIdentifier.should.equal('this is 2nd job');
657 |
658 | });
659 |
660 | //
661 | // FULL QUEUE UNIT TESTING
662 | //
663 |
664 | it('#constructor() sets values correctly', async () => {
665 |
666 | const queueNotInitialized = new Queue();
667 |
668 | queueNotInitialized.should.have.properties({
669 | realm: null,
670 | worker: new Worker(),
671 | status: 'inactive'
672 | });
673 |
674 | });
675 |
676 | it('QueueFactory initializes Realm', async () => {
677 |
678 | const queue = await QueueFactory();
679 |
680 | queue.realm.constructor.name.should.equal('Realm');
681 |
682 | });
683 |
684 | it('init() Calling init() multiple times will only set queue.realm once.', async () => {
685 |
686 | const queue = await QueueFactory();
687 |
688 | queue.realm.constructor.name.should.equal('Realm');
689 |
690 | // Overwrite realm instance to test it doesn't get set to the actual
691 | // Realm singleton instance again in init() since queue.realm is no longer null.
692 | queue.realm = 'arbitrary-string';
693 |
694 | queue.init();
695 |
696 | queue.realm.should.equal('arbitrary-string');
697 |
698 | });
699 |
700 | it('#addWorker() and removeWorker() should pass calls through to Worker class', async () => {
701 |
702 | const queue = await QueueFactory();
703 | const workerOptions = {
704 | concurrency: 4,
705 | onSuccess: async () => {}
706 | };
707 |
708 | queue.addWorker('job-name', () => {}, workerOptions);
709 |
710 | // first worker is added with default options.
711 | Worker.workers['job-name'].should.be.a.Function();
712 | Worker.workers['job-name'].options.should.deepEqual({
713 | concurrency: workerOptions.concurrency,
714 | onStart: null,
715 | onSuccess: workerOptions.onSuccess,
716 | onFailure: null,
717 | onFailed: null,
718 | onComplete: null
719 | });
720 |
721 | queue.removeWorker('job-name');
722 |
723 | // Worker has been removed.
724 | should.not.exist(Worker.workers['job-name']);
725 |
726 | });
727 |
728 | it('#createJob() requires job name at minimum', async () => {
729 |
730 | const queue = await QueueFactory();
731 |
732 | try {
733 | await queue.createJob();
734 | throw new Error('Job with no name should have thrown error.');
735 | } catch (error) {
736 | error.should.deepEqual(new Error('Job name must be supplied.'));
737 | }
738 |
739 | });
740 |
741 | it('#createJob() should validate job options.', async () => {
742 |
743 | const queue = await QueueFactory();
744 | const jobName = 'job-name';
745 |
746 | queue.addWorker(jobName, () => {});
747 |
748 | try {
749 | await queue.createJob(jobName, {}, {
750 | timeout: -100
751 | }, false);
752 | throw new Error('createJob() should validate job timeout option.');
753 | } catch (error) {
754 | error.should.deepEqual(new Error('Invalid job option.'));
755 | }
756 |
757 | try {
758 | await queue.createJob(jobName, {}, {
759 | attempts: -100
760 | }, false);
761 | throw new Error('createJob() should validate job attempts option.');
762 | } catch (error) {
763 | error.should.deepEqual(new Error('Invalid job option.'));
764 | }
765 |
766 | });
767 |
768 | it('#createJob() should apply defaults correctly', async () => {
769 |
770 | const queue = await QueueFactory();
771 | const jobName = 'job-name';
772 |
773 | queue.addWorker(jobName, () => {});
774 |
775 | queue.createJob(jobName, {}, {}, false);
776 |
777 | // startQueue is false so queue should not have started.
778 | queue.status.should.equal('inactive');
779 |
780 | const jobs = await queue.getJobs(true);
781 |
782 | // Check job has default values.
783 | jobs[0].should.have.properties({
784 | name: jobName,
785 | payload: JSON.stringify({}),
786 | data: JSON.stringify({attempts: 1}),
787 | priority: 0,
788 | active: false,
789 | timeout: 25000
790 | });
791 |
792 | });
793 |
794 | it('#createJob() should create a new job on the queue', async () => {
795 |
796 | const queue = await QueueFactory();
797 | const jobName = 'job-name';
798 | const payload = { data: 'example-data' };
799 | const jobOptions = { priority: 4, timeout: 3000, attempts: 3};
800 |
801 | queue.addWorker(jobName, () => {});
802 |
803 | queue.createJob(jobName, payload, jobOptions, false);
804 |
805 | // startQueue is false so queue should not have started.
806 | queue.status.should.equal('inactive');
807 |
808 | const jobs = await queue.getJobs(true);
809 |
810 | jobs[0].should.have.properties({
811 | name: jobName,
812 | payload: JSON.stringify(payload),
813 | data: JSON.stringify({attempts: jobOptions.attempts}),
814 | priority: jobOptions.priority,
815 | active: false,
816 | timeout: jobOptions.timeout
817 | });
818 |
819 | });
820 |
821 | it('#createJob() should default to starting queue. stop() should stop queue.', async () => {
822 |
823 | const queue = await QueueFactory();
824 | const jobName = 'job-name';
825 | const payload = { data: 'example-data' };
826 | const jobOptions = { priority: 4, timeout: 3000, attempts: 3};
827 |
828 | queue.addWorker(jobName, () => {});
829 |
830 | queue.createJob(jobName, payload, jobOptions, true);
831 | queue.status.should.equal('active');
832 |
833 | queue.stop();
834 |
835 | queue.status.should.equal('inactive');
836 |
837 | });
838 |
839 | it('#start() should start queue.', async () => {
840 |
841 | const queue = await QueueFactory();
842 | const jobName = 'job-name';
843 | const payload = { data: 'example-data' };
844 | const jobOptions = { priority: 4, timeout: 3000, attempts: 3};
845 |
846 | let counter = 0; // Incrementing this will be our job "work".
847 |
848 | queue.addWorker(jobName, () => {
849 | counter++;
850 | });
851 |
852 | // Create a couple jobs
853 | queue.createJob(jobName, payload, jobOptions, false);
854 | queue.createJob(jobName, payload, jobOptions, false);
855 |
856 | // startQueue is false so queue should not have started.
857 | queue.status.should.equal('inactive');
858 |
859 | queue.start();
860 |
861 | queue.status.should.equal('active');
862 |
863 | // Give queue 1000ms to churn through all the jobs.
864 | await new Promise((resolve) => {
865 | setTimeout(() => {
866 | resolve(true);
867 | }, 1000);
868 | });
869 |
870 | // Queue should be finished with no jobs left.
871 | queue.status.should.equal('inactive');
872 |
873 | const jobs = await queue.getJobs(true);
874 |
875 | jobs.length.should.equal(0);
876 |
877 | // Counter should be updated to reflect worker execution.
878 | counter.should.equal(2);
879 |
880 | });
881 |
882 | it('#start() called when queue is already active should NOT fire up a concurrent queue.', async () => {
883 |
884 | const queue = await QueueFactory();
885 | const jobName = 'job-name';
886 | const payload = { data: 'example-data' };
887 | const jobOptions = { priority: 4, timeout: 3000, attempts: 3};
888 |
889 | queue.addWorker(jobName, async () => {
890 |
891 | // Make queue take some time to process.
892 | await new Promise( resolve => {
893 | setTimeout(resolve, 1000);
894 | });
895 |
896 | });
897 |
898 | // Create a couple jobs
899 | queue.createJob(jobName, payload, jobOptions, false);
900 | queue.createJob(jobName, payload, jobOptions, false);
901 |
902 | // startQueue is false so queue should not have started.
903 | queue.status.should.equal('inactive');
904 |
905 | // Start queue, don't await so this test can continue while queue processes.
906 | queue.start();
907 |
908 | queue.status.should.equal('active');
909 |
910 | // Calling queue.start() on already running queue should cause start() to return
911 | // early with false bool indicating concurrent start did not occur.
912 | const falseStart = await queue.start(); //Must be awaited to resolve async func promise into false value.
913 |
914 | falseStart.should.be.False();
915 |
916 | });
917 |
918 | it('#getJobs() should grab all jobs in queue.', async () => {
919 |
920 | const queue = await QueueFactory();
921 | const jobName = 'job-name';
922 | const payload = { data: 'example-data' };
923 | const jobOptions = { priority: 4, timeout: 3000, attempts: 3};
924 |
925 | queue.addWorker(jobName, () => {});
926 |
927 | // Create a couple jobs
928 | queue.createJob(jobName, payload, jobOptions, false);
929 | queue.createJob(jobName, payload, jobOptions, false);
930 | queue.createJob(jobName, payload, jobOptions, false);
931 | queue.createJob(jobName, payload, jobOptions, false);
932 |
933 | const jobs = await queue.getJobs(true);
934 |
935 | jobs.length.should.equal(4);
936 |
937 | const mvccJobs = await queue.getJobs(); // Test non-blocking read version as well.
938 |
939 | mvccJobs.length.should.equal(4);
940 |
941 | });
942 |
943 | it('#getConcurrentJobs(queueLifespanRemaining) should work as expected for queues started with a lifespan.', async () => {
944 |
945 | const queue = await QueueFactory();
946 | const jobName = 'job-name';
947 |
948 | queue.addWorker(jobName, () => {}, {
949 | concurrency: 3
950 | });
951 |
952 | // Test that jobs with no timeout set will not be returned by getConcurrentJobs() if queueLifespanRemaining is passed.
953 | queue.createJob(jobName, {}, {
954 | timeout: 0
955 | }, false);
956 | queue.createJob(jobName, {}, {
957 | timeout: 0
958 | }, false);
959 | queue.createJob(jobName, {}, {
960 | timeout: 0
961 | }, false);
962 |
963 | const jobs = await queue.getConcurrentJobs(2000);
964 |
965 | // No jobs should be grabbed
966 | jobs.length.should.equal(0);
967 |
968 | // Reset DB
969 | queue.flushQueue();
970 |
971 | // Test that jobs with timeout not at least 500ms less than queueLifespanRemaining are not grabbed.
972 | queue.createJob(jobName, {}, {
973 | timeout: 500
974 | }, false);
975 | queue.createJob(jobName, {}, {
976 | timeout: 500
977 | }, false);
978 | queue.createJob(jobName, {}, {
979 | timeout: 500
980 | }, false);
981 |
982 | const notEnoughBufferJobs = await queue.getConcurrentJobs(600);
983 |
984 | // No jobs should be grabbed
985 | notEnoughBufferJobs.length.should.equal(0);
986 |
987 | // Reset DB
988 | queue.flushQueue();
989 |
990 | //Lower bound edge case test
991 | queue.createJob(jobName, {}, {
992 | timeout: 0
993 | }, false);
994 | queue.createJob(jobName, {}, {
995 | timeout: 1
996 | }, false);
997 | queue.createJob(jobName, {}, {
998 | timeout: 1
999 | }, false);
1000 |
1001 | // startQueue is false so queue should not have started.
1002 | queue.status.should.equal('inactive');
1003 |
1004 | const lowerBoundEdgeCaseJobs = await queue.getConcurrentJobs(501);
1005 |
1006 | // Only the jobs with the timeouts set should be grabbed.
1007 | lowerBoundEdgeCaseJobs.length.should.equal(2);
1008 |
1009 | // Reset DB
1010 | queue.flushQueue();
1011 |
1012 | //Test concurrency is working as expected with lifespans.
1013 | queue.createJob(jobName, {}, {
1014 | timeout: 800
1015 | }, false);
1016 | queue.createJob(jobName, {}, {
1017 | timeout: 1000
1018 | }, false);
1019 | queue.createJob(jobName, {}, {
1020 | timeout: 1000
1021 | }, false);
1022 | queue.createJob(jobName, {}, {
1023 | timeout: 1000
1024 | }, false);
1025 |
1026 | // startQueue is false so queue should not have started.
1027 | queue.status.should.equal('inactive');
1028 |
1029 | const lifespanConcurrencyJobs = await queue.getConcurrentJobs(2000);
1030 |
1031 | // Only 3 jobs should be grabbed in this test even though all jobs
1032 | // have valid timeouts because worker concurrency is set to 3
1033 | lifespanConcurrencyJobs.length.should.equal(3);
1034 |
1035 | // Reset DB
1036 | queue.flushQueue();
1037 |
1038 | });
1039 |
1040 | it('#getConcurrentJobs() If worker concurrency is set to 3, getConcurrentJobs() should get up to 3 of same type of job as next job on top of queue.', async () => {
1041 |
1042 | const queue = await QueueFactory();
1043 | const jobName = 'job-name';
1044 | const jobOptions = { priority: 4, timeout: 3000, attempts: 3};
1045 |
1046 | queue.addWorker(jobName, () => {}, {
1047 | concurrency: 3
1048 | });
1049 | queue.addWorker('a-different-job', () => {});
1050 |
1051 | // Create a couple jobs
1052 | queue.createJob(jobName, { random: 'this is 1st random data' }, jobOptions, false);
1053 | queue.createJob('a-different-job', { dummy: 'data' }, {}, false); // This should not be returned by concurrentJobs() should all be of the 'job-name' type.
1054 | queue.createJob(jobName, { random: 'this is 2nd random data' }, jobOptions, false);
1055 | queue.createJob(jobName, { random: 'this is 3rd random data' }, jobOptions, false);
1056 | queue.createJob(jobName, { random: 'this is 4th random data' }, jobOptions, false);
1057 |
1058 | const concurrentJobs = await queue.getConcurrentJobs();
1059 |
1060 | // Verify correct jobs retrieved.
1061 | concurrentJobs.length.should.equal(3);
1062 | JSON.parse(concurrentJobs[0].payload).should.deepEqual({ random: 'this is 1st random data' });
1063 | JSON.parse(concurrentJobs[1].payload).should.deepEqual({ random: 'this is 2nd random data' });
1064 | JSON.parse(concurrentJobs[2].payload).should.deepEqual({ random: 'this is 3rd random data' });
1065 |
1066 | // Ensure that other jobs also got created, but not returned by getConcurrentJobs().
1067 | const jobs = await queue.getJobs(true);
1068 | jobs.length.should.equal(5);
1069 |
1070 | });
1071 |
1072 | it('#getConcurrentJobs() If worker concurrency is set to 10, but only 4 jobs of next job type exist, getConcurrentJobs() should only return 4 jobs.', async () => {
1073 |
1074 | const queue = await QueueFactory();
1075 | const jobName = 'job-name';
1076 | const jobOptions = { priority: 4, timeout: 3000, attempts: 3};
1077 |
1078 | queue.addWorker(jobName, () => {}, {
1079 | concurrency: 10
1080 | });
1081 | queue.addWorker('a-different-job', () => {});
1082 |
1083 | // Create a couple jobs
1084 | queue.createJob(jobName, { random: 'this is 1st random data' }, jobOptions, false);
1085 | queue.createJob('a-different-job', { dummy: 'data' }, {}, false); // This should not be returned by concurrentJobs() should all be of the 'job-name' type.
1086 | queue.createJob(jobName, { random: 'this is 2nd random data' }, jobOptions, false);
1087 | queue.createJob(jobName, { random: 'this is 3rd random data' }, jobOptions, false);
1088 | queue.createJob(jobName, { random: 'this is 4th random data' }, jobOptions, false);
1089 |
1090 | const concurrentJobs = await queue.getConcurrentJobs();
1091 |
1092 | // Verify correct jobs retrieved.
1093 | concurrentJobs.length.should.equal(4);
1094 | JSON.parse(concurrentJobs[0].payload).should.deepEqual({ random: 'this is 1st random data' });
1095 | JSON.parse(concurrentJobs[1].payload).should.deepEqual({ random: 'this is 2nd random data' });
1096 | JSON.parse(concurrentJobs[2].payload).should.deepEqual({ random: 'this is 3rd random data' });
1097 | JSON.parse(concurrentJobs[3].payload).should.deepEqual({ random: 'this is 4th random data' });
1098 |
1099 | // Ensure that other jobs also got created, but not returned by getConcurrentJobs().
1100 | const jobs = await queue.getJobs(true);
1101 | jobs.length.should.equal(5);
1102 |
1103 | });
1104 |
1105 | it('#getConcurrentJobs() Ensure that priority is respected.', async () => {
1106 |
1107 | const queue = await QueueFactory();
1108 | const jobName = 'job-name';
1109 | const jobOptions = { priority: 0, timeout: 3000, attempts: 3};
1110 |
1111 | queue.addWorker(jobName, () => {}, {
1112 | concurrency: 3
1113 | });
1114 | queue.addWorker('a-different-job', () => {}, {
1115 | concurrency: 2
1116 | });
1117 |
1118 | // Create a couple jobs
1119 | queue.createJob(jobName, { random: 'this is 1st random data' }, jobOptions, false);
1120 | queue.createJob('a-different-job', { dummy: '1 data' }, { priority: 3 }, false);
1121 | queue.createJob(jobName, { random: 'this is 2nd random data' }, jobOptions, false);
1122 | queue.createJob('a-different-job', { dummy: '2 data' }, { priority: 5 }, false);
1123 | queue.createJob('a-different-job', { dummy: '3 data' }, { priority: 3 }, false);
1124 | queue.createJob(jobName, { random: 'this is 3rd random data' }, jobOptions, false);
1125 | queue.createJob(jobName, { random: 'this is 4th random data' }, jobOptions, false);
1126 |
1127 | const concurrentJobs = await queue.getConcurrentJobs();
1128 |
1129 | // Verify correct jobs retrieved.
1130 | // 'a-different-job' should be the jobs returned since job with payload "2 data" has highest priority
1131 | // since the other 'a-different-job' jobs have same priority, "1 data" should get preference for 2nd concurrent job due
1132 | // to timestamp order.
1133 | concurrentJobs.length.should.equal(2);
1134 | JSON.parse(concurrentJobs[0].payload).should.deepEqual({ dummy: '2 data' });
1135 | JSON.parse(concurrentJobs[1].payload).should.deepEqual({ dummy: '1 data' });
1136 |
1137 | // Ensure that other jobs also got created, but not returned by getConcurrentJobs().
1138 | const jobs = await queue.getJobs(true);
1139 | jobs.length.should.equal(7);
1140 |
1141 | });
1142 |
1143 | it('#getConcurrentJobs() Marks selected jobs as "active"', async () => {
1144 |
1145 | const queue = await QueueFactory();
1146 | const jobName = 'job-name';
1147 | const jobOptions = { priority: 0, timeout: 3000, attempts: 3};
1148 |
1149 | queue.addWorker(jobName, () => {}, {
1150 | concurrency: 3
1151 | });
1152 | queue.addWorker('a-different-job', () => {}, {
1153 | concurrency: 2
1154 | });
1155 |
1156 | // Create a couple jobs
1157 | queue.createJob(jobName, { random: 'this is 1st random data' }, jobOptions, false);
1158 | queue.createJob('a-different-job', { dummy: '1 data' }, { priority: 3 }, false);
1159 | queue.createJob(jobName, { random: 'this is 2nd random data' }, jobOptions, false);
1160 | queue.createJob('a-different-job', { dummy: '2 data' }, { priority: 5 }, false);
1161 | queue.createJob('a-different-job', { dummy: '3 data' }, { priority: 3 }, false);
1162 | queue.createJob(jobName, { random: 'this is 3rd random data' }, jobOptions, false);
1163 | queue.createJob(jobName, { random: 'this is 4th random data' }, jobOptions, false);
1164 |
1165 | // Jobs returned by getConcurrentJobs() are marked "active" so they won't be returned by future getConcurrentJobs() calls.
1166 | const concurrentJobs = await queue.getConcurrentJobs();
1167 |
1168 | // Get all the jobs in the DB and check that the "concurrentJobs" are marked "active."
1169 | const jobs = await queue.getJobs(true);
1170 | jobs.length.should.equal(7);
1171 |
1172 | const activeJobs = jobs.filter( job => job.active);
1173 | activeJobs.length.should.equal(2);
1174 | JSON.parse(concurrentJobs[0].payload).should.deepEqual({ dummy: '2 data' });
1175 | JSON.parse(concurrentJobs[1].payload).should.deepEqual({ dummy: '1 data' });
1176 |
1177 | });
1178 |
1179 | it('#getConcurrentJobs() consecutive calls to getConcurrentJobs() gets new non-active jobs (and marks them active).', async () => {
1180 |
1181 | const queue = await QueueFactory();
1182 | const jobName = 'job-name';
1183 | const jobOptions = { priority: 0, timeout: 3000, attempts: 3};
1184 |
1185 | queue.addWorker(jobName, () => {}, {
1186 | concurrency: 3
1187 | });
1188 | queue.addWorker('a-different-job', () => {}, {
1189 | concurrency: 1
1190 | });
1191 |
1192 | // Create a couple jobs
1193 | queue.createJob(jobName, { random: 'this is 1st random data' }, jobOptions, false);
1194 | queue.createJob('a-different-job', { dummy: '1 data' }, { priority: 3 }, false);
1195 | queue.createJob(jobName, { random: 'this is 2nd random data' }, { priority: 4 }, false);
1196 | queue.createJob('a-different-job', { dummy: '2 data' }, { priority: 5 }, false);
1197 | queue.createJob('a-different-job', { dummy: '3 data' }, { priority: 3 }, false);
1198 | queue.createJob(jobName, { random: 'this is 3rd random data' }, jobOptions, false);
1199 | queue.createJob(jobName, { random: 'this is 4th random data' }, jobOptions, false);
1200 |
1201 | // Jobs returned by getConcurrentJobs() are marked "active" so they won't be returned by future getConcurrentJobs() calls.
1202 | const concurrentJobs = await queue.getConcurrentJobs();
1203 |
1204 | // Get all the jobs in the DB and check that the "concurrentJobs" are marked "active."
1205 | const jobs = await queue.getJobs(true);
1206 | jobs.length.should.equal(7);
1207 |
1208 | const activeJobs = jobs.filter( job => job.active);
1209 | activeJobs.length.should.equal(1);
1210 | JSON.parse(concurrentJobs[0].payload).should.deepEqual({ dummy: '2 data' });
1211 |
1212 | // Next call to getConcurrentJobs() should get the next jobs of the top of the queue as expected
1213 | // Next job in line should be type of job, then grab all the concurrents of that type and mark them active.
1214 | const moreConcurrentJobs = await queue.getConcurrentJobs();
1215 | moreConcurrentJobs.length.should.equal(3);
1216 | JSON.parse(moreConcurrentJobs[0].payload).should.deepEqual({ random: 'this is 2nd random data' });
1217 | JSON.parse(moreConcurrentJobs[1].payload).should.deepEqual({ random: 'this is 1st random data' });
1218 | JSON.parse(moreConcurrentJobs[2].payload).should.deepEqual({ random: 'this is 3rd random data' });
1219 |
1220 | // Now we should have 4 active jobs...
1221 | const allJobsAgain = await queue.getJobs(true);
1222 | const nextActiveJobs = allJobsAgain.filter( job => job.active);
1223 | nextActiveJobs.length.should.equal(4);
1224 |
1225 | // Next call to getConcurrentJobs() should work as expected
1226 | const thirdConcurrentJobs = await queue.getConcurrentJobs();
1227 | thirdConcurrentJobs.length.should.equal(1);
1228 | JSON.parse(thirdConcurrentJobs[0].payload).should.deepEqual({ dummy: '1 data' });
1229 |
1230 | // Next call to getConcurrentJobs() should work as expected
1231 | const fourthConcurrentJobs = await queue.getConcurrentJobs();
1232 | fourthConcurrentJobs.length.should.equal(1);
1233 | JSON.parse(fourthConcurrentJobs[0].payload).should.deepEqual({ dummy: '3 data' });
1234 |
1235 | // Next call to getConcurrentJobs() should be the last of the non-active jobs.
1236 | const fifthConcurrentJobs = await queue.getConcurrentJobs();
1237 | fifthConcurrentJobs.length.should.equal(1);
1238 | JSON.parse(fifthConcurrentJobs[0].payload).should.deepEqual({ random: 'this is 4th random data' });
1239 |
1240 | // Next call to getConcurrentJobs() should return an empty array.
1241 | const sixthConcurrentJobs = await queue.getConcurrentJobs();
1242 | sixthConcurrentJobs.length.should.equal(0);
1243 |
1244 | });
1245 |
1246 | it('#processJob() executes job worker then deletes job on success', async () => {
1247 |
1248 | const queue = await QueueFactory();
1249 | const jobName = 'job-name';
1250 | const jobOptions = { priority: 0, timeout: 3000, attempts: 3};
1251 |
1252 | let counter = 0; // Incrementing this will be our job "work"
1253 |
1254 | queue.addWorker(jobName, () => {}, {
1255 | concurrency: 3
1256 | });
1257 | queue.addWorker('a-different-job', () => {
1258 | counter++;
1259 | }, {
1260 | concurrency: 1
1261 | });
1262 |
1263 | // Create a couple jobs
1264 | queue.createJob(jobName, { random: 'this is 1st random data' }, jobOptions, false);
1265 | queue.createJob('a-different-job', { dummy: '1 data' }, { priority: 3 }, false);
1266 | queue.createJob(jobName, { random: 'this is 2nd random data' }, jobOptions, false);
1267 | queue.createJob('a-different-job', { dummy: '2 data' }, { priority: 5 }, false);
1268 | queue.createJob('a-different-job', { dummy: '3 data' }, { priority: 3 }, false);
1269 | queue.createJob(jobName, { random: 'this is 3rd random data' }, jobOptions, false);
1270 | queue.createJob(jobName, { random: 'this is 4th random data' }, jobOptions, false);
1271 |
1272 | // Jobs returned by getConcurrentJobs() are marked "active" so they won't be returned by future getConcurrentJobs() calls.
1273 | const concurrentJobs = await queue.getConcurrentJobs();
1274 | concurrentJobs.length.should.equal(1);
1275 | JSON.parse(concurrentJobs[0].payload).should.deepEqual({ dummy: '2 data' });
1276 |
1277 | // Process the job
1278 | await queue.processJob(concurrentJobs[0]);
1279 |
1280 | // Ensure job work was performed.
1281 | counter.should.equal(1);
1282 |
1283 | // Ensure completed job has been removed.
1284 | const jobs = await queue.getJobs(true);
1285 | jobs.length.should.equal(6);
1286 |
1287 | const jobExists = jobs.reduce((exists, job) => {
1288 | const payload = JSON.parse(job.payload);
1289 | if (payload.dummy && payload.dummy == '2 data') {
1290 | exists = true;
1291 | }
1292 | return exists;
1293 | }, false);
1294 |
1295 | jobExists.should.be.False();
1296 |
1297 | });
1298 |
1299 | it('#processJob() increments failedAttempts counter until max attempts then fails on job failure.', async () => {
1300 |
1301 | const queue = await QueueFactory();
1302 | queue.flushQueue();
1303 | const jobName = 'job-name';
1304 | const jobOptions = { priority: 0, timeout: 3000, attempts: 3};
1305 |
1306 | let counter = 0; // Incrementing this will be our job "work"
1307 |
1308 | queue.addWorker(jobName, () => {}, {
1309 | concurrency: 3
1310 | });
1311 | queue.addWorker('a-different-job', (id, payload) => {
1312 |
1313 | if (payload.dummy && payload.dummy == '2 data') {
1314 | throw new Error('Fake job failure!');
1315 | }
1316 |
1317 | counter++;
1318 |
1319 | }, {
1320 | concurrency: 2
1321 | });
1322 |
1323 | // Create a couple jobs
1324 | queue.createJob(jobName, { random: 'this is 1st random data' }, jobOptions, false);
1325 | queue.createJob('a-different-job', { dummy: '1 data' }, { priority: 3 }, false);
1326 | queue.createJob(jobName, { random: 'this is 2nd random data' }, { priority: 1, timeout: 3000, attempts: 3}, false);
1327 | queue.createJob('a-different-job', { dummy: '2 data' }, { priority: 5, attempts: 3 }, false);
1328 | queue.createJob('a-different-job', { dummy: '3 data' }, { priority: 3 }, false);
1329 | queue.createJob(jobName, { random: 'this is 3rd random data' }, jobOptions, false);
1330 | queue.createJob(jobName, { random: 'this is 4th random data' }, jobOptions, false);
1331 |
1332 | // Jobs returned by getConcurrentJobs() are marked "active" so they won't be returned by future getConcurrentJobs() calls.
1333 | const concurrentJobs = await queue.getConcurrentJobs();
1334 | concurrentJobs.length.should.equal(2);
1335 | JSON.parse(concurrentJobs[0].payload).should.deepEqual({ dummy: '2 data' });
1336 | JSON.parse(concurrentJobs[1].payload).should.deepEqual({ dummy: '1 data' });
1337 |
1338 | // Process the jobs
1339 | await Promise.all([queue.processJob(concurrentJobs[0]), queue.processJob(concurrentJobs[1])]);
1340 |
1341 | // Ensure job work was performed by ONE job (first should have failed).
1342 | counter.should.equal(1);
1343 |
1344 | // Ensure other job was deleted on job completion and ensure failedAttempts incremented on failed job.
1345 | const jobs = await queue.getJobs(true);
1346 | jobs.length.should.equal(6);
1347 | let failedJob = jobs.find((job) => {
1348 | const payload = JSON.parse(job.payload);
1349 | return (payload.dummy && payload.dummy == '2 data');
1350 | });
1351 | let failedJobData = JSON.parse(failedJob.data);
1352 | failedJobData.failedAttempts.should.equal(1);
1353 |
1354 |
1355 | // Next getConcurrentJobs() batch should get 2 jobs again, the failed job and remaining job of this job type.
1356 | const secondConcurrentJobs = await queue.getConcurrentJobs();
1357 | secondConcurrentJobs.length.should.equal(2);
1358 | JSON.parse(secondConcurrentJobs[0].payload).should.deepEqual({ dummy: '2 data' });
1359 | JSON.parse(secondConcurrentJobs[1].payload).should.deepEqual({ dummy: '3 data' });
1360 |
1361 | // Process the jobs
1362 | await Promise.all([queue.processJob(secondConcurrentJobs[0]), queue.processJob(secondConcurrentJobs[1])]);
1363 |
1364 | // Ensure more job work was performed by ONE job (first should have failed).
1365 | counter.should.equal(2);
1366 |
1367 | // Ensure other job was deleted on job completion and ensure failedAttempts incremented again on failed job.
1368 | const secondJobs = await queue.getJobs(true);
1369 | secondJobs.length.should.equal(5);
1370 | failedJob = secondJobs.find((job) => {
1371 | const payload = JSON.parse(job.payload);
1372 | return (payload.dummy && payload.dummy == '2 data');
1373 | });
1374 | failedJobData = JSON.parse(failedJob.data);
1375 | failedJobData.failedAttempts.should.equal(2);
1376 |
1377 | // Next getConcurrentJobs() batch should should get the one remaining job of this type that can fail one more time.
1378 | const thirdConcurrentJobs = await queue.getConcurrentJobs();
1379 | thirdConcurrentJobs.length.should.equal(1);
1380 | JSON.parse(thirdConcurrentJobs[0].payload).should.deepEqual({ dummy: '2 data' });
1381 |
1382 | // Process the jobs
1383 | await queue.processJob(thirdConcurrentJobs[0]);
1384 |
1385 | // Ensure new job work didn't happen because this job failed a 3rd time.
1386 | counter.should.equal(2);
1387 |
1388 | // Ensure other job was deleted on job completion and ensure failedAttempts incremented again on failed job.
1389 | const thirdJobs = await queue.getJobs(true);
1390 | thirdJobs.length.should.equal(5); // Failed job still exists, it is just marked as failure.
1391 | failedJob = thirdJobs.find((job) => {
1392 | const payload = JSON.parse(job.payload);
1393 | return (payload.dummy && payload.dummy == '2 data');
1394 | });
1395 | failedJobData = JSON.parse(failedJob.data);
1396 | failedJobData.failedAttempts.should.equal(3);
1397 |
1398 | // Ensure job marked as failed.
1399 | failedJob.failed.should.be.a.Date();
1400 |
1401 | // Next getConcurrentJobs() should now finally return 'job-name' type jobs.
1402 | const fourthConcurrentJobs = await queue.getConcurrentJobs();
1403 | fourthConcurrentJobs.length.should.equal(3);
1404 |
1405 | });
1406 |
1407 | it('#processJob() logs errors on job failure', async () => {
1408 |
1409 | const queue = await QueueFactory();
1410 | const jobName = 'job-name';
1411 | const jobOptions = { priority: 0, timeout: 5000, attempts: 3};
1412 |
1413 | let counter = 0; // Incrementing this will be our job "work"
1414 |
1415 | queue.addWorker(jobName, () => {
1416 |
1417 | counter++;
1418 |
1419 | throw new Error('Example Error number: ' + counter);
1420 |
1421 | }, {});
1422 |
1423 | queue.createJob(jobName, {}, jobOptions, false);
1424 |
1425 | const jobs = await queue.getConcurrentJobs();
1426 |
1427 | await queue.processJob(jobs[0]);
1428 |
1429 | const logCheckOneJob = await queue.getJobs(true);
1430 |
1431 | logCheckOneJob[0].data.should.equal(JSON.stringify({
1432 | attempts: 3,
1433 | failedAttempts: 1,
1434 | errors: ['Example Error number: 1']
1435 | }));
1436 |
1437 | await queue.processJob(jobs[0]);
1438 |
1439 | const logCheckTwoJob = await queue.getJobs(true);
1440 |
1441 | logCheckTwoJob[0].data.should.equal(JSON.stringify({
1442 | attempts: 3,
1443 | failedAttempts: 2,
1444 | errors: ['Example Error number: 1', 'Example Error number: 2']
1445 | }));
1446 |
1447 | await queue.processJob(jobs[0]);
1448 |
1449 | const logCheckThreeJob = await queue.getJobs(true);
1450 |
1451 | logCheckThreeJob[0].data.should.equal(JSON.stringify({
1452 | attempts: 3,
1453 | failedAttempts: 3,
1454 | errors: ['Example Error number: 1', 'Example Error number: 2', 'Example Error number: 3']
1455 | }));
1456 |
1457 | const noAvailableJobCheck = await queue.getConcurrentJobs();
1458 |
1459 | noAvailableJobCheck.length.should.equal(0);
1460 |
1461 | });
1462 |
1463 | it('#processJob() handles a job timeout as expected', async () => {
1464 |
1465 | const queue = await QueueFactory();
1466 | const jobName = 'job-name';
1467 | const jobOptions = { priority: 0, timeout: 500, attempts: 1};
1468 |
1469 | queue.addWorker(jobName, async () => {
1470 |
1471 | await new Promise((resolve) => {
1472 | setTimeout(() => {
1473 | resolve(true);
1474 | }, 2000);
1475 | });
1476 |
1477 | });
1478 |
1479 | queue.createJob(jobName, {}, jobOptions, false);
1480 |
1481 | const jobs = await queue.getConcurrentJobs();
1482 |
1483 | const jobId = jobs[0].id;
1484 |
1485 | await queue.processJob(jobs[0]);
1486 |
1487 | const logCheckOneJob = await queue.getJobs(true);
1488 |
1489 | logCheckOneJob[0].data.should.equal(JSON.stringify({
1490 | attempts: 1,
1491 | failedAttempts: 1,
1492 | errors: ['TIMEOUT: Job id: '+ jobId +' timed out in 500ms.']
1493 | }));
1494 |
1495 | const noAvailableJobCheck = await queue.getConcurrentJobs();
1496 |
1497 | noAvailableJobCheck.length.should.equal(0);
1498 |
1499 | });
1500 |
1501 | it('#flushQueue(name) should delete all jobs in the queue of type "name".', async () => {
1502 |
1503 | const queue = await QueueFactory();
1504 | const jobName = 'job-name';
1505 | const jobOptions = { priority: 0, timeout: 3000, attempts: 3};
1506 |
1507 | queue.addWorker(jobName, () => {}, {
1508 | concurrency: 3
1509 | });
1510 | queue.addWorker('a-different-job', () => {}, {
1511 | concurrency: 1
1512 | });
1513 |
1514 | // Create a couple jobs
1515 | queue.createJob(jobName, { random: 'this is 1st random data' }, jobOptions, false);
1516 | queue.createJob('a-different-job', { dummy: '1 data' }, { priority: 3 }, false);
1517 | queue.createJob(jobName, { random: 'this is 2nd random data' }, { priority: 4 }, false);
1518 | queue.createJob('a-different-job', { dummy: '2 data' }, { priority: 5 }, false);
1519 | queue.createJob('a-different-job', { dummy: '3 data' }, { priority: 3 }, false);
1520 | queue.createJob(jobName, { random: 'this is 3rd random data' }, jobOptions, false);
1521 | queue.createJob(jobName, { random: 'this is 4th random data' }, jobOptions, false);
1522 |
1523 | // Check all jobs created
1524 | const jobs = await queue.getJobs(true);
1525 | jobs.length.should.equal(7);
1526 |
1527 | queue.flushQueue(jobName);
1528 |
1529 | // Remaining 3 jobs should be of type 'a-different-job'
1530 | const remainingJobs = await queue.getJobs(true);
1531 | remainingJobs.length.should.equal(3);
1532 |
1533 | const jobNameTypeExist = remainingJobs.reduce((exists, job) => {
1534 | if (job.name == jobName) {
1535 | exists = true;
1536 | }
1537 | return exists;
1538 | }, false);
1539 |
1540 | jobNameTypeExist.should.be.False();
1541 |
1542 | });
1543 |
1544 | it('#flushQueue() should delete all jobs in the queue.', async () => {
1545 |
1546 | const queue = await QueueFactory();
1547 | const jobName = 'job-name';
1548 | const jobOptions = { priority: 0, timeout: 3000, attempts: 3};
1549 |
1550 | queue.addWorker(jobName, () => {}, {
1551 | concurrency: 3
1552 | });
1553 | queue.addWorker('a-different-job', () => {}, {
1554 | concurrency: 1
1555 | });
1556 |
1557 | // Create a couple jobs
1558 | queue.createJob(jobName, { random: 'this is 1st random data' }, jobOptions, false);
1559 | queue.createJob('a-different-job', { dummy: '1 data' }, { priority: 3 }, false);
1560 | queue.createJob(jobName, { random: 'this is 2nd random data' }, { priority: 4 }, false);
1561 | queue.createJob('a-different-job', { dummy: '2 data' }, { priority: 5 }, false);
1562 | queue.createJob('a-different-job', { dummy: '3 data' }, { priority: 3 }, false);
1563 | queue.createJob(jobName, { random: 'this is 3rd random data' }, jobOptions, false);
1564 | queue.createJob(jobName, { random: 'this is 4th random data' }, jobOptions, false);
1565 |
1566 | // Check all jobs created
1567 | const jobs = await queue.getJobs(true);
1568 | jobs.length.should.equal(7);
1569 |
1570 | queue.flushQueue();
1571 |
1572 | // All jobs should be deleted.
1573 | const remainingJobs = await queue.getJobs(true);
1574 | remainingJobs.length.should.equal(0);
1575 |
1576 | });
1577 |
1578 | it('#flushQueue(name) does not bother with delete query if no jobs exist already.', async () => {
1579 |
1580 | const queue = await QueueFactory();
1581 |
1582 | // Mock queue.realm.delete() so we can test that it has not been called.
1583 | let hasDeleteBeenCalled = false;
1584 | queue.realm.delete = () => {
1585 | hasDeleteBeenCalled = true; // Switch flag if function gets called.
1586 | };
1587 |
1588 | queue.flushQueue('no-jobs-exist-for-this-job-name');
1589 |
1590 | hasDeleteBeenCalled.should.be.False();
1591 |
1592 | });
1593 |
1594 | ////
1595 | //// JOB LIFECYCLE CALLBACK TESTING
1596 | ////
1597 |
1598 | it('onStart lifecycle callback fires before job begins processing.', async () => {
1599 |
1600 | // This test will intermittently fail in CI environments like travis-ci.
1601 | // Intermittent failure is a result of the poor performance of CI environments
1602 | // causing the timeouts in this test to become really flakey (setTimeout can't
1603 | // guarantee exact time of function execution, and in a high load env execution can
1604 | // be significantly delayed.
1605 | if (process.env.COVERALLS_ENV == 'production') {
1606 | return true;
1607 | }
1608 |
1609 | const queue = await QueueFactory();
1610 | queue.flushQueue();
1611 | const jobName = 'job-name';
1612 | let jobProcessed = false;
1613 | let testFailed = false;
1614 |
1615 | queue.addWorker(jobName, async () => {
1616 |
1617 | // Timeout needed because onStart runs async so we need to ensure this function gets
1618 | // executed last.
1619 | await new Promise((resolve) => {
1620 | setTimeout(() => {
1621 | jobProcessed = true;
1622 | resolve();
1623 | }, 0);
1624 | });
1625 |
1626 | }, {
1627 | onStart: () => {
1628 |
1629 | // If onStart runs after job has processed, fail test.
1630 | if (jobProcessed) {
1631 | testFailed = true;
1632 | throw new Error('ERROR: onStart fired after job began processing.');
1633 | }
1634 |
1635 | }
1636 | });
1637 |
1638 | // Create a job
1639 | queue.createJob(jobName, { random: 'this is 1st random data' }, {}, false);
1640 |
1641 | jobProcessed.should.equal(false);
1642 | testFailed.should.equal(false);
1643 | await queue.start();
1644 | jobProcessed.should.equal(true);
1645 | testFailed.should.equal(false);
1646 |
1647 | });
1648 |
1649 | it('onSuccess, onComplete lifecycle callbacks fire after job begins processing.', async () => {
1650 |
1651 | // This test will intermittently fail in CI environments like travis-ci.
1652 | // Intermittent failure is a result of the poor performance of CI environments
1653 | // causing the timeouts in this test to become really flakey (setTimeout can't
1654 | // guarantee exact time of function execution, and in a high load env execution can
1655 | // be significantly delayed.
1656 | if (process.env.COVERALLS_ENV == 'production') {
1657 | return true;
1658 | }
1659 |
1660 | const queue = await QueueFactory();
1661 | queue.flushQueue();
1662 | const jobName = 'job-name';
1663 | let jobProcessed = false;
1664 | let testFailed = false;
1665 | let onSuccessFired = false;
1666 | let onCompleteFired = false;
1667 |
1668 | queue.addWorker(jobName, async () => {
1669 |
1670 | // Simulate work
1671 | await new Promise((resolve) => {
1672 | setTimeout(() => {
1673 | jobProcessed = true;
1674 | resolve();
1675 | }, 300);
1676 | });
1677 |
1678 | }, {
1679 | onSuccess: () => {
1680 |
1681 | onSuccessFired = true;
1682 |
1683 | // If onSuccess runs before job has processed, fail test.
1684 | if (!jobProcessed) {
1685 | testFailed = true;
1686 | throw new Error('ERROR: onSuccess fired before job began processing.');
1687 | }
1688 |
1689 | },
1690 | onComplete: () => {
1691 |
1692 | onCompleteFired = true;
1693 |
1694 | // If onComplete runs before job has processed, fail test.
1695 | if (!jobProcessed) {
1696 | testFailed = true;
1697 | throw new Error('ERROR: onComplete fired before job began processing.');
1698 | }
1699 |
1700 | }
1701 | });
1702 |
1703 | // Create a job
1704 | queue.createJob(jobName, { random: 'this is 1st random data' }, {}, false);
1705 |
1706 | jobProcessed.should.equal(false);
1707 | testFailed.should.equal(false);
1708 | onSuccessFired.should.equal(false);
1709 | onCompleteFired.should.equal(false);
1710 | await queue.start();
1711 | jobProcessed.should.equal(true);
1712 | testFailed.should.equal(false);
1713 | onSuccessFired.should.equal(true);
1714 | onCompleteFired.should.equal(true);
1715 |
1716 | });
1717 |
1718 | it('onFailure, onFailed lifecycle callbacks fire after job begins processing.', async () => {
1719 |
1720 | // This test will intermittently fail in CI environments like travis-ci.
1721 | // Intermittent failure is a result of the poor performance of CI environments
1722 | // causing the timeouts in this test to become really flakey (setTimeout can't
1723 | // guarantee exact time of function execution, and in a high load env execution can
1724 | // be significantly delayed.
1725 | if (process.env.COVERALLS_ENV == 'production') {
1726 | return true;
1727 | }
1728 |
1729 | const queue = await QueueFactory();
1730 | queue.flushQueue();
1731 | const jobName = 'job-name';
1732 | let jobProcessStarted = false;
1733 | let testFailed = false;
1734 |
1735 | queue.addWorker(jobName, async () => {
1736 |
1737 | // Simulate work
1738 | await new Promise((resolve, reject) => {
1739 | setTimeout(() => {
1740 | jobProcessStarted = true;
1741 | reject(new Error('Job failed.'));
1742 | }, 300);
1743 | });
1744 |
1745 | }, {
1746 | onFailure: () => {
1747 |
1748 | // If onFailure runs before job has processed, fail test.
1749 | if (!jobProcessStarted) {
1750 | testFailed = true;
1751 | throw new Error('ERROR: onFailure fired before job began processing.');
1752 | }
1753 |
1754 | },
1755 | onFailed: () => {
1756 |
1757 | // If onFailed runs before job has processed, fail test.
1758 | if (!jobProcessStarted) {
1759 | testFailed = true;
1760 | throw new Error('ERROR: onFailed fired before job began processing.');
1761 | }
1762 |
1763 | }
1764 | });
1765 |
1766 | // Create a job
1767 | queue.createJob(jobName, { random: 'this is 1st random data' }, {}, false);
1768 |
1769 | jobProcessStarted.should.equal(false);
1770 | testFailed.should.equal(false);
1771 | await queue.start();
1772 | jobProcessStarted.should.equal(true);
1773 | testFailed.should.equal(false);
1774 |
1775 | });
1776 |
1777 | it('onFailure, onFailed lifecycle callbacks work as expected.', async () => {
1778 |
1779 | // This test will intermittently fail in CI environments like travis-ci.
1780 | // Intermittent failure is a result of the poor performance of CI environments
1781 | // causing the timeouts in this test to become really flakey (setTimeout can't
1782 | // guarantee exact time of function execution, and in a high load env execution can
1783 | // be significantly delayed.
1784 | if (process.env.COVERALLS_ENV == 'production') {
1785 | return true;
1786 | }
1787 |
1788 | const queue = await QueueFactory();
1789 | queue.flushQueue();
1790 | const jobName = 'job-name';
1791 | let jobAttemptCounter = 0;
1792 | let onFailureFiredCounter = 0;
1793 | let onFailedFiredCounter = 0;
1794 |
1795 | queue.addWorker(jobName, async () => {
1796 |
1797 | // Simulate work
1798 | await new Promise((resolve, reject) => {
1799 | setTimeout(() => {
1800 | jobAttemptCounter++;
1801 | reject(new Error('Job failed.'));
1802 | }, 0);
1803 | });
1804 |
1805 | }, {
1806 |
1807 | onFailure: () => {
1808 |
1809 | onFailureFiredCounter++;
1810 |
1811 | },
1812 | onFailed: () => {
1813 |
1814 | onFailedFiredCounter++;
1815 |
1816 | }
1817 | });
1818 |
1819 | const attempts = 3;
1820 |
1821 | // Create a job
1822 | queue.createJob(jobName, { random: 'this is 1st random data' }, {
1823 | attempts
1824 | }, false);
1825 |
1826 | jobAttemptCounter.should.equal(0);
1827 | await queue.start();
1828 | onFailureFiredCounter.should.equal(attempts);
1829 | onFailedFiredCounter.should.equal(1);
1830 | jobAttemptCounter.should.equal(attempts);
1831 |
1832 | });
1833 |
1834 | it('onComplete fires only once on job with multiple attempts that ends in success.', async () => {
1835 |
1836 | // This test will intermittently fail in CI environments like travis-ci.
1837 | // Intermittent failure is a result of the poor performance of CI environments
1838 | // causing the timeouts in this test to become really flakey (setTimeout can't
1839 | // guarantee exact time of function execution, and in a high load env execution can
1840 | // be significantly delayed.
1841 | if (process.env.COVERALLS_ENV == 'production') {
1842 | return true;
1843 | }
1844 |
1845 | const queue = await QueueFactory();
1846 | queue.flushQueue();
1847 | const jobName = 'job-name';
1848 | let jobAttemptCounter = 0;
1849 | let onFailureFiredCounter = 0;
1850 | let onFailedFiredCounter = 0;
1851 | let onCompleteFiredCounter = 0;
1852 | const attempts = 3;
1853 |
1854 | queue.addWorker(jobName, async () => {
1855 |
1856 | jobAttemptCounter++;
1857 |
1858 | // Keep failing attempts until last attempt then success.
1859 | if (jobAttemptCounter < attempts) {
1860 |
1861 | // Simulate work that fails
1862 | await new Promise((resolve, reject) => {
1863 | setTimeout(() => {
1864 | reject(new Error('Job failed.'));
1865 | }, 0);
1866 | });
1867 |
1868 | } else {
1869 |
1870 | // Simulate work that succeeds
1871 | await new Promise((resolve) => {
1872 | setTimeout(() => {
1873 | resolve();
1874 | }, 0);
1875 | });
1876 |
1877 | }
1878 |
1879 | }, {
1880 |
1881 | onFailure: () => {
1882 |
1883 | onFailureFiredCounter++;
1884 |
1885 | },
1886 | onFailed: () => {
1887 |
1888 | onFailedFiredCounter++;
1889 |
1890 | },
1891 | onComplete: () => {
1892 |
1893 | onCompleteFiredCounter++;
1894 |
1895 | }
1896 | });
1897 |
1898 | // Create a job
1899 | queue.createJob(jobName, { random: 'this is 1st random data succes' }, {
1900 | attempts
1901 | }, false);
1902 |
1903 | jobAttemptCounter.should.equal(0);
1904 | await queue.start();
1905 | onFailureFiredCounter.should.equal(attempts - 1);
1906 | onFailedFiredCounter.should.equal(0);
1907 | jobAttemptCounter.should.equal(attempts);
1908 | onCompleteFiredCounter.should.equal(1);
1909 |
1910 | });
1911 |
1912 | it('onComplete fires only once on job with multiple attempts that ends in failure.', async () => {
1913 |
1914 | // This test will intermittently fail in CI environments like travis-ci.
1915 | // Intermittent failure is a result of the poor performance of CI environments
1916 | // causing the timeouts in this test to become really flakey (setTimeout can't
1917 | // guarantee exact time of function execution, and in a high load env execution can
1918 | // be significantly delayed.
1919 | if (process.env.COVERALLS_ENV == 'production') {
1920 | return true;
1921 | }
1922 |
1923 | const queue = await QueueFactory();
1924 | queue.flushQueue();
1925 | const jobName = 'job-name';
1926 | let jobAttemptCounter = 0;
1927 | let onFailureFiredCounter = 0;
1928 | let onFailedFiredCounter = 0;
1929 | let onCompleteFiredCounter = 0;
1930 | const attempts = 3;
1931 |
1932 | queue.addWorker(jobName, async () => {
1933 |
1934 | jobAttemptCounter++;
1935 |
1936 | // Simulate work that fails
1937 | await new Promise((resolve, reject) => {
1938 | setTimeout(() => {
1939 | reject(new Error('Job failed.'));
1940 | }, 0);
1941 | });
1942 |
1943 | }, {
1944 |
1945 | onFailure: () => {
1946 |
1947 | onFailureFiredCounter++;
1948 |
1949 | },
1950 | onFailed: () => {
1951 |
1952 | onFailedFiredCounter++;
1953 |
1954 | },
1955 | onComplete: () => {
1956 |
1957 | onCompleteFiredCounter++;
1958 |
1959 | }
1960 | });
1961 |
1962 | // Create a job
1963 | queue.createJob(jobName, { random: 'this is 1st random data' }, {
1964 | attempts
1965 | }, false);
1966 |
1967 | jobAttemptCounter.should.equal(0);
1968 | await queue.start();
1969 | onFailureFiredCounter.should.equal(attempts);
1970 | onFailedFiredCounter.should.equal(1);
1971 | jobAttemptCounter.should.equal(attempts);
1972 | onCompleteFiredCounter.should.equal(1);
1973 |
1974 | });
1975 |
1976 | it('onStart, onSuccess, onComplete Job lifecycle callbacks do not block job processing.', async () => {
1977 |
1978 | // This test will intermittently fail in CI environments like travis-ci.
1979 | // Intermittent failure is a result of the poor performance of CI environments
1980 | // causing the timeouts in this test to become really flakey (setTimeout can't
1981 | // guarantee exact time of function execution, and in a high load env execution can
1982 | // be significantly delayed.
1983 | if (process.env.COVERALLS_ENV == 'production') {
1984 | return true;
1985 | }
1986 |
1987 | const queue = await QueueFactory();
1988 | queue.flushQueue();
1989 | const jobName = 'job-name';
1990 | let workTracker = [];
1991 | let tracker = [];
1992 |
1993 | queue.addWorker(jobName, async (id, payload) => {
1994 |
1995 | // Simulate work
1996 | await new Promise((resolve) => {
1997 | workTracker.push(payload.random);
1998 | tracker.push('job processed');
1999 | setTimeout(resolve, 0);
2000 | });
2001 |
2002 | }, {
2003 |
2004 | onStart: async () => {
2005 |
2006 | // wait a bit
2007 | await new Promise((resolve) => {
2008 | setTimeout(() => {
2009 | tracker.push('onStart completed.');
2010 | resolve();
2011 | }, 1000);
2012 | });
2013 |
2014 | },
2015 | onSuccess: async () => {
2016 |
2017 | // wait a bit
2018 | await new Promise((resolve) => {
2019 | setTimeout(() => {
2020 | tracker.push('onSuccess completed.');
2021 | resolve();
2022 | }, 1000);
2023 | });
2024 |
2025 | },
2026 | onComplete: async () => {
2027 |
2028 | // wait a bit
2029 | await new Promise((resolve) => {
2030 | setTimeout(() => {
2031 | tracker.push('onComplete completed.');
2032 | resolve();
2033 | }, 1000);
2034 | });
2035 |
2036 | }
2037 | });
2038 |
2039 | // Create a job
2040 | queue.createJob(jobName, { random: 'this is 1st random data' }, {}, false);
2041 | queue.createJob(jobName, { random: 'this is 2nd random data' }, {}, false);
2042 | queue.createJob(jobName, { random: 'this is 3rd random data' }, {}, false);
2043 | queue.createJob(jobName, { random: 'this is 4th random data' }, {}, false);
2044 | queue.createJob(jobName, { random: 'this is 5th random data' }, {}, false);
2045 |
2046 | await queue.start();
2047 |
2048 | // Ensure all jobs processed.
2049 | workTracker.should.containDeep([
2050 | 'this is 1st random data',
2051 | 'this is 2nd random data',
2052 | 'this is 4th random data',
2053 | 'this is 3rd random data',
2054 | 'this is 5th random data'
2055 | ]);
2056 |
2057 | // Since lifecycle callbacks take a second to process,
2058 | // queue should churn through all jobs well before any of the lifecycle
2059 | // callbacks complete.
2060 | const firstFive = tracker.slice(0, 5);
2061 | firstFive.should.deepEqual([
2062 | 'job processed',
2063 | 'job processed',
2064 | 'job processed',
2065 | 'job processed',
2066 | 'job processed'
2067 | ]);
2068 |
2069 | });
2070 |
2071 | it('onFailure, onFailed Job lifecycle callbacks do not block job processing.', async () => {
2072 |
2073 | // This test will intermittently fail in CI environments like travis-ci.
2074 | // Intermittent failure is a result of the poor performance of CI environments
2075 | // causing the timeouts in this test to become really flakey (setTimeout can't
2076 | // guarantee exact time of function execution, and in a high load env execution can
2077 | // be significantly delayed.
2078 | if (process.env.COVERALLS_ENV == 'production') {
2079 | return true;
2080 | }
2081 |
2082 | const queue = await QueueFactory();
2083 | queue.flushQueue();
2084 | const jobName = 'job-name';
2085 | let workTracker = [];
2086 | let tracker = [];
2087 |
2088 | queue.addWorker(jobName, async (id, payload) => {
2089 |
2090 | // Simulate failure
2091 | await new Promise((resolve, reject) => {
2092 | workTracker.push(payload.random);
2093 | setTimeout(() => {
2094 | tracker.push('job attempted');
2095 | reject(new Error('job failed'));
2096 | }, 0);
2097 | });
2098 |
2099 | }, {
2100 | onFailure: async () => {
2101 |
2102 | // wait a bit
2103 | await new Promise((resolve) => {
2104 | setTimeout(() => {
2105 | tracker.push('onFailure completed.');
2106 | resolve();
2107 | }, 1000);
2108 | });
2109 |
2110 | },
2111 | onFailed: async () => {
2112 |
2113 | // wait a bit
2114 | await new Promise((resolve) => {
2115 | setTimeout(() => {
2116 | tracker.push('onFailed completed.');
2117 | resolve();
2118 | }, 1000);
2119 | });
2120 |
2121 | }
2122 | });
2123 |
2124 | // Create a job
2125 | queue.createJob(jobName, { random: 'this is 1st random data' }, {}, false);
2126 | queue.createJob(jobName, { random: 'this is 2nd random data' }, {}, false);
2127 | queue.createJob(jobName, { random: 'this is 3rd random data' }, {}, false);
2128 | queue.createJob(jobName, { random: 'this is 4th random data' }, {}, false);
2129 | queue.createJob(jobName, { random: 'this is 5th random data' }, {}, false);
2130 |
2131 | await queue.start();
2132 |
2133 | // Ensure all jobs started to process (even though they are failed).
2134 | workTracker.should.containDeep([
2135 | 'this is 1st random data',
2136 | 'this is 2nd random data',
2137 | 'this is 4th random data',
2138 | 'this is 3rd random data',
2139 | 'this is 5th random data'
2140 | ]);
2141 |
2142 | // Since lifecycle callbacks take a second to process,
2143 | // queue should churn through all jobs well before any of the lifecycle
2144 | // callbacks complete.
2145 | const firstFive = tracker.slice(0, 5);
2146 | firstFive.should.deepEqual([
2147 | 'job attempted',
2148 | 'job attempted',
2149 | 'job attempted',
2150 | 'job attempted',
2151 | 'job attempted'
2152 | ]);
2153 |
2154 | });
2155 |
2156 | /**
2157 | *
2158 | * Regression test for issue 15: Indefinite job Timeout is broken
2159 | *
2160 | * https://github.com/billmalarky/react-native-queue/issues/15
2161 | *
2162 | */
2163 | it('does not override an explicitly set job timeout value of 0 with the default value of 25000.', async () => {
2164 |
2165 | const queue = await QueueFactory();
2166 | queue.flushQueue();
2167 | const jobName = 'job-name';
2168 |
2169 | // Attach the worker.
2170 | queue.addWorker(jobName, async () => {});
2171 |
2172 | // Create a job
2173 | queue.createJob(jobName, { random: 'this is 1st random data' }, {
2174 | timeout: 0
2175 | }, false);
2176 |
2177 | // Check that the created job has a timeout value of 0 instead of 25000.
2178 | const jobs = await queue.getJobs(true);
2179 | const job = jobs[0];
2180 | job.timeout.should.equal(0);
2181 |
2182 | // Flush jobs
2183 | queue.flushQueue();
2184 |
2185 | });
2186 |
2187 | });
--------------------------------------------------------------------------------
/tests/Worker.test.js:
--------------------------------------------------------------------------------
1 |
2 | // Define globals for eslint.
3 | /* global describe it */
4 |
5 | // Load dependencies
6 | import should from 'should'; // eslint-disable-line no-unused-vars
7 | import Worker from '../Models/Worker';
8 |
9 | describe('Models/Worker', function() {
10 |
11 | it('#addWorker() should validate input', async () => {
12 |
13 | const worker = new Worker();
14 |
15 | try {
16 | worker.addWorker(null, async () => {});
17 | throw new Error('worker.addWorker() should throw error if no jobname supplied.');
18 | } catch (error) {
19 | error.should.deepEqual(new Error('Job name and associated worker function must be supplied.'));
20 | }
21 |
22 | try {
23 | worker.addWorker('test-job-one', null);
24 | throw new Error('worker.addWorker() should throw error if no worker function supplied.');
25 | } catch (error) {
26 | error.should.deepEqual(new Error('Job name and associated worker function must be supplied.'));
27 | }
28 |
29 | });
30 |
31 | it('#addWorker() should work as expected', async () => {
32 |
33 | const worker = new Worker();
34 |
35 | worker.addWorker('test-job-one', async () => {});
36 |
37 | const workerOptions = {
38 | concurrency: 3,
39 | onStart: async () => {}
40 | };
41 | worker.addWorker('test-job-two', async () => {}, workerOptions);
42 |
43 | // first worker is added with default options.
44 | Worker.workers['test-job-one'].should.be.a.Function();
45 | Worker.workers['test-job-one'].options.should.deepEqual({
46 | concurrency: 1,
47 | onStart: null,
48 | onSuccess: null,
49 | onFailure: null,
50 | onFailed: null,
51 | onComplete: null
52 | });
53 |
54 | // second worker is added with new concurrency option.
55 | Worker.workers['test-job-two'].should.be.a.Function();
56 | Worker.workers['test-job-two'].options.should.deepEqual({
57 | concurrency: workerOptions.concurrency,
58 | onStart: workerOptions.onStart,
59 | onSuccess: null,
60 | onFailure: null,
61 | onFailed: null,
62 | onComplete: null
63 | });
64 |
65 | });
66 |
67 | it('#removeWorker() should work as expected', async () => {
68 |
69 | const worker = new Worker();
70 |
71 | // Add workers.
72 | worker.addWorker('test-job-one', async () => {});
73 |
74 | const workerOptions = {
75 | concurrency: 3
76 | };
77 | worker.addWorker('test-job-two', async () => {}, workerOptions);
78 |
79 | worker.addWorker('test-job-three', async () => {});
80 |
81 | Object.keys(Worker.workers).should.deepEqual(['test-job-one', 'test-job-two', 'test-job-three']);
82 |
83 | worker.removeWorker('test-job-two');
84 |
85 | Object.keys(Worker.workers).should.deepEqual(['test-job-one', 'test-job-three']);
86 |
87 | });
88 |
89 | it('#getConcurrency() should throw error if no worker assigned to passed in job name.', async () => {
90 |
91 | const worker = new Worker();
92 | const jobName = 'no-worker-exists';
93 |
94 | try {
95 | worker.getConcurrency(jobName);
96 | throw new Error('getConcurrency() should have thrown an error due to no worker assigned to that job name.');
97 | } catch (error) {
98 | error.should.deepEqual(new Error('Job ' + jobName + ' does not have a worker assigned to it.'));
99 | }
100 |
101 | });
102 |
103 | it('#getConcurrency() should return worker job concurrency', async () => {
104 |
105 | const worker = new Worker();
106 |
107 | // Add worker.
108 | const workerOptions = {
109 | concurrency: 36
110 | };
111 | worker.addWorker('test-job-one', async () => {}, workerOptions);
112 |
113 | worker.getConcurrency('test-job-one').should.equal(36);
114 |
115 | });
116 |
117 | it('#executeJob() should error if worker not assigned to job yet.', async () => {
118 |
119 | const worker = new Worker();
120 |
121 | const jobName = 'this-worker-does-not-exist';
122 |
123 | try {
124 | await worker.executeJob({ name : jobName });
125 | throw new Error('execute job should have thrown an error due to no worker assigned to that job name.');
126 | } catch (error) {
127 | error.should.deepEqual(new Error('Job ' + jobName + ' does not have a worker assigned to it.'));
128 | }
129 |
130 | });
131 |
132 | it('#executeJob() timeout logic should work if timeout is set.', async () => {
133 |
134 | const jobTimeout = 100;
135 |
136 | const job = {
137 | id: 'd21dca87-435c-4533-b0af-ed9844e6b827',
138 | name: 'test-job-one',
139 | payload: JSON.stringify({
140 | key: 'value'
141 | }),
142 | data: JSON.stringify({
143 | attempts: 1
144 | }),
145 | priority: 0,
146 | active: false,
147 | timeout: jobTimeout,
148 | created: new Date(),
149 | failed: null
150 | };
151 |
152 | const worker = new Worker();
153 |
154 | worker.addWorker('test-job-one', async () => {
155 | return new Promise((resolve) => {
156 | setTimeout(() => {
157 | resolve(true);
158 | }, 1000);
159 | });
160 | });
161 |
162 | try {
163 | await worker.executeJob(job);
164 | throw new Error('execute job should have thrown an error due to timeout.');
165 | } catch (error) {
166 | error.should.deepEqual(new Error('TIMEOUT: Job id: ' + job.id + ' timed out in ' + jobTimeout + 'ms.'));
167 | }
168 |
169 | });
170 |
171 | it('#executeJob() should execute a job correctly.', async () => {
172 |
173 | let counter = 0;
174 |
175 | const job = {
176 | id: 'd21dca87-435c-4533-b0af-ed9844e6b827',
177 | name: 'test-job-one',
178 | payload: JSON.stringify({
179 | key: 'value'
180 | }),
181 | data: JSON.stringify({
182 | timeout: 0,
183 | attempts: 1
184 | }),
185 | priority: 0,
186 | active: false,
187 | created: new Date(),
188 | failed: null
189 | };
190 |
191 | const worker = new Worker();
192 |
193 | worker.addWorker('test-job-one', async () => {
194 |
195 | // Job increments counter.
196 | counter++;
197 |
198 | return new Promise((resolve) => {
199 | setTimeout(() => {
200 | resolve(true);
201 | }, 500);
202 | });
203 | });
204 |
205 | counter.should.equal(0);
206 | await worker.executeJob(job);
207 | counter.should.equal(1);
208 |
209 | });
210 |
211 | it('#executeJobLifecycleCallback() should execute a job lifecycle method correctly.', async () => {
212 |
213 | let onStartCalled = false;
214 | let testPassed = false;
215 |
216 | const job = {
217 | id: 'd21dca87-435c-4533-b0af-ed9844e6b827',
218 | name: 'test-job-one',
219 | payload: JSON.stringify({
220 | key: 'value'
221 | }),
222 | data: JSON.stringify({
223 | timeout: 0,
224 | attempts: 1
225 | }),
226 | priority: 0,
227 | active: false,
228 | created: new Date(),
229 | failed: null
230 | };
231 |
232 | const worker = new Worker();
233 |
234 | worker.addWorker('test-job-one', async () => {}, {
235 | onStart: (id, payload) => {
236 |
237 | onStartCalled = true;
238 |
239 | // Verify params passed correctly off job and payload JSON has been parsed.
240 | id.should.equal(job.id);
241 | payload.should.deepEqual({
242 | key: 'value'
243 | });
244 |
245 | // Explicitly mark test as passed because the assertions
246 | // directly above will be caught in the try/catch statement within
247 | // executeJobLifecycleCallback() if they throw an error. While any thrown errors will be
248 | // output to console, the test will still pass so won't be caught by CI testing.
249 | testPassed = true;
250 |
251 | }
252 | });
253 |
254 | onStartCalled.should.equal(false);
255 | const payload = JSON.parse(job.payload); // Payload JSON is always parsed by Queue model before passing to executeJobLifecycleCallback();
256 | await worker.executeJobLifecycleCallback('onStart', job.name, job.id, payload);
257 | onStartCalled.should.equal(true);
258 | testPassed.should.equal(true);
259 |
260 | });
261 |
262 | it('#executeJobLifecycleCallback() should throw an error on invalid job lifecycle name.', async () => {
263 |
264 | let onStartCalled = false;
265 | let testPassed = true;
266 |
267 | const job = {
268 | id: 'd21dca87-435c-4533-b0af-ed9844e6b827',
269 | name: 'test-job-one',
270 | payload: JSON.stringify({
271 | key: 'value'
272 | }),
273 | data: JSON.stringify({
274 | timeout: 0,
275 | attempts: 1
276 | }),
277 | priority: 0,
278 | active: false,
279 | created: new Date(),
280 | failed: null
281 | };
282 |
283 | const worker = new Worker();
284 |
285 | worker.addWorker('test-job-one', async () => {}, {
286 | onStart: () => {
287 |
288 | testPassed = false;
289 | throw new Error('Should not be called.');
290 |
291 | }
292 | });
293 |
294 | onStartCalled.should.equal(false);
295 | const payload = JSON.parse(job.payload); // Payload JSON is always parsed by Queue model before passing to executeJobLifecycleCallback();
296 | try {
297 | await worker.executeJobLifecycleCallback('onInvalidLifecycleName', job.name, job.id, payload);
298 | } catch (error) {
299 | error.should.deepEqual(new Error('Invalid job lifecycle callback name.'));
300 | }
301 | onStartCalled.should.equal(false);
302 | testPassed.should.equal(true);
303 |
304 | });
305 |
306 | it('#executeJobLifecycleCallback() job lifecycle callbacks that error out should gracefully degrade to console error.', async () => {
307 |
308 | let onStartCalled = false;
309 | let consoleErrorCalled = false;
310 |
311 | // Cache console error.
312 | const consoleErrorCache = console.error; // eslint-disable-line no-console
313 |
314 | // Overwrite console.error to make sure it gets called on job lifecycle
315 | // callback error and is passed the error object.
316 | console.error = (errorObject) => { // eslint-disable-line no-console
317 | consoleErrorCalled = true;
318 | errorObject.should.deepEqual(new Error('Something failed catastrophically!'));
319 | };
320 |
321 | const job = {
322 | id: 'd21dca87-435c-4533-b0af-ed9844e6b827',
323 | name: 'test-job-one',
324 | payload: JSON.stringify({
325 | key: 'value'
326 | }),
327 | data: JSON.stringify({
328 | timeout: 0,
329 | attempts: 1
330 | }),
331 | priority: 0,
332 | active: false,
333 | created: new Date(),
334 | failed: null
335 | };
336 |
337 | const worker = new Worker();
338 |
339 | worker.addWorker('test-job-one', async () => {}, {
340 | onStart: () => {
341 |
342 | onStartCalled = true;
343 | throw new Error('Something failed catastrophically!');
344 |
345 | }
346 | });
347 |
348 | onStartCalled.should.equal(false);
349 | consoleErrorCalled.should.equal(false);
350 | const payload = JSON.parse(job.payload); // Payload JSON is always parsed by Queue model before passing to executeJobLifecycleCallback();
351 | await worker.executeJobLifecycleCallback('onStart', job.name, job.id, payload);
352 | onStartCalled.should.equal(true);
353 | consoleErrorCalled.should.equal(true);
354 |
355 | // Re-apply console.error.
356 | console.error = consoleErrorCache; // eslint-disable-line no-console
357 |
358 | });
359 |
360 | });
--------------------------------------------------------------------------------