├── tsconfig.json ├── stubs ├── make │ └── job │ │ └── job.stub └── config │ └── resque.stub ├── base_plugin.ts ├── queue.ts ├── define_config.ts ├── .github └── FUNDING.yml ├── types.ts ├── index.ts ├── LICENSE ├── commands ├── make_job.ts └── resque_start.ts ├── services └── main.ts ├── jobs.ts ├── providers └── resque_provider.ts ├── configure.ts ├── scheduler.ts ├── .gitignore ├── package.json ├── plugin.ts ├── base_job.ts └── README.md /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@adonisjs/tsconfig/tsconfig.package.json", 3 | "compilerOptions": { 4 | "rootDir": "./", 5 | "outDir": "./build", 6 | "resolveJsonModule": false 7 | } 8 | } -------------------------------------------------------------------------------- /stubs/make/job/job.stub: -------------------------------------------------------------------------------- 1 | 2 | {{{ 3 | exports({ to: app.makePath('app/jobs/', jobFileName) }) 4 | }}} 5 | 6 | import { BaseJob } from 'adonis-resque' 7 | 8 | export default class {{jobName}} extends BaseJob { 9 | perform() { 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /base_plugin.ts: -------------------------------------------------------------------------------- 1 | import { Plugin } from "node-resque" 2 | 3 | export default abstract class BasePlugin extends Plugin { 4 | static create(this: T, options: T['prototype']['options'] = {}): [T, T['prototype']['options']] { 5 | return [this, options] 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /queue.ts: -------------------------------------------------------------------------------- 1 | import { Queue } from "node-resque"; 2 | import { getConnection } from "./services/main.js"; 3 | import { NodeResqueJob } from "./types.js"; 4 | 5 | /** 6 | * Create a NodeResque Queue 7 | * @docs https://github.com/actionhero/node-resque?tab=readme-ov-file#queues 8 | * @param jobs 9 | * @returns Queue 10 | */ 11 | export function createQueue(jobs: Record) { 12 | return new Queue({ 13 | connection: getConnection() 14 | }, jobs); 15 | } 16 | -------------------------------------------------------------------------------- /define_config.ts: -------------------------------------------------------------------------------- 1 | import type { RedisConnections } from './types.js' 2 | import { MultiWorker, Worker } from 'node-resque' 3 | 4 | export function defineConfig(config: { 5 | redisConnection: keyof Connections 6 | runWorkerInWebEnv: boolean 7 | runScheduler: boolean 8 | isMultiWorkerEnabled: boolean 9 | multiWorkerOption: MultiWorker['options'] 10 | workerOption: Worker['options'] 11 | queueNameForJobs: string 12 | queueNameForWorkers: string 13 | logger: string | null 14 | verbose: boolean 15 | }) { 16 | return config 17 | } -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [tealight-uk, shiny] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 12 | polar: # Replace with a single Polar username 13 | buy_me_a_coffee: # Replace with a single Buy Me a Coffee username 14 | thanks_dev: # Replace with a single thanks.dev username 15 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 16 | -------------------------------------------------------------------------------- /types.ts: -------------------------------------------------------------------------------- 1 | export { Worker, Plugins, Scheduler, Queue } from "node-resque" 2 | export type { RedisConnections } from '@adonisjs/redis/types' 3 | import { Plugin } from "node-resque" 4 | import BaseJob from "./base_job.js" 5 | import { defineConfig } from "./define_config.js" 6 | export type ResqueConfig = ReturnType 7 | 8 | export interface NodeResqueJob { 9 | perform(..._args: any[]): any 10 | job: BaseJob 11 | plugins: typeof Plugin[], 12 | pluginOptions: Record 13 | args?: any[] 14 | } 15 | export interface JobSchedule { 16 | interval?: string | number 17 | cron?: string 18 | } 19 | 20 | export interface ResqueFailure { 21 | workerId?: number 22 | queue: string 23 | job: NodeResqueJob 24 | failure: Error 25 | duration: number 26 | args: any[] 27 | pluginOptions?: Record 28 | } 29 | declare module '@adonisjs/core/types' { 30 | interface EventsList { 31 | 'resque:failure': ResqueFailure 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /index.ts: -------------------------------------------------------------------------------- 1 | export const packageName = 'adonis-resque' 2 | 3 | export { configure } from './configure.js' 4 | export { defineConfig } from './define_config.js' 5 | 6 | export { Worker } from 'node-resque' 7 | export { Queue } from 'node-resque' 8 | export { default as BaseJob } from './base_job.js' 9 | export { default as BasePlugin } from './base_plugin.js' 10 | export { Plugin } from './plugin.js' 11 | export type { RetryOptions, JobLockOptions, NoopOptions, QueueLockOptions } from './plugin.js' 12 | import { joinToURL } from '@poppinss/utils' 13 | import { ResqueConfig } from './types.js' 14 | export * from './types.js' 15 | import app from '@adonisjs/core/services/app' 16 | 17 | export const stubsRoot = joinToURL(import.meta.url, 'stubs') 18 | 19 | /** 20 | * Get a resque config 21 | * @param name key name 22 | */ 23 | export function getConfig(name: key): ResqueConfig[key] { 24 | return getConfigAll()[name] 25 | } 26 | 27 | /** 28 | * Get all resque config 29 | */ 30 | export function getConfigAll() { 31 | return app.config.get('resque') 32 | } 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Dai Jie 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /commands/make_job.ts: -------------------------------------------------------------------------------- 1 | import { BaseCommand, args } from '@adonisjs/core/ace' 2 | import type { CommandOptions } from '@adonisjs/core/types/ace' 3 | import { stubsRoot } from '../index.js' 4 | 5 | import StringBuilder from '@poppinss/utils/string_builder' 6 | 7 | export default class MakeJob extends BaseCommand { 8 | static commandName = 'make:job' 9 | static description = 'Make a new job class' 10 | 11 | static options: CommandOptions = { 12 | startApp: true, 13 | } 14 | 15 | /** 16 | * The name of the job file. 17 | */ 18 | @args.string({ description: 'Name of the job file' }) 19 | declare name: string 20 | 21 | 22 | async run() { 23 | const codemods = await this.createCodemods() 24 | const jobName = new StringBuilder(this.name) 25 | .removeExtension() 26 | .removeSuffix('service') 27 | .removeSuffix('model') 28 | .singular() 29 | .pascalCase() 30 | .toString() 31 | const jobFileName = new StringBuilder(jobName).snakeCase().ext('.ts').toString() 32 | await codemods.makeUsingStub(stubsRoot, 'make/job/job.stub', { 33 | flags: this.parsed.flags, 34 | jobName, 35 | jobFileName 36 | }) 37 | } 38 | } -------------------------------------------------------------------------------- /services/main.ts: -------------------------------------------------------------------------------- 1 | import app from "@adonisjs/core/services/app" 2 | import { Jobs, MultiWorker, Worker } from "node-resque" 3 | import { RedisConnections } from '@adonisjs/redis/types' 4 | import { defineConfig } from "../define_config.js" 5 | 6 | export function getConnection() { 7 | const resqueConfig = app.config.get>('resque') 8 | const connections = app.config.get('redis.connections') 9 | const connection = connections[resqueConfig.redisConnection] as any 10 | return { 11 | host: connection.host, 12 | port: Number.parseInt(connection.port ?? '6379'), 13 | options: connection, 14 | pkg: 'ioredis', 15 | database: connection.db ?? 0, 16 | } 17 | } 18 | 19 | export function createWorker(jobs: Jobs, queues: string[]) { 20 | const workerOption = app.config.get('resque.workerOption') 21 | return new Worker({ 22 | connection: getConnection(), 23 | queues, 24 | ...workerOption 25 | }, jobs) 26 | } 27 | 28 | export function createMultiWorker(jobs: Jobs, queues: string[]) { 29 | const multiWorkerOption = app.config.get('resque.multiWorkerOption') 30 | return new MultiWorker({ 31 | queues, 32 | connection: getConnection(), 33 | ...multiWorkerOption, 34 | }, jobs) 35 | } 36 | 37 | export function isMultiWorkerEnabled() { 38 | return app.config.get('resque.isMultiWorkerEnabled') 39 | } -------------------------------------------------------------------------------- /stubs/config/resque.stub: -------------------------------------------------------------------------------- 1 | {{{ 2 | exports({ to: app.configPath('resque.ts') }) 3 | }}} 4 | import { defineConfig } from 'adonis-resque' 5 | 6 | const resqueConfig = defineConfig({ 7 | redisConnection: 'main', 8 | /** 9 | * run web & worker in same process, if enabled 10 | * You need to run command node ace resque:start if it is turned off 11 | * 12 | * it's convenient but NOT Recommanded in production 13 | * also, DO NOT enable for math-heavy jobs, even in the dev or staging environment. 14 | * 15 | */ 16 | runWorkerInWebEnv: true, 17 | /** 18 | * when runScheduler enabled, it starts with worker 19 | * if you'd like to run scheduler in the separated processes 20 | * please turn runScheduler off, and run command 21 | * node ace resque:start --scheduler 22 | */ 23 | runScheduler: true, 24 | isMultiWorkerEnabled: true, 25 | multiWorkerOption: { 26 | minTaskProcessors: 1, 27 | maxTaskProcessors: 10 28 | }, 29 | workerOption: { 30 | }, 31 | /** 32 | * the default queue name for jobs to enqueue 33 | */ 34 | queueNameForJobs: 'default', 35 | /** 36 | * queue name for workers to listen, 37 | * is a string or an array of string 38 | * setting a proper queue name could change their priorities 39 | * e.g. queueNameForWorkers: "high-priority, medium-priority, low-priority" 40 | * All the jobs in high-priority will be worked before any of the jobs in the other queues. 41 | */ 42 | queueNameForWorkers: '*', 43 | /** 44 | * logger name in config/logger.ts 45 | * null means the default 46 | */ 47 | logger: null, 48 | verbose: true, 49 | }) 50 | 51 | export default resqueConfig 52 | -------------------------------------------------------------------------------- /jobs.ts: -------------------------------------------------------------------------------- 1 | import app from "@adonisjs/core/services/app" 2 | import { fsImportAll } from "@poppinss/utils" 3 | import Job from "./base_job.js" 4 | import { NodeResqueJob } from './types.js' 5 | 6 | export async function importAllJobs() { 7 | const jobs: Record = await fsImportAll(app.makePath('app/jobs'), { 8 | ignoreMissingRoot: true 9 | }) 10 | /** 11 | * Duck typing check 12 | * @param job 13 | * @returns 14 | */ 15 | const isValidJob = (job: any): job is typeof Job => { 16 | if (!job) { 17 | return false 18 | } 19 | if (typeof job?.prototype?.perform !=='function') { 20 | return false 21 | } 22 | if (typeof job?.prototype?.enqueue !== 'function') { 23 | return false 24 | } 25 | return true 26 | } 27 | const Jobs = Object.values(jobs).filter(isValidJob) 28 | return Jobs.reduce(async (initlizedAccumulator, Job) => { 29 | let accumulator = await initlizedAccumulator 30 | const job = await app.container.make(Job) 31 | if (!Array.isArray(job.plugins)) { 32 | job.plugins = [] 33 | } 34 | const plugins = job.plugins.map(([plugin]) => plugin) 35 | const pluginOptions = job.plugins.reduce((acc, [plugin, options]) => { 36 | acc[plugin.name] = options 37 | return acc 38 | }, {} as Record) 39 | accumulator[Job.name] = { 40 | perform: async (...args: any[]) => { 41 | try { 42 | const jobResult = await job.perform.call(job, ...args) 43 | return jobResult 44 | } catch (error) { 45 | return job.handleError.call(job, error) 46 | } 47 | }, 48 | job, 49 | plugins, 50 | pluginOptions 51 | } 52 | return accumulator 53 | }, Promise.resolve>({})) 54 | } 55 | -------------------------------------------------------------------------------- /providers/resque_provider.ts: -------------------------------------------------------------------------------- 1 | import { ApplicationService } from '@adonisjs/core/types' 2 | import { Queue } from '../types.js' 3 | import { importAllJobs } from '../jobs.js' 4 | import { BaseCommand } from '@adonisjs/core/ace' 5 | import { createQueue } from '../queue.js' 6 | import { getConfig } from '../index.js' 7 | 8 | declare module '@adonisjs/core/types' { 9 | interface ContainerBindings { 10 | queue: Queue 11 | } 12 | } 13 | 14 | export default class ResqueProvider { 15 | command?: BaseCommand 16 | constructor(protected app: ApplicationService) { 17 | } 18 | 19 | async register() { 20 | this.app.container.singleton('queue', async () => { 21 | const jobs = await importAllJobs() 22 | const queue = createQueue(jobs) 23 | await queue.connect() 24 | return queue 25 | }) 26 | const emitter = await this.app.container.make('emitter') 27 | emitter.on('resque:failure', async failure => { 28 | if (!failure?.job?.job) { 29 | throw failure.failure 30 | } 31 | return failure.job.job.onFailure(failure) 32 | }) 33 | } 34 | 35 | async start() { 36 | if (this.app.getEnvironment() === 'web' && getConfig('runWorkerInWebEnv')) { 37 | const ace = await this.app.container.make('ace') 38 | await ace.boot() 39 | if (ace.getCommand('resque:start')) { 40 | this.command = await ace.exec('resque:start', []) 41 | if (this.command.exitCode !== 0 || this.command.error) { 42 | const error = this.command.error || new Error(`Failed to start command resque:start`) 43 | throw error 44 | } 45 | } 46 | } 47 | } 48 | 49 | async shutdown() { 50 | if (this.command) { 51 | await this.command.terminate() 52 | } 53 | const queue = await this.app.container.make('queue') 54 | await queue.end() 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /configure.ts: -------------------------------------------------------------------------------- 1 | import type Configure from '@adonisjs/core/commands/configure' 2 | import { packageName, stubsRoot } from './index.js' 3 | import { access } from 'fs/promises' 4 | /** 5 | * Configures the package 6 | */ 7 | export async function configure(command: Configure) { 8 | const codemods = await command.createCodemods() 9 | /** 10 | * Check if @adonisjs/redis installed, or provide some suggestion. 11 | */ 12 | if (await doesDependencyPackageMissing(command)) { 13 | const dependencyPackage = '@adonisjs/redis' 14 | const shouldInstallPackages = await command.prompt.confirm( 15 | `Do you want to install dependencies ${dependencyPackage}?`, 16 | { name: 'install', default: true } 17 | ) 18 | const packagesToInstall = [ 19 | { 20 | name: dependencyPackage, 21 | isDevDependency: false 22 | } 23 | ] 24 | if (shouldInstallPackages) { 25 | await codemods.installPackages(packagesToInstall) 26 | command.logger.warning(`Run ${command.colors.bgMagenta(command.colors.red(`node ace configure ${dependencyPackage}`))} after installing`) 27 | } else { 28 | await codemods.listPackagesToInstall(packagesToInstall) 29 | } 30 | command.logger.warning(`and configure ${command.colors.red(dependencyPackage)} correctly before using this package`) 31 | command.logger.action 32 | } 33 | await codemods.makeUsingStub(stubsRoot, 'config/resque.stub', {}) 34 | 35 | /** 36 | * Register provider 37 | */ 38 | await codemods.updateRcFile((rcFile) => { 39 | rcFile.addProvider(`${packageName}/providers/resque_provider`) 40 | rcFile.addCommand(`${packageName}/commands`) 41 | }) 42 | } 43 | 44 | async function doesDependencyPackageMissing(command: Configure) { 45 | const redisConfigFile = command.app.configPath('redis.ts') 46 | try { 47 | await access(redisConfigFile) 48 | return false 49 | } catch (err: any) { 50 | return true 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /scheduler.ts: -------------------------------------------------------------------------------- 1 | import { Scheduler } from 'node-resque' 2 | import { getConnection } from './services/main.js'; 3 | import { NodeResqueJob } from './types.js'; 4 | import Cron from 'croner' 5 | import ms from 'ms' 6 | 7 | /** 8 | * Create a NodeResque Scheduler 9 | * @docs https://github.com/actionhero/node-resque?tab=readme-ov-file#scheduler 10 | * @returns 11 | */ 12 | export function createScheduler() { 13 | return new Scheduler({ 14 | connection: getConnection(), 15 | }) 16 | } 17 | export type Interval = NodeJS.Timeout | Cron 18 | export async function startJobSchedules(resqueScheduler: Scheduler, jobs: Record): Promise { 19 | const intervals: Interval[] = [] 20 | /** 21 | * check whether is a leader sheduler or not 22 | * @returns boolean isLeader 23 | */ 24 | const isLeader = () => { 25 | return resqueScheduler.leader 26 | } 27 | 28 | /** 29 | * Create a croner if job.cron exists 30 | * @param job 31 | * @returns 32 | */ 33 | const createCronerFor = (job: NodeResqueJob['job']) => { 34 | if (job.cron) { 35 | return Cron(job.cron, async () => { 36 | if (isLeader()) { 37 | return await job.enqueue() 38 | } 39 | }) 40 | } 41 | } 42 | 43 | /** 44 | * let job repeat for every ${job.interval} 45 | * @param job 46 | * @returns 47 | */ 48 | const createRepeaterFor = (job: NodeResqueJob['job']) => { 49 | if (!job.interval) { 50 | return 51 | } 52 | let milliseconds 53 | if (typeof job.interval === 'number') { 54 | milliseconds = job.interval 55 | } else { 56 | milliseconds = ms(job.interval) 57 | } 58 | const intervalId = setInterval(async () => { 59 | if (isLeader()) { 60 | await job.enqueue() 61 | } 62 | }, milliseconds) 63 | return intervalId 64 | } 65 | for (const { job } of Object.values(jobs)) { 66 | const croner = createCronerFor(job) 67 | if (croner) { 68 | intervals.push(croner) 69 | } 70 | const intervalId = createRepeaterFor(job) 71 | if (intervalId) { 72 | intervals.push(intervalId) 73 | } 74 | } 75 | return intervals 76 | } 77 | 78 | export function cancelSchedules(intervals?: Interval[]) { 79 | if (!intervals) { 80 | return 81 | } 82 | for(const inteval of intervals) { 83 | if (inteval instanceof Cron) { 84 | inteval.stop() 85 | } else { 86 | clearInterval(inteval) 87 | } 88 | } 89 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | .pnpm-debug.log* 9 | 10 | # Diagnostic reports (https://nodejs.org/api/report.html) 11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 12 | 13 | # Runtime data 14 | pids 15 | *.pid 16 | *.seed 17 | *.pid.lock 18 | 19 | # Directory for instrumented libs generated by jscoverage/JSCover 20 | lib-cov 21 | 22 | # Coverage directory used by tools like istanbul 23 | coverage 24 | *.lcov 25 | 26 | # nyc test coverage 27 | .nyc_output 28 | 29 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 30 | .grunt 31 | 32 | # Bower dependency directory (https://bower.io/) 33 | bower_components 34 | 35 | # node-waf configuration 36 | .lock-wscript 37 | 38 | # Compiled binary addons (https://nodejs.org/api/addons.html) 39 | build/Release 40 | 41 | # Dependency directories 42 | node_modules/ 43 | jspm_packages/ 44 | 45 | # Snowpack dependency directory (https://snowpack.dev/) 46 | web_modules/ 47 | 48 | # TypeScript cache 49 | *.tsbuildinfo 50 | 51 | # Optional npm cache directory 52 | .npm 53 | 54 | # Optional eslint cache 55 | .eslintcache 56 | 57 | # Optional stylelint cache 58 | .stylelintcache 59 | 60 | # Microbundle cache 61 | .rpt2_cache/ 62 | .rts2_cache_cjs/ 63 | .rts2_cache_es/ 64 | .rts2_cache_umd/ 65 | 66 | # Optional REPL history 67 | .node_repl_history 68 | 69 | # Output of 'npm pack' 70 | *.tgz 71 | 72 | # Yarn Integrity file 73 | .yarn-integrity 74 | 75 | # dotenv environment variable files 76 | .env 77 | .env.development.local 78 | .env.test.local 79 | .env.production.local 80 | .env.local 81 | 82 | # parcel-bundler cache (https://parceljs.org/) 83 | .cache 84 | .parcel-cache 85 | 86 | # Next.js build output 87 | .next 88 | out 89 | 90 | # Nuxt.js build / generate output 91 | .nuxt 92 | dist 93 | 94 | # Gatsby files 95 | .cache/ 96 | # Comment in the public line in if your project uses Gatsby and not Next.js 97 | # https://nextjs.org/blog/next-9-1#public-directory-support 98 | # public 99 | 100 | # vuepress build output 101 | .vuepress/dist 102 | 103 | # vuepress v2.x temp and cache directory 104 | .temp 105 | .cache 106 | 107 | # Docusaurus cache and generated files 108 | .docusaurus 109 | 110 | # Serverless directories 111 | .serverless/ 112 | 113 | # FuseBox cache 114 | .fusebox/ 115 | 116 | # DynamoDB Local files 117 | .dynamodb/ 118 | 119 | # TernJS port file 120 | .tern-port 121 | 122 | # Stores VSCode versions used for testing VSCode extensions 123 | .vscode-test 124 | 125 | # yarn v2 126 | .yarn/cache 127 | .yarn/unplugged 128 | .yarn/build-state.yml 129 | .yarn/install-state.gz 130 | .pnp.* 131 | 132 | build/ -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "adonis-resque", 3 | "version": "2.0.2", 4 | "type": "module", 5 | "description": " Resque Queue for AdonisJS v6 ", 6 | "main": "build/index.js", 7 | "files": [ 8 | "build" 9 | ], 10 | "exports": { 11 | ".": "./build/index.js", 12 | "./services/main": "./build/services/main.js", 13 | "./commands": "./build/commands/main.js", 14 | "./providers/resque_provider": "./build/providers/resque_provider.js", 15 | "./types": "./build/types.js" 16 | }, 17 | "engines": { 18 | "node": ">=18.16.0" 19 | }, 20 | "scripts": { 21 | "build": "npm run compile", 22 | "dev": "tsup-node --watch --onSuccess 'tsc -d && npm run postcompile'", 23 | "typecheck": "tsc --noEmit", 24 | "copy:templates": "copyfiles \"stubs/**/**/*.stub\" build", 25 | "compile": "tsup-node && tsc --emitDeclarationOnly --declaration", 26 | "postcompile": "npm run copy:templates && npm run index:commands", 27 | "test": "node --loader ts-node/esm --enable-source-maps bin/test.ts", 28 | "index:commands": "adonis-kit index build/commands" 29 | }, 30 | "bugs": { 31 | "url": "https://github.com/shiny/adonis-resque/issues" 32 | }, 33 | "keywords": [ 34 | "adonisjs", 35 | "adonis6", 36 | "resque", 37 | "node-resque", 38 | "queue" 39 | ], 40 | "author": "Chieh", 41 | "license": "MIT", 42 | "homepage": "https://github.com/shiny/adonis-resque", 43 | "repository": { 44 | "type": "git", 45 | "url": "git+https://github.com/shiny/adonis-resque.git" 46 | }, 47 | "devDependencies": { 48 | "@adonisjs/assembler": "^7.7.0", 49 | "@adonisjs/core": "^6.12.1", 50 | "@adonisjs/eslint-config": "^1.3.0", 51 | "@adonisjs/prettier-config": "^1.3.0", 52 | "@adonisjs/redis": "8.0.1", 53 | "@adonisjs/tsconfig": "^1.3.0", 54 | "@japa/expect": "3.0.2", 55 | "@japa/expect-type": "2.0.2", 56 | "@japa/runner": "3.1.4", 57 | "@types/ms": "^0.7.34", 58 | "@types/node": "^22.5.0", 59 | "copyfiles": "^2.4.1", 60 | "eslint": "^8.57.0", 61 | "nock": "^13.5.5", 62 | "ts-node": "^10.9.2", 63 | "tsup": "^8.2.4", 64 | "typescript": "^5.5.4" 65 | }, 66 | "peerDependencies": { 67 | "@adonisjs/core": "^6.12.1", 68 | "@adonisjs/redis": "^9.1.0" 69 | }, 70 | "tsup": { 71 | "entry": [ 72 | "./index.ts", 73 | "./base_job.ts", 74 | "commands/make_job.ts", 75 | "commands/resque_start.ts", 76 | "./src/types.ts", 77 | "./services/main.ts", 78 | "./providers/resque_provider.ts" 79 | ], 80 | "outDir": "./build", 81 | "clean": true, 82 | "format": "esm", 83 | "dts": false, 84 | "sourcemap": true, 85 | "target": "esnext" 86 | }, 87 | "dependencies": { 88 | "@poppinss/utils": "^6.7.3", 89 | "croner": "^8.1.1", 90 | "ms": "^2.1.3", 91 | "node-resque": "^9.3.5" 92 | } 93 | } -------------------------------------------------------------------------------- /plugin.ts: -------------------------------------------------------------------------------- 1 | import { Plugins } from "node-resque" 2 | type LockKey = string | (() => string) 3 | export interface JobLockOptions { 4 | /** 5 | * should we re-enqueue the job if it is already locked? 6 | */ 7 | reEnqueue?: boolean 8 | /** 9 | * reEnqueue the job, delay it by milliseconds 10 | */ 11 | enqueueTimeout?: number 12 | /** 13 | * lock the job for seconds 14 | * default: 3600 (1 hour) 15 | */ 16 | lockTimeout?: number 17 | /** 18 | * the key, for identifying the lock 19 | */ 20 | key?: LockKey 21 | } 22 | export interface NoopOptions { 23 | logger?: (error: Error) => unknown 24 | } 25 | export interface QueueLockOptions { 26 | /** 27 | * in seconds 28 | */ 29 | lockTimeout?: number 30 | key?: LockKey 31 | } 32 | export interface RetryOptions { 33 | retryLimit?: number 34 | /** 35 | * in milliseconds, delay for next retry attempt. 36 | */ 37 | retryDelay?: number 38 | /** 39 | * define an array containing the delay milliseconds number for each retry attempt. 40 | * if the array is shorter than the retryLimit, 41 | * the last value will be used for all remaining attempts. 42 | */ 43 | backoffStrategy?: number[] 44 | } 45 | export class Plugin { 46 | /** 47 | * If a job with the same name, queue 48 | * and args is already running 49 | * put this job back in the queue and try later 50 | * @param options.reEnqueue 51 | * @param options.enqueueTimeout 52 | * @param options.lockTimeout 53 | * @param options.key 54 | * If true, re-enqueue the job if it is already running. If false, do not enqueue the job if it is already running. 55 | * @docs Source code: https://github.com/actionhero/node-resque/blob/main/src/plugins/JobLock.ts 56 | */ 57 | static jobLock (options: JobLockOptions = {}): [typeof Plugins.JobLock, JobLockOptions] { 58 | return [Plugins.JobLock, options] 59 | } 60 | /** 61 | * If a job with the same name, queue 62 | * and args is already in the delayed queue(s) 63 | * do not enqueue it again 64 | * @docs Source code: https://github.com/actionhero/node-resque/blob/main/src/plugins/DelayQueueLock.ts 65 | */ 66 | static delayQueueLock(): [typeof Plugins.DelayQueueLock, {}] { 67 | return [Plugins.DelayQueueLock, {}] 68 | } 69 | /** 70 | * Log the error and do not throw it 71 | * @param options.logger 72 | * @docs Source code: https://github.com/actionhero/node-resque/blob/main/src/plugins/Noop.ts 73 | * A function to log the error. 74 | */ 75 | static noop(options: NoopOptions = {}): [typeof Plugins.Noop, NoopOptions] { 76 | return [Plugins.Noop, options] 77 | } 78 | /** 79 | * If a job with the same name, queue 80 | * and args is already in the queue 81 | * do not enqueue it again 82 | * @param options.lockTimeout 83 | * @param options.key 84 | * @docs Source Code: https://github.com/actionhero/node-resque/blob/main/src/plugins/QueueLock.ts 85 | */ 86 | static queueLock(options: QueueLockOptions = {}): [typeof Plugins.QueueLock, QueueLockOptions] { 87 | return [Plugins.QueueLock, options] 88 | } 89 | /** 90 | * If a job fails, retry it N times 91 | * before finally placing it into the failed queue 92 | * 93 | * @param options.retryLimit 94 | * @param options.retryDelay 95 | * @param options.backoffStrategy 96 | * @docs Source code: https://github.com/actionhero/node-resque/blob/main/src/plugins/Retry.ts 97 | */ 98 | static retry(options: RetryOptions = {}): [typeof Plugins.Retry, RetryOptions] { 99 | return [Plugins.Retry, options] 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /base_job.ts: -------------------------------------------------------------------------------- 1 | import app from "@adonisjs/core/services/app" 2 | import { ResqueConfig, ResqueFailure } from "./types.js" 3 | import { Plugin } from "node-resque" 4 | 5 | export default class BaseJob { 6 | 7 | interval?: string | number 8 | cron?: string 9 | 10 | plugins: [typeof Plugin, any][] = [] 11 | 12 | delayMs: number = 0 13 | runAtMs?: number 14 | /** 15 | * the default JobName is this class name 16 | * it **MUST be a unique name** 17 | */ 18 | jobName?: string 19 | /** 20 | * set a queueName for this job 21 | * default configured in `config/resque.ts` 22 | */ 23 | queueName?: string 24 | args: any[] = [] 25 | allArgs: any[][] = [] 26 | hasEnqueued: boolean = false 27 | hasEnqueuedAll: boolean = false 28 | app = app 29 | 30 | constructor(..._args: any[]) { 31 | 32 | } 33 | queue(queueName: string) { 34 | this.queueName = queueName 35 | return this 36 | } 37 | static async enqueueAll(this: T, args: Parameters[]) { 38 | const job = await app.container.make(this) 39 | return job.enqueueAll(args) 40 | } 41 | async enqueueAll(this: T, args: Parameters[]) { 42 | this.allArgs = args 43 | this.hasEnqueuedAll = true 44 | return this.execute() 45 | } 46 | perform(..._args: any[]): any { 47 | 48 | } 49 | handleError(error: unknown) { 50 | throw error 51 | } 52 | onFailure(_failure: ResqueFailure): void | Promise {} 53 | private async execute() { 54 | const resqueConfig = app.config.get('resque') 55 | const jobName = this.jobName ?? this.constructor.name 56 | const queueName = this.queueName ?? resqueConfig.queueNameForJobs 57 | const queue = await app.container.make('queue') 58 | let logger = await app.container.make('logger') 59 | if (resqueConfig.logger) { 60 | logger.use(resqueConfig.logger) 61 | } 62 | if (this.hasEnqueued) { 63 | const getTips = () => { 64 | if (!resqueConfig.verbose) { 65 | return undefined 66 | } 67 | const tips = `enqueued to queue ${queueName}, job ${jobName}` 68 | if (this.delayMs) { 69 | return `${tips}, delay ${this.delayMs}ms` 70 | } else if (this.runAtMs) { 71 | return `${tips}, run at ${this.runAtMs}` 72 | } else { 73 | return tips 74 | } 75 | } 76 | const tips = getTips() 77 | if (tips) { 78 | logger.info(tips) 79 | } 80 | if (this.delayMs) { 81 | return queue.enqueueIn(this.delayMs, queueName, jobName, this.args) 82 | } else if (this.runAtMs) { 83 | return queue.enqueueAt(this.runAtMs, queueName, jobName, this.args) 84 | } else { 85 | return queue.enqueue(queueName, jobName, this.args) 86 | } 87 | } else if (this.hasEnqueuedAll) { 88 | return Promise.all(this.allArgs.map(arg => queue.enqueue(queueName, jobName, arg))) 89 | } else { 90 | return false 91 | } 92 | } 93 | 94 | 95 | private push({ args, delayMs, runAtMs }: { 96 | args: Parameters; 97 | delayMs?: number 98 | runAtMs?: number 99 | }) { 100 | this.args = args 101 | this.hasEnqueued = true 102 | this.delayMs = delayMs ? delayMs : 0 103 | this.runAtMs = runAtMs 104 | return this.execute() 105 | } 106 | 107 | static async enqueue(this: T, ...args: Parameters['perform']>) { 108 | const job = await app.container.make(this) 109 | return job.enqueue(...args) 110 | } 111 | async enqueue(this: T, ...args: Parameters) { 112 | return this.push({ 113 | args 114 | }) 115 | } 116 | 117 | /** 118 | * 119 | * @param this 120 | * @param delayMs In ms, the number of ms to delay before this job is able to start being worked on 121 | * @param args 122 | * @returns 123 | */ 124 | static async enqueueIn(this: T, delayMs: number, ...args: Parameters['perform']>) { 125 | const job = await app.container.make(this) 126 | return job.enqueueIn(delayMs, ...args) 127 | } 128 | async enqueueIn(this: T, delayMs: number, ...args: Parameters) { 129 | return this.push({ 130 | args, 131 | delayMs, 132 | }) 133 | } 134 | 135 | static async enqueueAt(this: T, runAtMs: number, ...args: Parameters['perform']>) { 136 | const job = await app.container.make(this) 137 | return job.enqueueAt(runAtMs, ...args) 138 | } 139 | async enqueueAt(this: T, runAtMs: number, ...args: Parameters) { 140 | return this.push({ 141 | args, 142 | runAtMs, 143 | }) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /commands/resque_start.ts: -------------------------------------------------------------------------------- 1 | import { BaseCommand, flags } from '@adonisjs/core/ace' 2 | import type { CommandOptions } from '@adonisjs/core/types/ace' 3 | import { createWorker, createMultiWorker, isMultiWorkerEnabled } from 'adonis-resque/services/main' 4 | import { importAllJobs } from '../jobs.js' 5 | import { cancelSchedules, createScheduler, Interval, startJobSchedules } from '../scheduler.js' 6 | import { MultiWorker, ParsedJob, Scheduler, Worker } from 'node-resque' 7 | import { getConfig } from '../index.js' 8 | 9 | export default class ResqueStart extends BaseCommand { 10 | static commandName = 'resque:start' 11 | static description = 'Start workers / schedules for resque' 12 | 13 | static options: CommandOptions = { 14 | startApp: true, 15 | staysAlive: true, 16 | } 17 | 18 | @flags.boolean({ 19 | description: 'start job schedule' 20 | }) 21 | declare schedule: boolean 22 | 23 | @flags.boolean({ 24 | description: 'start workers', 25 | default: true 26 | }) 27 | declare worker: boolean 28 | 29 | @flags.boolean({ 30 | description: 'multi workers' 31 | }) 32 | declare isMulti: boolean 33 | 34 | @flags.boolean({ 35 | description: 'enable log verbose' 36 | }) 37 | declare verbose: boolean 38 | 39 | @flags.array({ 40 | description: 'queue names for worker to listen' 41 | }) 42 | declare queueName: string[] 43 | 44 | intervals?: Interval[] 45 | workerInstance?: MultiWorker | Worker 46 | schedulerInstance?: Scheduler 47 | 48 | async run() { 49 | const pid = process.pid 50 | const jobs = await importAllJobs() 51 | const emitter = await this.app.container.make('emitter') 52 | const verbose = this.verbose?? getConfig('verbose') 53 | 54 | const inConsole = this.app.getEnvironment() === 'console' 55 | if (inConsole) { 56 | const router = await this.app.container.make('router') 57 | router.commit() 58 | } 59 | if (this.worker) { 60 | const queueNames = 61 | this.queueName ?? getConfig('queueNameForWorkers') 62 | .split(',') 63 | .map((value) => value.trim()) 64 | .filter((value) => value !== '') 65 | const isMultiWorker = this.isMulti ?? isMultiWorkerEnabled() 66 | if (isMultiWorker) { 67 | this.workerInstance = createMultiWorker(jobs, queueNames) as MultiWorker 68 | this.logger.info(`Resque multiWorker:${pid} started with ${Object.keys(jobs).length} jobs`) 69 | this.workerInstance.on('failure', (workerId: number, queue: string, job: ParsedJob, failure: Error, duration: number) => { 70 | if (verbose) 71 | this.logger.info(`Job ${job.class} in queue ${queue} failed on worker ${workerId} in ${duration}ms`) 72 | emitter.emit('resque:failure', { 73 | workerId, 74 | queue, 75 | job: jobs[job.class], 76 | failure, 77 | duration, 78 | args: job.args, 79 | pluginOptions: job.pluginOptions 80 | }) 81 | }) 82 | this.workerInstance.on('cleaning_worker', (workerId: number, _worker: Worker, pid: number) => { 83 | if (verbose) 84 | this.logger.info(`Worker ${workerId} (PID ${pid}) cleaning up...`) 85 | }) 86 | this.workerInstance.on('end', (workerId: number) => { 87 | if (verbose) 88 | this.logger.info(`Worker ${workerId} ended.`) 89 | }) 90 | this.workerInstance.on('success', (workerId: number, queue: string, job: ParsedJob, result: any, duration: number) => { 91 | if (verbose) 92 | this.logger.success(`Job ${job.class} in queue ${queue} completed on worker ${workerId} in ${duration}ms, result: ${JSON.stringify(result)}`) 93 | }) 94 | 95 | } else { 96 | this.workerInstance = createWorker(jobs, queueNames) 97 | this.workerInstance.on('failure', (queue: string, job: ParsedJob, failure: Error, duration: number) => { 98 | if (verbose) 99 | this.logger.info(`Job ${job.class} in queue ${queue} failed in ${duration}ms`) 100 | emitter.emit('resque:failure', { 101 | queue, 102 | job: jobs[job.class], 103 | failure, 104 | duration, 105 | args: job.args, 106 | pluginOptions: job.pluginOptions 107 | }) 108 | }) 109 | this.workerInstance.on('success', (queue: string, job: ParsedJob, result: any, duration: number) => { 110 | if (verbose) 111 | this.logger.success(`Job ${job.class} in queue ${queue} completed in ${duration}ms, result: ${JSON.stringify(result)}`) 112 | }) 113 | await this.workerInstance.connect() 114 | } 115 | await this.workerInstance.start() 116 | 117 | } 118 | const runScheduler = getConfig('runScheduler') 119 | if (this.schedule ?? runScheduler) { 120 | this.schedulerInstance = createScheduler() 121 | await this.schedulerInstance.connect() 122 | await this.schedulerInstance.start() 123 | this.schedulerInstance.on('end', () => { 124 | if (verbose) 125 | this.logger.info(`Scheduler ended.`) 126 | }) 127 | this.intervals = await startJobSchedules(this.schedulerInstance, jobs) 128 | if (verbose) 129 | this.logger.info(`Scheduler:${pid} started`) 130 | } 131 | } 132 | 133 | prepare() { 134 | const isVerbose = this.verbose ?? getConfig('verbose') 135 | const terminate = async (signal: NodeJS.Signals) => { 136 | if (isVerbose) 137 | this.logger.info('Receive ' + signal) 138 | await this.terminate() 139 | } 140 | const cleanup = async () => { 141 | this.logger.info('Resque worker terminating...') 142 | if (this.workerInstance) { 143 | await this.workerInstance.end() 144 | } 145 | if (this.schedulerInstance) { 146 | await this.schedulerInstance.end() 147 | } 148 | cancelSchedules(this.intervals) 149 | } 150 | this.app.listen('SIGINT', terminate).listen('SIGTERM', terminate) 151 | this.app.terminating(cleanup) 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 |
3 | 4 |

Node Resque Queue for AdonisJS v6

5 |

A third-party wrapper for `node-resque` in AdonisJS v6.

6 | 7 | 8 | 9 |
10 | 11 | > [!CAUTION] 12 | > This package is not compatible with AdonisJS v5. 13 | 14 | 15 | 16 | - [Introduction](#introduction) 17 | - [Installation](#installation) 18 | - [Folders](#folders) 19 | - [Job Usage](#job-usage) 20 | - [Basic](#basic) 21 | - [Batch enqueue](#batch-enqueue) 22 | - [Delayed enqueue](#delayed-enqueue) 23 | - [Repeated Enqueue](#repeated-enqueue) 24 | - [Handle Failure Jobs](#handle-failure-jobs) 25 | - [Job.onFailure](#jobonfailure) 26 | - [Failure Event](#failure-event) 27 | - [Dependency Injection in Job Classes](#dependency-injection-in-job-classes) 28 | - [Demonstration](#demonstration) 29 | - [Send Mail Job](#send-mail-job) 30 | - [Plugin](#plugin) 31 | - [Plugin.jobLock](#pluginjoblock) 32 | - [Plugin.queueLock](#pluginqueuelock) 33 | - [Plugin.delayQueueLock](#plugindelayqueuelock) 34 | - [Plugin.retry](#pluginretry) 35 | - [Plugin.noop](#pluginnoop) 36 | - [Custom Your Own Plugin](#custom-your-own-plugin) 37 | - [Configuration](#configuration) 38 | - [Concepts \& Components](#concepts--components) 39 | - [Queue](#queue) 40 | - [Example: Getting a total count of pending jobs](#example-getting-a-total-count-of-pending-jobs) 41 | - [Worker](#worker) 42 | - [Multi Worker](#multi-worker) 43 | - [Scheduler](#scheduler) 44 | - [Deployment](#deployment) 45 | - [Development](#development) 46 | - [Production](#production) 47 | - [Web UI](#web-ui) 48 | - [Notice 1: for the graceful exit](#notice-1-for-the-graceful-exit) 49 | - [Notice 2: Job does not `@inject`able](#notice-2-job-does-not-injectable) 50 | - [Who's Using](#whos-using) 51 | - [Contributors](#contributors) 52 | - [Reference](#reference) 53 | - [Migration from v1 to v2](#migration-from-v1-to-v2) 54 | - [Changes in ver 2](#changes-in-ver-2) 55 | - [this.logger Removed](#thislogger-removed) 56 | - [Chainable Methods Removed](#chainable-methods-removed) 57 | - [Dependency Injection Now Available in Job Files](#dependency-injection-now-available-in-job-files) 58 | - [License](#license) 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | ## Introduction 75 | `adonis-resque` add the queue ability to adonis base on node-resque. Resque is a background job system backed by Redis. 76 | 77 | ## Installation 78 | 79 | ```bash 80 | npm i adonis-resque 81 | node ace configure adonis-resque 82 | ``` 83 | > [!IMPORTANT] 84 | > `@adonisjs/redis` is required for resque redis connection. 85 | > 86 | ## Folders 87 | 88 | Jobs are placed in folder `app/jobs` by default. 89 | You can import job through sub-path. 90 | ```typescript 91 | import Example from '#jobs/example' 92 | ``` 93 | 94 | > [!TIP] 95 | > Please follow this instruction: [The sub-path imports](https://docs.adonisjs.com/guides/folder-structure#the-sub-path-imports). 96 | > 97 | > Both `package.json` and `tsconfig.json` are required to add the job path: 98 | > - add to `package.json` 99 | > ```json 100 | > "#jobs/*": "./app/jobs/*.js" 101 | > ``` 102 | > - add to field `compilerOptions.paths` in `tsconfig.json` 103 | > ```json 104 | > "#jobs/*": ["./app/jobs/*.js"] 105 | > ``` 106 | 107 | 108 | ## Job Usage 109 | You can create a resque job by adonis command: `node ace make:job ` 110 | 111 | ### Basic 112 | 113 | Create a basic example job: `node ace make:job BasicExample`. 114 | Every job has a perform method. It runs in the background, which consumer from the node-resque queue. 115 | 116 | ```typescript 117 | import { BaseJob } from 'adonis-resque' 118 | import logger from '@adonisjs/core/services/logger' 119 | export default class BasicExample extends BaseJob { 120 | async perform(name: string) { 121 | logger.info(`Hello ${name}`) 122 | } 123 | } 124 | ``` 125 | 126 | Now you can enqueue this job. 127 | ```typescript 128 | import BasicExample from '#jobs/basic_example' 129 | await BasicExample.enqueue('Bob') 130 | ``` 131 | 132 | Console would print `Hello Bob`. 133 | 134 | > [!WARNING] 135 | > Make sure your Job ClassName is unique in the queue. and the arguments of it's method `perform` would be serialize to a json string to redis under the hood. 136 | 137 | ### Batch enqueue 138 | ```typescript 139 | await BasicExample.enqueueAll([ 140 | ['Alice'], 141 | ['Bob'], 142 | ['Carol'], 143 | ['Dave'], 144 | ['Eve'] 145 | ]) 146 | ``` 147 | 148 | ### Delayed enqueue 149 | ```typescript 150 | const oneSecondLater = 1000 151 | await BasicExample.enqueueIn(oneSecondLater, 'Bob') 152 | ``` 153 | Or enqueue at a specify timestamp 154 | ```typescript 155 | const fiveSecondsLater = new Date().getTime() + 5000 156 | await BasicExample.enqueueAt(fiveSecondsLater, 'Bob') 157 | ``` 158 | 159 | ### Repeated Enqueue 160 | 161 | class Job has the schedule properties. 162 | - `interval`, .e.g `5s`, `15m`, `2h` and `1d`. [package ms](https://github.com/vercel/ms) for more details. 163 | - `cron`, for cron syntax, look up the [croner package](https://github.com/hexagon/croner) 164 | 165 | The example below enqueue in both every 1 second and 5 minutes, since it's `cron`/`interval` settings. 166 | 167 | ```typescript 168 | import logger from '@adonisjs/core/services/logger' 169 | import { BaseJob } from 'adonis-resque' 170 | export default class Repeater extends BaseJob { 171 | // enqueue job cronly 172 | cron = '*/1 * * * * *' 173 | // enqueue every five minutes 174 | interval = '5m' 175 | async perform() { 176 | logger.info(`Repeater every 5 minutes / every seconds`) 177 | } 178 | } 179 | ``` 180 | 181 | 182 | ## Handle Failure Jobs 183 | 184 | ### Job.onFailure 185 | 186 | You can handle failure jobs by defining a `onFailure` method. 187 | Once a job fails, it would be called before moving to the `failed` queue. 188 | 189 | ```typescript 190 | import { BaseJob, type ResqueFailure } from 'adonis-resque' 191 | class Job extends BaseJob { 192 | async onFailure(failure: ResqueFailure) { 193 | console.log('resque job failured:', failure) 194 | } 195 | } 196 | ``` 197 | 198 | The definition of interface ResqueFailure 199 | 200 | ```typescript 201 | export interface ResqueFailure { 202 | // Only the failure emitted by MultiWorker has a workerId 203 | workerId?: number 204 | queue: string 205 | job: NodeResqueJob 206 | failure: Error 207 | duration: number 208 | } 209 | ``` 210 | 211 | > [!TIP] 212 | > If you are using `retry` plugin, the `onFailure` method will be called only if the job has exceeded the retry limit. 213 | 214 | ### Failure Event 215 | 216 | Another way to handle failure jobs is to listen to the `resque:failure` event. 217 | 218 | go `start/events.ts` to handle. 219 | ```typescript 220 | import emitter from '@adonisjs/core/services/emitter' 221 | 222 | emitter.on('resque:failure', (failure) => { 223 | console.error('resque:failure', failure) 224 | }) 225 | ``` 226 | 227 | ## Dependency Injection in Job Classes 228 | Starting from version 2+, dependency injection is now supported in class-based jobs. Since jobs run in the background, you cannot inject the current HttpContext. 229 | 230 | Here’s an example of a job that injects the logger: 231 | 232 | ```typescript 233 | import { inject } from '@adonisjs/core' 234 | import type { Logger } from '@adonisjs/core/logger' 235 | import { BaseJob } from 'adonis-resque' 236 | 237 | @inject() 238 | export default class BasicExample extends BaseJob { 239 | constructor(public logger: Logger) { 240 | super() 241 | } 242 | 243 | perform() { 244 | this.logger.info('Basic example') 245 | return 'gogogo' 246 | } 247 | } 248 | ``` 249 | 250 | ## Demonstration 251 | ### Send Mail Job 252 | 253 | In the Adonis Documentation, they use bullmq as mail queueing example. 254 | But if we want to use `adonis-resque` for `mail.sendLater`, how to do? 255 | 256 | 1. Create a Mail Job 257 | Run `node ace make:job Mail` to create the mail job, then edit it in `app/jobs/mail.ts` 258 | 259 | ```typescript 260 | import { BaseJob } from 'adonis-resque' 261 | import logger from '@adonisjs/core/services/logger' 262 | import mail from '@adonisjs/mail/services/main' 263 | import { MessageBodyTemplates, NodeMailerMessage } from '@adonisjs/mail/types' 264 | 265 | interface Options { 266 | mailMessage: { 267 | message: NodeMailerMessage; 268 | views: MessageBodyTemplates; 269 | } 270 | config: any 271 | } 272 | export default class Mail extends BaseJob { 273 | async perform(option: Options) { 274 | const { messageId } = await mail.use('smtp') 275 | .sendCompiled(option.mailMessage, option.config) 276 | logger.info(`Email sent, id is ${messageId}`) 277 | } 278 | } 279 | ``` 280 | 281 | 2. Custom `mail.setMessenger` in a service provider 282 | You can add the below code snippet to a boot method of any service provider. 283 | 284 | ```typescript 285 | const mail = await this.app.container.make('mail.manager') 286 | mail.setMessenger(() => { 287 | return { 288 | async queue(mailMessage, sendConfig) { 289 | return Mail.enqueue({ mailMessage, config: sendConfig }) 290 | } 291 | } 292 | }) 293 | ``` 294 | 295 | 3. `mail.sendLater` is available now! Let's have a try. :shipit: 296 | ```typescript 297 | await mail.sendLater((message) => { 298 | message.to('your-address@example.com', 'Your Name') 299 | .subject('Hello from adonis-resque') 300 | .html(`Congratulations!`) 301 | }) 302 | ``` 303 | 304 | > [!CAUTION] 305 | > You should ensure `@adonisjs/mail` has a correct config, you'd better to test it first. 306 | 307 | 308 | ## Plugin 309 | 310 | Adonis-resque encapsulated the default node-resque plugins in a smooth way, with the better typing support. 311 | 312 | The default node-resque plugins are: 313 | 314 | - `Retry` 315 | - `JobLock` 316 | - `Retry` 317 | - `QueueLock` 318 | - `DelayQueueLock` 319 | - `Noop` 320 | 321 | You can adding them to the `plugins` property of your job class, by `Plugin.({...pluginOptions})` 322 | 323 | ### Plugin.jobLock 324 | 325 | JobLock plugin is used to prevent a job to be performed (hook in `beforePerform`), which has the same name, queue and args is already running. 326 | 327 | If reEnqueue is `true`, the job will be put back (re-enqueue) with a delay . 328 | 329 | ```typescript 330 | import { BaseJob, Plugin } from "adonis-resque" 331 | import logger from '@adonisjs/core/services/logger' 332 | 333 | export default class ExampleJob extends BaseJob { 334 | plugins = [ 335 | Plugin.jobLock({ reEnqueue: false, enqueueTimeout: 3000 }) 336 | ] 337 | async perform() { 338 | logger.info('Example job started') 339 | await sleep(60000) 340 | logger.info('Example job done') 341 | } 342 | } 343 | 344 | function sleep(ms: number) { 345 | return new Promise(resolve => setTimeout(resolve, ms)); 346 | } 347 | ``` 348 | 349 | ### Plugin.queueLock 350 | 351 | Similar to jobLock, but it would be prevented before enqueue. 352 | If a job with the same name, queue, and args is already in the queue, do not enqueue it again. 353 | 354 | QueueLock Options: 355 | 356 | ```typescript 357 | export interface QueueLockOptions { 358 | /** 359 | * in seconds 360 | */ 361 | lockTimeout?: number 362 | key?: LockKey 363 | } 364 | ``` 365 | ### Plugin.delayQueueLock 366 | 367 | Same as queueLock, but it is for the delay queue. 368 | 369 | > [!IMPORTANT] 370 | > **How "Prevented Before Enqueue" Works?** 371 | > The `delayQueueLock` only prevents immediate enqueue operations; it doesn't prevent delayed enqueue operations, such as those using `enqueueIn` or `enqueueAt`. 372 | > In `node-resque`, only non-delayed enqueue operations trigger the `beforeEnqueue` hook. If you use `enqueueIn` or `enqueueAt`, the hook is not triggered. 373 | > [Here is source code](https://github.com/actionhero/node-resque/blob/a7eb5742df427aaf338efcc40579534ac458f57b/src/core/queue.ts#L86) 374 | > 375 | > Therefore, in the following code, the second operation won't be enqueued: 376 | > ```typescript 377 | > 378 | > export default class ExampleJob extends BaseJob { 379 | > plugins = [ 380 | > Plugin.delayQueueLock() 381 | > ] 382 | > async perform() {} 383 | > } 384 | > await ExampleJob.enqueueIn(1000) 385 | > // won't enqueue to queue 386 | > await ExampleJob.enqueue() 387 | > ``` 388 | > However, in the following code, the job will perform twice, no matter if the `delayQueueLock` plugin is enabled or not: 389 | > ```typescript 390 | > await ExampleJob.enqueueIn(1000) 391 | > await ExampleJob.enqueueIn(1500) 392 | > ``` 393 | 394 | 395 | ### Plugin.retry 396 | 397 | If a job fails, retry it times before finally placing it into the failed queue 398 | You can specify the retryDelay option to delay, or set the backoffStrategy to the delay ms array. 399 | Note: 400 | It will retry by key, composed of queueName, jobName and args. 401 | Not retry by the enqueuing action. 402 | 403 | ```typescript 404 | import { BaseJob, Plugin } from "adonis-resque" 405 | 406 | export default class ExampleJob extends BaseJob { 407 | plugins = [ 408 | Plugin.retry({ 409 | retryLimit: 3, 410 | backoffStrategy: [1000, 3000, 8000] 411 | }) 412 | ] 413 | async perform() { 414 | const res = await fetch(`https://dummyjson.com/products/1`) 415 | this.logger.info(`Response status is ${res.status}`) 416 | } 417 | } 418 | ``` 419 | 420 | ### Plugin.noop 421 | 422 | you can customize the error logger by option `logger` 423 | 424 | ### Custom Your Own Plugin 425 | 426 | You can create your own plugin by extending the `BasePlugin` class, 427 | it also implements the `Plugin` abstract class from `node-resque`. 428 | See more details in [node-resque plugins](https://github.com/actionhero/node-resque?tab=readme-ov-file#plugins). 429 | 430 | ```typescript 431 | import { BasePlugin } from "adonis-resque" 432 | export default class MyPlugin extends BasePlugin { 433 | /** 434 | * Plugin options here 435 | * with default values 436 | */ 437 | options = { 438 | foo: 'bar' 439 | } 440 | async beforeEnqueue() { 441 | } 442 | async afterEnqueue() { 443 | } 444 | async beforePerform(){ 445 | } 446 | async afterPerform() { 447 | } 448 | } 449 | ``` 450 | 451 | Applying it in your job class. 452 | 453 | ```typescript 454 | import { BaseJob } from "adonis-resque" 455 | // import MyPlugin from './my-plugin' 456 | 457 | class ExampleJob extends BaseJob { 458 | plugins = [ 459 | MyPlugin.create({ foo: 'baz' }) 460 | ] 461 | } 462 | ``` 463 | 464 | 465 | ## Configuration 466 | 467 | Here is an example of `config/resque.ts` 468 | 469 | ```typescript 470 | { 471 | /** 472 | * redis connection config from @adonisjs/redis 473 | */ 474 | redisConnection: 'main', 475 | /** 476 | * run web & worker in same process, if enabled 477 | * You need to run command node ace resque:start if it is turned off 478 | * 479 | * it's convenient but NOT Recommanded in production 480 | * also, DO NOT enable for math-heavy jobs, even in the dev or staging environment. 481 | * 482 | */ 483 | runWorkerInWebEnv: true, 484 | /** 485 | * when runScheduler enabled, it starts with worker 486 | * if you'd like to run scheduler in the separated processes 487 | * please turn runScheduler off, and run command 488 | * node ace resque:start --scheduler 489 | */ 490 | runScheduler: true, 491 | /** 492 | * enable node-resque multiworker 493 | * @docs https://github.com/actionhero/node-resque?tab=readme-ov-file#multi-worker 494 | */ 495 | isMultiWorkerEnabled: true, 496 | /** 497 | * the first argument in MultiWorker constructor 498 | */ 499 | multiWorkerOption: { 500 | minTaskProcessors: 1, 501 | maxTaskProcessors: 10 502 | }, 503 | /** 504 | * the argument for Worker constructor, if multiWorker is not enabled 505 | */ 506 | workerOption: { 507 | }, 508 | /** 509 | * the default queue name for jobs to enqueue 510 | */ 511 | queueNameForJobs: 'default', 512 | /** 513 | * queue name for workers to listen, 514 | * is a string or an array of string 515 | * setting a proper queue name could change their priorities 516 | * e.g. queueNameForWorkers: "high-priority, medium-priority, low-priority" 517 | * All the jobs in high-priority will be worked before any of the jobs in the other queues. 518 | */ 519 | queueNameForWorkers: '*', 520 | queueNameForWorkers: '*', 521 | /** 522 | * set null to use the default logger 523 | */ 524 | logger: null, 525 | // verbose mode for debugging 526 | verbose: true 527 | } 528 | ``` 529 | 530 | ## Concepts & Components 531 | ### Queue 532 | 533 | Queue is a redis list under the hood, consumed(redis pop) in worker. 534 | Each job has a queueName, defined in Job class, default is `default`. 535 | 536 | ```typescript 537 | class Example extends BaseJob { 538 | queueName = 'default' 539 | } 540 | ``` 541 | You can enqueue jobs into queue from anywhere, .e.g, from a Adonis web controller or from another Job class: `Example.enqueue(...)`. 542 | 543 | That means a redis push. 544 | 545 | For failed jobs, they have been move to queue `failed`. 546 | 547 | The `node-resque` queue object is exposed in a container: 548 | ```typescript 549 | const queue = await app.container.make('queue') 550 | ``` 551 | 552 | You can interact with it, find API here: https://node-resque.actionherojs.com/classes/Queue.html. 553 | 554 | #### Example: Getting a total count of pending jobs 555 | 556 | ```typescript 557 | const pendingJobsTotal = await queue.length() 558 | ``` 559 | 560 | ### Worker 561 | 562 | The jobs you created are pulling and executed by workers actually. 563 | It is a adonis ace command, long-running under the console environment. 564 | 565 | You can start a worker by command: 566 | ```bash 567 | node ace resque:start --worker 568 | ``` 569 | 570 | It has been executed by web server in default, so you don't need to run it. 571 | 572 | > [!IMPORTANT] 573 | > In the production environment, we don't recommend you to use the web server to run the worker. 574 | > Instead, you should run the worker in the separated process, or multiple server. 575 | > 576 | 577 | Every worker has its own workerName, which composed of `os.hostname() + ":" + process.pid + "+" + counter;` 578 | 579 | #### Multi Worker 580 | MultiWorker [comes from `node-resque`](https://github.com/actionhero/node-resque/tree/main?tab=readme-ov-file#multi-worker), is a pool manager for workers. It starts multiple workers in the same process, not in child process. Threfore, those workers have a same pid in their workerName. 581 | 582 | This is designed for I/O intensive jobs, not CPU. 583 | So don't get confused by the word `multi`. 584 | 585 | > [!TIP] 586 | > If you'd like to handle CPU intensive jobs, you can start multipe processes(by running the ace command), even on multipe machines. 587 | 588 | Multi worker is enabled by `isMultiWorkerEnabled` in config by default. If you'd like start a single worker, you can set the config to false, or use the flag `--is-multi` to control: 589 | 590 | ```bash 591 | node ace resque:start --worker --is-multi=false 592 | ``` 593 | 594 | 595 | ### Scheduler 596 | 597 | Scheduler is a kind of specialized internal worker. It is a coordinator of the jobs. 598 | 599 | the Utilities are: 600 | - Support croner & interval jobs 601 | - Process delayed job (`job.enqueueIn`/`job.enqueueAt`) 602 | - Checking stuck workers 603 | - General cluster cleanup 604 | - ... 605 | 606 | > [!TIP] 607 | > 608 | > Scheduler could and should be run in many processes(machines), only one will be elected to be the `leader`, and actually do work; the others are backup. 609 | > For more informations, see node-resque leader scheduler: https://github.com/actionhero/node-resque?tab=readme-ov-file#job-schedules 610 | 611 | By default, scheduler starts with an worker together. You can modify `runScheduler` to `false` in config, to changing this behavior. 612 | 613 | To start a stand alone scheduler, you can run 614 | ```bash 615 | node ace resque:start --worker=false --schedule 616 | ``` 617 | 618 | ## Deployment 619 | 620 | Resque is a distributed job queue system, it can be deloyed to separated processes or servers. 621 | 622 | ### Development 623 | 624 | The scheduler and multiworker are started with web server together. 625 | You don't need to take any additional steps to make it work. 626 | 627 | ### Production 628 | 629 | You should disable the config `runWorkerInWebEnv`, and run `node ace resque:start` command in separated processes. 630 | 631 | It is recommended to start mutiple schedules and workers. 632 | For I/O intensive jobs, you should start multiWorker with `--is-multi` on; for CPU intensive jobs, no benefits from that. 633 | 634 | Usually, we build a docker image for both web server and jobs, start worker and scheduler by setting the entrypoint. 635 | They could share the same other configures, like environment variables. 636 | 637 | docker-compose.yml 638 | ```yaml 639 | ... 640 | entrypoint: ["node", "ace", "resque:start", "--worker", "--schedule"] 641 | ``` 642 | 643 | For people who don't use docker, you can also use `supervisord` or `pm2` to manage the processes. 644 | 645 | 646 | ## Web UI 647 | node-resque also compatible with some Resque Web UI, .e.g [resque-web](https://github.com/resque/resque-web) 648 | 649 | Here is `docker-compose.yml` an example 650 | ```yaml 651 | services: 652 | redis: 653 | image: redis 654 | resque-web: 655 | image: appwrite/resque-web:1.1.0 656 | ports: 657 | - "5678:5678" 658 | environment: 659 | - RESQUE_WEB_HOST=redis # (OPTIONAL - Use only if different than the default 127.0.0.1) 660 | - RESQUE_WEB_PORT=6379 # (OPTIONAL - Use only if different the default 6379) 661 | - RESQUE_WEB_HTTP_BASIC_AUTH_USER= # (OPTIONAL - if not set no password used) 662 | - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD= # (OPTIONAL - if not set no password used) 663 | depends_on: 664 | - redis 665 | restart: unless-stopped 666 | ``` 667 | 668 | if redis server has a password, you can add a entrypoint 669 | 670 | ```yaml 671 | entrypoint: 672 | - resque-web 673 | - -FL 674 | - -r 675 | # Change your redis password here, 676 | # default redis user is default 677 | - "redis://:@:" 678 | - /config.ru 679 | ``` 680 | 681 | ![Web UI](https://i.imgur.com/nN2d9ak.png) 682 | 683 | ## Notice 1: for the graceful exit 684 | resque require the graceful exit, or schedulers would waiting for a leader election. 685 | 686 | node-resque may not exit in dev environment if you exit by `ctrl+c`. 687 | you can change `bin/server.ts` 688 | ```typescript 689 | app.listenIf(app.managedByPm2, 'SIGINT', () => app.terminate()) 690 | ``` 691 | to 692 | ```typescript 693 | app.listen('SIGINT', () => app.terminate()) 694 | ``` 695 | 696 | if adonis dev server terminated but process still there, you can send SIGTERM signal to all node process(on macOS) `killall node` 697 | 698 | You can also check the redis key `resque:resque_scheduler_leader_lock`, which value is scheduler name contains pid of the leader process. it should be release once server terminated. 699 | 700 | ## Notice 2: Job does not `@inject`able 701 | Since our job is a [Thenable class](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise#thenables), that made it not able be inject([related issue](https://github.com/adonisjs/fold/issues/63)), which means `@inject` decorator won't work. 702 | 703 | You can use `app.container.make()` in job instead. 704 | 705 | ## Who's Using 706 | [Create an issue](https://github.com/shiny/adonis-resque/issues/new) to submit your project. 707 | 708 |
709 | 710 |
711 | 712 | ## Contributors 713 | 714 | This project is contributed by u301 team for giving back to the AdonisJS community. 715 | 716 | ## Reference 717 | 718 | - [node-resque](https://github.com/actionhero/node-resque) 719 | 720 | ## Migration from v1 to v2 721 | In the latest version of AdonisJS, the previous method of directly creating an instance of @adonisjs/logger (used in v1) is no longer compatible. 722 | To address this, adonis-resque has been upgraded to v2, with changes to the API interface. 723 | 724 | ### Changes in ver 2 725 | 726 | #### this.logger Removed 727 | `this.logger` removed in Job. You can use `import logger from '@adonisjs/core/services/logger'` instead. 728 | 729 | #### Chainable Methods Removed 730 | In v1, you could use chainable calls like `await ExampleJob.enqueue(...args).in(1000)`. However, this feature has been removed in v2. Now, you should use `await ExampleJob.enqueueIn(1000, ...args)` instead. Additionally, these methods now return a Promise, whereas they previously only returned this. 731 | 732 | The following methods have been removed: 733 | 734 | - in 735 | - at 736 | - The static method queue() 737 | 738 | Since Resque's functionality is relatively simple, the benefits of chainable calls are minimal, and they also hinder Adonis's dependency injection capabilities. 739 | 740 | #### Dependency Injection Now Available in Job Files 741 | With the removal of chainable calls, dependency injection is now available in job files. However, keep in mind that because jobs run in the background, the `HttpContext` cannot be injected. 742 | 743 | ## License 744 | MIT 745 | 746 | [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fshiny%2Fadonis-resque.svg?type=large&issueType=license)](https://app.fossa.com/projects/git%2Bgithub.com%2Fshiny%2Fadonis-resque?ref=badge_large&issueType=license) 747 | --------------------------------------------------------------------------------