├── .gitignore
├── LICENSE
├── README.md
├── package.js
└── src
├── Cluster
├── ChildProcess.js
├── MasterCluster.js
├── StaticCluster.js
└── _index.js
├── InMemoryTaskQueue.js
├── TaskQueue.js
├── Worker
├── ChildWorker.js
├── MasterWorker.js
└── statuses.js
├── index.js
├── logs.js
└── tests
├── _index.js
├── ipcTests.js
├── simpleTests.js
└── taskMap.js
/.gitignore:
--------------------------------------------------------------------------------
1 | .vscode
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 Nathan Schwarz
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # meteor-cluster
2 |
3 | Meteor Package enabling users to create a Worker Pool on the server to handle heavy jobs.
4 | It can run synchronous and asynchronous tasks from a persitent / in-memory queue.
5 | It can also run recurring and scheduled tasks.
6 |
7 | # TaskQueue
8 |
9 | `TaskQueue` is both a Mongodb and an in-memory backed job queue.
10 | It enables to add, update, remove jobs consistently between processes.
11 |
12 | You can attach event listeners to handle the tasks results / errors
13 |
14 | ## prototype
15 |
16 | `TaskQueue.addTask({ taskType: String, data: Object, priority: Integer, _id: String, dueDate: Date, inMemory: Boolean })`
17 | - `taskType` is mandatory
18 | - `data` is mandatory but you can pass an empty object
19 | - `priority` is mandatory, default is set to 1
20 | - `_id` is optional
21 | - `dueDate` is mandatory, default is set to `new Date()`
22 | - `inMemory` is optional, default is set to `false`
23 |
24 | ### Event listeners (Master only) :
25 |
26 | `TaskQueue.addEventListener(eventType: String, callback: function)`
27 | - `eventType` is one of `[ 'done', 'error' ]`
28 | - `callback` is a function prototyped as `callback({ value: Any, task: Task })`, `value` contains the result / error.
29 |
30 | `TaskQueue.removeEventListener(eventType: String)`
31 | - `eventType` is one of `[ 'done', 'error' ]`
32 |
33 | note : you can only attach one event listener by eventType.
34 |
35 | ### In-Memory Queue (Master only) :
36 |
37 | `TaskQueue.inMemory.findById(_id: String)`
38 | `TaskQueue.inMemory.removeById(_id: String)`
39 | `TaskQueue.inMemory.tasks()` : returns all in-memory tasks
40 | `TaskQueue.inMemory.availableTasks()` : returns available in-memory tasks
41 |
42 | ## note on the in-memory / persistent task queue
43 |
44 | Both in-memory and persistent tasks are available at the same time, and can be used altogether but :
45 |
46 | - in-memory tasks can only be created on the Master (which is because it's non persistent...)
47 | - in-memory tasks will always be called first over persistent tasks even if their respective `priority` are greater.
48 | - if you use both in-memory and persistent tasks at the same time, the persistent tasks will be called only when no in-memory tasks are available (may change later).
49 |
50 | # Cluster
51 |
52 | `Cluster` is an isomorphic class to handle both the Worker and the Master
53 | on the Master it :
54 | - verifies if jobs are in the queue
55 | - verifies if workers are available, or create them
56 | - dispatches jobs to the workers
57 | - removes the task from the queue once the job is done
58 | - closes the workers when no jobs are available(behavior can be overriden with `keepAlive`)
59 |
60 | on the Worker it :
61 | - starts the job
62 | - when the job's done, tell the Master that it's available and to remove previous task.
63 |
64 | ## prototype
65 |
66 | `constructor(taskMap: Object, { port: Integer, maxAvailableWorkers: Integer, refreshRate: Integer, inMemoryOnly: Boolean, messageBroker: function, logs: String, keepAlive: String | Integer, autoInitialize: Boolean })`
67 | - `taskMap`: a map of functions associated to a `taskType`
68 | - `maxAvailableWorkers`: maximum number of child process (cores), default is set to system maximum
69 | - `port`: server port for child process servers, default set to `3008`
70 | - `refreshRate`: Worker pool refresh rate (in ms), default set to `1000`
71 | - `inMemoryOnly`: force the cluster to only pull jobs from the in-memory task queue.
72 | - `messageBroker` is optional, default set to null (see IPC section)
73 | - `logs`: is one of `['all', 'error']`, default sets to `all` : if set to `'error'`, will only show the errors and warning logs.
74 | - `keepAlive`: an optional parameter that can be set to either:
75 | - `'always'` to have the system start up the `maxAvailableWorkers` number of workers immediately and keep them all alive always
76 | - some `Integer` value will have the system not shutdown workers until the number passed in milliseconds has passed since last a job was available to be picked up by a worker
77 |
78 | **NOTE:** default behavior when `keepAlive` is not set is to only keep alive workers when there are jobs available to be picked up by them.
79 | - `autoInitialize`: an optional parameter that controls whether the system will automatically initialize the cluster's polling for jobs when the `MasterCluster` is created. Default is set to `true`.
80 |
81 | `Cluster.isMaster()`: `true` if this process is the master
82 |
83 | `Cluster.maxWorkers()`: returns the maximum number of workers available at the same time
84 |
85 | `setRefreshRate(refreshRate: Integer)`: change the refresh rate on the master
86 |
87 | if the Master process crashes or restarts, all the unfinished jobs will be restarted from the beginning.
88 | Each job is logged when started / finished with the format : `${timestamp}:nschwarz:cluster:${taskType}:${taskId}`
89 |
90 | ## IPC (advanced usage)
91 |
92 | Introduced in version 2.0.0, you can communicate between the child processes and the Master.
93 | To do so, you must provide the Master Cluster instance with a `messageBroker` function.
94 | this function will handle (on the master) all custom messages from the child processes.
95 |
96 | the function should be prototype as follow :
97 | `messageBroker(respond: function, msg: { status: Int > 1, data: Any })`
98 | - `respond` enables you to answer to a message from a child
99 |
100 | All communications between the master and a child must be started by the child.
101 | To do so you can use the second parameter passed in all functions provided to the taskMap `toggleIPC` which is prototyped as follow :
102 |
103 | `toggleIPC(messageBroker: function, initalize: function): Promise`
104 | - `messageBroker` is prototyped as `messageBroker(msg: Any)`
105 | - `initialize` is prototyped as `initialize(sendMessageToMaster: function)`
106 |
107 | because `toggleIPC` returns a promise you must return it (recursively), otherwise the job will be considered done, and the worker Idle.
108 | Not returning it will result in unwanted, non expected behavior.
109 |
110 |
111 | # CPUS allocation
112 |
113 | You should not use the default `maxAvailableWorkers` (cpus allocation number) value.
114 | The default value is set to your system cpus number, but it's a reference value.
115 | It's up to you to understand your needs and allocate cpus accordingly.
116 |
117 | ## how can I calculate the maximum number of cpus I can allocate ?
118 |
119 | for example, if you're running on a 8 core machine :
120 |
121 | - The app you're running is using 1 cpu to run.
122 | - You should have a reverse proxy on your server, you should at least save 1 cpu (may be more depending on your traffic).
123 | - the database you're using is hosted on the same server, you should save 1 cpu for it.
124 | - you're running an external service such as Redis or Elastic Search, so that's 1 down.
125 |
126 | so you should have `maxAvailableWorkers = Cluster.maxWorkers() - 4 === 4`
127 |
128 | ## what if I allocated too much CPUS ?
129 |
130 | You can't allocate more than your maximum system cpu number.
131 | You still can outrange the theoretical maximum process number :
132 |
133 | in such case your overall system should be **slowed down** because some of the processes execution will be deferred.
134 | **It will drastically reduce the multi-core performance gain**.
135 |
136 | # examples
137 | ## basic usage
138 |
139 | ```
140 | import { Meteor } from 'meteor/meteor'
141 | import { Cluster, TaskQueue } from 'meteor/nschwarz:cluster'
142 |
143 | const taskMap = {
144 | 'TEST': job => console.log(`testing ${job._id} at position ${job.data.position}`),
145 | 'SYNC': (job) => console.log("this is a synchronous task"),
146 | 'ASYNC': (job) => new Promise((resolve, reject) => Meteor.setTimeout(() => {
147 | console.log("this is an asynchronous task")
148 | resolve()
149 | }, job.data.timeout))
150 | }
151 |
152 | function onJobsDone({ value, task }) {
153 | console.log('do something with the result')
154 | }
155 |
156 | function onJobsError({ value, task }) {
157 | console.log('do something with the error')
158 | }
159 |
160 | function syncTask() {
161 | return TaskQueue.addTask({ taskType: 'SYNC', data: {}})
162 | }
163 |
164 | function asyncTask() {
165 | return TaskQueue.addTask({ taskType: 'ASYNC', data: { timeout: 5000 }, priority: 6 })
166 | }
167 |
168 | function inMemoryTask(priority, position) {
169 | return TaskQueue.addTask({ taskType: 'TEST', priority, data: { position }, inMemory: true })
170 | }
171 |
172 | function persistentTask(priority, position) {
173 | return TaskQueue.addTask({ taskType: 'TEST', priority, data: { position }, inMemory: false })
174 | }
175 |
176 | const cluster = new Cluster(taskMap)
177 | Meteor.startup(() => {
178 | if (Cluster.isMaster()) {
179 | TaskQueue.addEventListener('done', onJobsDone)
180 | TaskQueue.addEventListener('error', onJobsError)
181 |
182 | syncTask()
183 | asyncTask()
184 | inMemoryTask(8, 1)
185 | inMemoryTask(1, 2)
186 |
187 | persistentTask(8, 1)
188 | persistentTask(1, 2)
189 | }
190 | })
191 | ```
192 |
193 | ## scheduled task example : run a task in ten minutes
194 | ```
195 | import { add } from 'date-fns/date' // external library to handle date objects
196 |
197 | const dueDate = add(new Date(), { minutes: 10 })
198 | TaskQueue.addTask({ taskType: 'sometype', priority: 1, data: {}, dueDate })
199 | ```
200 |
201 | ## scheduled task example : run a recurring task every ten minutes
202 | ```
203 | import { add } from 'date-fns/date' // external library to handle date objects
204 |
205 | function recurringTask(job) {
206 | // do something
207 | const dueDate = add(new Date(), { minutes: 10 })
208 | TaskQueue.addTask({ taskType: 'recurringTask', priority: 1, data: {}, dueDate })
209 | }
210 |
211 | const taskMap = {
212 | recurringTask
213 | }
214 | ```
215 |
216 | ## simple IPC example (advanced usage)
217 | ```
218 | function ipcPingTest(job, toggleIPC) {
219 | return toggleIPC(
220 | (msg) => {
221 | console.log(msg)
222 | return 'result you eventually want to pass to the master'
223 | }, (smtm) => smtm({ status: 4, data: 'ping' })
224 | )
225 | }
226 |
227 | const taskMap = {
228 | ipcPingTest
229 | }
230 |
231 | function messageBroker(respond, msg) {
232 | if (msg.data === 'ping') {
233 | respond('pong')
234 | }
235 | }
236 |
237 | const cluster = new Cluster(taskMap, { messageBroker })
238 | ```
239 |
240 | ## multiple IPC example (advanced usage)
241 | ```
242 | function ipcPingTest(job, toggleIPC) {
243 | return toggleIPC(
244 | (msg) => {
245 | console.log(msg)
246 | return toggleIPC(
247 | (msg) => console.log(msg),
248 | (smtm) => smtm({ status: 4, data: 'ping' })
249 | )
250 | }, (smtm) => smtm({ status: 4, data: 'ping' }))
251 | }
252 |
253 | const taskMap = {
254 | ipcPingTest
255 | }
256 |
257 | function messageBroker(respond, msg) {
258 | if (msg.data === 'ping') {
259 | respond('pong')
260 | }
261 | }
262 |
263 | const cluster = new Cluster(taskMap, { messageBroker })
264 | ```
265 |
266 | # common mistakes and good practices
267 |
268 | ## secure your imports
269 |
270 | Because the worker will only work on tasks, you should remove the unnecessary imports to avoid resources consumption and longer startup time.
271 | As a good practice you should put all your Master imports logic in the same file, and import it only on the master.
272 | What I mean by "Master imports Logic" is :
273 |
274 | - all your publications
275 | - all your REST endpoints declarations
276 | - graphql server and such...
277 | - SSR / front related code
278 |
279 | It could be summarized as such :
280 |
281 | ```
282 | // in your entry file
283 |
284 | if (Cluster.isMaster()) {
285 | import './MasterImports.js'
286 | }
287 | // ...rest of your cluster logic
288 | ```
289 |
290 | ## recurring tasks
291 |
292 | Because recurring tasks are created "recursively", there will always be a task in the queue.
293 | If the server is restarted, it will start the recurring task because it's still in the queue.
294 | Be sure to remove all recurring task *on the master* before starting others, or secure the insert.
295 | Otherwise you will have multiple identical recurring tasks running at the same time.
296 |
297 | You can either do :
298 |
299 | ```
300 | Meteor.startup(() => {
301 | if (Cluster.isMaster()) {
302 | TaskQueue.remove({ taskType: 'recurringTask' })
303 | }
304 | })
305 | ```
306 |
307 | or at task *initialization* :
308 |
309 | ```
310 | const recurringTaskExists = TaskQueue.findOne({ taskType: 'recurringTask' }) !== undefined
311 | if (!recurringTaskExists) {
312 | TaskQueue.addtask({ taskType: 'recurringTask', priority: 1, data: {}, dueDate })
313 | }
314 | ```
315 |
316 | ## task uniqueness
317 |
318 | If you want to be sure to have unique tasks, you should set a unique Id with `TaskQueue.addTask`.
319 | A good model could be : `${taskType}${associated_Model_ID}`
320 |
321 | ## multiple apps
322 |
323 | There's no way right now to know from which app the task is started (may change later) :
324 | you should only run the Cluster on **one of the app** to avoid other apps to run a task which is not included in its taskMap.
325 | You can still use the TaskQueue in all the apps of course.
326 | If your apps have different domain names / configurations (for the mailer for example), you should pass these through the `data` field.
327 |
328 | For example if you're using `Meteor.absoluteUrl` or such in a task it will have the value associated with the app running the Cluster.
329 |
--------------------------------------------------------------------------------
/package.js:
--------------------------------------------------------------------------------
1 | Package.describe({
2 | name: 'nschwarz:cluster',
3 | version: '2.3.0',
4 | summary: 'native nodejs clusterization for meteor server',
5 | git: 'https://github.com/nathanschwarz/meteor-cluster.git',
6 | documentation: 'README.md',
7 | })
8 |
9 | Package.onUse(api => {
10 | api.versionsFrom('2.4')
11 | api.use(['mongo', 'ecmascript', 'random'])
12 | api.mainModule('src/index.js', 'server')
13 | })
14 |
15 | Npm.depends({
16 | debug: '4.3.1',
17 | })
18 |
19 | Package.onTest(api => {
20 | api.use('nschwarz:cluster')
21 | api.use(['ecmascript'])
22 | api.mainModule('src/tests/_index.js')
23 | })
24 |
--------------------------------------------------------------------------------
/src/Cluster/ChildProcess.js:
--------------------------------------------------------------------------------
1 | const debug = Npm.require('debug')
2 | import StaticCluster from './StaticCluster'
3 | const process = require('process')
4 | import TaskQueue from '../TaskQueue'
5 | import ChildWorker from '../Worker/ChildWorker'
6 |
7 | class ChildProcess extends StaticCluster {
8 | constructor(taskMap, { logs = 'all' } = {}) {
9 | super()
10 | Meteor.startup(() => {
11 | TaskQueue.registerTaskMap(taskMap)
12 | })
13 | // enable / disable logs
14 | if (logs === 'all') {
15 | debug.enable('nschwarz:cluster:*')
16 | } else {
17 | debug.enable('nschwarz:cluster:ERROR*,nschwarz:cluster:WARNING*')
18 | }
19 | // register listeners if this process is a worker
20 | process.on('message', ChildWorker.onMessageFromMaster)
21 | process.on('disconnect', ChildWorker.onDisconnect)
22 | }
23 | }
24 |
25 | export default ChildProcess
26 |
--------------------------------------------------------------------------------
/src/Cluster/MasterCluster.js:
--------------------------------------------------------------------------------
1 | import StaticCluster from './StaticCluster'
2 | import { Meteor } from 'meteor/meteor'
3 | import TaskQueue from '../TaskQueue'
4 | import MasterWorker from '../Worker/MasterWorker'
5 | import { warnLogger } from '../logs'
6 |
7 | const process = require('process')
8 | const MAX_CPUS = StaticCluster.maxWorkers()
9 |
10 | class MasterCluster extends StaticCluster {
11 |
12 | lastJobAvailableMilliseconds = Date.now()
13 |
14 | /**
15 | * initialize Cluster on the master
16 | *
17 | * @param { Object } taskMap
18 | * @param { Object } masterProps
19 | * - port?: Integer
20 | * - refreshRate?: Integer
21 | * - inMemoryOnly?: Boolean
22 | * - messageBroker?: Function
23 | * - keepAlive?: String | number
24 | * - autoInitialize?: Boolean
25 | */
26 | constructor(
27 | taskMap,
28 | {
29 | port = 3008,
30 | maxAvailableWorkers = MAX_CPUS,
31 | refreshRate = 1000,
32 | inMemoryOnly = false,
33 | messageBroker = null,
34 | keepAlive = null,
35 | autoInitialize = true
36 | } = {}
37 | ) {
38 | super()
39 | Meteor.startup(() => {
40 | if (maxAvailableWorkers > MAX_CPUS) {
41 | warnLogger(`cannot have ${maxAvailableWorkers} workers, setting max system available: ${MAX_CPUS}`)
42 | this._cpus = MAX_CPUS
43 | } else if (maxAvailableWorkers <= 0) {
44 | warnLogger(`cannot have ${maxAvailableWorkers} workers, setting initial value to 1`)
45 | this._cpus = 1
46 | } else {
47 | this._cpus = maxAvailableWorkers
48 | }
49 | if (this._cpus === MAX_CPUS) {
50 | warnLogger(`you should not use all the cpus, read more https://github.com/nathanschwarz/meteor-cluster/blob/main/README.md#cpus-allocation`)
51 | }
52 | if (keepAlive && !keepAlive === `always` && !(Number.isInteger(keepAlive) && keepAlive > 0)) {
53 | warnLogger(`keepAlive should be either be "always" or some Integer greater than 0 specifying a time in milliseconds to remain on;`
54 | + ` ignoring keepAlive configuration and falling back to default behavior of only spinning up and keeping workers when the jobs are available`)
55 | }
56 | if (typeof autoInitialize !== `boolean`) {
57 | warnLogger(`autoInitialize should be a boolean(was passed as: ${typeof autoInitialize}),`
58 | + ` ignoring autoInitialize configuration and falling back to default behavior of autoInitialize: true`)
59 | }
60 | this._port = port
61 | this._workers = []
62 | this.inMemoryOnly = inMemoryOnly
63 | this.messageBroker = messageBroker
64 | this.refreshRate = refreshRate
65 |
66 | // find worker by process id
67 | this.getWorkerIndex = (id) => this._workers.findIndex(w => w.id === id)
68 | this.getWorker = (id) => this._workers[this.getWorkerIndex(id)]
69 |
70 | // update all previous undone task, to restart them (if the master server has crashed or was stopped)
71 | TaskQueue.update({ onGoing: true }, { $set: { onGoing: false } }, { multi: true })
72 |
73 | // initializing interval
74 | this.setIntervalHandle = null
75 |
76 | if (autoInitialize) {
77 | this.initialize()
78 | }
79 | })
80 | }
81 |
82 | /**
83 | * add workers if tasks > current workers
84 | * remove workers if tasks < current workers
85 | *
86 | * @param { Integer } wantedWorkers
87 | * @returns non idle workers
88 | */
89 | _getAvailableWorkers (wantedWorkers) {
90 | const workerToCreate = wantedWorkers - this._workers.length
91 | if (workerToCreate > 0) {
92 | for (let i = 0; i < workerToCreate; i++) {
93 | const worker = new MasterWorker(this.messageBroker)
94 | worker.register({ ...process.env, PORT: this.port })
95 | this._workers.push(worker)
96 | }
97 | } else if (workerToCreate < 0) {
98 | this._workers.filter(w => w.isIdle && w.isReady).slice(workerToCreate).forEach(w => w.close())
99 | }
100 | this._workers = this._workers.filter(w => !w.removed)
101 | return this._workers.filter(w => w.isIdle && w.isReady)
102 | }
103 |
104 | /**
105 | * Dispatch jobs to idle workers
106 | *
107 | * @param { Worker } availableWorkers
108 | */
109 | async _dispatchJobs (availableWorkers) {
110 | for (const worker of availableWorkers) {
111 | const job = await TaskQueue.pull(this.inMemoryOnly)
112 | if (job !== undefined) {
113 | worker.startJob(job)
114 | }
115 | }
116 | }
117 |
118 | /**
119 | * Called at the interval set by Cluster.setRefreshRate
120 | *
121 | * - gets jobs from the list
122 | * - if jobs are available update the lastJobAvailableMilliseconds to current time
123 | * - calculates the desired number of workers
124 | * - gets available workers
125 | * - dispatch the jobs to the workers
126 | */
127 | async _run () {
128 | const currentMs = Date.now()
129 |
130 | const jobsCount = TaskQueue.count(this.inMemoryOnly)
131 |
132 | // if there are jobs that are pending, update the lastJobAvailableMilliseconds to current time
133 | // and keep the wantedWorkers
134 | if (jobsCount > 0) {
135 | this.lastJobAvailableMilliseconds = currentMs
136 | }
137 | // default behavior is to keep the workers alive in line with the number of jobs available
138 | let wantedWorkers = Math.min(this._cpus, jobsCount)
139 | if (this.keepAlive === `always`) {
140 | // always keep the number of workers at the max requested
141 | wantedWorkers = this._cpus
142 | } else if (Number.isInteger(this.keepAlive)) {
143 | // don't start shutting down workers till keepAlive milliseconds has elapsed since a job was available
144 | if (currentMs - this.lastJobAvailableMilliseconds >= this.keepAlive) {
145 | // still with the threshold of keepAlive milliseconds, keep the number of workers at the current worker
146 | // count or the requested jobs count whichever is bigger
147 | wantedWorkers = Math.min(this._cpus, Math.max(jobsCount, this._workers.length))
148 | }
149 | }
150 |
151 | const availableWorkers = this._getAvailableWorkers(wantedWorkers)
152 | await this._dispatchJobs(availableWorkers)
153 | }
154 |
155 | /**
156 | * Starts the Cluster._run interval call at the rate determined by this.refreshRate
157 | */
158 | initialize () {
159 | this.setRefreshRate(this.refreshRate)
160 | }
161 |
162 | /**
163 | * Set the refresh rate at which Cluster._run is called and restart the interval call at the new
164 | * rate
165 | *
166 | * @param { Integer } delay
167 | */
168 | setRefreshRate (delay) {
169 | this.refreshRate = delay
170 |
171 | if (this.interval != null) {
172 | Meteor.clearInterval(this.interval)
173 | }
174 | this.interval = Meteor.setInterval(() => this._run(), delay)
175 | }
176 | }
177 |
178 | export default MasterCluster
179 |
--------------------------------------------------------------------------------
/src/Cluster/StaticCluster.js:
--------------------------------------------------------------------------------
1 | const cluster = require('cluster')
2 | const MAX_CPUS = require('os').cpus().length
3 |
4 | class StaticCluster {
5 | static maxWorkers() {
6 | return MAX_CPUS
7 | }
8 | static isMaster() {
9 | return cluster.isMaster
10 | }
11 | }
12 |
13 | export default StaticCluster
--------------------------------------------------------------------------------
/src/Cluster/_index.js:
--------------------------------------------------------------------------------
1 | const cluster = require('cluster')
2 | import MasterCluster from './MasterCluster'
3 | import ChildProcess from './ChildProcess'
4 |
5 | export default (cluster.isMaster ? MasterCluster : ChildProcess)
6 |
--------------------------------------------------------------------------------
/src/InMemoryTaskQueue.js:
--------------------------------------------------------------------------------
1 | import { Random } from 'meteor/random'
2 |
3 | class InMemoryTaskQueue {
4 | static compareJobs(a, b) {
5 | if (a.dueDate < b.dueDate || a.priority > b.priority) {
6 | return -1
7 | }
8 | if (a.dueDate > b.dueDate || a.priority < b.priority) {
9 | return 1
10 | }
11 | if (a.createdAt > b.createdAt) {
12 | return -1
13 | }
14 | if (a.createdAt < b.createdAt) {
15 | return 1
16 | }
17 | return 0
18 | }
19 | constructor() {
20 | this._data = []
21 | }
22 | _findIndex(_id) {
23 | return this._data.findIndex(job => job._id === _id)
24 | }
25 | insert(doc) {
26 | doc._id = `inMemory_${Random.id()}`
27 | this._data = [ ...this._data, doc ].sort(InMemoryTaskQueue.compareJobs)
28 | }
29 | findById(_id) {
30 | const _idx = this._findIndex(_id)
31 | if (_idx === -1) {
32 | return undefined
33 | }
34 | return this._data[_idx]
35 | }
36 | removeById(_id) {
37 | const idx = this._findIndex(_id)
38 | if (idx === -1) {
39 | return undefined
40 | }
41 | return this._data.splice(idx, 1)[0]
42 | }
43 | // get all jobs
44 | tasks() {
45 | return this._data
46 | }
47 | // get available jobs (onGoing: false)
48 | availableTasks() {
49 | const now = new Date()
50 | return this._data.filter(job => !job.onGoing && now >= job.dueDate)
51 | }
52 | // count available jobs (onGoing: false)
53 | count() {
54 | return this.availableTasks().length
55 | }
56 | // pull available jobs from the queue
57 | pull() {
58 | const availableTasks = this.availableTasks()
59 | if (availableTasks.length) {
60 | availableTasks[0].onGoing = true
61 | return availableTasks[0]
62 | }
63 | return undefined
64 | }
65 | }
66 |
67 | export default InMemoryTaskQueue
68 |
--------------------------------------------------------------------------------
/src/TaskQueue.js:
--------------------------------------------------------------------------------
1 | import { Meteor } from 'meteor/meteor'
2 | import { Mongo } from 'meteor/mongo'
3 | import { Match } from 'meteor/check'
4 |
5 | const cluster = require('cluster')
6 |
7 | import InMemoryTaskQueue from './InMemoryTaskQueue'
8 | import { logger, errorLogger } from './logs'
9 |
10 |
11 | class MongoTaskQueue extends Mongo.Collection {
12 | // verify that the collection indexes are set
13 | _setIndexes() {
14 | this.rawCollection().createIndex({ taskType: 1 })
15 | this.rawCollection().createIndex({ onGoing: 1 })
16 | // remove post @1.2 index if it exists
17 | this.rawCollection().dropIndex({ priority: -1, createdAt: 1 }).catch(e => e)
18 |
19 | // add dueDate index for scheduled tasks in @1.2; add dueDate field to prior @1.2 tasks
20 | this.update({ dueDate: null }, { $set: { dueDate: new Date() }}, { multi: true })
21 | this.rawCollection().createIndex({ dueDate: 1, priority: -1, createdAt: 1 })
22 | }
23 | constructor(props) {
24 | super(props)
25 | this.taskMap = {}
26 | if (cluster.isMaster) {
27 | Meteor.startup(() => this._setIndexes())
28 | this.inMemory = new InMemoryTaskQueue()
29 |
30 | // event listeners
31 | this.listeners = {
32 | done: null,
33 | error: null
34 | }
35 | this.addEventListener = (type, cb) => {
36 | if (this.listeners[type] !== undefined) {
37 | this.listeners[type] = cb
38 | } else {
39 | throw new Error(`TaskQueue: can't listen to ${type} event doesn't exists`)
40 | }
41 | }
42 | this.removeEventListener = (type) => this.addEventListener(type, null)
43 |
44 | // remove the job from the queue when completed, pass the result to the done listener
45 | this.onJobDone = Meteor.bindEnvironment(async ({ result, taskId }) => {
46 | let doc = null
47 | if (taskId.startsWith('inMemory_')) {
48 | doc = this.inMemory.removeById(taskId)
49 | } else {
50 | doc = await this.rawCollection().findOneAndDelete({ _id: taskId }).then(res => res.value)
51 | }
52 | if (this.listeners.done !== null) {
53 | this.listeners.done({ value: result, task: doc })
54 | }
55 | return doc._id
56 | })
57 |
58 | // log job errors to the error stream, pass the error and the task to the error listener
59 | this.onJobError = Meteor.bindEnvironment(({ error, taskId }) => {
60 | let doc = null
61 | if (taskId.startsWith('inMemory_')) {
62 | doc = this.inMemory.findById(taskId)
63 | } else {
64 | doc = this.findOne({ _id: taskId })
65 | }
66 | if (this.listeners.error !== null) {
67 | this.listeners.error({ value: error, task: doc })
68 | }
69 | errorLogger(error)
70 | return doc._id
71 | })
72 |
73 | // pull available jobs from the queue
74 | this.pull = (inMemoryOnly = false) => {
75 | const inMemoryCount = this.inMemory.count()
76 | if (inMemoryCount > 0 || inMemoryOnly) {
77 | return this.inMemory.pull()
78 | }
79 | return this.rawCollection().findOneAndUpdate({ onGoing: false, dueDate: { $lte: new Date() }}, { $set: { onGoing: true }}, { sort: { priority: -1, createdAt: 1, dueDate: 1 }}).then(res => {
80 | if (res.value != null) {
81 | return res.value
82 | }
83 | return undefined
84 | })
85 | }
86 |
87 | // count available jobs (onGoing: false)
88 | this.count = (inMemoryOnly = false) => {
89 | const inMemoryCount = this.inMemory.count()
90 | if (inMemoryCount > 0 || inMemoryOnly) {
91 | return inMemoryCount
92 | }
93 | return this.find({ onGoing: false }).count()
94 | }
95 | } else {
96 | // execute the task on the child process
97 | this.execute = async (job, toggleIPC) => {
98 | const begin = Date.now()
99 | const isInMemory = typeof(job) === 'object'
100 | const task = isInMemory ? job : this.findOne({ _id: job })
101 | logger(`\b:${task.taskType}:${task._id}:\tstarted`)
102 | const result = await this.taskMap[task.taskType](task, toggleIPC)
103 | const end = Date.now()
104 | const totalTime = end - begin
105 | logger(`\b:${task.taskType}:${task._id}:\tdone in ${totalTime}ms`)
106 | return result
107 | }
108 | }
109 | }
110 | registerTaskMap(map = {}) {
111 | this.taskMap = map
112 | }
113 | addTask({ taskType, priority = 1, data = {}, _id = null, inMemory = false, dueDate = new Date() }, cb = null) {
114 | const tests = [
115 | { name: 'taskType', value: taskType, type: String, typeLabel: 'String' },
116 | { name: 'priority', value: priority, type: Match.Integer, typeLabel: 'Integer' },
117 | { name: 'data', value: data, type: [ Match.Object, [ Match.Any ]], typeLabel: 'Object|Array' },
118 | { name: 'inMemory', value: inMemory, type: Boolean, typeLabel: 'Boolean' },
119 | { name: 'dueDate', value: dueDate, type: Date, typeLabel: 'Date' }
120 | ]
121 | const error = tests.some(t => Array.isArray(t.type) ? !Match.OneOf(t.value, t.type) : !Match.test(t.value, t.type))
122 | if (error) {
123 | throw new Error(`nschwarz:cluster:addTask\twrong value ${t.value} for ${t.name}, expecting ${t.typeLabel}`)
124 | }
125 |
126 | let doc = { taskType, priority, data, createdAt: new Date(), onGoing: false, dueDate }
127 | if (_id != null) {
128 | doc._id = _id
129 | }
130 | if (inMemory) {
131 | if (!cluster.isMaster) {
132 | throw new Error('cannot insert inMemory job from child process')
133 | }
134 | return this.inMemory.insert(doc)
135 | }
136 | if (cb) {
137 | return this.insert(doc, cb)
138 | } else {
139 | return this.insert(doc)
140 | }
141 | }
142 | }
143 |
144 | const TaskQueue = new MongoTaskQueue('taskQueue')
145 | export default TaskQueue
146 |
--------------------------------------------------------------------------------
/src/Worker/ChildWorker.js:
--------------------------------------------------------------------------------
1 | const cluster = require('cluster')
2 | const process = require('process')
3 | import WORKER_STATUSES from './statuses'
4 | import TaskQueue from '../TaskQueue'
5 |
6 | class ChildWorker {
7 | // sends an identifyed msg to the Master
8 | static sendMsg(workerStatus) {
9 | const msg = { id: [cluster.worker.id], ...workerStatus }
10 | process.send(msg)
11 | }
12 | // worker events
13 | static toggleIPC(messageBroker, initialize) {
14 | return new Promise((resolve, reject) => {
15 | process.removeAllListeners('message')
16 | process.on('message', (msg) => resolve(messageBroker(msg)))
17 | initialize(ChildWorker.sendMsg)
18 | }).catch(e => {
19 | throw new Error(e)
20 | })
21 | }
22 | // default msg handler, used to start the task issued by the master
23 | static onMessageFromMaster(task) {
24 | const taskId = task._id
25 | TaskQueue.execute(task, ChildWorker.toggleIPC)
26 | .then(res => ChildWorker.onJobDone(res, taskId))
27 | .catch(error => ChildWorker.onJobFailed(error, taskId))
28 | }
29 | // exit the process when disconected event is issued by the master
30 | static onDisconnect() {
31 | process.exit(0)
32 | }
33 | // task events
34 | static onJobDone(result, taskId) {
35 | process.removeAllListeners('message')
36 | process.on('message', ChildWorker.onMessageFromMaster)
37 | const msg = { result, taskId, status: WORKER_STATUSES.IDLE }
38 | ChildWorker.sendMsg(msg)
39 | }
40 | static onJobFailed(error, taskId) {
41 | process.removeAllListeners('message')
42 | process.on('message', ChildWorker.onMessageFromMaster)
43 | const msg = { taskId, status: WORKER_STATUSES.IDLE_ERROR, error: {
44 | message: error.message,
45 | stack: error.stack,
46 | type: error.type,
47 | arguments: error.arguments
48 | }}
49 | ChildWorker.sendMsg(msg)
50 | }
51 | }
52 |
53 | export default ChildWorker
54 |
--------------------------------------------------------------------------------
/src/Worker/MasterWorker.js:
--------------------------------------------------------------------------------
1 | const cluster = require('cluster')
2 | const process = require('process')
3 | import WORKER_STATUSES from './statuses'
4 | import TaskQueue from '../TaskQueue'
5 |
6 | class MasterWorker {
7 | constructor(messageBroker = null) {
8 | this.isIdle = true
9 | this.isReady = false
10 | this.removed = false
11 | this.worker = null
12 | this.messageBroker = messageBroker
13 | }
14 | setIdle({ taskId, result, error = undefined }) {
15 | this.isIdle = true
16 | if (error !== undefined) {
17 | TaskQueue.onJobError({ error, taskId })
18 | } else {
19 | TaskQueue.onJobDone({ result, taskId })
20 | }
21 | }
22 | // events
23 | onListening() {
24 | this.isReady = true
25 | }
26 | onExit() {
27 | this.removed = true
28 | }
29 | onMessage(msg) {
30 | if (msg.status === WORKER_STATUSES.IDLE || msg.status === WORKER_STATUSES.IDLE_ERROR) {
31 | this.setIdle(msg)
32 | } else if (this.messageBroker !== null) {
33 | this.messageBroker((msg) => this.worker.send(msg), msg)
34 | }
35 | }
36 | register(env) {
37 | this.worker = cluster.fork(env)
38 | this.worker.on('listening', () => this.onListening())
39 | this.worker.on('message', (msg) => this.onMessage(msg))
40 | this.worker.on('exit', () => this.onExit())
41 | }
42 | startJob(task) {
43 | this.isIdle = false
44 | this.worker.send(task)
45 | }
46 | close() {
47 | if (this.worker === null) {
48 | throw new Error('cannot disconnect worker has not started yet')
49 | }
50 | this.worker.disconnect()
51 | this.isIdle = true
52 | this.isReady = false
53 | }
54 | }
55 |
56 | export default MasterWorker
57 |
--------------------------------------------------------------------------------
/src/Worker/statuses.js:
--------------------------------------------------------------------------------
1 | const WORKER_STATUSES = {
2 | IDLE: 0,
3 | IDLE_ERROR: 1
4 | }
5 |
6 | export default WORKER_STATUSES
--------------------------------------------------------------------------------
/src/index.js:
--------------------------------------------------------------------------------
1 | import Cluster from './Cluster/_index'
2 | import TaskQueue from './TaskQueue'
3 |
4 | export { Cluster, TaskQueue }
5 |
--------------------------------------------------------------------------------
/src/logs.js:
--------------------------------------------------------------------------------
1 | const debug = Npm.require('debug')
2 |
3 | const logger = debug('nschwarz:cluster:TASK')
4 | const warnLogger = debug('nschwarz:cluster:WARNING\t')
5 | const errorLogger = debug('nschwarz:cluster:ERROR\t')
6 |
7 | logger.log = console.log.bind(console)
8 | warnLogger.log = console.warn.bind(console)
9 |
10 | debug.enable('nschwarz:cluster:ERROR*,nschwarz:cluster:WARNING*')
11 |
12 | export { logger, warnLogger, errorLogger }
13 |
--------------------------------------------------------------------------------
/src/tests/_index.js:
--------------------------------------------------------------------------------
1 | import { Meteor } from 'meteor/meteor'
2 | import { Cluster, TaskQueue } from '../index.js'
3 | import taskMap from './taskMap'
4 |
5 | function mongoJobs() {
6 | console.log('\n\n####### MONGO TASKS TESTS #######\n\n')
7 | TaskQueue.addTask({ taskType: 'simpleTest', data: {}})
8 | TaskQueue.addTask({ taskType: 'simpleAsyncTest', data: {}})
9 | TaskQueue.addTask({ taskType: 'ipcSinglePingTest', data: {}})
10 | const dueDate = new Date()
11 | dueDate.setSeconds(dueDate.getSeconds() + 5)
12 | TaskQueue.addTask({ taskType: 'simpleSchedTest', data: {}, dueDate })
13 | TaskQueue.addTask({ taskType: 'simpleRecuringTest', data: { }, dueDate })
14 | TaskQueue.addTask({ taskType: 'ipcMultiPingTest', data: {}})
15 | }
16 |
17 | function inMemoryJobs() {
18 | console.log('\n\n####### IN_MEMORY TESTS #######\n\n')
19 | TaskQueue.addTask({ taskType: 'simpleTest', data: {}, inMemory: true })
20 | TaskQueue.addTask({ taskType: 'simpleAsyncTest', data: {}, inMemory: true })
21 | TaskQueue.addTask({ taskType: 'simpleSchedTest', data: { isLastJob: true }, inMemory: true })
22 | }
23 |
24 | function cleanup() {
25 | console.log('\n\n####### CLEANING UP #######\n\n')
26 | TaskQueue.remove({ taskType: { $in: [
27 | 'simpleTest',
28 | 'simpleAsyncTest',
29 | 'simpleSchedTest',
30 | 'simpleRecuringTest',
31 | 'ipcSinglePingTest',
32 | 'ipcMultiPingTest'
33 | ]}})
34 | }
35 |
36 | function handleOtherEvents({ startsInMemory, isLastJob }) {
37 | if (startsInMemory) {
38 | inMemoryJobs()
39 | } else if (isLastJob) {
40 | Meteor.setTimeout(() => {
41 | cleanup()
42 | console.log('\n\n####### TEST SUITE DONE #######\n\n')
43 | }, 1000 * 10)
44 | }
45 | }
46 | function onJobDone(job) {
47 | const task = job.task
48 | const inMemory = task._id.startsWith('inMemory_')
49 | const exists = inMemory ? TaskQueue.inMemory.findById(job._id) : TaskQueue.findOne({ _id: job._id })
50 |
51 | if (exists === undefined) {
52 | console.log(`[${task.taskType}][${task._id}]: removed succesfully after execution`)
53 | } else {
54 | console.error(`[${task.taskType}][${task._id}]: still in queue after execution`)
55 | }
56 | handleOtherEvents(task.data)
57 | }
58 |
59 | function onJobError(job) {
60 | const task = job.task
61 | handleOtherEvents(task.data)
62 | }
63 |
64 | function messageBroker(respond, msg) {
65 | console.log(`\n\n${msg.data}\n\n`)
66 | if (msg.data === 'ping') {
67 | respond('pong')
68 | }
69 | }
70 |
71 | const cluster = new Cluster(taskMap, { refreshRate: 500, messageBroker })
72 |
73 | function testSuite() {
74 | if (Cluster.isMaster()) {
75 | cleanup()
76 | TaskQueue.addEventListener('done', onJobDone)
77 | TaskQueue.addEventListener('error', onJobError)
78 | mongoJobs()
79 | }
80 | }
81 |
82 | Meteor.startup(testSuite)
83 |
--------------------------------------------------------------------------------
/src/tests/ipcTests.js:
--------------------------------------------------------------------------------
1 | function ipcSinglePingTest(job, toggleIPC) {
2 | return toggleIPC(
3 | (msg) => console.log(`\n\n${msg}\n\n`),
4 | (smtm) => smtm({ status: 4, data: 'ping' })
5 | )
6 | }
7 |
8 | function ipcMultiPingTest(job, toggleIPC) {
9 | return toggleIPC((msg) => {
10 | console.log(`\n\n${msg}\n\n`)
11 | return toggleIPC(
12 | (msg) => console.log(`\n\n${msg}\n\n`),
13 | (smtm) => smtm({ status: 4, data: 'ping' })
14 | )
15 | }, (smtm) => smtm({ status: 4, data: 'ping' })
16 | )
17 | }
18 |
19 | export { ipcSinglePingTest, ipcMultiPingTest }
20 |
--------------------------------------------------------------------------------
/src/tests/simpleTests.js:
--------------------------------------------------------------------------------
1 | import { Meteor } from 'meteor/meteor'
2 | import { TaskQueue } from '../index.js'
3 |
4 | function simpleTest(job) {
5 | console.log(`\n\n[simpleTest][${job._id}]: running succesfully\n\n`)
6 | }
7 |
8 | function simpleAsyncTest(job) {
9 | return new Promise((resolve, reject) => {
10 | Meteor.setTimeout(() => resolve(
11 | console.log(`\n\n[simpleAsyncTest][${job._id}]: running succesfully\n\n`)
12 | ), 1000)
13 | })
14 | }
15 |
16 |
17 | function logDiff(msDiff, taskType, _id) {
18 | if (msDiff < 0) {
19 | throw new Error(`\n\n[${taskType}][${_id}]: called too soon: diff is ${msDiff}ms\n\n`)
20 | } else if (msDiff > 5000) {
21 | throw new Error(`\n\n[${taskType}][${_id}]: called too late: diff is ${msDiff}ms\n\n`)
22 | }
23 | console.log(`\n\n[${taskType}][${_id}]: called on time: diff is ${msDiff}ms\n\n`)
24 | }
25 |
26 | function handleMsDiff(job) {
27 | const now = Date.now()
28 | const expectedDate = new Date(job.dueDate).getTime()
29 | const createdAt = new Date(job.createdAt).getTime()
30 | const wantedTime = expectedDate - createdAt
31 | const msDiff = Math.abs(now - expectedDate)
32 | logDiff(msDiff, job.taskType, job._id)
33 | }
34 |
35 | function simpleSchedTest(job) {
36 | handleMsDiff(job)
37 | }
38 |
39 | function simpleRecuringTest(job) {
40 | handleMsDiff(job)
41 | if (job.data.completed < 3 || job.data.completed === undefined) {
42 | const dueDate = new Date()
43 | dueDate.setSeconds(dueDate.getSeconds() + 5)
44 | TaskQueue.addTask({ taskType: 'simpleRecuringTest', data: { ...job.data, completed: (job.data.completed || 0) + 1, startsInMemory: job.data.completed === 2 }, dueDate })
45 | }
46 | }
47 |
48 | export {
49 | simpleTest,
50 | simpleAsyncTest,
51 | simpleSchedTest,
52 | simpleRecuringTest
53 | }
54 |
--------------------------------------------------------------------------------
/src/tests/taskMap.js:
--------------------------------------------------------------------------------
1 | import { ipcSinglePingTest, ipcMultiPingTest } from './ipcTests'
2 | import { simpleTest, simpleAsyncTest, simpleSchedTest, simpleRecuringTest } from './simpleTests'
3 |
4 | const taskMap = {
5 | simpleTest,
6 | simpleAsyncTest,
7 | simpleSchedTest,
8 | simpleRecuringTest,
9 | ipcSinglePingTest,
10 | ipcMultiPingTest
11 | }
12 |
13 | export default taskMap
14 |
--------------------------------------------------------------------------------