├── .nvmrc ├── .dockerignore ├── .gitignore ├── media └── bullmq-monitor.png ├── .eslintrc ├── configs └── config-local.json ├── src ├── logger.ts ├── controllers │ ├── metrics.ts │ ├── views │ │ └── login.ts │ └── dashboard.ts ├── config.ts ├── server.ts ├── utils.ts ├── monitor │ ├── promQueue.ts │ └── promMetricsCollector.ts └── app.ts ├── Dockerfile ├── .github ├── dependabot.yml └── workflows │ ├── test.yml │ └── release.yml ├── CHANGELOG.md ├── LICENSE ├── tsconfig.json ├── package.json └── README.md /.nvmrc: -------------------------------------------------------------------------------- 1 | v18.12.0 -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .npmrc 3 | dist -------------------------------------------------------------------------------- /media/bullmq-monitor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ron96g/bullmq-exporter/HEAD/media/bullmq-monitor.png -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "es2021": true 5 | }, 6 | "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"], 7 | "overrides": [], 8 | "parser": "@typescript-eslint/parser", 9 | "parserOptions": { 10 | "ecmaVersion": "latest", 11 | "sourceType": "module" 12 | }, 13 | "plugins": ["@typescript-eslint"], 14 | "rules": {} 15 | } 16 | -------------------------------------------------------------------------------- /configs/config-local.json: -------------------------------------------------------------------------------- 1 | { 2 | "redis": { 3 | "host": "localhost:6379/", 4 | "username": "default", 5 | "password": "redispw", 6 | "ssl": false 7 | }, 8 | "cookieSecret": "myCookieSecret123!", 9 | "cookieMaxAge": "1h", 10 | "users": [ 11 | { 12 | "username": "admin", 13 | "password": "password", 14 | "role": "admin" 15 | }, 16 | { 17 | "username": "user", 18 | "password": "password", 19 | "role": "user" 20 | } 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /src/logger.ts: -------------------------------------------------------------------------------- 1 | import winston, { transports } from "winston"; 2 | 3 | const logLevel = process.env.LOG_LEVEL || "info"; 4 | 5 | export const winstonLoggerOpts: winston.LoggerOptions = { 6 | level: "info", 7 | format: winston.format.json(), 8 | defaultMeta: { service: "bullmq-exporter" }, 9 | transports: [ 10 | new transports.Console({ 11 | level: logLevel, 12 | }), 13 | ], 14 | }; 15 | 16 | const logger = winston.createLogger(winstonLoggerOpts); 17 | 18 | export default logger; 19 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG NODE_VERSION="18.12.0" 2 | ARG ALPINE_VERSION="3.16" 3 | 4 | FROM node:$NODE_VERSION as builder 5 | 6 | WORKDIR /app 7 | 8 | COPY ["package.json", "package-lock.json*", "./"] 9 | 10 | RUN npm install 11 | COPY . . 12 | RUN npm run build 13 | 14 | 15 | FROM node:$NODE_VERSION-alpine$ALPINE_VERSION 16 | 17 | WORKDIR /app 18 | 19 | COPY ["package.json", "package-lock.json*", "./"] 20 | 21 | RUN npm install --omit=dev 22 | 23 | COPY --from=builder /app/dist /app 24 | COPY ./configs /app/configs 25 | 26 | CMD [ "node", "server.js" ] 27 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "npm" 9 | directory: "/" 10 | schedule: 11 | interval: "daily" 12 | - package-ecosystem: "docker" 13 | directory: "/" 14 | schedule: 15 | interval: "daily" 16 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 1.0.0 (2023-05-30) 4 | 5 | 6 | ### Features 7 | 8 | * logging; refactoring; formatting; updated dependencies; updated docs ([4efedb1](https://github.com/ron96G/bullmq-exporter/commit/4efedb118fd79ba5d5740801e78ed973ebd98c87)) 9 | * redis config may be set using env; updated readme ([8a2dcd2](https://github.com/ron96G/bullmq-exporter/commit/8a2dcd21012bc144060e7b6cd6cb4d3b5257627c)) 10 | 11 | 12 | ### Bug Fixes 13 | 14 | * **typo:** fixed typo in readme ([f6030a1](https://github.com/ron96G/bullmq-exporter/commit/f6030a1801d93562b02ca316a191e5ae5b2e3557)) 15 | -------------------------------------------------------------------------------- /src/controllers/metrics.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | 3 | export interface IFMetricsCollector { 4 | collectSerialized: () => Promise; 5 | discoverAllQueues: () => Promise; 6 | } 7 | 8 | export function ConfigureRoutes( 9 | app: Router, 10 | metricsCollector: IFMetricsCollector 11 | ) { 12 | app.get(`/prometheus/metrics`, async (req, res) => { 13 | const metrics = await metricsCollector.collectSerialized(); 14 | res.header("content-type", "text/plain"); 15 | res.header("Pragma", "no-cache"); 16 | res.header( 17 | "Cache-Control", 18 | "no-store, no-cache, must-revalidate, proxy-revalidate" 19 | ); 20 | res.header("Content-Type-Options", "nosniff"); 21 | res.status(200).send(metrics); 22 | }); 23 | 24 | app.get(`/discoverQueues`, async (req, res) => { 25 | const queues = await metricsCollector.discoverAllQueues(); 26 | res.status(200).json(queues); 27 | }); 28 | } 29 | -------------------------------------------------------------------------------- /src/config.ts: -------------------------------------------------------------------------------- 1 | import { readFileSync } from "fs"; 2 | 3 | export interface Config { 4 | redis: { 5 | host: string; 6 | username: string; 7 | password: string; 8 | ssl: boolean; 9 | }; 10 | cookieSecret: string; 11 | cookieMaxAge: string; 12 | users?: Array; 13 | } 14 | 15 | function setFromEnv(key: string, def: any) { 16 | const val = process.env[key.toUpperCase()]; 17 | return val === undefined ? def : val; 18 | } 19 | 20 | const prefix = process.env.NODE_ENV?.toLowerCase() || "local"; 21 | const jsonRaw = readFileSync(`./configs/config-${prefix}.json`); 22 | const config = JSON.parse(jsonRaw.toString()) as Config; 23 | 24 | config.redis.host = setFromEnv("REDIS_HOST", config.redis.host); 25 | config.redis.username = setFromEnv("REDIS_USERNAME", config.redis.username); 26 | config.redis.password = setFromEnv("REDIS_PASSWORD", config.redis.password); 27 | config.redis.ssl = setFromEnv("REDIS_SSL", config.redis.ssl); 28 | 29 | export default config; 30 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - renovate/** 8 | - dependabot/** 9 | pull_request: 10 | types: 11 | - opened 12 | - synchronize 13 | 14 | permissions: 15 | contents: read 16 | 17 | jobs: 18 | test_matrix: 19 | strategy: 20 | matrix: 21 | node-version: 22 | - 18 23 | - 19 24 | runs-on: ubuntu-latest 25 | steps: 26 | - uses: actions/checkout@v3 27 | - name: Use Node.js ${{ matrix.node-version }} 28 | uses: actions/setup-node@v3 29 | with: 30 | node-version: ${{ matrix.node-version }} 31 | cache: npm 32 | - run: npm clean-install 33 | - run: npm run test 34 | test: 35 | runs-on: ubuntu-latest 36 | needs: test_matrix 37 | steps: 38 | - uses: actions/checkout@v3 39 | - uses: actions/setup-node@v3 40 | with: 41 | node-version: lts/* 42 | cache: npm 43 | - run: npm clean-install 44 | - run: npm audit signatures 45 | - run: npm run lint 46 | -------------------------------------------------------------------------------- /src/server.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import http from "http"; 3 | import https from "https"; 4 | import { app } from "./app"; 5 | import logger from "./logger"; 6 | import { handleShutdown } from "./utils"; 7 | 8 | const keyPath = process.env.TLS_KEY_FILE || "/etc/tls/tls.key"; 9 | const crtPath = process.env.TLS_CRT_FILE || "/etc/tls/tls.crt"; 10 | 11 | const tlsCredentials = { 12 | key: fs.existsSync(keyPath) ? fs.readFileSync(keyPath, "utf8") : "", 13 | cert: fs.existsSync(crtPath) ? fs.readFileSync(crtPath, "utf8") : "", 14 | }; 15 | 16 | const tlsEnabled = tlsCredentials.cert && tlsCredentials.key; 17 | 18 | let server; 19 | let port: number; 20 | 21 | if (tlsEnabled) { 22 | server = https.createServer(tlsCredentials, app); 23 | port = +(process.env.HTTPS_PORT || 8443); 24 | } else { 25 | server = http.createServer(app); 26 | port = +(process.env.HTTP_PORT || 8080); 27 | } 28 | handleShutdown(server); 29 | 30 | if (process.env.NODE_ENV !== "test") { 31 | server.listen(port, () => 32 | logger.info( 33 | `Service listening on ${tlsEnabled ? "https" : "http"}://0.0.0.0:${port}` 34 | ) 35 | ); 36 | } 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022-2023 Ron Gummich 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "include": ["src/**/*.ts"], 3 | "compilerOptions": { 4 | /* Visit https://aka.ms/tsconfig.json to read more about this file */ 5 | 6 | /* Basic Options */ 7 | "resolveJsonModule": true, 8 | "target": "es2020" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, 9 | "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, 10 | "declaration": true /* Generates corresponding '.d.ts' file. */, 11 | "outDir": "dist" /* Redirect output structure to the directory. */, 12 | "rootDir": "src" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */, 13 | "strict": true /* Enable all strict type-checking options. */, 14 | "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, 15 | "experimentalDecorators": true /* Enables experimental support for ES7 decorators. */, 16 | "emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */, 17 | 18 | /* Advanced Options */ 19 | "skipLibCheck": true /* Skip type checking of declaration files. */, 20 | "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | tags: 8 | - "*" 9 | 10 | permissions: 11 | contents: write 12 | pull-requests: write 13 | 14 | jobs: 15 | test: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v3 19 | - uses: actions/setup-node@v3 20 | with: 21 | node-version: lts/* 22 | cache: npm 23 | - run: npm clean-install 24 | - run: npm audit signatures 25 | release-please: 26 | runs-on: ubuntu-latest 27 | needs: test 28 | steps: 29 | - uses: google-github-actions/release-please-action@v3 30 | with: 31 | release-type: node 32 | package-name: release-please-action 33 | dockerize: 34 | name: Build and push Docker image to Docker Hub 35 | environment: default 36 | needs: release-please 37 | runs-on: ubuntu-latest 38 | steps: 39 | - uses: actions/checkout@v3 40 | - id: meta 41 | uses: docker/metadata-action@v4 42 | with: 43 | images: docker.io/rgummich/bullmq-exporter 44 | flavor: latest=true 45 | tags: | 46 | type=ref,event=branch 47 | type=ref,event=pr 48 | type=semver,pattern={{version}} 49 | - uses: docker/login-action@v2 50 | with: 51 | username: ${{ secrets.DOCKER_USERNAME }} 52 | password: ${{ secrets.DOCKER_PASSWORD }} 53 | - uses: docker/build-push-action@v4 54 | with: 55 | push: true 56 | tags: ${{ steps.meta.outputs.tags }} 57 | labels: ${{ steps.meta.outputs.labels }} 58 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | interface Closeable { 2 | close(cb?: (err?: Error) => void): void; 3 | } 4 | 5 | export function handleShutdown(closable: Closeable) { 6 | const exitCB = (err?: Error) => { 7 | if (err) console.error(`Exit failed with ${err}`); 8 | }; 9 | process.on("SIGINT", () => { 10 | closable.close(exitCB); 11 | }); 12 | process.on("SIGQUIT", () => { 13 | closable.close(exitCB); 14 | }); 15 | process.on("SIGTERM", () => { 16 | closable.close(exitCB); 17 | }); 18 | } 19 | 20 | interface FutureCloseable { 21 | close(cb?: (err?: Error) => void): Promise; 22 | } 23 | 24 | export function handleFutureShutdown(closable: FutureCloseable) { 25 | const exitCB = (err?: Error) => { 26 | if (err) console.error(`Exit failed with ${err}`); 27 | }; 28 | 29 | process.on("SIGINT", async () => { 30 | console.log("received SIGINT"); 31 | try { 32 | await closable.close(exitCB); 33 | } catch (e: unknown) { 34 | console.log(e); 35 | } 36 | console.log("closed"); 37 | }); 38 | process.on("SIGQUIT", async () => { 39 | console.log("received SIGQUIT"); 40 | try { 41 | await closable.close(exitCB); 42 | } catch (e: unknown) { 43 | console.log(e); 44 | } 45 | console.log("closed"); 46 | }); 47 | process.on("SIGTERM", async () => { 48 | console.log("received SIGTERM"); 49 | try { 50 | await closable.close(exitCB); 51 | } catch (e: unknown) { 52 | console.log(e); 53 | } 54 | console.log("closed"); 55 | }); 56 | } 57 | 58 | export function formatConnectionString( 59 | url: string, 60 | username?: string, 61 | password?: string, 62 | ssl = true 63 | ): string { 64 | const accessData = 65 | username && password 66 | ? `${username}:${password}@` 67 | : username 68 | ? `${username}@` 69 | : ""; 70 | return `${ssl ? "rediss" : "redis"}://${accessData}${url}`; 71 | } 72 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bullmq-exporter", 3 | "version": "1.0.0", 4 | "description": "Service that can be used to monitor BullMQ by providing Prometheus metrics and a Bullmq dashboard secured behind a login wall.", 5 | "main": "src/server.ts", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 0", 8 | "lint": "eslint . --ext ts", 9 | "prettier": "prettier -w src/**/*.ts", 10 | "build": "tsc", 11 | "clean": "rm -rf node_modules dist", 12 | "run": "node dist/server.js", 13 | "dev": "ts-node src/server.ts", 14 | "ts-node": "ts-node src/server.ts", 15 | "nodemon": "nodemon" 16 | }, 17 | "keywords": [], 18 | "author": "", 19 | "license": "ISC", 20 | "dependencies": { 21 | "@bull-board/api": "^4.11.0", 22 | "@bull-board/express": "^4.11.0", 23 | "bullmq": "^3.6.2", 24 | "connect-ensure-login": "^0.1.1", 25 | "ejs": "^3.1.8", 26 | "express": "^4.18.2", 27 | "express-session": "^1.17.3", 28 | "express-winston": "^4.2.0", 29 | "ioredis": "^5.3.0", 30 | "parse-duration": "^1.0.2", 31 | "passport": "^0.6.0", 32 | "passport-local": "^1.0.0", 33 | "prom-client": "^14.1.1", 34 | "winston": "^3.8.2" 35 | }, 36 | "devDependencies": { 37 | "@types/connect-ensure-login": "^0.1.7", 38 | "@types/ejs": "^3.1.1", 39 | "@types/express": "^4.17.17", 40 | "@types/express-session": "^1.17.5", 41 | "@types/passport": "^1.0.11", 42 | "@types/passport-local": "^1.0.35", 43 | "@typescript-eslint/eslint-plugin": "^5.50.0", 44 | "@typescript-eslint/parser": "^5.50.0", 45 | "eslint": "^8.33.0", 46 | "husky": "^8.0.3", 47 | "nodemon": "^2.0.20", 48 | "prettier": "^2.8.3", 49 | "ts-node": "^10.9.1", 50 | "typescript": "^4.9.5" 51 | }, 52 | "nodemonConfig": { 53 | "watch": [ 54 | "src" 55 | ], 56 | "ext": "ts", 57 | "exec": "npx ts-node src/server.ts" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/monitor/promQueue.ts: -------------------------------------------------------------------------------- 1 | import * as bullmq from "bullmq"; 2 | import { PrometheusMetrics } from "./promMetricsCollector"; 3 | 4 | interface MonitoredQueueOptions { 5 | bullmqOpts: bullmq.QueueBaseOptions; 6 | name: string; 7 | metricsPrefix?: string; 8 | } 9 | 10 | async function sleep(ms: number) { 11 | return new Promise((resolve) => setTimeout(resolve, ms)); 12 | } 13 | 14 | /** 15 | * @see https://github.com/taskforcesh/bullmq/blob/master/docs/gitbook/api/bullmq.queueeventslistener.md 16 | */ 17 | export class PrometheusMonitoredQueue extends bullmq.QueueEvents { 18 | metrics: PrometheusMetrics; 19 | queue: bullmq.Queue; 20 | canceled = false; 21 | 22 | constructor( 23 | name: string, 24 | metrics: PrometheusMetrics, 25 | opts: MonitoredQueueOptions 26 | ) { 27 | super(name, opts.bullmqOpts); 28 | this.queue = new bullmq.Queue(name, opts.bullmqOpts); 29 | this.metrics = metrics; 30 | this.on("completed", this.onCompleted); 31 | this.loop(2000); 32 | } 33 | 34 | async onCompleted(completedJob: { jobId: string }) { 35 | const job = await this.queue.getJob(completedJob.jobId); 36 | if (!job) { 37 | return; 38 | } 39 | 40 | const completedDuration = job.finishedOn! - job.timestamp!; // both cannot be null 41 | const processedDuration = job.finishedOn! - job.processedOn!; // both cannot be null 42 | this.metrics.completedDuration 43 | .labels({ queue: this.name }) 44 | .observe(completedDuration); 45 | this.metrics.processedDuration 46 | .labels({ queue: this.name }) 47 | .observe(processedDuration); 48 | } 49 | 50 | async loop(ms = 5000) { 51 | while (this.canceled === false) { 52 | await this.updateGauges(); 53 | await sleep(ms); 54 | } 55 | console.log("Stopped updating gauges for " + this.name); 56 | } 57 | 58 | async updateGauges() { 59 | const { completed, active, delayed, failed, waiting } = 60 | await this.queue.getJobCounts(); 61 | this.metrics.activeGauge.labels({ queue: this.name }).set(active); 62 | this.metrics.completedGauge.labels({ queue: this.name }).set(completed); 63 | this.metrics.delayedGauge.labels({ queue: this.name }).set(delayed); 64 | this.metrics.failedGauge.labels({ queue: this.name }).set(failed); 65 | this.metrics.waitingGauge.labels({ queue: this.name }).set(waiting); 66 | } 67 | 68 | async close() { 69 | this.canceled = true; 70 | await super.close(); 71 | await this.queue.close(); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/controllers/views/login.ts: -------------------------------------------------------------------------------- 1 | import ejs from "ejs"; 2 | 3 | const loginPage = ` 4 | 75 | 76 | 88 | `; 89 | 90 | export function renderLoginPage(invalid: boolean, loginPath: string) { 91 | return ejs.render(loginPage, { invalid: invalid, loginPath: loginPath }); 92 | } 93 | -------------------------------------------------------------------------------- /src/app.ts: -------------------------------------------------------------------------------- 1 | import express from "express"; 2 | import Redis from "ioredis"; 3 | import expressWinston, { 4 | LoggerOptions as ExpressWinstonOpts, 5 | } from "express-winston"; 6 | import config from "./config"; 7 | import { 8 | ConfigureRoutes as ConfigureDashboardRoutes, 9 | User, 10 | } from "./controllers/dashboard"; 11 | import { ConfigureRoutes as ConfigureMetricsRoute } from "./controllers/metrics"; 12 | import logger, { winstonLoggerOpts } from "./logger"; 13 | import { PrometheusMetricsCollector } from "./monitor/promMetricsCollector"; 14 | import { formatConnectionString, handleFutureShutdown } from "./utils"; 15 | 16 | export const app = express(); 17 | app.disable("x-powered-by"); 18 | 19 | const expressWinstonOpts: ExpressWinstonOpts = { 20 | ...(winstonLoggerOpts as ExpressWinstonOpts), 21 | meta: false, 22 | ignoreRoute: function (req, _res) { 23 | return req.path.includes("/health"); 24 | }, 25 | }; 26 | app.use(expressWinston.logger(expressWinstonOpts)); 27 | 28 | app.get("/health", async (_req, res) => { 29 | res.status(200).send("OK"); 30 | }); 31 | 32 | const username = config.redis.username; 33 | const password = config.redis.password; 34 | const host = config.redis.host; 35 | 36 | if (username === undefined || password === undefined || host === undefined) { 37 | process.exit(125); 38 | } 39 | 40 | const enableSsl = config.redis.ssl; 41 | const prefix = process.env.NODE_ENV?.toLowerCase() || "local"; 42 | const cookieSecret = config.cookieSecret; 43 | const cookieMaxAge = config.cookieMaxAge; 44 | const defaultUsers: Array = [ 45 | { username: "admin", password: "secret", role: "admin" }, 46 | { username: "user", password: "secret", role: "user" }, 47 | ]; 48 | 49 | const users = config.users || defaultUsers; 50 | 51 | const redisConnString = formatConnectionString( 52 | host, 53 | username, 54 | password, 55 | enableSsl 56 | ); 57 | 58 | export const metricsCollector = new PrometheusMetricsCollector("monitor", { 59 | bullmqOpts: { 60 | prefix: prefix, 61 | }, 62 | client: new Redis(redisConnString, { maxRetriesPerRequest: null }), 63 | queues: [], 64 | }); 65 | 66 | handleFutureShutdown(metricsCollector); 67 | 68 | const dashboardRouter = express.Router(); 69 | app.use("/bullmq", dashboardRouter); 70 | 71 | metricsCollector 72 | .discoverAllQueues() 73 | .then((queues) => { 74 | logger.info(`Discovered ${queues.length} queues`); 75 | ConfigureDashboardRoutes(dashboardRouter, { 76 | basePath: "/bullmq", 77 | queues: metricsCollector.monitoredQueues.map((q) => q.queue), 78 | cookieSecret: cookieSecret, 79 | cookieMaxAge: cookieMaxAge, 80 | users: users, 81 | }); 82 | ConfigureMetricsRoute(app, metricsCollector); 83 | }) 84 | .catch((err) => { 85 | console.error(err); 86 | process.exit(125); 87 | }); 88 | -------------------------------------------------------------------------------- /src/controllers/dashboard.ts: -------------------------------------------------------------------------------- 1 | import { Router } from "express"; 2 | import express from "express"; 3 | import session from "express-session"; 4 | import { ExpressAdapter } from "@bull-board/express"; 5 | import { createBullBoard } from "@bull-board/api"; 6 | import { BullMQAdapter } from "@bull-board/api/bullMQAdapter"; 7 | import passport from "passport"; 8 | import { Strategy as LocalStrategy } from "passport-local"; 9 | import { ensureLoggedIn } from "connect-ensure-login"; 10 | import { Queue } from "bullmq"; 11 | import { renderLoginPage } from "./views/login"; 12 | import logger from "../logger"; 13 | import parse from "parse-duration"; 14 | 15 | export interface User { 16 | username: string; 17 | password: string; 18 | role: string; 19 | } 20 | 21 | export interface DashboardOptions { 22 | basePath: string; 23 | users: User[]; 24 | cookieSecret: string; 25 | cookieMaxAge: string; 26 | queues: Array; 27 | } 28 | 29 | let users: Map = new Map(); 30 | 31 | passport.use( 32 | new LocalStrategy(function (username, password, cb) { 33 | const user = users.get(username); 34 | if (user !== undefined && user.password === password) { 35 | return cb(null, { user: user.username, role: user.role }); 36 | } 37 | return cb(null, false); 38 | }) 39 | ); 40 | 41 | passport.serializeUser((user, cb) => { 42 | cb(null, user); 43 | }); 44 | 45 | passport.deserializeUser((user, cb) => { 46 | cb(null, user as any); 47 | }); 48 | 49 | const ensureRole = (options: any): express.RequestHandler => { 50 | return (req, res, next) => { 51 | const user = (req.session as any)?.passport.user; 52 | if (user.role == options.role) return next(); 53 | res.redirect(options.failureRedirect); 54 | }; 55 | }; 56 | 57 | export function ConfigureRoutes(app: Router, opts: DashboardOptions) { 58 | const basePath = opts.basePath; 59 | const cookieSecret = opts.cookieSecret; 60 | const queues = opts.queues; 61 | const cookieMaxAge = parse(opts.cookieMaxAge); 62 | users = new Map(opts.users.map((u) => [u.username, u])); 63 | 64 | logger.info( 65 | `Setting up routes for dashboard with basePath ${ 66 | basePath == "" ? "/" : basePath 67 | }` 68 | ); 69 | const failedLoginRedirect = basePath + "/ui/login?invalid=true"; 70 | const requireLoginRedirect = basePath + "/ui/login"; 71 | 72 | app.use(passport.initialize()); 73 | app.use( 74 | session({ 75 | secret: cookieSecret, 76 | saveUninitialized: true, 77 | resave: true, 78 | cookie: { maxAge: cookieMaxAge }, 79 | }) 80 | ); 81 | app.use(express.urlencoded({ extended: false })); 82 | app.use(passport.session()); 83 | 84 | app.get(`/ui/login`, (req, res) => { 85 | res.send( 86 | renderLoginPage(req.query.invalid === "true", requireLoginRedirect) 87 | ); 88 | }); 89 | 90 | app.post( 91 | `/ui/login`, 92 | passport.authenticate("local", { failureRedirect: failedLoginRedirect }), 93 | (req, res) => { 94 | const user = (req.session as any)?.passport.user; 95 | if (user.role == "admin") return res.redirect(`${basePath}/ui/admin`); 96 | return res.redirect(`${basePath}/ui`); 97 | } 98 | ); 99 | 100 | // readOnly bull board 101 | const readOnlyAdapter = new ExpressAdapter(); 102 | readOnlyAdapter.setBasePath(`${basePath}/ui`); 103 | createBullBoard({ 104 | queues: queues.map((q) => new BullMQAdapter(q, { readOnlyMode: true })), 105 | serverAdapter: readOnlyAdapter, 106 | }); 107 | 108 | app.use( 109 | `/ui`, 110 | ensureLoggedIn({ redirectTo: requireLoginRedirect }), 111 | readOnlyAdapter.getRouter() 112 | ); 113 | 114 | // admin bull board 115 | const adminAdapter = new ExpressAdapter(); 116 | adminAdapter.setBasePath(`${basePath}/ui/admin`); 117 | createBullBoard({ 118 | queues: queues.map((q) => new BullMQAdapter(q)), 119 | serverAdapter: adminAdapter, 120 | }); 121 | 122 | app.use( 123 | `/ui/admin`, 124 | ensureLoggedIn({ redirectTo: requireLoginRedirect }), 125 | ensureRole({ role: "admin", failureRedirect: `${basePath}/ui` }), 126 | adminAdapter.getRouter() 127 | ); 128 | } 129 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Docker](https://img.shields.io/badge/docker-%230db7ed.svg?style=for-the-badge&logo=docker&logoColor=white)](https://hub.docker.com/r/rgummich/bullmq-exporter) 2 | 3 |
4 | 5 |
6 | 7 | Service that acts as a central component to monitor BullMQ: 8 | 9 | - Exposes a BullMQ dashboard which is per default behind a login page 10 | - Acts as a [Prometheus-Exporter](https://prometheus.io/docs/instrumenting/exporters/) to collect metrics about queues in BullMQ 11 | 12 | ## Implementation 13 | 14 | The following section will provide a brief overview of the libraries and practices used in the implementation of this service. 15 | 16 | ### BullMQ Dashboard 17 | 18 | Implemented by using [@bull-board](https://github.com/felixmosh/bull-board) and secured using [passport](https://www.passportjs.org/). 19 | 20 | ### Prometheus Exporter 21 | 22 | Strongly influenced by [bull_exporter](https://github.com/UpHabit/bull_exporter). Which uses the old bull library. 23 | 24 | Implemented by using the [bullmq](https://docs.bullmq.io/) library (specifically the [QueueEvents](https://docs.bullmq.io/guide/events) class) and [prom-client](https://github.com/siimon/prom-client). 25 | 26 | For each queue a class extending the QueueEvents class is created. This class listens for the following events: `completed`. Whenever an eventListener is triggered, a [histogram](https://prometheus.io/docs/concepts/metric_types/#histogram) is updated with 27 | 28 | 1. the duration between the start of the processing and the end of the job 29 | 2. the duration between the creation of the job and the end of its processing. 30 | 31 | Furthermore, a cron job is executed every n seconds which collects the current status of the queues (`completed`, `active`, `delayed`, `failed`, `waiting` jobs) and writes them to a [gauge](https://prometheus.io/docs/concepts/metric_types/#gauge). 32 | 33 | Thus, the following metrics are collected: 34 | 35 | | Metric | Type | Description | 36 | | ------------------------- | --------- | ------------------------------------------------------- | 37 | | bullmq_processed_duration | histogram | Processing time for completed jobs | 38 | | bullmq_completed_duration | histogram | Completion time for jobs | 39 | | bullmq_completed | gauge | Total number of completed jobs | 40 | | bullmq_active | gauge | Total number of active jobs (currently being processed) | 41 | | bullmq_delayed | gauge | Total number of jobs that will run in the future | 42 | | bullmq_failed | gauge | Total number of failed jobs | 43 | | bullmq_waiting | gauge | Total number of jobs waiting to be processed | 44 | 45 | Each metric also has the attribute `queue` which indicated which queue the metric is associated with. 46 | 47 | ## How to use 48 | 49 | ### Variables 50 | 51 | These environment variables may be set to overwrite the values in the config file. 52 | Note that not all values are supported. 53 | 54 | | Name | Description | 55 | | -------------- | ----------------------------------- | 56 | | REDIS_HOST | Redis host, e. g. "localhost:6379/" | 57 | | REDIS_USERNAME | Redis username | 58 | | REDIS_PASSWORD | Redis password | 59 | | REDIS_SSL | Whether to use ssl | 60 | 61 | ### Local 62 | 63 | 1. Install the dependencies 64 | 65 | ```bash 66 | npm install 67 | ``` 68 | 69 | 2. Default environment is `local`. This can be set using the `NODE_ENV` variable. 70 | 71 | ```bash 72 | export NODE_ENV=production 73 | ``` 74 | 75 | 3. Make sure that the required config file is present: `./configs/config-${NODE_ENV}.json` (see [local](./configs/config-local.json)). 76 | 4. Start the server 77 | 78 | ```bash 79 | npm run dev 80 | ``` 81 | 82 | 5. Access the resources `http://localhost:8080/bullmq/ui/login` or `http://localhost:8080/prometheus/metrics` 83 | 84 | ### Docker 85 | 86 | The Dockerimage is published using the [local](./configs/config-local.json) configuration. In most cases that will not be sufficient and should be overwritten. 87 | This can be done using environment variables (see [here](#variables)) or by mounting a separate file. 88 | 89 | ```bash 90 | # This needs a config file under ./configs/config-dev.json 91 | docker run \ 92 | -it \ 93 | --mount type=bind,source=$(pwd)/configs,target=/app/configs \ 94 | --env=NODE_ENV=dev \ 95 | --env=REDIS_HOST=some-host:6379/ \ 96 | rgummich/bullmq-exporter 97 | ``` 98 | 99 | ### Kubernetes 100 | 101 | In Kubernetes this may be done using [Secrets](https://kubernetes.io/docs/concepts/configuration/secret/). 102 | -------------------------------------------------------------------------------- /src/monitor/promMetricsCollector.ts: -------------------------------------------------------------------------------- 1 | import * as bullmq from "bullmq"; 2 | import Redis from "ioredis"; 3 | import * as prom_client from "prom-client"; 4 | import logger from "../logger"; 5 | import { PrometheusMonitoredQueue } from "./promQueue"; 6 | 7 | export interface MetricsCollectorOptions { 8 | bullmqOpts: bullmq.QueueBaseOptions; 9 | client: Redis; 10 | queues?: Array; 11 | } 12 | 13 | export interface PrometheusMetrics { 14 | completedGauge: prom_client.Gauge; 15 | activeGauge: prom_client.Gauge; 16 | delayedGauge: prom_client.Gauge; 17 | failedGauge: prom_client.Gauge; 18 | waitingGauge: prom_client.Gauge; 19 | completedDuration: prom_client.Histogram; 20 | processedDuration: prom_client.Histogram; 21 | } 22 | 23 | export class PrometheusMetricsCollector { 24 | registry: prom_client.Registry; 25 | monitoredQueues: Array = []; 26 | 27 | name: string; 28 | bullmqOpts: bullmq.QueueBaseOptions; 29 | defaultRedisClient: Redis; 30 | 31 | metrics: PrometheusMetrics | undefined; 32 | 33 | constructor(name: string, opts: MetricsCollectorOptions) { 34 | this.registry = new prom_client.Registry(); 35 | this.name = name; 36 | this.bullmqOpts = opts.bullmqOpts ?? { 37 | connection: { maxRetriesPerRequest: null }, 38 | }; 39 | this.defaultRedisClient = opts.client; 40 | this.registerMetrics(this.registry); 41 | 42 | if (opts.queues) { 43 | this.registerQueues(opts.queues); 44 | } else { 45 | this.discoverAllQueues() 46 | .then((queues) => { 47 | logger.info(`Discovered ${queues.length} queues`); 48 | }) 49 | .catch((err) => { 50 | logger.error(`Failed to discover queues: ${err}`); 51 | process.exit(125); 52 | }); 53 | } 54 | } 55 | 56 | registerMetrics(reg: prom_client.Registry, prefix = "") { 57 | this.metrics = { 58 | completedGauge: new prom_client.Gauge({ 59 | name: `${prefix}bullmq_completed`, 60 | help: "Total number of completed jobs", 61 | labelNames: ["queue"], 62 | }), 63 | activeGauge: new prom_client.Gauge({ 64 | name: `${prefix}bullmq_active`, 65 | help: "Total number of active jobs (currently being processed)", 66 | labelNames: ["queue"], 67 | }), 68 | failedGauge: new prom_client.Gauge({ 69 | name: `${prefix}bullmq_failed`, 70 | help: "Total number of failed jobs", 71 | labelNames: ["queue"], 72 | }), 73 | delayedGauge: new prom_client.Gauge({ 74 | name: `${prefix}bullmq_delayed`, 75 | help: "Total number of jobs that will run in the future", 76 | labelNames: ["queue"], 77 | }), 78 | waitingGauge: new prom_client.Gauge({ 79 | name: `${prefix}bullmq_waiting`, 80 | help: "Total number of jobs waiting to be processed", 81 | labelNames: ["queue"], 82 | }), 83 | processedDuration: new prom_client.Histogram({ 84 | name: `${prefix}bullmq_processed_duration`, 85 | help: "Processing time for completed jobs (processing until completed)", 86 | buckets: [5, 50, 100, 250, 500, 750, 1000, 2500], 87 | labelNames: ["queue"], 88 | }), 89 | completedDuration: new prom_client.Histogram({ 90 | name: `${prefix}bullmq_completed_duration`, 91 | help: "Completion time for jobs (created until completed)", 92 | buckets: [5, 50, 100, 250, 500, 750, 1000, 2500, 5000, 10000], 93 | labelNames: ["queue"], 94 | }), 95 | }; 96 | 97 | Object.values(this.metrics).forEach((metric) => reg.registerMetric(metric)); 98 | } 99 | 100 | async discoverAllQueues() { 101 | const keyPattern = new RegExp( 102 | `^${this.bullmqOpts.prefix}:([^:]+):(id|failed|active|waiting|stalled-check)$` 103 | ); 104 | const keyStream = await this.defaultRedisClient.scanStream({ 105 | match: `${this.bullmqOpts.prefix}:*:*`, 106 | }); 107 | 108 | const queues = new Set(); 109 | for await (const keyChunk of keyStream) { 110 | for (const key of keyChunk) { 111 | const match = keyPattern.exec(key); 112 | if (match && match[1]) { 113 | queues.add(match[1]); 114 | } 115 | } 116 | } 117 | this.registerQueues(Array.from(queues)); 118 | return Array.from(queues); 119 | } 120 | 121 | registerQueues(queues: Array) { 122 | this.monitoredQueues = queues.map( 123 | (queueName) => 124 | new PrometheusMonitoredQueue(queueName, this.metrics!, { 125 | bullmqOpts: { 126 | ...this.bullmqOpts, 127 | connection: this.defaultRedisClient, 128 | }, 129 | name: queueName, 130 | }) 131 | ); 132 | } 133 | 134 | async collect() { 135 | return await this.registry.metrics(); 136 | } 137 | 138 | async collectSerialized() { 139 | return await this.collect(); 140 | } 141 | 142 | async close() { 143 | logger.debug("Closing metrics collector"); 144 | try { 145 | const val = await this.defaultRedisClient.quit(); 146 | logger.debug("Successfully quit redis connection", "response", val); 147 | } catch (e: unknown) { 148 | logger.warn("Failed to quit redis connection", "error", e); 149 | } 150 | return this.monitoredQueues.forEach(async (q) => await q.close()); 151 | } 152 | } 153 | --------------------------------------------------------------------------------