├── .dockerignore ├── docs ├── img │ └── grafana-1.png └── k8s-sample.yaml ├── setup ├── ci │ ├── travis_deploy.sh │ └── build_and_push.sh └── docker │ └── main.sh ├── __tests__ ├── tsconfig.json ├── setup.util.ts ├── create.util.ts ├── queueGauges.ts └── __snapshots__ │ └── queueGauges.ts.snap ├── .gitignore ├── .babelrc ├── renovate.json ├── jest.config.js ├── .releaserc.yaml ├── Dockerfile ├── tsconfig.json ├── src ├── logger.ts ├── options.ts ├── index.ts ├── queueGauges.ts ├── server.ts └── metricCollector.ts ├── LICENSE ├── .travis.yml ├── .github └── pull_request_template.md ├── package.json ├── tslint.json ├── .circleci └── config.yml ├── CHANGELOG.md ├── README.md └── bull.dashboard.py /.dockerignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | node_modules/ 3 | .idea/ 4 | -------------------------------------------------------------------------------- /docs/img/grafana-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Phonbopit/bull_exporter/master/docs/img/grafana-1.png -------------------------------------------------------------------------------- /setup/ci/travis_deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | cd "$(dirname "${BASH_SOURCE[0]}")/" 5 | 6 | ./build_and_push.sh --push 7 | -------------------------------------------------------------------------------- /__tests__/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "noEmit": true, 5 | "inlineSourceMap": false 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | npm-debug.log* 3 | yarn-debug.log* 4 | yarn-error.log* 5 | node_modules/ 6 | dist/ 7 | .idea/ 8 | src/**/*.js 9 | __tests__/**/*.js 10 | /.venv/ 11 | /__pycache__/ 12 | -------------------------------------------------------------------------------- /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "@babel/preset-env", 5 | { 6 | "targets": { 7 | "node": "current" 8 | } 9 | } 10 | ] 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base" 4 | ], 5 | "docker": { 6 | "enabled": false 7 | }, 8 | "lockFileMaintenance": { "enabled": true }, 9 | "timezone": "America/Toronto", 10 | "prHourlyLimit": 6, 11 | "rebaseStalePrs": true, 12 | "reviewers": ["GabrielCastro"], 13 | "schedule": [ 14 | "after 9am and before 3pm", 15 | "every weekday" 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /__tests__/setup.util.ts: -------------------------------------------------------------------------------- 1 | import * as crypto from 'crypto'; 2 | 3 | let currentTest: any; 4 | 5 | (jasmine as any).getEnv().addReporter({ 6 | specStarted: (result: any) => currentTest = result, 7 | }); 8 | 9 | export function getCurrentTest(): string { 10 | return currentTest.description; 11 | } 12 | 13 | export function getCurrentTestHash(): string { 14 | return crypto.createHash('md5') 15 | .update(getCurrentTest()) 16 | .digest('hex') 17 | .substr(0, 16); 18 | } 19 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | // preset: 'ts-jest/presets/js-with-babel', 4 | testEnvironment: 'node', 5 | collectCoverage: false, 6 | testPathIgnorePatterns: [ 7 | '/(dist|node_modules)/', 8 | '[.]js$', 9 | '[.]util[.][jt]s$', 10 | '[.]d[.][jt]s$', 11 | ], 12 | setupTestFrameworkScriptFile: '/__tests__/setup.util.ts', 13 | globals: { 14 | 'ts-jest': { 15 | tsConfig: '__tests__/tsconfig.json' 16 | } 17 | } 18 | }; 19 | -------------------------------------------------------------------------------- /.releaserc.yaml: -------------------------------------------------------------------------------- 1 | repositoryUrl: git@github.com:UpHabit/bull_exporter.git 2 | branch: master 3 | dryRun: false 4 | debug: false 5 | plugins: 6 | - '@semantic-release/commit-analyzer' 7 | - '@semantic-release/release-notes-generator' 8 | - '@semantic-release/changelog' 9 | - - '@semantic-release/npm' 10 | - npmPublish: false 11 | - '@semantic-release/git' 12 | - - '@semantic-release/exec' 13 | # TODO: split this into prepare & publish 14 | - successCmd: 'bash -e ./setup/ci/build_and_push.sh --push' 15 | - - '@semantic-release/github' 16 | - failComment: false 17 | -------------------------------------------------------------------------------- /setup/docker/main.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | url="${EXPORTER_REDIS_URL:-redis://localhost:6379/0}" 5 | prefix="${EXPORTER_PREFIX:-bull}" 6 | metric_prefix="${EXPORTER_STAT_PREFIX:-bull_queue_}" 7 | queues="${EXPORTER_QUEUES:-}" 8 | EXPORTER_AUTODISCOVER="${EXPORTER_AUTODISCOVER:-}" 9 | 10 | flags=( 11 | --url "$url" 12 | --prefix "$prefix" 13 | --metric-prefix "$metric_prefix" 14 | ) 15 | 16 | if [[ "$EXPORTER_AUTODISCOVER" != 0 && "$EXPORTER_AUTODISCOVER" != 'false' ]] ; then 17 | flags+=(-a) 18 | fi 19 | 20 | # shellcheck disable=2206 21 | flags+=($queues) 22 | 23 | exec node dist/src/index.js "${flags[@]}" 24 | -------------------------------------------------------------------------------- /__tests__/create.util.ts: -------------------------------------------------------------------------------- 1 | import Bull = require('bull'); 2 | import { Registry } from 'prom-client'; 3 | 4 | import { makeGuages, QueueGauges } from '../src/queueGauges'; 5 | 6 | export interface TestData { 7 | name: string; 8 | queue: Bull.Queue; 9 | prefix: string; 10 | guages: QueueGauges; 11 | registry: Registry; 12 | } 13 | 14 | export function makeQueue(name: string = 'TestQueue', prefix: string = 'test-queue'): TestData { 15 | 16 | const registry = new Registry(); 17 | const queue = new Bull(name, { prefix }); 18 | 19 | return { 20 | name, 21 | queue, 22 | prefix, 23 | registry, 24 | guages: makeGuages('test_stat_', [registry]), 25 | }; 26 | } 27 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:10-alpine as build-env 2 | 3 | RUN mkdir -p /src 4 | WORKDIR /src 5 | 6 | COPY package.json . 7 | COPY yarn.lock . 8 | RUN yarn install --pure-lockfile 9 | 10 | COPY . . 11 | RUN node_modules/.bin/tsc -p . 12 | RUN yarn install --pure-lockfile --production 13 | 14 | FROM node:10-alpine 15 | RUN apk --no-cache add tini bash 16 | ENTRYPOINT ["/sbin/tini", "--"] 17 | 18 | RUN mkdir -p /src 19 | RUN chown -R nobody:nogroup /src 20 | WORKDIR /src 21 | USER nobody 22 | 23 | COPY /setup/docker/main.sh /src/ 24 | COPY --chown=nobody:nogroup --from=build-env /src/node_modules /src/node_modules 25 | COPY --chown=nobody:nogroup --from=build-env /src/dist /src/dist 26 | 27 | CMD /src/main.sh 28 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2018", 4 | "module": "commonjs", 5 | "lib": ["es2018"], 6 | "declaration": true, 7 | "declarationMap": true, 8 | "sourceMap": true, 9 | "outDir": "dist", 10 | "importHelpers": true, 11 | "strict": true, 12 | "noImplicitAny": true, 13 | "noImplicitThis": true, 14 | "alwaysStrict": true, 15 | "noUnusedLocals": true, 16 | "noUnusedParameters": true, 17 | "noImplicitReturns": true, 18 | "noFallthroughCasesInSwitch": true, 19 | "allowSyntheticDefaultImports": true, 20 | "esModuleInterop": true, 21 | "resolveJsonModule": true 22 | }, 23 | "include": [ 24 | "src/**.ts" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /src/logger.ts: -------------------------------------------------------------------------------- 1 | import * as Logger from 'bunyan'; 2 | 3 | const streams = [{ 4 | stream: process.stdout, 5 | level: 'info' as Logger.LogLevelString, 6 | }]; 7 | 8 | export function create(name: string): Logger { 9 | return Logger.createLogger({ 10 | name, 11 | streams, 12 | serializers: { 13 | err(err: any): any { 14 | if (!err) { return err; } 15 | return { 16 | name: err.name || err.constructor.name || 'Error', 17 | message: err.message, 18 | stack: err.stack, 19 | errors: err.errors, 20 | }; 21 | }, 22 | req: Logger.stdSerializers.req, 23 | res: Logger.stdSerializers.res, 24 | }, 25 | }); 26 | } 27 | 28 | export const logger = create('bull-prom-metrics'); 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 UpHabit 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | sudo: required 3 | 4 | node_js: 5 | - 10 6 | 7 | env: 8 | global: 9 | - KUBECTL_VERSION=v1.11.0 10 | 11 | branches: 12 | only: 13 | - master 14 | - develop 15 | - /^release\/.+/ 16 | - /^ci\/.+/ 17 | 18 | services: 19 | - redis-server 20 | - docker 21 | 22 | 23 | before_script: 24 | - yarn install --pure-lockfile 25 | - yarn run build 26 | 27 | cache: 28 | directories: 29 | - node_modules 30 | - $HOME/.local 31 | 32 | script: 33 | - shellcheck -x $(git ls-files | grep '[.]sh$') 34 | - ./node_modules/.bin/madge -c . 35 | - yarn run test 36 | - yarn run lint 37 | 38 | before_deploy: 39 | - pip install --user awscli 40 | - export PATH=$PATH:$HOME/.local/bin:$HOME/bin 41 | - > 42 | yarn global add 43 | semantic-release 44 | @semantic-release/changelog 45 | @semantic-release/exec 46 | @semantic-release/git 47 | @semantic-release/github 48 | @semantic-release/npm 49 | - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin 50 | 51 | deploy: 52 | - provider: script 53 | skip_cleanup: true 54 | 'on': 55 | branch: master 56 | script: npx semantic-release 57 | 58 | - provider: script 59 | skip_cleanup: true 60 | 'on': 61 | branch: ci/semantic-release 62 | script: npx semantic-release --branch ci/semantic-release --dry-run 63 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 4 | ### Description 5 | 6 | 7 | ### This is a 8 | 9 | - [ ] Bug Fix 10 | - [ ] Feature 11 | - [ ] Documentation 12 | - [ ] Other 13 | 14 | ## Checklists 15 | #### Commit style 16 | 17 | - [ ] Changes are on a branch with a descriptive name eg. `fix/missing-queue`, `docs/setup-guide` 18 | 19 | - [ ] Commits start with one of `feat:` `fix:` `docs:` `chore:` or similar 20 | 21 | - [ ] No excessive commits, eg: there should be no `fix:` commits for bugs that existed only on the PR branch (see [guide-to-interactive-rebasing](https://hackernoon.com/beginners-guide-to-interactive-rebasing-346a3f9c3a6d)) 22 | 23 | #### Protected files 24 | 25 | The following files should not change unless they are directly a part of your change. 26 | 27 | - [ ] `yarn.lock` (unless package.json is also modified, then only the new/updated package should be changed here) 28 | 29 | - [ ] `package.json` (renovate bot should handle all routine updates) 30 | 31 | - [ ] `package-lock.json` (Should not exist as this project uses yarn) 32 | 33 | - [ ] `tsconfig.json` (only make it stricter, making it more lenient requires more discussion) 34 | 35 | - [ ] `tslint.json` (only make it stricter, making it more lenient requires more discussion) 36 | -------------------------------------------------------------------------------- /src/options.ts: -------------------------------------------------------------------------------- 1 | import yargs from 'yargs'; 2 | 3 | import { version } from '../package.json'; 4 | 5 | export interface Options { 6 | url: string; 7 | prefix: string; 8 | metricPrefix: string; 9 | once: boolean; 10 | port: number; 11 | bindAddress: string; 12 | autoDiscover: boolean; 13 | _: string[]; 14 | } 15 | 16 | export function getOptionsFromArgs(...args: string[]): Options { 17 | return yargs 18 | .version(version) 19 | .alias('V', 'version') 20 | .options({ 21 | url: { 22 | alias: 'u', 23 | describe: 'A redis connection url', 24 | default: 'redis://127.0.0.1:6379', 25 | demandOption: true, 26 | }, 27 | prefix: { 28 | alias: 'p', 29 | default: 'bull', 30 | demandOption: true, 31 | }, 32 | metricPrefix: { 33 | alias: 'm', 34 | default: 'bull_queue_', 35 | defaultDescription: 'prefix for all exported metrics', 36 | demandOption: true, 37 | }, 38 | once: { 39 | alias: 'n', 40 | default: false, 41 | type: 'boolean', 42 | description: 'Print stats and exit without starting a server', 43 | }, 44 | port: { 45 | default: 9538, 46 | }, 47 | autoDiscover: { 48 | default: false, 49 | alias: 'a', 50 | type: 'boolean', 51 | }, 52 | bindAddress: { 53 | alias: 'b', 54 | description: 'Address to listen on', 55 | default: '0.0.0.0', 56 | }, 57 | }).parse(args); 58 | } 59 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bull_exporter", 3 | "version": "1.3.7", 4 | "description": "Promethus Exporter for Bull Queues", 5 | "main": "dist/src/index.js", 6 | "repository": "git@github.com:UpHabit/bull_exporter.git", 7 | "author": "Gabriel Castro ", 8 | "license": "MIT", 9 | "scripts": { 10 | "build": "tsc -p .", 11 | "lint": "tslint -p .", 12 | "test": "jest --ci" 13 | }, 14 | "devDependencies": { 15 | "@babel/core": "7.8.3", 16 | "@babel/preset-env": "7.8.3", 17 | "@babel/preset-typescript": "7.8.3", 18 | "@types/express": "4.17.2", 19 | "@types/jest": "24.9.0", 20 | "@types/node": "10.17.13", 21 | "jest": "25.1.0", 22 | "madge": "3.6.0", 23 | "ts-jest": "24.3.0", 24 | "tslint": "5.20.1", 25 | "tslint-config-airbnb": "5.11.2", 26 | "tslint-eslint-rules": "5.4.0", 27 | "tslint-microsoft-contrib": "6.2.0", 28 | "typescript": "3.7.4" 29 | }, 30 | "dependencies": { 31 | "@types/bull": "^3.5.4", 32 | "@types/bunyan": "^1.8.5", 33 | "@types/uuid": "^3.4.4", 34 | "@types/yargs": "^15.0.0", 35 | "bull": "^3.6.0", 36 | "bunyan": "^1.8.12", 37 | "express": "^4.16.4", 38 | "prom-client": "^11.2.1", 39 | "tslib": "^1.9.3", 40 | "uuid": "^3.3.2", 41 | "yargs": "^15.1.0" 42 | }, 43 | "resolutions": { 44 | "acorn": ">=7.1.1", 45 | "kind-of": ">=6.0.3", 46 | "mkdirp": "^0.5.3", 47 | "minimist": ">=1.2.5", 48 | "@types/node": "10.17.13", 49 | "typescript": "3.7.4" 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /setup/ci/build_and_push.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | cd "$(dirname "${BASH_SOURCE[0]}")/../.." 4 | 5 | if [[ "$#" -gt 1 ]] || [[ "$#" -eq 1 && "${1:-}" != "--push" ]] ; then 6 | cat << USAGE 7 | usage ${BASH_SOURCE[0]} [--push] 8 | 9 | build and tag the docker image then optionally push it 10 | USAGE 11 | exit 1 12 | fi 13 | 14 | 15 | # shellcheck disable=2001 16 | BRANCH_NAME="$(echo "${TRAVIS_BRANCH:-}" | sed 's,/,_,g')" 17 | hash="$(git log --pretty=format:'%h' -n 1)" 18 | for tag in $(git tag -l --contains HEAD) ; do 19 | hash="${hash}-$tag" 20 | done 21 | if ! git diff --quiet ; then 22 | hash="$hash-dirty" 23 | fi 24 | 25 | docker build . -t bull_exporter:latest 26 | 27 | printf ' \xF0\x9F\x90\xB3 \xF0\x9F\x94\xA8 Done building\n' 28 | echo " uphabit/bull_exporter:latest" 29 | echo " uphabit/bull_exporter:git-$hash" 30 | 31 | docker tag bull_exporter:latest uphabit/bull_exporter:latest 32 | docker tag bull_exporter:latest uphabit/bull_exporter:"git-$hash" 33 | if [[ -n "${BRANCH_NAME:-}" ]] ; then 34 | docker tag bull_exporter:latest uphabit/bull_exporter:"branch-$BRANCH_NAME-latest" 35 | echo " uphabit/bull_exporter:branch-$BRANCH_NAME-latest" 36 | fi 37 | 38 | if [[ "${1:-}" != "--push" ]]; then 39 | exit 0 40 | fi 41 | 42 | docker push uphabit/bull_exporter:latest 43 | docker push uphabit/bull_exporter:"git-$hash" 44 | 45 | if [[ -n "${BRANCH_NAME:-}" ]] ; then 46 | docker push uphabit/bull_exporter:"branch-$BRANCH_NAME-latest" 47 | fi 48 | 49 | printf ' \xF0\x9F\x90\xB3 \xE2\xAC\x86\xEF\xB8\x8F Upload Complete\n' 50 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "tslint:recommended", 4 | "tslint-config-airbnb" 5 | ], 6 | "linterOptions": { 7 | "format": "stylish", 8 | "exclude": [ 9 | "**/*.json", 10 | "**/*.js" 11 | ] 12 | }, 13 | "rules": { 14 | "ter-arrow-parens": [true, "as-needed"], 15 | "ter-indent": [true, 2, { 16 | "SwitchCase": 1 17 | }], 18 | "import-name": false, 19 | "curly": [true], 20 | "max-line-length": [true, 255], 21 | "ordered-imports": [ 22 | true, 23 | { 24 | "import-sources-order": "case-insensitive", 25 | "named-imports-order": "case-insensitive", 26 | "grouped-imports": true, 27 | "module-source-path": "full" 28 | } 29 | ], 30 | "quotemark": [true, "single", "avoid-escape"], 31 | "variable-name": [true, "ban-keywords", "check-format", "allow-leading-underscore"], 32 | "typedef": [true, "call-signature", "parameter", "property-declaration", "member-variable-declaration"], 33 | "await-promise": [true, "RequestPromise", "Bluebird"], 34 | "no-floating-promises": [true, "RequestPromise", "Bluebird"], 35 | "object-literal-sort-keys": false, 36 | "prefer-for-of": true, 37 | "no-for-in-array": true, 38 | "deprecation": true, 39 | "ban-comma-operator": true, 40 | "interface-name": false, 41 | "interface-over-type-literal": false, 42 | "no-unnecessary-initializer": false, 43 | "array-type": false, 44 | "member-ordering": false, 45 | "max-classes-per-file": false, 46 | "unified-signatures": false, 47 | "no-namespace": false, 48 | "prefer-object-spread": true, 49 | "triple-equals": true, 50 | "eofline": true 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import promClient from 'prom-client'; 2 | 3 | import { logger } from './logger'; 4 | import { MetricCollector } from './metricCollector'; 5 | import { getOptionsFromArgs, Options } from './options'; 6 | import { startServer } from './server'; 7 | 8 | // because we explicitly want just the metrics here 9 | // tslint:disable:no-console 10 | 11 | export async function printOnce(opts: Options): Promise { 12 | const collector = new MetricCollector(opts._, { 13 | logger, 14 | metricPrefix: opts.metricPrefix, 15 | redis: opts.url, 16 | prefix: opts.prefix, 17 | autoDiscover: opts.autoDiscover, 18 | }); 19 | if (opts.autoDiscover) { 20 | await collector.discoverAll(); 21 | } 22 | await collector.updateAll(); 23 | await collector.close(); 24 | 25 | console.log(promClient.register.metrics()); 26 | } 27 | 28 | export async function runServer(opts: Options): Promise { 29 | const { done } = await startServer(opts); 30 | await done; 31 | } 32 | 33 | export async function main(...args: string[]): Promise { 34 | const opts = getOptionsFromArgs(...args); 35 | if (opts.once) { 36 | await printOnce(opts); 37 | } else { 38 | await runServer(opts); 39 | } 40 | } 41 | 42 | if (require.main === module) { 43 | const args = process.argv.slice(2); 44 | 45 | let exitCode = 0; 46 | main(...args) 47 | .catch(() => process.exitCode = exitCode = 1) 48 | .then(() => { 49 | setTimeout( 50 | () => { 51 | logger.error('No clean exit after 5 seconds, force exit'); 52 | process.exit(exitCode); 53 | }, 54 | 5000, 55 | ).unref(); 56 | }) 57 | .catch(err => { 58 | console.error('Double error'); 59 | console.error(err.stack); 60 | process.exit(-1); 61 | }); 62 | } 63 | -------------------------------------------------------------------------------- /docs/k8s-sample.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | 3 | kind: Deployment 4 | metadata: 5 | name: bull-exporter 6 | labels: 7 | app: bull 8 | role: exporter 9 | 10 | spec: 11 | selector: 12 | matchLabels: 13 | app: bull 14 | role: exporter 15 | replicas: 1 16 | template: 17 | metadata: 18 | labels: 19 | app: bull 20 | role: exporter 21 | 22 | spec: 23 | automountServiceAccountToken: false 24 | containers: 25 | - name: bull-exporter 26 | image: uphabit/bull_exporter:latest 27 | resources: 28 | requests: 29 | cpu: 100m 30 | memory: 128M 31 | limits: 32 | cpu: 200m 33 | memory: 512M 34 | 35 | securityContext: 36 | privileged: false 37 | allowPrivilegeEscalation: false 38 | readOnlyRootFilesystem: true 39 | capabilities: 40 | drop: 41 | - all 42 | env: 43 | # space delimited list of queues 44 | - name: EXPORTER_QUEUES 45 | value: "mail job_one video audio" 46 | 47 | # find the redis service in the cluster 48 | - name: EXPORTER_REDIS_URL 49 | value: redis://redis:6379/0 50 | 51 | livenessProbe: 52 | initialDelaySeconds: 30 53 | periodSeconds: 15 54 | httpGet: 55 | path: /healthz 56 | port: 5959 57 | 58 | --- 59 | apiVersion: v1 60 | kind: Service 61 | 62 | metadata: 63 | name: bull-exporter 64 | labels: 65 | app: bull 66 | role: exporter 67 | annotations: 68 | prometheus.io/scrape: 'true' 69 | prometheus.io/port: '5959' 70 | spec: 71 | type: ClusterIP 72 | ports: 73 | - name: http 74 | port: 5959 75 | targetPort: 5959 76 | selector: 77 | app: bull 78 | role: exporter 79 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.0 2 | 3 | jobs: 4 | checkout_code_with_deps: 5 | docker: 6 | - image: circleci/node:12 7 | working_directory: ~/src 8 | steps: 9 | - checkout 10 | - restore_cache: 11 | keys: 12 | - npmdeps-{{ checksum "yarn.lock" }} 13 | - run: yarn install --pure-lockfile 14 | - save_cache: 15 | key: npmdeps-{{ checksum "yarn.lock" }} 16 | paths: 17 | - ~/src/node_modules 18 | - save_cache: 19 | key: repo-with-deps-{{ .Environment.CIRCLE_SHA1 }} 20 | paths: 21 | - ~/src/ 22 | 23 | test: 24 | docker: 25 | - image: circleci/node:12 26 | - image: circleci/redis 27 | working_directory: ~/src 28 | steps: 29 | - restore_cache: 30 | keys: 31 | - repo-with-deps-{{ .Environment.CIRCLE_SHA1 }} 32 | - run: yarn run lint 33 | - run: yarn run build 34 | - run: yarn run test 35 | 36 | deploy: 37 | docker: 38 | - image: circleci/node:12 39 | working_directory: ~/src 40 | steps: 41 | - restore_cache: 42 | keys: 43 | - repo-with-deps-{{ .Environment.CIRCLE_SHA1 }} 44 | - run: | 45 | yarn global add \ 46 | semantic-release \ 47 | @semantic-release/changelog \ 48 | @semantic-release/exec \ 49 | @semantic-release/git \ 50 | @semantic-release/github \ 51 | @semantic-release/npm 52 | - run: | 53 | PATH="$PATH:$(yarn global bin)" 54 | semantic-release --dry-run 55 | 56 | workflows: 57 | version: 2 58 | all: 59 | jobs: 60 | - checkout_code_with_deps: 61 | filters: 62 | branches: 63 | only: 64 | - master 65 | - /^circleci\/.*/ 66 | - test: 67 | requires: 68 | - checkout_code_with_deps 69 | 70 | - deploy: 71 | requires: 72 | - test 73 | -------------------------------------------------------------------------------- /src/queueGauges.ts: -------------------------------------------------------------------------------- 1 | import bull from 'bull'; 2 | import { Gauge, Registry, Summary } from 'prom-client'; 3 | 4 | export interface QueueGauges { 5 | completed: Gauge; 6 | active: Gauge; 7 | delayed: Gauge; 8 | failed: Gauge; 9 | waiting: Gauge; 10 | completeSummary: Summary; 11 | } 12 | 13 | export function makeGuages(statPrefix: string, registers: Registry[]): QueueGauges { 14 | return { 15 | completed: new Gauge({ 16 | registers, 17 | name: `${statPrefix}completed`, 18 | help: 'Number of completed messages', 19 | labelNames: ['queue', 'prefix'], 20 | }), 21 | completeSummary: new Summary({ 22 | registers, 23 | name: `${statPrefix}complete_duration`, 24 | help: 'Time to complete jobs', 25 | labelNames: ['queue', 'prefix'], 26 | maxAgeSeconds: 300, 27 | ageBuckets: 13, 28 | }), 29 | active: new Gauge({ 30 | registers, 31 | name: `${statPrefix}active`, 32 | help: 'Number of active messages', 33 | labelNames: ['queue', 'prefix'], 34 | }), 35 | delayed: new Gauge({ 36 | registers, 37 | name: `${statPrefix}delayed`, 38 | help: 'Number of delayed messages', 39 | labelNames: ['queue', 'prefix'], 40 | }), 41 | failed: new Gauge({ 42 | registers, 43 | name: `${statPrefix}failed`, 44 | help: 'Number of failed messages', 45 | labelNames: ['queue', 'prefix'], 46 | }), 47 | waiting: new Gauge({ 48 | registers, 49 | name: `${statPrefix}waiting`, 50 | help: 'Number of waiting messages', 51 | labelNames: ['queue', 'prefix'], 52 | }), 53 | }; 54 | } 55 | 56 | export async function getJobCompleteStats(prefix: string, name: string, job: bull.Job, gauges: QueueGauges): Promise { 57 | if (!job.finishedOn) { 58 | return; 59 | } 60 | const duration = job.finishedOn - job.processedOn!; 61 | gauges.completeSummary.observe({ prefix, queue: name }, duration); 62 | } 63 | 64 | export async function getStats(prefix: string, name: string, queue: bull.Queue, gauges: QueueGauges): Promise { 65 | const { completed, active, delayed, failed, waiting } = await queue.getJobCounts(); 66 | 67 | gauges.completed.set({ prefix, queue: name }, completed); 68 | gauges.active.set({ prefix, queue: name }, active); 69 | gauges.delayed.set({ prefix, queue: name }, delayed); 70 | gauges.failed.set({ prefix, queue: name }, failed); 71 | gauges.waiting.set({ prefix, queue: name }, waiting); 72 | } 73 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## [1.3.7](https://github.com/UpHabit/bull_exporter/compare/v1.3.6...v1.3.7) (2020-03-31) 2 | 3 | 4 | ### Bug Fixes 5 | 6 | * **security:** Upgrade dependencies to fix security issues ([5c564be](https://github.com/UpHabit/bull_exporter/commit/5c564bec1bf4697e368313f4dcaa6a9e81faf1af)) 7 | 8 | ## [1.3.6](https://github.com/UpHabit/bull_exporter/compare/v1.3.5...v1.3.6) (2020-01-15) 9 | 10 | 11 | ### Bug Fixes 12 | 13 | * **deps:** update dependency @types/yargs to v15 ([8919b5a](https://github.com/UpHabit/bull_exporter/commit/8919b5a6e4122b9a3a05f657393b85b149d56370)) 14 | 15 | ## [1.3.5](https://github.com/UpHabit/bull_exporter/compare/v1.3.4...v1.3.5) (2019-12-19) 16 | 17 | 18 | ### Bug Fixes 19 | 20 | * **deps:** update dependency yargs to v15 ([9dc49a2](https://github.com/UpHabit/bull_exporter/commit/9dc49a28e40f43968fb5c41b72caf588129b02c0)) 21 | 22 | ## [1.3.4](https://github.com/UpHabit/bull_exporter/compare/v1.3.3...v1.3.4) (2019-11-14) 23 | 24 | 25 | ### Bug Fixes 26 | 27 | * dependency updates ([b645a3e](https://github.com/UpHabit/bull_exporter/commit/b645a3e4b9921743aeb6921ac780c1cc76ecc47b)) 28 | 29 | ## [1.3.3](https://github.com/UpHabit/bull_exporter/compare/v1.3.2...v1.3.3) (2019-10-30) 30 | 31 | 32 | ### Bug Fixes 33 | 34 | * lock file maintenance ([96b0310](https://github.com/UpHabit/bull_exporter/commit/96b031098fc97c0714643410a9f362b8bc8bd965)) 35 | 36 | ## [1.3.2](https://github.com/UpHabit/bull_exporter/compare/v1.3.1...v1.3.2) (2019-10-30) 37 | 38 | 39 | ### Bug Fixes 40 | 41 | * call collectJobCompletions after discover ([2e57d79](https://github.com/UpHabit/bull_exporter/commit/2e57d79ad7435ebdfdd4fd23979601fed0a60b22)) 42 | 43 | ## [1.3.1](https://github.com/UpHabit/bull_exporter/compare/v1.3.0...v1.3.1) (2019-08-21) 44 | 45 | 46 | ### Bug Fixes 47 | 48 | * add more patterns to match for queue discovery ([285a263](https://github.com/UpHabit/bull_exporter/commit/285a263)) 49 | 50 | # [1.3.0](https://github.com/UpHabit/bull_exporter/compare/v1.2.1...v1.3.0) (2019-08-21) 51 | 52 | 53 | ### Features 54 | 55 | * add queue discovery ([1afb598](https://github.com/UpHabit/bull_exporter/commit/1afb598)) 56 | 57 | ## [1.2.1](https://github.com/UpHabit/bull_exporter/compare/v1.2.0...v1.2.1) (2019-08-20) 58 | 59 | 60 | ### Bug Fixes 61 | 62 | * **deps:** update dependency yargs to v14 ([1014aff](https://github.com/UpHabit/bull_exporter/commit/1014aff)) 63 | 64 | # [1.2.0](https://github.com/UpHabit/bull_exporter/compare/v1.1.0...v1.2.0) (2019-07-12) 65 | 66 | 67 | ### Features 68 | 69 | * **deps:** lock file maintenance - lodash CVE ([cc785e3](https://github.com/UpHabit/bull_exporter/commit/cc785e3)) 70 | 71 | # [1.1.0](https://github.com/UpHabit/bull_exporter/compare/v1.0.0...v1.1.0) (2019-05-01) 72 | 73 | 74 | ### Features 75 | 76 | * Add grafana generator ([263f3a4](https://github.com/UpHabit/bull_exporter/commit/263f3a4)) 77 | 78 | # 1.0.0 (2019-01-25) 79 | 80 | 81 | ### Features 82 | 83 | * **ci:** Add .releaserc.yaml ([746dbf2](https://github.com/UpHabit/bull_exporter/commit/746dbf2)) 84 | -------------------------------------------------------------------------------- /src/server.ts: -------------------------------------------------------------------------------- 1 | import express from 'express'; 2 | import * as http from 'http'; 3 | import promClient from 'prom-client'; 4 | import { v4 as uuid } from 'uuid'; 5 | 6 | import { logger } from './logger'; 7 | import { MetricCollector } from './metricCollector'; 8 | import { Options } from './options'; 9 | 10 | function calcDuration(start: [number, number]): number { 11 | const diff = process.hrtime(start); 12 | return diff[0] * 1e3 + diff[1] * 1e-6; 13 | } 14 | 15 | export async function makeServer(opts: Options): Promise { 16 | const app = express(); 17 | app.disable('x-powered-by'); 18 | 19 | app.use((_req: express.Request, res: express.Response, next: express.NextFunction) => { 20 | res.header('Content-Security-Policy', `default-src 'none'; form-action 'none'`); 21 | res.header('X-Permitted-Cross-Domain-Policies', 'none'); 22 | res.header('Pragma', 'no-cache'); 23 | res.header('Cache-Control', 'no-store, no-cache, must-revalidate, proxy-revalidate'); 24 | res.header('Content-Type-Options', 'nosniff'); 25 | res.header('XSS-Protection', '1; mode=block'); 26 | next(); 27 | }); 28 | 29 | app.use((req: express.Request, res: express.Response, next: express.NextFunction) => { 30 | const start = process.hrtime(); 31 | const id = uuid(); 32 | const reqLog = logger.child({ 33 | req, 34 | req_id: id, 35 | }); 36 | 37 | res.on('finish', () => { 38 | const data = { 39 | res, 40 | duration: calcDuration(start), 41 | }; 42 | reqLog.info(data, 'request finish'); 43 | }); 44 | 45 | res.on('close', () => { 46 | const data = { 47 | res, 48 | duration: calcDuration(start), 49 | 50 | }; 51 | reqLog.warn(data, 'request socket closed'); 52 | }); 53 | 54 | next(); 55 | 56 | }); 57 | 58 | const collector = new MetricCollector(opts._, { 59 | logger, 60 | metricPrefix: opts.metricPrefix, 61 | redis: opts.url, 62 | prefix: opts.prefix, 63 | autoDiscover: opts.autoDiscover, 64 | }); 65 | 66 | if (opts.autoDiscover) { 67 | await collector.discoverAll(); 68 | } 69 | 70 | collector.collectJobCompletions(); 71 | 72 | app.post('/discover_queues', (_req: express.Request, res: express.Response, next: express.NextFunction) => { 73 | collector.discoverAll() 74 | .then(() => { 75 | res.send({ 76 | ok: true, 77 | }); 78 | }) 79 | .catch((err: any) => next(err)); 80 | }); 81 | 82 | app.get('/healthz', (_req: express.Request, res: express.Response, next: express.NextFunction) => { 83 | collector.ping() 84 | .then(() => { 85 | res.send({ 86 | ok: true, 87 | }); 88 | }) 89 | .catch((err: any) => next(err)); 90 | }); 91 | 92 | app.get('/metrics', (_req: express.Request, res: express.Response, next: express.NextFunction) => { 93 | collector.updateAll() 94 | .then(() => { 95 | res.contentType(promClient.register.contentType); 96 | res.send(promClient.register.metrics()); 97 | }) 98 | .catch(err => next(err)); 99 | }); 100 | 101 | app.use((err: any, _req: express.Request, res: express.Response, _next: express.NextFunction) => { 102 | res.status(500); 103 | res.send({ 104 | err: (err && err.message) || 'Unknown error', 105 | }); 106 | }); 107 | 108 | return app; 109 | } 110 | 111 | export async function startServer(opts: Options): Promise<{ done: Promise }> { 112 | const app = await makeServer(opts); 113 | 114 | let server: http.Server; 115 | await new Promise((resolve, reject) => { 116 | server = app.listen(opts.port, opts.bindAddress, (err: any) => { 117 | if (err) { 118 | reject(err); 119 | return; 120 | } 121 | logger.info(`Running on ${opts.bindAddress}:${opts.port}`); 122 | resolve(); 123 | }); 124 | }); 125 | 126 | process.on('SIGTERM', () => server.close()); 127 | 128 | const done = new Promise((resolve, reject) => { 129 | server.on('close', () => resolve()); 130 | server.on('error', (err: any) => reject(err)); 131 | }); 132 | 133 | return { done }; 134 | } 135 | -------------------------------------------------------------------------------- /__tests__/queueGauges.ts: -------------------------------------------------------------------------------- 1 | import * as bull from 'bull'; 2 | 3 | import { getJobCompleteStats, getStats } from '../src/queueGauges'; 4 | 5 | import { TestData } from './create.util'; 6 | import { getCurrentTestHash } from './setup.util'; 7 | 8 | let testData: TestData; 9 | 10 | beforeEach(async () => { 11 | jest.resetModuleRegistry(); 12 | const { makeQueue } = await import('./create.util'); 13 | const hash = getCurrentTestHash(); 14 | testData = makeQueue(hash); 15 | }); 16 | 17 | afterEach(async () => { 18 | await testData.queue.clean(0, 'completed'); 19 | await testData.queue.clean(0, 'active'); 20 | await testData.queue.clean(0, 'delayed'); 21 | await testData.queue.clean(0, 'failed'); 22 | await testData.queue.empty(); 23 | await testData.queue.close(); 24 | }); 25 | 26 | it('should list 1 queued job', async () => { 27 | 28 | const { 29 | name, 30 | queue, 31 | prefix, 32 | guages, 33 | registry, 34 | } = testData; 35 | 36 | await queue.add({ a: 1 }); 37 | 38 | await getStats(prefix, name, queue, guages); 39 | 40 | expect(registry.metrics()).toMatchSnapshot(); 41 | }); 42 | 43 | it('should list 1 completed job', async () => { 44 | const { 45 | name, 46 | queue, 47 | prefix, 48 | guages, 49 | registry, 50 | } = testData; 51 | 52 | queue.process(async (jobInner: bull.Job) => { 53 | expect(jobInner).toMatchObject({ data: { a: 1 } }); 54 | }); 55 | const job = await queue.add({ a: 1 }); 56 | await job.finished(); 57 | 58 | await getStats(prefix, name, queue, guages); 59 | await getJobCompleteStats(prefix, name, job, guages); 60 | 61 | expect(registry.metrics()).toMatchSnapshot(); 62 | }); 63 | 64 | it('should list 1 completed job with delay', async () => { 65 | const { 66 | name, 67 | queue, 68 | prefix, 69 | guages, 70 | registry, 71 | } = testData; 72 | 73 | queue.process(async (jobInner: bull.Job) => { 74 | expect(jobInner).toMatchObject({ data: { a: 1 } }); 75 | }); 76 | const job = await queue.add({ a: 1 }); 77 | await job.finished(); 78 | 79 | // TODO: https://github.com/DefinitelyTyped/DefinitelyTyped/pull/31567 80 | // TODO: file bug with bull? finishedOn and processedOn are not set when we call finish 81 | const doneJob: any = await queue.getJob(job.id); 82 | // lie about job duration 83 | doneJob.finishedOn = doneJob.processedOn + 1000; 84 | 85 | await getStats(prefix, name, queue, guages); 86 | await getJobCompleteStats(prefix, name, doneJob, guages); 87 | 88 | expect(registry.metrics()).toMatchSnapshot(); 89 | }); 90 | 91 | it('should list 1 failed job', async () => { 92 | const { 93 | name, 94 | queue, 95 | prefix, 96 | guages, 97 | registry, 98 | } = testData; 99 | 100 | queue.process(async (jobInner: bull.Job) => { 101 | expect(jobInner).toMatchObject({ data: { a: 1 } }); 102 | throw new Error('expected'); 103 | }); 104 | const job = await queue.add({ a: 1 }); 105 | 106 | await expect(job.finished()).rejects.toThrow(/expected/); 107 | 108 | await getStats(prefix, name, queue, guages); 109 | 110 | expect(registry.metrics()).toMatchSnapshot(); 111 | }); 112 | 113 | it('should list 1 delayed job', async () => { 114 | const { 115 | name, 116 | queue, 117 | prefix, 118 | guages, 119 | registry, 120 | } = testData; 121 | 122 | await queue.add({ a: 1 }, { delay: 100_000 }); 123 | 124 | await getStats(prefix, name, queue, guages); 125 | 126 | expect(registry.metrics()).toMatchSnapshot(); 127 | }); 128 | 129 | it('should list 1 active job', async () => { 130 | const { 131 | name, 132 | queue, 133 | prefix, 134 | guages, 135 | registry, 136 | } = testData; 137 | 138 | let jobStartedResolve!: () => void; 139 | let jobDoneResolve!: () => void; 140 | const jobStartedPromise = new Promise(resolve => jobStartedResolve = resolve); 141 | const jobDonePromise = new Promise(resolve => jobDoneResolve = resolve); 142 | 143 | queue.process(async () => { 144 | jobStartedResolve(); 145 | await jobDonePromise; 146 | }); 147 | const job = await queue.add({ a: 1 }); 148 | 149 | await jobStartedPromise; 150 | await getStats(prefix, name, queue, guages); 151 | expect(registry.metrics()).toMatchSnapshot(); 152 | jobDoneResolve(); 153 | await job.finished(); 154 | 155 | await getStats(prefix, name, queue, guages); 156 | expect(registry.metrics()).toMatchSnapshot(); 157 | }); 158 | -------------------------------------------------------------------------------- /src/metricCollector.ts: -------------------------------------------------------------------------------- 1 | import bull from 'bull'; 2 | import * as Logger from 'bunyan'; 3 | import { EventEmitter } from 'events'; 4 | import IoRedis from 'ioredis'; 5 | import { register as globalRegister, Registry } from 'prom-client'; 6 | 7 | import { logger as globalLogger } from './logger'; 8 | import { getJobCompleteStats, getStats, makeGuages, QueueGauges } from './queueGauges'; 9 | 10 | export interface MetricCollectorOptions extends Omit { 11 | metricPrefix: string; 12 | redis: string; 13 | autoDiscover: boolean; 14 | logger: Logger; 15 | } 16 | 17 | export interface QueueData { 18 | queue: bull.Queue; 19 | name: string; 20 | prefix: string; 21 | } 22 | 23 | export class MetricCollector { 24 | 25 | private readonly logger: Logger; 26 | 27 | private readonly defaultRedisClient: IoRedis.Redis; 28 | private readonly redisUri: string; 29 | private readonly bullOpts: Omit; 30 | private readonly queuesByName: Map> = new Map(); 31 | 32 | private get queues(): QueueData[] { 33 | return [...this.queuesByName.values()]; 34 | } 35 | 36 | private readonly myListeners: Set<(id: string) => Promise> = new Set(); 37 | 38 | private readonly guages: QueueGauges; 39 | 40 | constructor( 41 | queueNames: string[], 42 | opts: MetricCollectorOptions, 43 | registers: Registry[] = [globalRegister], 44 | ) { 45 | const { logger, autoDiscover, redis, metricPrefix, ...bullOpts } = opts; 46 | this.redisUri = redis; 47 | this.defaultRedisClient = new IoRedis(this.redisUri); 48 | this.defaultRedisClient.setMaxListeners(32); 49 | this.bullOpts = bullOpts; 50 | this.logger = logger || globalLogger; 51 | this.addToQueueSet(queueNames); 52 | this.guages = makeGuages(metricPrefix, registers); 53 | } 54 | 55 | private createClient(_type: 'client' | 'subscriber' | 'bclient', redisOpts?: IoRedis.RedisOptions): IoRedis.Redis { 56 | if (_type === 'client') { 57 | return this.defaultRedisClient!; 58 | } 59 | return new IoRedis(this.redisUri, redisOpts); 60 | } 61 | 62 | private addToQueueSet(names: string[]): void { 63 | for (const name of names) { 64 | if (this.queuesByName.has(name)) { 65 | continue; 66 | } 67 | this.logger.info('added queue', name); 68 | this.queuesByName.set(name, { 69 | name, 70 | queue: new bull(name, { 71 | ...this.bullOpts, 72 | createClient: this.createClient.bind(this), 73 | }), 74 | prefix: this.bullOpts.prefix || 'bull', 75 | }); 76 | } 77 | } 78 | 79 | public async discoverAll(): Promise { 80 | const keyPattern = new RegExp(`^${this.bullOpts.prefix}:([^:]+):(id|failed|active|waiting|stalled-check)$`); 81 | this.logger.info({ pattern: keyPattern.source }, 'running queue discovery'); 82 | 83 | const keyStream = this.defaultRedisClient.scanStream({ 84 | match: `${this.bullOpts.prefix}:*:*`, 85 | }); 86 | // tslint:disable-next-line:await-promise tslint does not like Readable's here 87 | for await (const keyChunk of keyStream) { 88 | for (const key of keyChunk) { 89 | const match = keyPattern.exec(key); 90 | if (match && match[1]) { 91 | this.addToQueueSet([match[1]]); 92 | } 93 | } 94 | } 95 | } 96 | 97 | private async onJobComplete(queue: QueueData, id: string): Promise { 98 | try { 99 | const job = await queue.queue.getJob(id); 100 | if (!job) { 101 | this.logger.warn({ job: id }, 'unable to find job from id'); 102 | return; 103 | } 104 | await getJobCompleteStats(queue.prefix, queue.name, job, this.guages); 105 | } catch (err) { 106 | this.logger.error({ err, job: id }, 'unable to fetch completed job'); 107 | } 108 | } 109 | 110 | public collectJobCompletions(): void { 111 | for (const q of this.queues) { 112 | const cb = this.onJobComplete.bind(this, q); 113 | this.myListeners.add(cb); 114 | q.queue.on('global:completed', cb); 115 | } 116 | } 117 | 118 | public async updateAll(): Promise { 119 | const updatePromises = this.queues.map(q => getStats(q.prefix, q.name, q.queue, this.guages)); 120 | await Promise.all(updatePromises); 121 | } 122 | 123 | public async ping(): Promise { 124 | await this.defaultRedisClient.ping(); 125 | } 126 | 127 | public async close(): Promise { 128 | this.defaultRedisClient.disconnect(); 129 | for (const q of this.queues) { 130 | for (const l of this.myListeners) { 131 | (q.queue as any as EventEmitter).removeListener('global:completed', l); 132 | } 133 | } 134 | await Promise.all(this.queues.map(q => q.queue.close())); 135 | } 136 | 137 | } 138 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bull Queue Exporter 2 | **Prometheus exporter for Bull metrics.** 3 | 4 |

5 | 6 | 7 | 8 |
9 |

10 |

11 | 12 | 13 | 14 | 15 | 16 | 17 |

18 | 19 | ___ 20 | 21 | 22 | ## UI 23 | ![Grafana Dashboard](./docs/img/grafana-1.png) 24 | 25 | ## Setup 26 | #### Prometheus 27 | **An existing prometheus server is required to use this project** 28 | 29 | To learn more about how to setup promethues and grafana see: https://eksworkshop.com/monitoring/ 30 | 31 | #### Grafana 32 | The dashboard pictured above is [available to download from grafana](https://grafana.com/grafana/dashboards/10128). 33 | It will work aslong as EXPORTER_STAT_PREFIX is not changed. 34 | 35 | ## Queue Discovery 36 | Queues are discovered at start up by running `KEYS bull:*:id` 37 | this can also be triggered manually from the `/discover_queues` endpoint 38 | `curl -XPOST localhost:9538/discover_queues` 39 | 40 | ## Metrics 41 | 42 | | Metric | type | description | 43 | |------------------------------|---------|-------------| 44 | | bull_queue_completed | counter | Total number of completed jobs | 45 | | bull_queue_complete_duration | summary | Processing time for completed jobs | 46 | | bull_queue_active | counter | Total number of active jobs (currently being processed) | 47 | | bull_queue_delayed | counter | Total number of jobs that will run in the future | 48 | | bull_queue_failed | counter | Total number of failed jobs | 49 | | bull_queue_waiting | counter | Total number of jobs waiting to be processed | 50 | 51 | ## Kubernetes Usage 52 | 53 | ### Environment variables for default docker image 54 | 55 | | variable | default | description | 56 | |-----------------------|--------------------------|-------------------------------------------------| 57 | | EXPORTER_REDIS_URL | redis://localhost:6379/0 | Redis uri to connect | 58 | | EXPORTER_PREFIX | bull | prefix for queues | 59 | | EXPORTER_STAT_PREFIX | bull_queue_ | prefix for exported metrics | 60 | | EXPORTER_QUEUES | - | a space separated list of queues to check | 61 | | EXPORTER_AUTODISCOVER | - | set to '0' or 'false' to disable queue discovery| 62 | 63 | 64 | ### Example deployment 65 | 66 | see: [k8s-sample.yaml](./docs/k8s-sample.yaml) for more options 67 | 68 | ```yaml 69 | apiVersion: apps/v1 70 | 71 | kind: Deployment 72 | metadata: 73 | name: bull-exporter 74 | labels: 75 | app: bull 76 | role: exporter 77 | 78 | spec: 79 | selector: 80 | matchLabels: 81 | app: bull 82 | role: exporter 83 | replicas: 1 84 | template: 85 | metadata: 86 | labels: 87 | app: bull 88 | role: exporter 89 | spec: 90 | containers: 91 | - name: bull-exporter 92 | image: uphabit/bull_exporter:latest 93 | securityContext: 94 | runAsGroup: 65534 # nobody 95 | runAsUser: 65534 # nobody 96 | runAsNonRoot: true 97 | privileged: false 98 | allowPrivilegeEscalation: false 99 | readOnlyRootFilesystem: true 100 | capabilities: 101 | drop: 102 | - all 103 | resources: 104 | requests: 105 | cpu: 100m 106 | memory: 128M 107 | limits: 108 | cpu: 200m 109 | memory: 512M 110 | env: 111 | # space delimited list of queues 112 | - name: EXPORTER_QUEUES 113 | value: "mail job_one video audio" 114 | 115 | # find the redis service in the cluster 116 | - name: EXPORTER_REDIS_URL 117 | value: redis://redis:6379/0 118 | --- 119 | apiVersion: v1 120 | kind: Service 121 | metadata: 122 | name: bull-exporter 123 | labels: 124 | app: bull 125 | role: exporter 126 | annotations: 127 | prometheus.io/scrape: 'true' 128 | prometheus.io/port: '9538' 129 | spec: 130 | type: ClusterIP 131 | ports: 132 | - name: http 133 | port: 9538 134 | targetPort: 9538 135 | selector: 136 | app: bull 137 | role: exporter 138 | 139 | ``` 140 | -------------------------------------------------------------------------------- /bull.dashboard.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ''':' 3 | 4 | TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 5 | FILE="$TOP/$(basename "${BASH_SOURCE[0]}")" 6 | 7 | if [[ ! -d "$TOP/.venv" ]] ; then 8 | echo "Creating venv in $TOP/.venv" 9 | python3 -m venv "$TOP/.venv" 10 | fi 11 | 12 | source "$TOP/.venv/bin/activate" 13 | 14 | if ! pip show grafanalib >/dev/null 2>&1 ; then 15 | echo "Installing grafanalib" 16 | pip install grafanalib 17 | fi 18 | 19 | exec generate-dashboard "$FILE" "$@" 20 | ''' 21 | 22 | from grafanalib.core import * 23 | 24 | prefix = 'bull' 25 | 26 | dashboard = Dashboard( 27 | title="Bull Queue", 28 | links=[DashboardLink( 29 | uri='https://github.com/UpHabit/bull_exporter', 30 | title='About Bull', 31 | type='link', 32 | dashboard=None, 33 | )], 34 | rows=[ 35 | Row(panels=[ 36 | Graph( 37 | title="Queue Length", 38 | dataSource='Prometheus', 39 | transparent=True, 40 | targets=[ 41 | Target( 42 | expr=f'sum({prefix}_queue_waiting) by (prefix, queue)', 43 | legendFormat="{{ queue }}", 44 | refId='A', 45 | ), 46 | ], 47 | yAxes=[ 48 | YAxis(format=OPS_FORMAT), 49 | YAxis(format=OPS_FORMAT), 50 | ], 51 | ), 52 | Graph( 53 | title="Queue Length", 54 | dataSource='Prometheus', 55 | transparent=True, 56 | targets=[ 57 | Target( 58 | expr=f'sum({prefix}_queue_waiting / rate({prefix}_queue_completed[5m])) by (queue, prefix)', 59 | legendFormat="{{ queue }}", 60 | refId='A', 61 | ), 62 | ], 63 | yAxes=[ 64 | YAxis(format=OPS_FORMAT), 65 | YAxis(format=OPS_FORMAT), 66 | ], 67 | ), 68 | ]), 69 | Row(panels=[ 70 | Graph( 71 | title="Queue States", 72 | dataSource='Prometheus', 73 | transparent=True, 74 | seriesOverrides=[{ 75 | "alias": "Complete Rate", 76 | "yaxis": 2 77 | }, { 78 | "alias": "Fail Rate", 79 | "yaxis": 2 80 | }], 81 | yAxes=[ 82 | YAxis(format='short'), 83 | YAxis(format='opm'), 84 | ], 85 | targets=[ 86 | Target( 87 | expr=f'sum(rate({prefix}_queue_completed[5m])) * 60', 88 | legendFormat="Complete Rate", 89 | refId='A', 90 | ), 91 | Target( 92 | expr=f'sum(rate({prefix}_queue_failed[5m])) * 60', 93 | legendFormat="Fail Rate", 94 | refId='B', 95 | ), 96 | Target( 97 | expr=f'sum({prefix}_queue_active)', 98 | legendFormat="Active", 99 | refId='C', 100 | ), 101 | Target( 102 | expr=f'sum({prefix}_queue_waiting)', 103 | legendFormat="Waiting", 104 | refId='D', 105 | ), 106 | Target( 107 | expr=f'sum({prefix}_queue_delayed)', 108 | legendFormat="Delayed", 109 | refId='E', 110 | ), 111 | Target( 112 | expr=f'sum({prefix}_queue_failed)', 113 | legendFormat="Failed", 114 | refId='F', 115 | ), 116 | ], 117 | ), 118 | Graph( 119 | title="Failures By Queue", 120 | dataSource='Prometheus', 121 | transparent=True, 122 | targets=[ 123 | Target( 124 | expr=f'max({prefix}_queue_failed) by (queue)', 125 | legendFormat="{{ queue }}", 126 | refId='A', 127 | ) 128 | ] 129 | ), 130 | ]), 131 | Row(panels=[ 132 | Graph( 133 | title="Job Duration 90th Percentile", 134 | dataSource='Prometheus', 135 | transparent=True, 136 | yAxes=[ 137 | YAxis(format=MILLISECONDS_FORMAT), 138 | YAxis(format=MILLISECONDS_FORMAT), 139 | ], 140 | targets=[ 141 | Target( 142 | expr=f'sum({prefix}_queue_complete_duration {{ quantile = "0.9" }}) by (queue)', 143 | legendFormat="{{ queue }}", 144 | refId='A', 145 | ) 146 | ] 147 | ), 148 | ]), 149 | ], 150 | ).auto_panel_ids() 151 | -------------------------------------------------------------------------------- /__tests__/__snapshots__/queueGauges.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`should list 1 active job 1`] = ` 4 | "# HELP test_stat_completed Number of completed messages 5 | # TYPE test_stat_completed gauge 6 | test_stat_completed{prefix=\\"test-queue\\",queue=\\"2dc07c0ff9d27ef5\\"} 0 7 | 8 | # HELP test_stat_complete_duration Time to complete jobs 9 | # TYPE test_stat_complete_duration summary 10 | 11 | # HELP test_stat_active Number of active messages 12 | # TYPE test_stat_active gauge 13 | test_stat_active{prefix=\\"test-queue\\",queue=\\"2dc07c0ff9d27ef5\\"} 1 14 | 15 | # HELP test_stat_delayed Number of delayed messages 16 | # TYPE test_stat_delayed gauge 17 | test_stat_delayed{prefix=\\"test-queue\\",queue=\\"2dc07c0ff9d27ef5\\"} 0 18 | 19 | # HELP test_stat_failed Number of failed messages 20 | # TYPE test_stat_failed gauge 21 | test_stat_failed{prefix=\\"test-queue\\",queue=\\"2dc07c0ff9d27ef5\\"} 0 22 | 23 | # HELP test_stat_waiting Number of waiting messages 24 | # TYPE test_stat_waiting gauge 25 | test_stat_waiting{prefix=\\"test-queue\\",queue=\\"2dc07c0ff9d27ef5\\"} 0 26 | " 27 | `; 28 | 29 | exports[`should list 1 active job 2`] = ` 30 | "# HELP test_stat_completed Number of completed messages 31 | # TYPE test_stat_completed gauge 32 | test_stat_completed{prefix=\\"test-queue\\",queue=\\"2dc07c0ff9d27ef5\\"} 1 33 | 34 | # HELP test_stat_complete_duration Time to complete jobs 35 | # TYPE test_stat_complete_duration summary 36 | 37 | # HELP test_stat_active Number of active messages 38 | # TYPE test_stat_active gauge 39 | test_stat_active{prefix=\\"test-queue\\",queue=\\"2dc07c0ff9d27ef5\\"} 0 40 | 41 | # HELP test_stat_delayed Number of delayed messages 42 | # TYPE test_stat_delayed gauge 43 | test_stat_delayed{prefix=\\"test-queue\\",queue=\\"2dc07c0ff9d27ef5\\"} 0 44 | 45 | # HELP test_stat_failed Number of failed messages 46 | # TYPE test_stat_failed gauge 47 | test_stat_failed{prefix=\\"test-queue\\",queue=\\"2dc07c0ff9d27ef5\\"} 0 48 | 49 | # HELP test_stat_waiting Number of waiting messages 50 | # TYPE test_stat_waiting gauge 51 | test_stat_waiting{prefix=\\"test-queue\\",queue=\\"2dc07c0ff9d27ef5\\"} 0 52 | " 53 | `; 54 | 55 | exports[`should list 1 completed job 1`] = ` 56 | "# HELP test_stat_completed Number of completed messages 57 | # TYPE test_stat_completed gauge 58 | test_stat_completed{prefix=\\"test-queue\\",queue=\\"7cc276b43bb43656\\"} 1 59 | 60 | # HELP test_stat_complete_duration Time to complete jobs 61 | # TYPE test_stat_complete_duration summary 62 | 63 | # HELP test_stat_active Number of active messages 64 | # TYPE test_stat_active gauge 65 | test_stat_active{prefix=\\"test-queue\\",queue=\\"7cc276b43bb43656\\"} 0 66 | 67 | # HELP test_stat_delayed Number of delayed messages 68 | # TYPE test_stat_delayed gauge 69 | test_stat_delayed{prefix=\\"test-queue\\",queue=\\"7cc276b43bb43656\\"} 0 70 | 71 | # HELP test_stat_failed Number of failed messages 72 | # TYPE test_stat_failed gauge 73 | test_stat_failed{prefix=\\"test-queue\\",queue=\\"7cc276b43bb43656\\"} 0 74 | 75 | # HELP test_stat_waiting Number of waiting messages 76 | # TYPE test_stat_waiting gauge 77 | test_stat_waiting{prefix=\\"test-queue\\",queue=\\"7cc276b43bb43656\\"} 0 78 | " 79 | `; 80 | 81 | exports[`should list 1 completed job with delay 1`] = ` 82 | "# HELP test_stat_completed Number of completed messages 83 | # TYPE test_stat_completed gauge 84 | test_stat_completed{prefix=\\"test-queue\\",queue=\\"672d03e07bff9cd3\\"} 1 85 | 86 | # HELP test_stat_complete_duration Time to complete jobs 87 | # TYPE test_stat_complete_duration summary 88 | test_stat_complete_duration{quantile=\\"0.01\\",prefix=\\"test-queue\\",queue=\\"672d03e07bff9cd3\\"} 1000 89 | test_stat_complete_duration{quantile=\\"0.05\\",prefix=\\"test-queue\\",queue=\\"672d03e07bff9cd3\\"} 1000 90 | test_stat_complete_duration{quantile=\\"0.5\\",prefix=\\"test-queue\\",queue=\\"672d03e07bff9cd3\\"} 1000 91 | test_stat_complete_duration{quantile=\\"0.9\\",prefix=\\"test-queue\\",queue=\\"672d03e07bff9cd3\\"} 1000 92 | test_stat_complete_duration{quantile=\\"0.95\\",prefix=\\"test-queue\\",queue=\\"672d03e07bff9cd3\\"} 1000 93 | test_stat_complete_duration{quantile=\\"0.99\\",prefix=\\"test-queue\\",queue=\\"672d03e07bff9cd3\\"} 1000 94 | test_stat_complete_duration{quantile=\\"0.999\\",prefix=\\"test-queue\\",queue=\\"672d03e07bff9cd3\\"} 1000 95 | test_stat_complete_duration_sum{prefix=\\"test-queue\\",queue=\\"672d03e07bff9cd3\\"} 1000 96 | test_stat_complete_duration_count{prefix=\\"test-queue\\",queue=\\"672d03e07bff9cd3\\"} 1 97 | 98 | # HELP test_stat_active Number of active messages 99 | # TYPE test_stat_active gauge 100 | test_stat_active{prefix=\\"test-queue\\",queue=\\"672d03e07bff9cd3\\"} 0 101 | 102 | # HELP test_stat_delayed Number of delayed messages 103 | # TYPE test_stat_delayed gauge 104 | test_stat_delayed{prefix=\\"test-queue\\",queue=\\"672d03e07bff9cd3\\"} 0 105 | 106 | # HELP test_stat_failed Number of failed messages 107 | # TYPE test_stat_failed gauge 108 | test_stat_failed{prefix=\\"test-queue\\",queue=\\"672d03e07bff9cd3\\"} 0 109 | 110 | # HELP test_stat_waiting Number of waiting messages 111 | # TYPE test_stat_waiting gauge 112 | test_stat_waiting{prefix=\\"test-queue\\",queue=\\"672d03e07bff9cd3\\"} 0 113 | " 114 | `; 115 | 116 | exports[`should list 1 delayed job 1`] = ` 117 | "# HELP test_stat_completed Number of completed messages 118 | # TYPE test_stat_completed gauge 119 | test_stat_completed{prefix=\\"test-queue\\",queue=\\"4f2249f79c9d68b5\\"} 0 120 | 121 | # HELP test_stat_complete_duration Time to complete jobs 122 | # TYPE test_stat_complete_duration summary 123 | 124 | # HELP test_stat_active Number of active messages 125 | # TYPE test_stat_active gauge 126 | test_stat_active{prefix=\\"test-queue\\",queue=\\"4f2249f79c9d68b5\\"} 0 127 | 128 | # HELP test_stat_delayed Number of delayed messages 129 | # TYPE test_stat_delayed gauge 130 | test_stat_delayed{prefix=\\"test-queue\\",queue=\\"4f2249f79c9d68b5\\"} 1 131 | 132 | # HELP test_stat_failed Number of failed messages 133 | # TYPE test_stat_failed gauge 134 | test_stat_failed{prefix=\\"test-queue\\",queue=\\"4f2249f79c9d68b5\\"} 0 135 | 136 | # HELP test_stat_waiting Number of waiting messages 137 | # TYPE test_stat_waiting gauge 138 | test_stat_waiting{prefix=\\"test-queue\\",queue=\\"4f2249f79c9d68b5\\"} 0 139 | " 140 | `; 141 | 142 | exports[`should list 1 failed job 1`] = ` 143 | "# HELP test_stat_completed Number of completed messages 144 | # TYPE test_stat_completed gauge 145 | test_stat_completed{prefix=\\"test-queue\\",queue=\\"dd304535bf11c117\\"} 0 146 | 147 | # HELP test_stat_complete_duration Time to complete jobs 148 | # TYPE test_stat_complete_duration summary 149 | 150 | # HELP test_stat_active Number of active messages 151 | # TYPE test_stat_active gauge 152 | test_stat_active{prefix=\\"test-queue\\",queue=\\"dd304535bf11c117\\"} 0 153 | 154 | # HELP test_stat_delayed Number of delayed messages 155 | # TYPE test_stat_delayed gauge 156 | test_stat_delayed{prefix=\\"test-queue\\",queue=\\"dd304535bf11c117\\"} 0 157 | 158 | # HELP test_stat_failed Number of failed messages 159 | # TYPE test_stat_failed gauge 160 | test_stat_failed{prefix=\\"test-queue\\",queue=\\"dd304535bf11c117\\"} 1 161 | 162 | # HELP test_stat_waiting Number of waiting messages 163 | # TYPE test_stat_waiting gauge 164 | test_stat_waiting{prefix=\\"test-queue\\",queue=\\"dd304535bf11c117\\"} 0 165 | " 166 | `; 167 | 168 | exports[`should list 1 queued job 1`] = ` 169 | "# HELP test_stat_completed Number of completed messages 170 | # TYPE test_stat_completed gauge 171 | test_stat_completed{prefix=\\"test-queue\\",queue=\\"54cceb2dc1d55935\\"} 0 172 | 173 | # HELP test_stat_complete_duration Time to complete jobs 174 | # TYPE test_stat_complete_duration summary 175 | 176 | # HELP test_stat_active Number of active messages 177 | # TYPE test_stat_active gauge 178 | test_stat_active{prefix=\\"test-queue\\",queue=\\"54cceb2dc1d55935\\"} 0 179 | 180 | # HELP test_stat_delayed Number of delayed messages 181 | # TYPE test_stat_delayed gauge 182 | test_stat_delayed{prefix=\\"test-queue\\",queue=\\"54cceb2dc1d55935\\"} 0 183 | 184 | # HELP test_stat_failed Number of failed messages 185 | # TYPE test_stat_failed gauge 186 | test_stat_failed{prefix=\\"test-queue\\",queue=\\"54cceb2dc1d55935\\"} 0 187 | 188 | # HELP test_stat_waiting Number of waiting messages 189 | # TYPE test_stat_waiting gauge 190 | test_stat_waiting{prefix=\\"test-queue\\",queue=\\"54cceb2dc1d55935\\"} 1 191 | " 192 | `; 193 | --------------------------------------------------------------------------------