├── .github └── workflows │ ├── build.yml │ ├── osv-scanner.yml │ └── release.yml ├── .gitignore ├── .npmignore ├── CHANGELOG.md ├── Dockerfile ├── HISTORY.md ├── LICENSE ├── README.md ├── app.js ├── lib ├── cmd.ts ├── index.ts ├── interfaces │ ├── integration.ts │ └── responders.ts ├── queue-factory.ts ├── queue-factory │ ├── bullmqv3-factory.ts │ ├── bullmqv4-factory.ts │ └── bullmqv5-factory.ts ├── queues-cache.ts ├── responders │ ├── bull-responders.ts │ ├── bullmq-responders.ts │ ├── index.ts │ └── respond.ts ├── socket.ts ├── utils.ts ├── version-checker.ts ├── ws-autoreconnect.ts └── ws-errors.enum.ts ├── package-lock.json ├── package.json ├── tests └── cli.spec.js ├── tsconfig.json └── yarn.lock /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build container 2 | 3 | on: 4 | push: 5 | 6 | jobs: 7 | run: 8 | name: Build and publish 9 | runs-on: ubuntu-latest 10 | steps: 11 | - 12 | name: Checkout 13 | uses: actions/checkout@v4 14 | - 15 | name: Set up Docker Buildx 16 | uses: docker/setup-buildx-action@v3 17 | - 18 | name: Cache Docker layers 19 | uses: actions/cache@v4 20 | with: 21 | path: /tmp/.buildx-cache 22 | key: ${{ runner.os }}-buildx-${{ github.sha }} 23 | restore-keys: | 24 | ${{ runner.os }}-buildx- 25 | - 26 | name: Login to GitHub Container Registry 27 | uses: docker/login-action@v3 28 | with: 29 | registry: ghcr.io 30 | username: ${{ github.repository_owner }} 31 | password: ${{ secrets.CR_PAT }} 32 | - 33 | name: Build and push 34 | id: docker_build 35 | uses: docker/build-push-action@v6 36 | with: 37 | push: true # push to registry 38 | pull: true # always fetch the latest base images 39 | tags: ghcr.io/taskforcesh/taskforce-connector:latest 40 | cache-from: type=local,src=/tmp/.buildx-cache 41 | cache-to: type=local,dest=/tmp/.buildx-cache-new 42 | - 43 | # Temp cache move fix until https://github.com/docker/buildx/pull/535 is released 44 | name: Move cache 45 | run: | 46 | rm -rf /tmp/.buildx-cache 47 | mv /tmp/.buildx-cache-new /tmp/.buildx-cache 48 | - 49 | name: Image digest 50 | run: echo ${{ steps.docker_build.outputs.digest }} -------------------------------------------------------------------------------- /.github/workflows/osv-scanner.yml: -------------------------------------------------------------------------------- 1 | name: OSV-Scanner Scheduled Scan 2 | 3 | on: 4 | schedule: 5 | - cron: '30 12 * * 1' 6 | pull_request: 7 | branches: [master] 8 | merge_group: 9 | branches: [master] 10 | 11 | permissions: 12 | # Required to upload SARIF file to CodeQL. See: https://github.com/github/codeql-action/issues/2117 13 | actions: read 14 | # Require writing security events to upload SARIF file to security tab 15 | security-events: write 16 | # Only need to read contents 17 | contents: read 18 | 19 | jobs: 20 | scan-scheduled: 21 | uses: 'google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@v2.0.0' 22 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | branches: 5 | - master 6 | jobs: 7 | release: 8 | name: Release 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@v2 13 | with: 14 | fetch-depth: 0 15 | - name: Setup Node.js 16 | uses: actions/setup-node@v3 17 | with: 18 | node-version: 20 19 | - name: Install dependencies 20 | run: yarn install --frozen-lockfile --non-interactive 21 | - name: Release 22 | env: 23 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 24 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 25 | run: npx semantic-release 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (http://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # Typescript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | dist/* 61 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (http://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # Typescript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # [1.35.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.34.0...v1.35.0) (2025-01-16) 2 | 3 | 4 | ### Features 5 | 6 | * updates Dockerfile and README with support for TLS flag usage ([#105](https://github.com/taskforcesh/taskforce-connector/issues/105)) ([ce68d17](https://github.com/taskforcesh/taskforce-connector/commit/ce68d17761430e1174daa76014f86546c1322ec9)) 7 | 8 | # [1.34.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.33.0...v1.34.0) (2025-01-15) 9 | 10 | 11 | ### Bug Fixes 12 | 13 | * remove undefined properties when using uri ([4a0dd9e](https://github.com/taskforcesh/taskforce-connector/commit/4a0dd9e5472701e4b922f196d556a5596a75e908)) 14 | 15 | 16 | ### Features 17 | 18 | * bump to latest bullmq version ([0ec5d4e](https://github.com/taskforcesh/taskforce-connector/commit/0ec5d4e8fa10675319c2c5489b6ccd5a084a616d)) 19 | 20 | # [1.33.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.32.0...v1.33.0) (2024-12-04) 21 | 22 | 23 | ### Features 24 | 25 | * add versioning support ([46395df](https://github.com/taskforcesh/taskforce-connector/commit/46395df4eb1f423de81ee5d9f38dddd96d2166c0)) 26 | 27 | # [1.32.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.31.5...v1.32.0) (2024-08-30) 28 | 29 | 30 | ### Features 31 | 32 | * add username for support with redis setups using ACL ([#99](https://github.com/taskforcesh/taskforce-connector/issues/99)) ([e8125ac](https://github.com/taskforcesh/taskforce-connector/commit/e8125acb84bab99c890df2345b3c155a0bb99c5c)) 33 | 34 | ## [1.31.5](https://github.com/taskforcesh/taskforce-connector/compare/v1.31.4...v1.31.5) (2024-08-30) 35 | 36 | 37 | ### Bug Fixes 38 | 39 | * prefix redis cache connection cache with checksum ([72e3deb](https://github.com/taskforcesh/taskforce-connector/commit/72e3deb5bc619ca1c9638916b689c7a0f6507ba2)) 40 | 41 | ## [1.31.4](https://github.com/taskforcesh/taskforce-connector/compare/v1.31.3...v1.31.4) (2024-08-04) 42 | 43 | 44 | ### Bug Fixes 45 | 46 | * **socket:** make sure cache is updated before responding ([8a5f8a1](https://github.com/taskforcesh/taskforce-connector/commit/8a5f8a1a92efe31113ac87ef83499eaa6370edc7)) 47 | 48 | ## [1.31.3](https://github.com/taskforcesh/taskforce-connector/compare/v1.31.2...v1.31.3) (2024-08-03) 49 | 50 | 51 | ### Bug Fixes 52 | 53 | * use a more compatible way to read package.json ([73d6804](https://github.com/taskforcesh/taskforce-connector/commit/73d68042ff9e8a16da213277aea003286d9e9209)) 54 | 55 | ## [1.31.2](https://github.com/taskforcesh/taskforce-connector/compare/v1.31.1...v1.31.2) (2024-07-06) 56 | 57 | 58 | ### Bug Fixes 59 | 60 | * fix package.json loading ([9998df3](https://github.com/taskforcesh/taskforce-connector/commit/9998df39ad6054f7336d94c045e2ef1bc7e766c2)) 61 | 62 | ## [1.31.1](https://github.com/taskforcesh/taskforce-connector/compare/v1.31.0...v1.31.1) (2024-07-04) 63 | 64 | 65 | ### Bug Fixes 66 | 67 | * upgrade dependencies for security patches ([f433335](https://github.com/taskforcesh/taskforce-connector/commit/f43333596723ebfd23abd251104140c587604aec)) 68 | 69 | # [1.31.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.30.0...v1.31.0) (2024-06-10) 70 | 71 | 72 | ### Features 73 | 74 | * upgrade to latest bull and bullmq ([955f968](https://github.com/taskforcesh/taskforce-connector/commit/955f968c85f4a6dc6f654119d874a353053ad6e2)) 75 | 76 | # [1.30.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.29.0...v1.30.0) (2024-06-10) 77 | 78 | 79 | ### Features 80 | 81 | * add support for specifying queue names programatically ([71adc07](https://github.com/taskforcesh/taskforce-connector/commit/71adc07a8ea5cc9829942fc945b818c6c26fdec3)) 82 | 83 | # [1.29.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.28.2...v1.29.0) (2024-06-10) 84 | 85 | 86 | ### Features 87 | 88 | * add support for specifying queue names ([aae22c5](https://github.com/taskforcesh/taskforce-connector/commit/aae22c5eec31b8f86400ef1d3a3ffc47d94226ef)) 89 | 90 | ## [1.28.2](https://github.com/taskforcesh/taskforce-connector/compare/v1.28.1...v1.28.2) (2024-06-09) 91 | 92 | 93 | ### Bug Fixes 94 | 95 | * **queue-factory:** slightly larger defaults for queue discovery iterations ([1df5ff8](https://github.com/taskforcesh/taskforce-connector/commit/1df5ff88c2beb37f6f12f65cb1f4ae6077fd1a46)) 96 | 97 | ## [1.28.1](https://github.com/taskforcesh/taskforce-connector/compare/v1.28.0...v1.28.1) (2024-06-09) 98 | 99 | 100 | ### Bug Fixes 101 | 102 | * **queue-factory:** use type for faster queue discovery ([150c1e1](https://github.com/taskforcesh/taskforce-connector/commit/150c1e121a68dc5cb37c5a8b4ee269aee5035052)) 103 | 104 | # [1.28.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.27.2...v1.28.0) (2024-05-20) 105 | 106 | 107 | ### Features 108 | 109 | * add response time to logs ([9b461f7](https://github.com/taskforcesh/taskforce-connector/commit/9b461f7a9a617506e1717901dd1576a5e7b0c426)) 110 | 111 | ## [1.27.2](https://github.com/taskforcesh/taskforce-connector/compare/v1.27.1...v1.27.2) (2024-05-20) 112 | 113 | 114 | ### Bug Fixes 115 | 116 | * improve scan max count ([dfaec16](https://github.com/taskforcesh/taskforce-connector/commit/dfaec16822695f2a31327a94971b1f2aefe66d09)) 117 | 118 | ## [1.27.1](https://github.com/taskforcesh/taskforce-connector/compare/v1.27.0...v1.27.1) (2024-05-20) 119 | 120 | 121 | ### Bug Fixes 122 | 123 | * **ws:** log send errors ([4726f7b](https://github.com/taskforcesh/taskforce-connector/commit/4726f7b52cd48a8a4ccb54447413aab6976fcaa8)) 124 | 125 | # [1.27.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.26.0...v1.27.0) (2024-05-20) 126 | 127 | 128 | ### Features 129 | 130 | * add support for getWorkersCount ([f07db90](https://github.com/taskforcesh/taskforce-connector/commit/f07db908737a3fb9938e2d2276266ff0c5e883e1)) 131 | * better logs when getting connection ([4dee10f](https://github.com/taskforcesh/taskforce-connector/commit/4dee10f7fd901caaf72a8f8a99a074551a051ec1)) 132 | * upgrade bull and bullmq dependencies ([67235bf](https://github.com/taskforcesh/taskforce-connector/commit/67235bf6a27883d4173bfaa1d4690bff3af4f621)) 133 | 134 | # [1.26.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.25.1...v1.26.0) (2023-12-21) 135 | 136 | 137 | ### Features 138 | 139 | * add support for bullmq flows ([42281f9](https://github.com/taskforcesh/taskforce-connector/commit/42281f9190149af7a6f91de670f99f6e353973b2)) 140 | 141 | ## [1.25.1](https://github.com/taskforcesh/taskforce-connector/compare/v1.25.0...v1.25.1) (2023-11-17) 142 | 143 | 144 | ### Bug Fixes 145 | 146 | * correctly discover queues in redis clusters ([781e963](https://github.com/taskforcesh/taskforce-connector/commit/781e963cca48dcde7063fab13cd4148522480848)) 147 | 148 | # [1.25.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.24.5...v1.25.0) (2023-11-05) 149 | 150 | 151 | ### Features 152 | 153 | * add support for env var REDIS_NODES ([ec32ad6](https://github.com/taskforcesh/taskforce-connector/commit/ec32ad6431d98681856849ed92d64127680ba33c)) 154 | 155 | ## [1.24.5](https://github.com/taskforcesh/taskforce-connector/compare/v1.24.4...v1.24.5) (2023-08-08) 156 | 157 | 158 | ### Bug Fixes 159 | 160 | * use different connections for bull and bullmq ([a7ebf24](https://github.com/taskforcesh/taskforce-connector/commit/a7ebf244903548c07ed3cc069a23918adbfc924f)) 161 | 162 | ## [1.24.4](https://github.com/taskforcesh/taskforce-connector/compare/v1.24.3...v1.24.4) (2023-08-08) 163 | 164 | 165 | ### Bug Fixes 166 | 167 | * upgrade bull and bullmq ([30e7b38](https://github.com/taskforcesh/taskforce-connector/commit/30e7b38d480a5683892c4995e3b9a83c7d3ca8ca)) 168 | 169 | ## [1.24.3](https://github.com/taskforcesh/taskforce-connector/compare/v1.24.2...v1.24.3) (2023-05-16) 170 | 171 | 172 | ### Bug Fixes 173 | 174 | * **cmdline:** fix teams and nodes options ([7981945](https://github.com/taskforcesh/taskforce-connector/commit/798194558067815bcb00030dfadcf4fcf089d886)) 175 | 176 | ## [1.24.2](https://github.com/taskforcesh/taskforce-connector/compare/v1.24.1...v1.24.2) (2023-05-16) 177 | 178 | 179 | ### Bug Fixes 180 | 181 | * **bullmq:** call clean with correct arguments ([0822374](https://github.com/taskforcesh/taskforce-connector/commit/08223745983b9ed1310e7668573cfe6faf27ea10)) 182 | 183 | ## [1.24.1](https://github.com/taskforcesh/taskforce-connector/compare/v1.24.0...v1.24.1) (2023-04-20) 184 | 185 | 186 | ### Bug Fixes 187 | 188 | * pick username from Connection options ([57fd2f5](https://github.com/taskforcesh/taskforce-connector/commit/57fd2f5fc958ea49adfd72c90d8cfd99def5e55f)) 189 | 190 | # [1.24.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.23.0...v1.24.0) (2023-04-20) 191 | 192 | 193 | ### Bug Fixes 194 | 195 | * **cli:** correct newest version reporting ([949dabb](https://github.com/taskforcesh/taskforce-connector/commit/949dabb19885e4fb75dc75f7259aa3cf0faf33ed)) 196 | * upgrade bull and bullmq packages ([72fd1f2](https://github.com/taskforcesh/taskforce-connector/commit/72fd1f27a4835ec96632f1088cda388a3a8ad773)) 197 | 198 | 199 | ### Features 200 | 201 | * add support for queue integrations ([f8d2067](https://github.com/taskforcesh/taskforce-connector/commit/f8d2067d2988922e921a3dafb32ba4a12633ed78)) 202 | 203 | # [1.23.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.22.0...v1.23.0) (2023-04-12) 204 | 205 | 206 | ### Features 207 | 208 | * add bullmq support ([1ff070b](https://github.com/taskforcesh/taskforce-connector/commit/1ff070ba57f480566f8c751e2c68444f7314a4e1)) 209 | 210 | # [1.22.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.21.2...v1.22.0) (2023-03-24) 211 | 212 | 213 | ### Features 214 | 215 | * add sentinel password support ([3ee89e0](https://github.com/taskforcesh/taskforce-connector/commit/3ee89e0c2e1f8c5e0c9d55426671e3d0f6b1feb3)) 216 | 217 | ## [1.21.2](https://github.com/taskforcesh/taskforce-connector/compare/v1.21.1...v1.21.2) (2023-03-10) 218 | 219 | 220 | ### Bug Fixes 221 | 222 | * better error handling ([6981600](https://github.com/taskforcesh/taskforce-connector/commit/69816006ce39a3638baa8751b0fc5b797cf52ae0)) 223 | 224 | ## [1.21.1](https://github.com/taskforcesh/taskforce-connector/compare/v1.21.0...v1.21.1) (2023-01-23) 225 | 226 | 227 | ### Bug Fixes 228 | 229 | * **queues:** use scan instead of keys fixes https://github.com/taskforcesh/issues/issues/65 ([3c017ec](https://github.com/taskforcesh/taskforce-connector/commit/3c017ec490e622b8054f5af6871ebb69a46638aa)) 230 | 231 | # [1.21.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.20.0...v1.21.0) (2023-01-23) 232 | 233 | 234 | ### Features 235 | 236 | * support custom backend programmatically ([87a0e60](https://github.com/taskforcesh/taskforce-connector/commit/87a0e60a6c06b757cebf9c1e0c0241ec65379726)) 237 | 238 | # [1.20.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.19.0...v1.20.0) (2022-06-09) 239 | 240 | 241 | ### Features 242 | 243 | * add experimental support to redis cluster ([e0e12bb](https://github.com/taskforcesh/taskforce-connector/commit/e0e12bb0e18d9781924f63dd57e61ca872e436b6)) 244 | 245 | # [1.19.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.18.0...v1.19.0) (2022-05-31) 246 | 247 | 248 | ### Bug Fixes 249 | 250 | * **dockerfile:** update pm2 fixes [#45](https://github.com/taskforcesh/taskforce-connector/issues/45) ([1aa1e5d](https://github.com/taskforcesh/taskforce-connector/commit/1aa1e5dc6d309ff3aae430c6a94e4c6322831d20)) 251 | 252 | 253 | ### Features 254 | 255 | * add ping support ([3805b5a](https://github.com/taskforcesh/taskforce-connector/commit/3805b5a6707e36cb493a3f097560892f19f23aa7)) 256 | 257 | # [1.18.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.17.0...v1.18.0) (2022-03-23) 258 | 259 | 260 | ### Features 261 | 262 | * add real-time metrics support. Fix [#46](https://github.com/taskforcesh/taskforce-connector/issues/46). ([7b65ee5](https://github.com/taskforcesh/taskforce-connector/commit/7b65ee5773eaddc222829ffbdef119751eb4f009)) 263 | 264 | # [1.17.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.16.0...v1.17.0) (2022-02-04) 265 | 266 | 267 | ### Features 268 | 269 | * add support for retry all failed jobs ([f61db93](https://github.com/taskforcesh/taskforce-connector/commit/f61db935dbf2910c73cb0ac8f3ee35def0ba9596)) 270 | 271 | # [1.16.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.15.0...v1.16.0) (2022-01-29) 272 | 273 | 274 | ### Bug Fixes 275 | 276 | * correct new version check ([f3b6437](https://github.com/taskforcesh/taskforce-connector/commit/f3b6437265c62ea9b3f65790d675704c8aa4e800)) 277 | 278 | 279 | ### Features 280 | 281 | * **commands:** add support for clean ([a56eadb](https://github.com/taskforcesh/taskforce-connector/commit/a56eadb93ed1ceffb8ea63214a5c4cbc2d0f2e3d)) 282 | 283 | # [1.15.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.14.2...v1.15.0) (2021-07-10) 284 | 285 | 286 | ### Features 287 | 288 | * add job#update and queue#add methods ([e1d7e15](https://github.com/taskforcesh/taskforce-connector/commit/e1d7e15f5c905ae28dae058f825665284e42eb12)) 289 | 290 | ## [1.14.2](https://github.com/taskforcesh/taskforce-connector/compare/v1.14.1...v1.14.2) (2021-06-29) 291 | 292 | 293 | ### Bug Fixes 294 | 295 | * **package:** add prepare script ([bc7f7c0](https://github.com/taskforcesh/taskforce-connector/commit/bc7f7c0eeddbe4d3b0850d9cf0a5c0329865fb8f)) 296 | 297 | ## [1.14.1](https://github.com/taskforcesh/taskforce-connector/compare/v1.14.0...v1.14.1) (2021-06-29) 298 | 299 | 300 | ### Bug Fixes 301 | 302 | * remove socket.d.ts from dist ([a7dfceb](https://github.com/taskforcesh/taskforce-connector/commit/a7dfcebd5ba9686f241cc958a3dd34d3956422ae)) 303 | 304 | # [1.14.0](https://github.com/taskforcesh/taskforce-connector/compare/v1.13.0...v1.14.0) (2021-06-29) 305 | 306 | 307 | ### Features 308 | 309 | * support using connector as a library ([fe07e2b](https://github.com/taskforcesh/taskforce-connector/commit/fe07e2bf63f1591f46664147e56c495131125bae)) 310 | 311 | # v1.13.0 and previous versions (2021-05-03) 312 | 313 | ### Bug Fixes 314 | 315 | * add client heartbeat for more robust connections ([517129f](https://github.com/taskforcesh/taskforce-connector/commit/517129f9b6479759b1bf42490fb1e023dc5c41af)) 316 | * allow old versions to run but generate warning fix [#8](https://github.com/taskforcesh/taskforce-connector/issues/8) ([3a0ccb9](https://github.com/taskforcesh/taskforce-connector/commit/3a0ccb990d5f6391f5ea3d156082e17d5a89cdd3)) 317 | * better handling of queue cache fixes [#12](https://github.com/taskforcesh/taskforce-connector/issues/12) ([75c9ede](https://github.com/taskforcesh/taskforce-connector/commit/75c9edea64160163f3c0b6ea99e7cae5ceda4741)) 318 | * better handling of queue cache fixes [#12](https://github.com/taskforcesh/taskforce-connector/issues/12) ([d94ff5b](https://github.com/taskforcesh/taskforce-connector/commit/d94ff5bc279704f17af31e4bab16c13d3fe44fd4)) 319 | * close redis connection after getting queues. ([88159d3](https://github.com/taskforcesh/taskforce-connector/commit/88159d3a20729c4415b5d63cd0e4bbcf5d6dd489)) 320 | * correct default value for the TLS option ([134a70d](https://github.com/taskforcesh/taskforce-connector/commit/134a70d79eda418ce6e8a49cca76c6e10b6377cf)) 321 | * do not reconnect on errors to avoid double connections ([24aa108](https://github.com/taskforcesh/taskforce-connector/commit/24aa108d700a6cb5eefd2899eae2dfb1d965ef3d)) 322 | * exit with error if missing token ([8720473](https://github.com/taskforcesh/taskforce-connector/commit/8720473d921f031c3a2693a6ddd5bfcc5508fd2f)) 323 | * read package.json with proper path ([39f6d7f](https://github.com/taskforcesh/taskforce-connector/commit/39f6d7fa770b50c92bb691fa0471710f3be264cf)) 324 | * upgrade bull version ([8e021ee](https://github.com/taskforcesh/taskforce-connector/commit/8e021eedbcd22122412039aa38c238834a0ac768)) 325 | * upgrade dependencies ([d3807be](https://github.com/taskforcesh/taskforce-connector/commit/d3807be641f848fd23a3054d10b6a6a5b71aba4b)) 326 | 327 | 328 | ### Features 329 | 330 | * add dockerfile ([ede57ec](https://github.com/taskforcesh/taskforce-connector/commit/ede57ec1a31bc72eb5f04d83bfcf08226610054b)) 331 | * add Dockerfile ([d221bb1](https://github.com/taskforcesh/taskforce-connector/commit/d221bb114302c82eeb97831a397219c9a4cebf0f)) 332 | * add get redis info support ([dc66204](https://github.com/taskforcesh/taskforce-connector/commit/dc6620416c7edfade5eb7f7c6b3d19917adacf40)) 333 | * add job logs support ([f7f86d3](https://github.com/taskforcesh/taskforce-connector/commit/f7f86d37d589f4e04d5d407e0948b36365c35e51)) 334 | * add moveToFailed job command ([e7c64e6](https://github.com/taskforcesh/taskforce-connector/commit/e7c64e6f68c8ad26e88614e3e387a633e3f93279)) 335 | * add pause, resume and isPaused commands ([cf34cc6](https://github.com/taskforcesh/taskforce-connector/commit/cf34cc6084222f051c89de8a95edf13e8ee6a40b)) 336 | * add support for empty queues ([7928d5a](https://github.com/taskforcesh/taskforce-connector/commit/7928d5a11ebca2bccbc0b0c15eb8776bd184f9f6)) 337 | * add support for getting jobs without data ([#19](https://github.com/taskforcesh/taskforce-connector/issues/19)) ([f838f89](https://github.com/taskforcesh/taskforce-connector/commit/f838f89d6472ae98f5fbd32516a389ffcdb1873e)) 338 | * add support for obliterate ([a7ca9a0](https://github.com/taskforcesh/taskforce-connector/commit/a7ca9a08e2682e5c30b878ad3e87d74dd22bd315)) 339 | * add support for prefixes ([7f56bba](https://github.com/taskforcesh/taskforce-connector/commit/7f56bba2520a914e81dd42189938344faa577f63)) 340 | * add support for prefixes ([#18](https://github.com/taskforcesh/taskforce-connector/issues/18)) ([0ec6e7a](https://github.com/taskforcesh/taskforce-connector/commit/0ec6e7a6ea66c45c3c3a5ba3c3e855277878b57c)) 341 | * add support for Redis Sentinel ([#15](https://github.com/taskforcesh/taskforce-connector/issues/15)) ([870fce3](https://github.com/taskforcesh/taskforce-connector/commit/870fce361e82447075c802b1c244573e015d2cbc)) 342 | * add support for removing jobs, discarding, promoting, and retrying ([75a26ac](https://github.com/taskforcesh/taskforce-connector/commit/75a26ace7ad87ff1b75947c79a846aebb2019f15)) 343 | * add support for TLS and teams ([6837da5](https://github.com/taskforcesh/taskforce-connector/commit/6837da5aace9cb9d66c4cb5c082661a20a414149)) 344 | * send version to server ([3ccf451](https://github.com/taskforcesh/taskforce-connector/commit/3ccf4516d812d49f22bd96fb44bdc08590dd66a4)) 345 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:20-alpine 2 | 3 | RUN apk --no-cache add curl 4 | 5 | RUN yarn global add --ignore-optional taskforce-connector pm2@5.2.0 && yarn cache clean 6 | 7 | CMD pm2-runtime taskforce -- -n "${TASKFORCE_CONNECTION}" --team "${TASKFORCE_TEAM}" `([ "$REDIS_USE_TLS" == "1" ] && echo --tls)` 8 | 9 | HEALTHCHECK --interval=30s --timeout=30s \ 10 | --start-period=5s --retries=3 CMD curl -f http://localhost || exit 1 11 | 12 | LABEL org.opencontainers.image.source="https://github.com/taskforcesh/taskforce-connector" 13 | -------------------------------------------------------------------------------- /HISTORY.md: -------------------------------------------------------------------------------- 1 | v1.8.0 2 | ====== 3 | 4 | - Support for connecting to sentinels 5 | 6 | v1.7.1 7 | ====== 8 | 9 | - chore: upgrade all dependencies 10 | 11 | v1.7.0 12 | ====== 13 | 14 | - feat: add pause, resume and isPaused commands 15 | - feat: send version to server 16 | 17 | v1.6.0 18 | ====== 19 | 20 | - feat: get redis info support 21 | 22 | v1.5.0 23 | ====== 24 | 25 | - feat: add moveToFailed command 26 | 27 | v1.4.4 28 | ====== 29 | 30 | - fix: fix queues cache fixes #12 31 | 32 | v1.1.1 33 | ====== 34 | 35 | - fix: close redis connection after getting queues. 36 | 37 | v1.1.0 38 | ====== 39 | 40 | - feat: add support for removing scheduled jobs. 41 | - feat: add support for removing, promoting, retrying and discarding jobs. 42 | 43 | v1.0.5 44 | ====== 45 | 46 | - chore: replace npmview by latest-version 47 | 48 | v1.0.4 49 | ====== 50 | 51 | - fix: read package.json with proper path 52 | 53 | v1.0.3 54 | ====== 55 | - exit with error if missing token option 56 | 57 | v1.0.2 58 | ====== 59 | - add database selection option. 60 | 61 | v1.0.1 62 | ====== 63 | - fix password option for redis connections. 64 | 65 | v1.0.0 66 | ====== 67 | 68 | - Initial Version 69 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 taskforcesh 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Taskforce Connector 2 | 3 | This small service allows you to connect queues to [Taskforce](https://taskforce.sh) acting as a proxy between your queues and the UI. It is useful for connecting local development queues as well as production grade queues without the need of sharing passwords or establishing SSH tunnels. 4 | 5 | Currently the connector supports [Bull](https://github.com/optimalbits/bull) and [BullMQ](https://github.com/taskforcesh/bullmq) queues. 6 | 7 | The connector is designed to be lightweight and using a minimal set of resources from the local queues. 8 | 9 | ## Install 10 | 11 | Using [yarn](https://yarnpkg.com) 12 | 13 | ```bash 14 | yarn global add taskforce-connector 15 | 16 | ``` 17 | 18 | Using npm: 19 | 20 | ```bash 21 | npm install -g taskforce-connector 22 | ``` 23 | 24 | ## Usage 25 | 26 | Call the tool and get a help on the options: 27 | 28 | ```bash 29 | ✗ taskforce --help 30 | 31 | Usage: taskforce [options] 32 | 33 | 34 | Options: 35 | 36 | -V, --version output the version number 37 | -n, --name [name] connection name [My Connection] (default: "My Connection") 38 | -t, --token [token] api token (get yours at https://taskforce.sh) 39 | -p, --port [port] redis port [6379] (default: "6379") 40 | --tls [tls] (default: "Activate secured TLS connection to Redis") 41 | -h, --host [host] redis host [localhost] (default: "localhost") 42 | -d, --database [db] redis database [0] (default: "0") 43 | --username [username] redis username 44 | --passwd [passwd] redis password 45 | --spasswd [spasswd] redis sentinel password 46 | -u, --uri [uri] redis uri 47 | --team [team] specify team where to put the connection 48 | -b, --backend [host] backend domain [api.taskforce.sh] (default: "wss://api.taskforce.sh") 49 | -s, --sentinels [host:port] comma-separated list of sentinel host/port pairs 50 | -m, --master [name] name of master node used in sentinel configuration 51 | -h, --help output usage information 52 | --nodes [nodes] comma-separated list of cluster nodes uris to connect to (Redis Cluster) 53 | --queues optional comma-separated list of queues to monitor 54 | --queuesFile optional file with queues to monitor 55 | ``` 56 | 57 | Example: 58 | 59 | ```bash 60 | ✗ taskforce -n "transcoder connection" -t 2cfe6a1b-5f0e-466f-99ad-12f51bea79a7 61 | ``` 62 | 63 | The token `2cfe6a1b-5f0e-466f-99ad-12f51bea79a7` is a private token that can be retrieved at your [Taskforce account](https://taskforce.sh/account). 64 | 65 | After running the command, you should be able to see the connection appear automatically on the dashboard. 66 | 67 | Sentinel Example: 68 | 69 | ```bash 70 | ✗ taskforce -n "transcoder connection" -t 2cfe6a1b-5f0e-466f-99ad-12f51bea79a7 -s sentinel1.mydomain:6379,sentinel2.mydomain:6379 -m mymaster 71 | ``` 72 | 73 | Note: You can also specify the following with environment variables. 74 | 75 | ```bash 76 | token TASKFORCE_TOKEN 77 | port REDIS_PORT 78 | host REDIS_HOST 79 | password REDIS_PASSWD 80 | sentinel-password REDIS_SENTINEL_PASSWD 81 | uri REDIS_URI 82 | sentinels REDIS_SENTINELS (comma separated list of sentinels) 83 | master REDIS_MASTER 84 | nodes REDIS_NODES (comma separated list of nodes for Redis Cluster) 85 | ``` 86 | 87 | To enable use if TLS when using the container set this environment variable: 88 | ```bash 89 | REDIS_USE_TLS=1 90 | ``` 91 | 92 | Note for Redis Cluster: You may also need to specify following with environment variables. 93 | ```bash 94 | Cluster TLS Certificate REDIS_CLUSTER_TLS 95 | ``` 96 | 97 | If your redis cluster still cannot connect due to failing certificate validation, you may need to pass this env to skip cert validation. 98 | ```bash 99 | NODE_TLS_REJECT_UNAUTHORIZED="0" 100 | ``` 101 | 102 | ## Secured TLS Connections 103 | 104 | Services that support TLS can also be used using the connector, use the `--tls` flag. Note that some services such as Heroku expects the port number to be "one more" than the normal unencrypted port [read more](https://devcenter.heroku.com/articles/securing-heroku-redis). 105 | 106 | ## Teams 107 | 108 | You can use the connector to spawn queue connections to any team that you created on your organization, just pass the team name 109 | as an option: 110 | 111 | ```bash 112 | ✗ taskforce -n "transcoder connection" -t 2cfe6a1b-5f0e-466f-99ad-12f51bea79a7 --team "my awesome team" 113 | 114 | ``` 115 | 116 | ## Use as a library 117 | 118 | It is also possible to add the connector as a library: 119 | 120 | As a commonjs dependency: 121 | 122 | ```js 123 | const { Connect } = require("taskforce-connector"); 124 | 125 | const taskforceConnection = Connect("my connection", "my token", { 126 | host: "my redis host", 127 | port: "my redis port", 128 | password: "my redis password", 129 | }); 130 | ``` 131 | 132 | or as a es6 module: 133 | 134 | ```ts 135 | import { Connect } from "taskforce-connector"; 136 | 137 | const taskforceConnection = Connect("my connection", "my token", { 138 | host: "my redis host", 139 | port: "my redis port", 140 | password: "my redis password", 141 | }); 142 | ``` 143 | 144 | If you are using the On Premises version of Taskforce, you can also specify the backend domain: 145 | 146 | ```ts 147 | const taskforceConnection = Connect( 148 | "my connection", 149 | "my token", 150 | { 151 | host: "my redis host", 152 | port: "my redis port", 153 | password: "my redis password", 154 | }, 155 | "My Prod Team", // optional team name 156 | "wss://mybackend.domain" 157 | ); 158 | ``` 159 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | const { name, version } = require(__dirname + "/package.json"); 3 | 4 | const { run } = require("./dist/cmd.js"); 5 | run(name, version); 6 | -------------------------------------------------------------------------------- /lib/cmd.ts: -------------------------------------------------------------------------------- 1 | import { Command, Option } from "commander"; 2 | import { blueBright, red } from "chalk"; 3 | import { readFileSync, existsSync } from "fs"; 4 | import { resolve } from "path"; 5 | 6 | import { Socket } from "./socket"; 7 | import { versionChecker } from "./version-checker"; 8 | 9 | export const run = (name: string, version: string) => { 10 | console.info( 11 | blueBright( 12 | "Taskforce Connector v" + version + " - (c) 2017-2024 Taskforce.sh Inc." 13 | ) 14 | ); 15 | 16 | const program = new Command(); 17 | program 18 | .version(version) 19 | 20 | .option( 21 | "-n, --name [name]", 22 | "connection name [My Connection]", 23 | "My Connection" 24 | ) 25 | .option( 26 | "-t, --token [token]", 27 | "api token (get yours at https://taskforce.sh)", 28 | process.env.TASKFORCE_TOKEN 29 | ) 30 | .option( 31 | "-p, --port [port]", 32 | "redis port [6379]", 33 | process.env.REDIS_PORT || "6379" 34 | ) 35 | .option("--tls [tls]", "Activate secured TLS connection to Redis") 36 | .option( 37 | "-h, --host [host]", 38 | "redis host [localhost]", 39 | process.env.REDIS_HOST || "localhost" 40 | ) 41 | .option("-d, --database [db]", "redis database [0]", "0") 42 | .option( 43 | "--username [username]", 44 | "redis username", 45 | process.env.REDIS_USERNAME 46 | ) 47 | .option("--passwd [passwd]", "redis password", process.env.REDIS_PASSWD) 48 | .option( 49 | "--spasswd [spasswd]", 50 | "redis sentinel password", 51 | process.env.REDIS_SENTINEL_PASSWD 52 | ) 53 | .option("-u, --uri [uri]", "redis uri", process.env.REDIS_URI) 54 | .option("--team [team]", "specify team where to put the connection") 55 | .option( 56 | "-b, --backend [host]", 57 | "backend domain [api.taskforce.sh]", 58 | "wss://api.taskforce.sh" 59 | ) 60 | .option( 61 | "-s, --sentinels [host:port]", 62 | "comma-separated list of sentinel host/port pairs", 63 | process.env.REDIS_SENTINELS 64 | ) 65 | .option( 66 | "-m, --master [name]", 67 | "name of master node used in sentinel configuration", 68 | process.env.REDIS_MASTER 69 | ) 70 | .option( 71 | "--nodes ", 72 | "comma-separated list of cluster nodes uris to connect to", 73 | process.env.REDIS_NODES ? process.env.REDIS_NODES : undefined 74 | ) 75 | .option("--queues ", "comma-separated list of queues to monitor") 76 | .addOption( 77 | new Option( 78 | "--queuesFile ", 79 | "file with queues to monitor" 80 | ).conflicts("queues") 81 | ) 82 | .parse(process.argv); 83 | 84 | const options = program.opts(); 85 | 86 | versionChecker(name, version).then(function () { 87 | /* 88 | lastestVersion(name).then(function (newestVersion) { 89 | if (semver.gt(newestVersion, version)) { 90 | console.error( 91 | chalk.yellow( 92 | "New version " + 93 | newestVersion + 94 | " of taskforce available, please upgrade with yarn global add taskforce-connector" 95 | ) 96 | ); 97 | } 98 | */ 99 | if (!options.token) { 100 | console.error( 101 | red( 102 | `ERROR: A valid token is required, use either TASKFORCE_TOKEN env or pass it with -t (get token at https://taskforce.sh)` 103 | ) 104 | ); 105 | process.exit(1); 106 | } 107 | 108 | const queueNames = options.queuesFile 109 | ? parseQueuesFile(options.queuesFile) 110 | : options.queues 111 | ? parseQueues(options.queues) 112 | : undefined; 113 | 114 | const connection = { 115 | port: options.port, 116 | host: options.host, 117 | username: options.username, 118 | password: options.passwd, 119 | sentinelPassword: options.spasswd, 120 | db: options.database, 121 | uri: options.uri, 122 | tls: options.tls 123 | ? { 124 | rejectUnauthorized: false, 125 | requestCert: true, 126 | agent: false, 127 | } 128 | : void 0, 129 | sentinels: 130 | options.sentinels && 131 | options.sentinels.split(",").map((hostPort: string) => { 132 | const [host, port] = hostPort.split(":"); 133 | return { host, port }; 134 | }), 135 | name: options.master, 136 | } as Record; 137 | 138 | // If uri is defined we should remove some port and host and also all the undefined 139 | // properties. 140 | if (options.uri) { 141 | delete connection.port; 142 | delete connection.host; 143 | 144 | Object.keys(connection).forEach((key) => { 145 | if (connection[key] === undefined) { 146 | delete connection[key]; 147 | } 148 | }); 149 | } 150 | 151 | Socket(options.name, options.backend, options.token, connection, { 152 | team: options.team, 153 | nodes: options.nodes ? options.nodes.split(",") : undefined, 154 | queueNames, 155 | }); 156 | }); 157 | 158 | // Catch uncaught exceptions and unhandled rejections 159 | process.on("uncaughtException", function (err) { 160 | console.error(err, "Uncaught exception"); 161 | }); 162 | 163 | process.on("unhandledRejection", (reason, promise) => { 164 | console.error({ promise, reason }, "Unhandled Rejection at: Promise"); 165 | }); 166 | 167 | function parseQueuesFile(file: string) { 168 | // Load the queues from the file. The file must be a list of queues separated by new lines 169 | const queuesFile = resolve(file); 170 | if (existsSync(queuesFile)) { 171 | return readFileSync(queuesFile, "utf8").split("\n").filter(Boolean); 172 | } else { 173 | console.error(red(`ERROR: File ${queuesFile} does not exist`)); 174 | process.exit(1); 175 | } 176 | } 177 | 178 | function parseQueues(queuesString: string) { 179 | return queuesString.split(","); 180 | } 181 | }; 182 | -------------------------------------------------------------------------------- /lib/index.ts: -------------------------------------------------------------------------------- 1 | import { Integration } from "./interfaces/integration"; 2 | import { Socket, Connection } from "./socket"; 3 | 4 | export { Integration } from "./interfaces/integration"; 5 | export { getRedisClient, FoundQueue } from "./queue-factory"; 6 | export { WebSocketClient } from "./ws-autoreconnect"; 7 | export { Connection } from "./socket"; 8 | export { respond } from "./responders/respond"; 9 | export { BullMQResponders } from "./responders/bullmq-responders"; 10 | 11 | export { versionChecker } from "./version-checker"; 12 | export { Socket } from "./socket"; 13 | 14 | export const Connect = ( 15 | name: string, 16 | token: string, 17 | connection: Connection, 18 | backend: string = "wss://api.taskforce.sh", 19 | opts: { 20 | team?: string; 21 | integrations?: { [key: string]: Integration }; 22 | nodes?: string[]; 23 | queueNames?: string[]; 24 | } = {} 25 | ) => { 26 | return Socket(name, backend, token, connection, opts); 27 | }; 28 | -------------------------------------------------------------------------------- /lib/interfaces/integration.ts: -------------------------------------------------------------------------------- 1 | import { RedisOptions } from "ioredis"; 2 | import { FoundQueue } from "../queue-factory"; 3 | import { Responders } from "./responders"; 4 | 5 | export interface Integration { 6 | responders: Responders; 7 | createQueue: ( 8 | foundQueue: FoundQueue, 9 | redisOpts: RedisOptions, 10 | nodes?: string[] 11 | ) => any; 12 | } 13 | -------------------------------------------------------------------------------- /lib/interfaces/responders.ts: -------------------------------------------------------------------------------- 1 | import { WebSocketClient } from "../ws-autoreconnect"; 2 | 3 | export interface Responders { 4 | respondJobCommand(ws: WebSocketClient, queue: any, msg: any): Promise; 5 | respondQueueCommand(ws: WebSocketClient, queue: any, msg: any): Promise; 6 | } 7 | -------------------------------------------------------------------------------- /lib/queue-factory.ts: -------------------------------------------------------------------------------- 1 | import { Redis, Cluster, RedisOptions } from "ioredis"; 2 | 3 | import { QueueType, getQueueType, redisOptsFromUrl } from "./utils"; 4 | import { Queue } from "bullmq"; 5 | import * as Bull from "bull"; 6 | import { BullMQResponders, BullResponders } from "./responders"; 7 | import { Responders } from "./interfaces/responders"; 8 | import { Integration } from "./interfaces/integration"; 9 | 10 | const chalk = require("chalk"); 11 | 12 | const queueNameRegExp = new RegExp("(.*):(.*):id"); 13 | const maxCount = 150000; 14 | const maxTime = 40000; 15 | 16 | // We keep a redis client that we can reuse for all the queues. 17 | let redisClients: Record = {} as any; 18 | 19 | export interface FoundQueue { 20 | prefix: string; 21 | name: string; 22 | type: QueueType; 23 | majorVersion: number; 24 | version?: string; 25 | } 26 | 27 | const scanForQueues = async (node: Redis | Cluster, startTime: number) => { 28 | let cursor = "0"; 29 | const keys = []; 30 | do { 31 | const [nextCursor, scannedKeys] = await node.scan( 32 | cursor, 33 | "MATCH", 34 | "*:*:id", 35 | "COUNT", 36 | maxCount, 37 | "TYPE", 38 | "string" 39 | ); 40 | cursor = nextCursor; 41 | 42 | keys.push(...scannedKeys); 43 | } while (Date.now() - startTime < maxTime && cursor !== "0"); 44 | 45 | return keys; 46 | }; 47 | 48 | const getQueueKeys = async (client: Redis | Cluster, queueNames?: string[]) => { 49 | let nodes = "nodes" in client ? client.nodes("master") : [client]; 50 | let keys = []; 51 | const startTime = Date.now(); 52 | const foundQueues = new Set(); 53 | 54 | for await (const node of nodes) { 55 | // If we have proposed queue names, lets check if they exist (including prefix) 56 | // Basically checking if there is a id key for the queue (prefix:name:id) 57 | if (queueNames) { 58 | const queueKeys = queueNames.map((queueName) => { 59 | // Separate queue name from prefix 60 | let [prefix, name] = queueName.split(":"); 61 | if (!name) { 62 | name = prefix; 63 | prefix = "bull"; 64 | } 65 | 66 | // If the queue name includes a prefix use that, otherwise use the default prefix "bull" 67 | return `${prefix}:${name}:id`; 68 | }); 69 | 70 | for (const key of queueKeys) { 71 | const exists = await node.exists(key); 72 | if (exists) { 73 | foundQueues.add(key); 74 | } 75 | } 76 | keys.push(...foundQueues); 77 | 78 | // Warn for missing queues 79 | for (const key of queueKeys) { 80 | if (!foundQueues.has(key)) { 81 | // Extract queue name from key 82 | const match = queueNameRegExp.exec(key); 83 | console.log( 84 | chalk.yellow("Redis:") + 85 | chalk.red( 86 | ` Queue "${match[1]}:${match[2]}" not found in Redis. Skipping...` 87 | ) 88 | ); 89 | } 90 | } 91 | } else { 92 | keys.push(...(await scanForQueues(node, startTime))); 93 | } 94 | } 95 | return keys; 96 | }; 97 | 98 | export async function getConnectionQueues( 99 | redisOpts: RedisOptions, 100 | clusterNodes?: string[], 101 | queueNames?: string[] 102 | ): Promise { 103 | const queues = await execRedisCommand( 104 | redisOpts, 105 | async (client) => { 106 | const keys = await getQueueKeys(client, queueNames); 107 | 108 | const queues = await Promise.all( 109 | keys 110 | .map(function (key) { 111 | var match = queueNameRegExp.exec(key); 112 | if (match) { 113 | return { 114 | prefix: match[1], 115 | name: match[2], 116 | type: "bull", // default to bull 117 | }; 118 | } 119 | }) 120 | .filter((queue) => queue !== undefined) 121 | .map(async function (queue) { 122 | const { type, majorVersion, version } = await getQueueType( 123 | queue.name, 124 | queue.prefix, 125 | client 126 | ); 127 | return { ...queue, type, majorVersion, version }; 128 | }) 129 | ); 130 | return queues; 131 | }, 132 | clusterNodes 133 | ); 134 | 135 | return queues; 136 | } 137 | 138 | export async function ping(redisOpts: RedisOptions, clusterNodes?: string[]) { 139 | return execRedisCommand(redisOpts, (client) => client.ping(), clusterNodes); 140 | } 141 | 142 | export async function getRedisInfo( 143 | redisOpts: RedisOptions, 144 | clusterNodes?: string[] 145 | ) { 146 | const info = await execRedisCommand( 147 | redisOpts, 148 | (client) => client.info(), 149 | clusterNodes 150 | ); 151 | return info; 152 | } 153 | 154 | export function getRedisClient( 155 | redisOpts: RedisOptions, 156 | type: "bull" | "bullmq", 157 | clusterNodes?: string[] 158 | ) { 159 | // Compute checksum for redisOpts 160 | const checksumJson = JSON.stringify(redisOpts); 161 | const checksum = require("crypto") 162 | .createHash("md5") 163 | .update(checksumJson) 164 | .digest("hex"); 165 | 166 | const key = `${type}-${checksum}`; 167 | 168 | if (!redisClients[key]) { 169 | if (clusterNodes && clusterNodes.length) { 170 | const { username, password } = redisOptsFromUrl(clusterNodes[0]); 171 | redisClients[key] = new Redis.Cluster(clusterNodes, { 172 | ...redisOpts, 173 | redisOptions: { 174 | username, 175 | password, 176 | tls: process.env.REDIS_CLUSTER_TLS 177 | ? { 178 | cert: Buffer.from( 179 | process.env.REDIS_CLUSTER_TLS ?? "", 180 | "base64" 181 | ).toString("ascii"), 182 | } 183 | : undefined, 184 | }, 185 | }); 186 | } else { 187 | redisClients[key] = new Redis(redisOpts); 188 | } 189 | 190 | redisClients[key].on("error", (err: Error) => { 191 | console.log( 192 | `${chalk.yellow("Redis:")} ${chalk.red("redis connection error")} ${ 193 | err.message 194 | }` 195 | ); 196 | }); 197 | 198 | redisClients[key].on("connect", () => { 199 | console.log( 200 | `${chalk.yellow("Redis:")} ${chalk.green("connected to redis server")}` 201 | ); 202 | }); 203 | 204 | redisClients[key].on("end", () => { 205 | console.log( 206 | `${chalk.yellow("Redis:")} ${chalk.blueBright( 207 | "disconnected from redis server" 208 | )}` 209 | ); 210 | }); 211 | } 212 | 213 | return redisClients[key]; 214 | } 215 | 216 | export async function execRedisCommand( 217 | redisOpts: RedisOptions, 218 | cb: (client: Redis | Cluster) => any, 219 | clusterNodes?: string[] 220 | ) { 221 | const redisClient = getRedisClient(redisOpts, "bull", clusterNodes); 222 | 223 | const result = await cb(redisClient); 224 | 225 | return result; 226 | } 227 | 228 | export function createQueue( 229 | foundQueue: FoundQueue, 230 | redisOpts: RedisOptions, 231 | opts: { 232 | nodes?: string[]; 233 | integrations?: { 234 | [key: string]: Integration; 235 | }; 236 | } = {} 237 | ): { queue: Bull.Queue | Queue; responders: Responders } { 238 | const { nodes, integrations } = opts; 239 | const createClient = function (type: "client" /*, redisOpts */) { 240 | switch (type) { 241 | case "client": 242 | return getRedisClient(redisOpts, "bull", nodes); 243 | default: 244 | throw new Error(`Unexpected connection type: ${type}`); 245 | } 246 | }; 247 | 248 | if (integrations && integrations[foundQueue.type]) { 249 | const integration = integrations[foundQueue.type]; 250 | return { 251 | queue: integration.createQueue(foundQueue, redisOpts, nodes), 252 | responders: integration.responders, 253 | }; 254 | } 255 | 256 | switch (foundQueue.type) { 257 | case "bullmq": 258 | const connection = getRedisClient(redisOpts, "bullmq", nodes); 259 | switch (foundQueue.majorVersion) { 260 | case 0: 261 | return { 262 | queue: new Queue(foundQueue.name, { 263 | connection, 264 | prefix: foundQueue.prefix, 265 | }), 266 | responders: BullMQResponders, 267 | }; 268 | case 3: 269 | const { createQueue } = require("./queue-factory/bullmqv3-factory"); 270 | return createQueue(foundQueue.name, foundQueue.prefix, connection); 271 | case 4: 272 | const { 273 | createQueue: createQueueV4, 274 | } = require("./queue-factory/bullmqv4-factory"); 275 | return createQueueV4(foundQueue.name, foundQueue.prefix, connection); 276 | case 5: 277 | const { 278 | createQueue: createQueueV5, 279 | } = require("./queue-factory/bullmqv5-factory"); 280 | return createQueueV5(foundQueue.name, foundQueue.prefix, connection); 281 | default: 282 | console.error( 283 | chalk.red(`ERROR:`) + 284 | `Unexpected major version: ${foundQueue.majorVersion} for queue ${foundQueue.name}` 285 | ); 286 | } 287 | 288 | case "bull": 289 | return { 290 | queue: new (Bull)(foundQueue.name, { 291 | createClient, 292 | prefix: foundQueue.prefix, 293 | }), 294 | responders: BullResponders, 295 | }; 296 | default: 297 | console.error( 298 | chalk.red(`ERROR:`) + 299 | `Unexpected queue type: ${foundQueue.type} for queue ${foundQueue.name}` 300 | ); 301 | } 302 | } 303 | -------------------------------------------------------------------------------- /lib/queue-factory/bullmqv3-factory.ts: -------------------------------------------------------------------------------- 1 | import { Queue } from "bullmq-v3"; 2 | import { Redis } from "ioredis"; 3 | import { BullMQResponders } from "../responders"; 4 | 5 | export const createQueue = ( 6 | name: string, 7 | prefix: string, 8 | connection: Redis, 9 | ) => ({ 10 | queue: new Queue(name, { 11 | connection, 12 | prefix, 13 | }), 14 | responders: BullMQResponders, 15 | }); 16 | -------------------------------------------------------------------------------- /lib/queue-factory/bullmqv4-factory.ts: -------------------------------------------------------------------------------- 1 | import { Queue } from "bullmq-v4"; 2 | import { Redis } from "ioredis"; 3 | import { BullMQResponders } from "../responders"; 4 | 5 | export const createQueue = ( 6 | name: string, 7 | prefix: string, 8 | connection: Redis 9 | ) => ({ 10 | queue: new Queue(name, { 11 | connection, 12 | prefix, 13 | }), 14 | responders: BullMQResponders, 15 | }); 16 | -------------------------------------------------------------------------------- /lib/queue-factory/bullmqv5-factory.ts: -------------------------------------------------------------------------------- 1 | import { Queue } from "bullmq-v5"; 2 | import { Redis } from "ioredis"; 3 | import { BullMQResponders } from "../responders"; 4 | 5 | export const createQueue = ( 6 | name: string, 7 | prefix: string, 8 | connection: Redis 9 | ) => ({ 10 | queue: new Queue(name, { 11 | connection, 12 | prefix, 13 | }), 14 | responders: BullMQResponders, 15 | }); 16 | -------------------------------------------------------------------------------- /lib/queues-cache.ts: -------------------------------------------------------------------------------- 1 | import * as Bull from "bull"; 2 | import { Queue } from "bullmq"; 3 | import { RedisOptions } from "ioredis"; 4 | import { keyBy } from "lodash"; 5 | import { FoundQueue, createQueue, getConnectionQueues } from "./queue-factory"; 6 | import { Responders } from "./interfaces/responders"; 7 | import { Integration } from "./interfaces/integration"; 8 | 9 | let queuesCache: { 10 | [index: string]: { queue: Bull.Queue | Queue; responders: Responders }; 11 | } = null; 12 | 13 | export const getCache = () => { 14 | return queuesCache; 15 | }; 16 | 17 | export function queueKey( 18 | queue: Omit 19 | ) { 20 | return `${queue.prefix}:${queue.name}`; 21 | } 22 | 23 | export async function updateQueuesCache( 24 | redisOpts: RedisOptions, 25 | opts: { 26 | nodes?: string[]; 27 | integrations?: { 28 | [key: string]: Integration; 29 | }; 30 | queueNames?: string[]; 31 | } = {} 32 | ) { 33 | const { nodes, integrations, queueNames } = opts; 34 | const newQueues = await getConnectionQueues(redisOpts, nodes, queueNames); 35 | 36 | queuesCache = queuesCache || {}; 37 | 38 | const oldQueues = Object.keys(queuesCache); 39 | const newQueuesObject = keyBy(newQueues, (queue) => queueKey(queue)); 40 | 41 | const toAdd = []; 42 | const toRemove = []; 43 | 44 | for (let i = 0; i < newQueues.length; i++) { 45 | const newQueue = newQueues[i]; 46 | const oldQueue = queuesCache[queueKey(newQueue)]; 47 | 48 | if (!oldQueue) { 49 | toAdd.push(newQueue); 50 | } 51 | } 52 | 53 | for (let i = 0; i < oldQueues.length; i++) { 54 | const oldQueue = oldQueues[i]; 55 | const newQueue = newQueuesObject[oldQueue]; 56 | 57 | if (!newQueue) { 58 | toRemove.push(queuesCache[oldQueue]); 59 | } 60 | } 61 | 62 | await Promise.all( 63 | toRemove.map(function ({ queue }: { queue: Bull.Queue | Queue }) { 64 | const closing = queue.close(); 65 | const name = (queue)["name"] as string; 66 | delete queuesCache[name]; 67 | return closing; 68 | }) 69 | ); 70 | 71 | toAdd.forEach(function (foundQueue: FoundQueue) { 72 | const key = queueKey(foundQueue); 73 | const queue = createQueue(foundQueue, redisOpts, { nodes, integrations }); 74 | if (queue) { 75 | queuesCache[key] = queue; 76 | } 77 | }); 78 | 79 | return newQueues.filter((queue) => !!queuesCache[queueKey(queue)]); 80 | } 81 | -------------------------------------------------------------------------------- /lib/responders/bull-responders.ts: -------------------------------------------------------------------------------- 1 | import * as Bull from "bull"; 2 | 3 | import { respond } from "./respond"; 4 | import { WebSocketClient } from "../ws-autoreconnect"; 5 | 6 | function paginate( 7 | ws: WebSocketClient, 8 | queue: Bull.Queue, 9 | messageId: string, 10 | start: number, 11 | end: number, 12 | method: string, 13 | opts?: { 14 | excludeData: boolean; 15 | } 16 | ) { 17 | start = start || 0; 18 | end = end || -1; 19 | return (queue) 20 | [method](start, end, opts) 21 | .then(function (jobs: Bull.Job[]) { 22 | respond(ws, Date.now(), messageId, jobs); 23 | }); 24 | } 25 | 26 | async function respondJobCommand( 27 | ws: WebSocketClient, 28 | queue: Bull.Queue, 29 | msg: any 30 | ) { 31 | const data = msg.data; 32 | const startTime = Date.now(); 33 | const job = await queue.getJob(data.jobId); 34 | 35 | switch (data.cmd) { 36 | case "retry": 37 | await job.retry(); 38 | break; 39 | case "promote": 40 | await job.promote(); 41 | break; 42 | case "remove": 43 | await job.remove(); 44 | break; 45 | case "discard": 46 | await job.discard(); 47 | break; 48 | case "moveToFailed": 49 | await job.moveToFailed({ message: "Failed manually" }); 50 | break; 51 | case "update": 52 | await job.update(data.data); 53 | default: 54 | console.error( 55 | `Missing command ${data.cmd}. Too old version of taskforce-connector?` 56 | ); 57 | } 58 | respond(ws, startTime, msg.id); 59 | } 60 | 61 | async function respondQueueCommand( 62 | ws: WebSocketClient, 63 | queue: Bull.Queue, 64 | msg: any 65 | ) { 66 | const startTime = Date.now(); 67 | const data = msg.data; 68 | switch (data.cmd) { 69 | case "getJob": 70 | const job = await queue.getJob(data.jobId); 71 | respond(ws, startTime, msg.id, job); 72 | break; 73 | case "getJobCounts": 74 | const jobCounts = await queue.getJobCounts(); 75 | respond(ws, startTime, msg.id, jobCounts); 76 | break; 77 | case "getMetrics": 78 | const metrics = await (queue).getMetrics( 79 | data.type, 80 | data.start, 81 | data.end 82 | ); 83 | respond(ws, startTime, msg.id, metrics); 84 | break; 85 | case "getWaiting": 86 | case "getActive": 87 | case "getDelayed": 88 | case "getCompleted": 89 | case "getFailed": 90 | case "getRepeatableJobs": 91 | case "getWorkers": 92 | paginate(ws, queue, msg.id, data.start, data.end, data.cmd, data.opts); 93 | break; 94 | 95 | case "getJobLogs": 96 | const logs = await queue.getJobLogs(data.jobId, data.start, data.end); 97 | respond(ws, startTime, msg.id, logs); 98 | 99 | case "getWaitingCount": 100 | case "getActiveCount": 101 | case "getDelayedCount": 102 | case "getCompletedCount": 103 | case "getFailedCount": 104 | case "getRepeatableCount": 105 | const count = await (queue)[data.cmd](); 106 | respond(ws, startTime, msg.id, count); 107 | break; 108 | case "getWorkersCount": 109 | const workers = await queue.getWorkers(); 110 | respond(ws, startTime, msg.id, workers.length); 111 | break; 112 | case "removeRepeatableByKey": 113 | await queue.removeRepeatableByKey(data.key); 114 | respond(ws, startTime, msg.id); 115 | break; 116 | case "add": 117 | await queue.add(...(data.args as [string, object, object])); 118 | respond(ws, startTime, msg.id); 119 | break; 120 | case "empty": 121 | await queue.empty(); 122 | respond(ws, startTime, msg.id); 123 | break; 124 | case "pause": 125 | await queue.pause(); 126 | respond(ws, startTime, msg.id); 127 | break; 128 | case "resume": 129 | await queue.resume(); 130 | respond(ws, startTime, msg.id); 131 | break; 132 | case "isPaused": 133 | const isPaused = await queue.isPaused(); 134 | respond(ws, startTime, msg.id, isPaused); 135 | break; 136 | case "obliterate": 137 | await queue.obliterate(); 138 | respond(ws, startTime, msg.id); 139 | break; 140 | case "clean": 141 | await queue.clean(data.grace, data.status, data.limit); 142 | respond(ws, startTime, msg.id); 143 | break; 144 | case "retryJobs": 145 | await (queue).retryJobs({ 146 | status: data.status, 147 | count: data.count, 148 | }); 149 | respond(ws, startTime, msg.id); 150 | break; 151 | default: 152 | console.error( 153 | `Missing command ${data.cmd}. Too old version of taskforce-connector?` 154 | ); 155 | respond(ws, msg.id, null); 156 | } 157 | } 158 | 159 | export const BullResponders = { 160 | respondJobCommand, 161 | respondQueueCommand, 162 | }; 163 | -------------------------------------------------------------------------------- /lib/responders/bullmq-responders.ts: -------------------------------------------------------------------------------- 1 | import { Queue, Job } from "bullmq"; 2 | 3 | import { respond } from "./respond"; 4 | import { WebSocketClient } from "../ws-autoreconnect"; 5 | 6 | function paginate( 7 | ws: WebSocketClient, 8 | queue: Queue, 9 | messageId: string, 10 | start: number, 11 | end: number, 12 | method: string, 13 | opts?: { 14 | excludeData: boolean; 15 | } 16 | ) { 17 | start = start || 0; 18 | end = end || -1; 19 | return (queue)[method](start, end, opts).then(function (jobs: Job[]) { 20 | respond(ws, Date.now(), messageId, jobs); 21 | }); 22 | } 23 | 24 | async function respondJobCommand(ws: WebSocketClient, queue: Queue, msg: any) { 25 | const data = msg.data; 26 | const startTime = Date.now(); 27 | const job = await queue.getJob(data.jobId); 28 | 29 | switch (data.cmd) { 30 | case "retry": 31 | await job.retry(); 32 | break; 33 | case "promote": 34 | await job.promote(); 35 | break; 36 | case "remove": 37 | await job.remove(); 38 | break; 39 | case "discard": 40 | await job.discard(); 41 | break; 42 | case "moveToFailed": 43 | await job.moveToFailed(new Error("Failed manually"), "0"); 44 | break; 45 | case "update": 46 | await job.updateData(data.data); 47 | default: 48 | console.error( 49 | `Missing command ${data.cmd}. Too old version of taskforce-connector?` 50 | ); 51 | } 52 | respond(ws, startTime, msg.id); 53 | } 54 | 55 | async function respondQueueCommand( 56 | ws: WebSocketClient, 57 | queue: Queue, 58 | msg: any 59 | ) { 60 | const data = msg.data; 61 | const startTime = Date.now(); 62 | switch (data.cmd) { 63 | case "getJob": 64 | const job = await queue.getJob(data.jobId); 65 | respond(ws, startTime, msg.id, job); 66 | break; 67 | case "getJobCounts": 68 | const jobCounts = await queue.getJobCounts(); 69 | respond(ws, startTime, msg.id, jobCounts); 70 | break; 71 | case "getMetrics": 72 | const metrics = await (queue).getMetrics( 73 | data.type, 74 | data.start, 75 | data.end 76 | ); 77 | respond(ws, startTime, msg.id, metrics); 78 | break; 79 | case "getDependencies": 80 | const dependencies = await queue.getDependencies( 81 | data.parentId, 82 | data.type, 83 | data.start, 84 | data.end 85 | ); 86 | respond(ws, startTime, msg.id, dependencies); 87 | break; 88 | 89 | case "getWaitingChildren": 90 | case "getWaiting": 91 | case "getActive": 92 | case "getDelayed": 93 | case "getCompleted": 94 | case "getFailed": 95 | case "getRepeatableJobs": 96 | case "getJobSchedulers": 97 | case "getWorkers": 98 | paginate(ws, queue, msg.id, data.start, data.end, data.cmd, data.opts); 99 | break; 100 | 101 | case "getJobLogs": 102 | const logs = await queue.getJobLogs(data.jobId, data.start, data.end); 103 | respond(ws, startTime, msg.id, logs); 104 | 105 | case "getRepeatableCount": 106 | case "getJobSchedulersCount": { 107 | const client = await queue.client; 108 | const jobSchedulersKey = queue.keys.repeat; 109 | const count = await client.zcard(jobSchedulersKey); 110 | respond(ws, startTime, msg.id, count); 111 | break; 112 | } 113 | 114 | case "getWaitingChildrenCount": 115 | case "getWaitingCount": 116 | case "getActiveCount": 117 | case "getDelayedCount": 118 | case "getCompletedCount": 119 | case "getFailedCount": 120 | const count = await (queue)[data.cmd](); 121 | respond(ws, startTime, msg.id, count); 122 | break; 123 | case "getWorkersCount": 124 | const workers = await queue.getWorkers(); 125 | respond(ws, startTime, msg.id, workers.length); 126 | break; 127 | case "removeRepeatableByKey": 128 | await queue.removeRepeatableByKey(data.key); 129 | respond(ws, startTime, msg.id); 130 | break; 131 | case "add": 132 | const [name, jobData, opts] = data.args as [string, object, object]; 133 | await queue.add(name, jobData, opts); 134 | respond(ws, startTime, msg.id); 135 | break; 136 | case "empty": 137 | await queue.drain(); 138 | respond(ws, startTime, msg.id); 139 | break; 140 | case "pause": 141 | await queue.pause(); 142 | respond(ws, startTime, msg.id); 143 | break; 144 | case "resume": 145 | await queue.resume(); 146 | respond(ws, startTime, msg.id); 147 | break; 148 | case "isPaused": 149 | const isPaused = await queue.isPaused(); 150 | respond(ws, startTime, msg.id, isPaused); 151 | break; 152 | case "obliterate": 153 | await queue.obliterate(); 154 | respond(ws, startTime, msg.id); 155 | break; 156 | case "clean": 157 | await queue.clean(data.grace, data.limit, data.status); 158 | respond(ws, startTime, msg.id); 159 | break; 160 | case "retryJobs": 161 | await (queue).retryJobs({ 162 | status: data.status, 163 | count: data.count, 164 | }); 165 | respond(ws, startTime, msg.id); 166 | break; 167 | default: 168 | console.error( 169 | `Missing command ${data.cmd}. Too old version of taskforce-connector?` 170 | ); 171 | respond(ws, startTime, msg.id, null); 172 | } 173 | } 174 | 175 | export const BullMQResponders = { 176 | respondJobCommand, 177 | respondQueueCommand, 178 | }; 179 | -------------------------------------------------------------------------------- /lib/responders/index.ts: -------------------------------------------------------------------------------- 1 | export * from './bull-responders'; 2 | export * from './bullmq-responders'; 3 | export * from './respond'; 4 | -------------------------------------------------------------------------------- /lib/responders/respond.ts: -------------------------------------------------------------------------------- 1 | import { WebSocketClient } from "../ws-autoreconnect"; 2 | 3 | export const respond = (ws: WebSocketClient, startTime: number, id: string, data: any = {}) => { 4 | const response = JSON.stringify({ 5 | id, 6 | data, 7 | }); 8 | ws.send(response, startTime); 9 | }; 10 | -------------------------------------------------------------------------------- /lib/socket.ts: -------------------------------------------------------------------------------- 1 | import { RedisOptions } from "ioredis"; 2 | import { pick } from "lodash"; 3 | import { getCache, updateQueuesCache, queueKey } from "./queues-cache"; 4 | import { WebSocketClient } from "./ws-autoreconnect"; 5 | import { 6 | FoundQueue, 7 | execRedisCommand, 8 | getRedisInfo, 9 | ping, 10 | } from "./queue-factory"; 11 | import { getQueueType, redisOptsFromUrl } from "./utils"; 12 | import { Integration } from "./interfaces/integration"; 13 | 14 | const { version } = require(`${__dirname}/../package.json`); 15 | 16 | const chalk = require("chalk"); 17 | 18 | export interface Connection { 19 | port?: number; 20 | host?: string; 21 | password?: string; 22 | db?: number; 23 | uri?: string; 24 | tls?: object; 25 | } 26 | 27 | export const Socket = ( 28 | name: string, 29 | server: string, 30 | token: string, 31 | connection: Connection, 32 | opts: { 33 | team?: string; 34 | nodes?: string[]; 35 | integrations?: { 36 | [key: string]: Integration; 37 | }; 38 | queueNames?: string[]; 39 | } = {} 40 | ) => { 41 | const { team, nodes } = opts; 42 | const ws = new WebSocketClient(); 43 | const redisOpts = redisOptsFromConnection(connection); 44 | 45 | ws.open(server, { 46 | headers: { 47 | Authorization: "Bearer " + token, 48 | Taskforce: "connector", 49 | }, 50 | }); 51 | 52 | console.log( 53 | `${chalk.yellow("WebSocket:")} ${chalk.blueBright( 54 | "opening connection to" 55 | )} ${chalk.gray("Taskforce.sh")} (${chalk.blueBright(server)})` 56 | ); 57 | 58 | ws.onopen = function open() { 59 | console.log( 60 | chalk.yellow("WebSocket:") + 61 | chalk.blueBright(" opened connection to ") + 62 | chalk.gray("Taskforce.sh") 63 | ); 64 | }; 65 | 66 | ws.onerror = function (err) { 67 | var msg; 68 | if (err.message === "Unexpected server response: 401") { 69 | msg = 70 | "Authorization failed, please check that you are using the correct token from your account page"; 71 | } else { 72 | msg = err.message; 73 | } 74 | console.log(chalk.yellow("WebSocket: ") + chalk.red(msg)); 75 | }; 76 | 77 | ws.onmessage = async function incoming(input: string) { 78 | const startTime = Date.now(); 79 | 80 | console.log( 81 | `${chalk.yellow("WebSocket:")} ${chalk.blueBright("received")} %s`, 82 | input 83 | ); 84 | 85 | try { 86 | if (input === "authorized") { 87 | console.log( 88 | chalk.yellow("WebSocket: ") + 89 | chalk.green("Succesfully authorized to taskforce.sh service") 90 | ); 91 | 92 | // 93 | // Send this connection. 94 | // 95 | const queues = await updateQueuesCache(redisOpts, opts); 96 | console.log( 97 | `${chalk.yellow("WebSocket:")} ${chalk.green( 98 | "sending connection:" 99 | )} ${chalk.blueBright(name)} ${ 100 | team ? chalk.green(" for team ") + chalk.blueBright(team) : "" 101 | }` 102 | ); 103 | ws.send( 104 | JSON.stringify({ 105 | res: "connection", 106 | cmd: "update", 107 | queues, 108 | connection: name, 109 | team, 110 | version, 111 | }), 112 | startTime 113 | ); 114 | } else { 115 | const msg = JSON.parse(input); 116 | 117 | if (!msg.data) { 118 | console.error( 119 | chalk.red("WebSocket:") + 120 | chalk.blueBright(" missing message data "), 121 | msg 122 | ); 123 | return; 124 | } 125 | 126 | const { res, queueName, queuePrefix } = msg.data; 127 | 128 | switch (res) { 129 | case "connections": 130 | await respondConnectionCommand(connection, msg); 131 | break; 132 | case "queues": 133 | case "jobs": 134 | let cache = getCache(); 135 | if (!cache) { 136 | await updateQueuesCache(redisOpts, opts); 137 | cache = getCache(); 138 | if (!cache) { 139 | throw new Error("Unable to update queues"); 140 | } 141 | } 142 | const { queue, responders } = 143 | cache[ 144 | queueKey({ name: queueName, prefix: queuePrefix || "bull" }) 145 | ]; 146 | 147 | if (!queue) { 148 | ws.send( 149 | JSON.stringify({ 150 | id: msg.id, 151 | err: "Queue not found", 152 | }), 153 | startTime 154 | ); 155 | } else { 156 | switch (res) { 157 | case "queues": 158 | await responders.respondQueueCommand(ws, queue, msg); 159 | break; 160 | case "jobs": 161 | await responders.respondJobCommand(ws, queue, msg); 162 | break; 163 | } 164 | } 165 | break; 166 | } 167 | } 168 | } catch (err) { 169 | console.error(err); 170 | } 171 | }; 172 | 173 | async function respondConnectionCommand(connection: Connection, msg: any) { 174 | const startTime = Date.now(); 175 | 176 | const data = msg.data; 177 | 178 | switch (data.cmd) { 179 | case "ping": 180 | const pong = await ping(redisOpts, nodes); 181 | respond(msg.id, startTime, pong); 182 | break; 183 | case "getConnection": 184 | { 185 | const queues = await updateQueuesCache(redisOpts, opts); 186 | 187 | console.log( 188 | `${chalk.yellow("WebSocket:")} ${chalk.green( 189 | "sending connection:" 190 | )} ${chalk.blueBright(name)} ${ 191 | team ? chalk.green(" for team ") + chalk.blueBright(team) : "" 192 | }` 193 | ); 194 | 195 | logSendingQueues(queues); 196 | 197 | respond(msg.id, startTime, { 198 | queues, 199 | connection: name, 200 | team, 201 | version, 202 | }); 203 | } 204 | break; 205 | case "getQueues": 206 | { 207 | const queues = await updateQueuesCache(redisOpts, opts); 208 | 209 | logSendingQueues(queues); 210 | 211 | respond(msg.id, startTime, queues); 212 | } 213 | break; 214 | case "getInfo": 215 | const info = await getRedisInfo(redisOpts, nodes); 216 | respond(msg.id, startTime, info); 217 | break; 218 | 219 | case "getQueueType": 220 | const queueType = await execRedisCommand( 221 | redisOpts, 222 | (client) => getQueueType(data.name, data.prefix, client), 223 | nodes 224 | ); 225 | respond(msg.id, startTime, { queueType }); 226 | break; 227 | } 228 | } 229 | 230 | function logSendingQueues(queues: FoundQueue[]) { 231 | for (const queue of queues) { 232 | const { name, prefix, type } = queue; 233 | console.log( 234 | `${chalk.yellow("WebSocket:")} ${chalk.blueBright( 235 | "Sending queue:" 236 | )} ${chalk.green(name)} ${chalk.blueBright("type:")} ${chalk.green( 237 | type 238 | )} ${chalk.blueBright("prefix:")} ${chalk.green(prefix)}` 239 | ); 240 | } 241 | } 242 | 243 | function respond(id: string, startTime: number, data: any = {}) { 244 | const response = JSON.stringify({ 245 | id, 246 | data, 247 | }); 248 | ws.send(response, startTime); 249 | } 250 | }; 251 | 252 | function redisOptsFromConnection(connection: Connection): RedisOptions { 253 | let opts: RedisOptions = { 254 | ...pick(connection, [ 255 | "host", 256 | "port", 257 | "username", 258 | "password", 259 | "family", 260 | "sentinelPassword", 261 | "db", 262 | "tls", 263 | "sentinels", 264 | "name", 265 | ]), 266 | }; 267 | 268 | if (connection.uri) { 269 | opts = { ...opts, ...redisOptsFromUrl(connection.uri) }; 270 | } 271 | 272 | opts.retryStrategy = function (times: number) { 273 | times = times % 8; 274 | const delay = Math.round(Math.pow(2, times + 8)); 275 | console.log(chalk.yellow("Redis: ") + `Reconnecting in ${delay} ms`); 276 | return delay; 277 | }; 278 | return opts; 279 | } 280 | -------------------------------------------------------------------------------- /lib/utils.ts: -------------------------------------------------------------------------------- 1 | import { Redis, Cluster } from "ioredis"; 2 | export type QueueType = "bull" | "bullmq" | "bullmq-pro"; 3 | import { RedisOptions } from "ioredis"; 4 | import * as url from "url"; 5 | 6 | export function redisOptsFromUrl(urlString: string) { 7 | const redisOpts: RedisOptions = {}; 8 | try { 9 | const redisUrl = url.parse(urlString); 10 | redisOpts.port = parseInt(redisUrl.port) || 6379; 11 | redisOpts.host = redisUrl.hostname; 12 | redisOpts.db = redisUrl.pathname 13 | ? parseInt(redisUrl.pathname.split("/")[1]) 14 | : 0; 15 | if (redisUrl.auth) { 16 | const username = redisUrl.auth.split(":")[0]; 17 | redisOpts.username = username ? username : undefined; 18 | redisOpts.password = redisUrl.auth.split(":")[1]; 19 | } 20 | } catch (e) { 21 | throw new Error(e.message); 22 | } 23 | return redisOpts; 24 | } 25 | 26 | export const getQueueType = async ( 27 | queueName: string, 28 | prefix: string, 29 | client: Redis | Cluster 30 | ): Promise<{ type: QueueType; majorVersion: number; version?: string }> => { 31 | // Check if queue includes the "meta" key, if so, it is a bullmq queue type. 32 | const metaKey = `${prefix}:${queueName}:meta`; 33 | 34 | // check if meta key includes the field "pro" 35 | // if so, it is a bullmq-pro queue type. 36 | const hasMeta = await client.exists(metaKey); 37 | if (hasMeta) { 38 | const longVersion = await client.hget(metaKey, "version"); 39 | const version = longVersion ? longVersion.split(":")[1] : ""; 40 | 41 | if (longVersion) { 42 | const type = longVersion.includes("bullmq-pro") ? "bullmq-pro" : "bullmq"; 43 | 44 | // Try to get the major version number from the version string (e.g. bullmq:3.20.0) 45 | const majorVersionStr = version?.split(".")[0]; 46 | const majorVersion = majorVersionStr ? parseInt(majorVersionStr, 10) : 0; 47 | if (majorVersion >= 3) { 48 | return { type, majorVersion, version }; 49 | } 50 | } 51 | 52 | const maxLenEvents = await client.hget(metaKey, "opts.maxLenEvents"); 53 | if (maxLenEvents) { 54 | return { type: "bullmq", majorVersion: 0, version }; 55 | } 56 | } 57 | 58 | // otherwise, it is a bull queue type. 59 | return { type: "bull", majorVersion: 0 }; 60 | }; 61 | -------------------------------------------------------------------------------- /lib/version-checker.ts: -------------------------------------------------------------------------------- 1 | import { gt } from 'semver'; 2 | import { yellow } from "chalk"; 3 | 4 | /** 5 | * Check if the current version of taskforce is the latest version 6 | * @param name - package name 7 | * @param version - current version of taskforce 8 | */ 9 | export const versionChecker = async (name: string, version: string) => { 10 | 11 | const latestVersion = (await import('latest-version')).default; 12 | 13 | try { 14 | const newestVersion = await latestVersion(name); 15 | 16 | if (gt(newestVersion, version)) { 17 | console.error( 18 | yellow( 19 | "New version " + 20 | newestVersion + 21 | " of taskforce available, please upgrade with yarn global add taskforce-connector" 22 | ) 23 | ); 24 | } 25 | } catch (err) { 26 | console.error( 27 | yellow( 28 | "Error checking for latest version of taskforce" 29 | ), 30 | err 31 | ); 32 | } 33 | }; 34 | -------------------------------------------------------------------------------- /lib/ws-autoreconnect.ts: -------------------------------------------------------------------------------- 1 | import * as WebSocket from "ws"; 2 | import * as chalk from "chalk"; 3 | import { WebsocketError } from "./ws-errors.enum"; 4 | 5 | const HEARTBEAT_INTERVAL = 15000; 6 | 7 | export class WebSocketClient { 8 | private number = 0; // Message number 9 | private autoReconnectInterval = 5 * 1000; // ms 10 | private url: string; 11 | private opts: object; 12 | private instance: WebSocket; 13 | private pingTimeout: NodeJS.Timeout; 14 | 15 | open(url: string, opts: object) { 16 | this.url = url; 17 | this.opts = opts; 18 | 19 | this.instance = new WebSocket(url, opts); 20 | this.instance.on("open", () => { 21 | this.heartbeat(); 22 | this.onopen(); 23 | }); 24 | 25 | this.instance.on("message", (data: string, flags: object) => { 26 | this.number++; 27 | this.onmessage(data, flags, this.number); 28 | }); 29 | 30 | this.instance.on("ping", () => this.heartbeat()); 31 | 32 | this.instance.on("close", (codeOrError: number | Error) => { 33 | clearTimeout(this.pingTimeout); 34 | 35 | switch (codeOrError) { 36 | case WebsocketError.NormalClosure: 37 | console.log( 38 | chalk.yellow("WebSocket:") + chalk.blue("normally closed") 39 | ); 40 | break; 41 | 42 | case 4000: 43 | console.log( 44 | chalk.yellow("WebSocket:") + chalk.red(" Invalid authentication") 45 | ); 46 | break; 47 | default: 48 | // Abnormal closure 49 | this.reconnect(codeOrError); 50 | break; 51 | } 52 | this.onclose(codeOrError); 53 | }); 54 | this.instance.on("error", (err: any) => { 55 | switch (err["code"]) { 56 | case "ECONNREFUSED": 57 | break; 58 | default: 59 | this.onerror(err); 60 | break; 61 | } 62 | }); 63 | } 64 | 65 | send(data: string, startTime: number, option?: { 66 | mask?: boolean; 67 | binary?: boolean; 68 | compress?: boolean; 69 | fin?: boolean; 70 | }) { 71 | try { 72 | this.instance.send(data, option, (err: Error) => { 73 | if (err) { 74 | console.log( 75 | `${chalk.yellow("WebSocket:")} ${chalk.red("send error", err)}` 76 | ); 77 | } else { 78 | console.log( 79 | `${chalk.yellow("WebSocket:")} ${chalk.blue("data sent successfully in")} ${chalk.green( 80 | Date.now() - startTime)}ms` 81 | ); 82 | } 83 | }); 84 | } catch (err) { 85 | this.instance.emit("error", err); 86 | } 87 | } 88 | 89 | reconnect(err: Error) { 90 | var msg = err.message || ""; 91 | console.log( 92 | chalk.yellow("WebSocket:") + 93 | chalk.red(` ${msg} retry in ${this.autoReconnectInterval}ms`) 94 | ); 95 | this.instance.removeAllListeners(); 96 | setTimeout(() => { 97 | console.log("WebSocket: reconnecting..."); 98 | this.open(this.url, this.opts); 99 | }, this.autoReconnectInterval); 100 | } 101 | 102 | onopen() { 103 | console.log("WebSocket: open", arguments); 104 | } 105 | 106 | heartbeat() { 107 | clearTimeout(this.pingTimeout); 108 | 109 | this.pingTimeout = setTimeout(() => { 110 | this.instance.terminate(); 111 | }, HEARTBEAT_INTERVAL); 112 | } 113 | 114 | onmessage = function (data: string, flags: object, num: number) { 115 | console.log("WebSocket: message", data, flags, num); 116 | }; 117 | onerror = function (e: Error) { 118 | console.log("WebSocket: error", arguments); 119 | }; 120 | onclose = function (e: Error) { 121 | console.log("WebSocket: closed", arguments); 122 | }; 123 | } 124 | -------------------------------------------------------------------------------- /lib/ws-errors.enum.ts: -------------------------------------------------------------------------------- 1 | export enum WebsocketError { 2 | NormalClosure = 1000, 3 | GoingAway = 1001, 4 | UnsupportedData = 1003, 5 | NoStatusDefined = 1005, 6 | AbnormalClosure = 1006, 7 | InvalidFramePayloadData = 1007, 8 | PolicyViolation = 1008, 9 | MessageTooBig = 1009, 10 | MissingExtension = 1010, 11 | InternalError = 1011, 12 | ServiceRestart = 1012, 13 | TryAgainLater = 1013, 14 | BadGateway = 1014, 15 | TLSHandshake = 1015 16 | } 17 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "taskforce-connector", 3 | "version": "1.35.0", 4 | "description": "Connects queues to Taskforce", 5 | "preferGlobal": true, 6 | "engine": { 7 | "node": ">=18" 8 | }, 9 | "main": "dist/index.js", 10 | "types": "dist/index.d.ts", 11 | "module": "./dist/index.js", 12 | "source": "lib/index.ts", 13 | "bin": { 14 | "taskforce": "app.js" 15 | }, 16 | "files": [ 17 | "dist" 18 | ], 19 | "dependencies": { 20 | "bull": "^4.15.1", 21 | "bullmq": "^5.34.10", 22 | "bullmq-v3": "npm:bullmq@^3.16.2", 23 | "bullmq-v4": "npm:bullmq@^4.18.2", 24 | "bullmq-v5": "npm:bullmq@^5.47.0", 25 | "chalk": "^4.1.0", 26 | "commander": "^12.1.0", 27 | "ioredis": "^5.4.1", 28 | "latest-version": "^9.0.0", 29 | "lodash": "^4.17.21", 30 | "semver": "^7.7.1", 31 | "ws": "^8.17.0" 32 | }, 33 | "devDependencies": { 34 | "@semantic-release/changelog": "^6.0.3", 35 | "@semantic-release/commit-analyzer": "^13.0.1", 36 | "@semantic-release/git": "^10.0.1", 37 | "@semantic-release/github": "^11.0.1", 38 | "@semantic-release/npm": "^12.0.1", 39 | "@semantic-release/release-notes-generator": "^14.0.3", 40 | "@types/bull": "^3.15.8", 41 | "@types/chalk": "^2.2.0", 42 | "@types/lodash": "^4.17.6", 43 | "@types/node": "^20.10.5", 44 | "@types/semver": "^7.5.8", 45 | "@types/ws": "^6.0.1", 46 | "jest": "^29.7.0", 47 | "typescript": "^5.3.3" 48 | }, 49 | "scripts": { 50 | "build": "tsc", 51 | "test": "jest", 52 | "start": "node app.js", 53 | "prepare": "npm run build", 54 | "remediate:relax": "osv-scanner fix --strategy=relax -M package.json -L package-lock.json && yarn install" 55 | }, 56 | "repository": { 57 | "type": "git", 58 | "url": "git+https://github.com/taskforcesh/taskforce-connector.git" 59 | }, 60 | "keywords": [ 61 | "taskforce", 62 | "bull", 63 | "queue" 64 | ], 65 | "author": "Taskforce.sh Inc.", 66 | "license": "MIT", 67 | "bugs": { 68 | "url": "https://github.com/taskforcesh/taskforce-connector/issues" 69 | }, 70 | "homepage": "https://github.com/taskforcesh/taskforce-connector#readme", 71 | "release": { 72 | "branches": [ 73 | "master" 74 | ], 75 | "plugins": [ 76 | "@semantic-release/commit-analyzer", 77 | "@semantic-release/release-notes-generator", 78 | [ 79 | "@semantic-release/changelog", 80 | { 81 | "changelogFile": "CHANGELOG.md" 82 | } 83 | ], 84 | [ 85 | "@semantic-release/npm", 86 | { 87 | "npmPublish": true 88 | } 89 | ], 90 | "@semantic-release/github", 91 | [ 92 | "@semantic-release/git", 93 | { 94 | "assets": [ 95 | "package.json", 96 | "yarn.lock", 97 | "CHANGELOG.md" 98 | ], 99 | "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}" 100 | } 101 | ] 102 | ] 103 | }, 104 | "jest": { 105 | "testEnvironment": "node" 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /tests/cli.spec.js: -------------------------------------------------------------------------------- 1 | const { program } = require("commander"); 2 | const { before } = require("lodash"); 3 | const mockExit = jest.spyOn(process, "exit").mockImplementation(() => {}); 4 | const consoleSpy = jest.spyOn(console, "error").mockImplementation(() => {}); 5 | 6 | describe("CLI Options", () => { 7 | beforeAll(() => { 8 | process.env.TASKFORCE_TOKEN = "1234567890"; 9 | }); 10 | 11 | afterEach(() => { 12 | delete process.env.TASKFORCE_TOKEN; 13 | jest.clearAllMocks(); 14 | }); 15 | 16 | it("should use environment variable for nodes if set", () => { 17 | process.env.REDIS_NODES = "node1:6379,node2:6379"; 18 | require("../app.js"); // Update the path as necessary 19 | 20 | expect(program.nodes).toEqual(["node1:6379", "node2:6379"]); 21 | }); 22 | 23 | it("should use default port if no environment variable is set", () => { 24 | delete process.env.REDIS_PORT; 25 | require("../app.js"); // Update the path as necessary 26 | 27 | expect(program.port).toBe("6379"); 28 | }); 29 | 30 | // Add more tests for each option and scenario... 31 | }); 32 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "module": "Node16", 5 | "outDir": "dist", 6 | "sourceMap": false, 7 | "declaration": true, 8 | "noImplicitAny": true, 9 | "removeComments": true, 10 | "moduleResolution": "Node16", 11 | "resolveJsonModule": true, 12 | "allowSyntheticDefaultImports": true, 13 | "esModuleInterop": false 14 | }, 15 | "compileOnSave": false, 16 | "include": ["./lib"], 17 | "exclude": ["node_modules"], 18 | "output": "./dist" 19 | } 20 | --------------------------------------------------------------------------------