├── .editorconfig ├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ └── main.yml ├── .gitignore ├── .npmrc ├── CHANGELOG.md ├── LICENSE.md ├── README.md ├── app.json ├── package.json └── src ├── constants.js ├── queue.js ├── receiver.js └── sender.js /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 2 8 | end_of_line = lf 9 | charset = utf-8 10 | trim_trailing_whitespace = true 11 | insert_final_newline = true 12 | max_line_length = 100 13 | indent_brace_style = 1TBS 14 | spaces_around_operators = true 15 | quote_type = auto 16 | 17 | [package.json] 18 | indent_style = space 19 | indent_size = 2 20 | 21 | [*.md] 22 | trim_trailing_whitespace = false 23 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | - package-ecosystem: "github-actions" 8 | directory: "/" 9 | schedule: 10 | # Check for updates to GitHub Actions every weekday 11 | interval: "daily" 12 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | 11 | jobs: 12 | test: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout 16 | uses: actions/checkout@v4 17 | - name: Setup Node.js 18 | uses: actions/setup-node@v4 19 | with: 20 | node-version: lts/* 21 | - name: Install 22 | run: npm install --no-package-lock 23 | - name: Test 24 | run: npm test 25 | # - name: Report 26 | # run: mkdir -p coverage && npx c8 report --reporter=text-lcov > coverage/lcov.info 27 | # - name: Coverage 28 | # uses: coverallsapp/github-action@master 29 | # with: 30 | # github-token: ${{ secrets.GITHUB_TOKEN }} 31 | - name: Release 32 | if: ${{ github.ref == 'refs/heads/master' && !startsWith(github.event.head_commit.message, 'chore(release):') && !startsWith(github.event.head_commit.message, 'docs:') }} 33 | env: 34 | CONVENTIONAL_GITHUB_RELEASER_TOKEN: ${{ secrets.GH_TOKEN }} 35 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 36 | run: | 37 | git config --global user.email ${{ secrets.GIT_EMAIL }} 38 | git config --global user.name ${{ secrets.GIT_USERNAME }} 39 | npm run release 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ############################ 2 | # npm 3 | ############################ 4 | node_modules 5 | npm-debug.log 6 | .node_history 7 | yarn.lock 8 | package-lock.json 9 | 10 | ############################ 11 | # tmp, editor & OS files 12 | ############################ 13 | .tmp 14 | *.swo 15 | *.swp 16 | *.swn 17 | *.swm 18 | .DS_Store 19 | *# 20 | *~ 21 | .idea 22 | *sublime* 23 | nbproject 24 | 25 | ############################ 26 | # Tests 27 | ############################ 28 | testApp 29 | coverage 30 | .nyc_output 31 | 32 | ############################ 33 | # Other 34 | ############################ 35 | .env 36 | .envrc 37 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | unsafe-perm=true 2 | save-prefix=~ 3 | save=false 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. 4 | 5 | ### 1.0.32 (2025-06-04) 6 | 7 | ### 1.0.31 (2024-07-31) 8 | 9 | ### 1.0.30 (2024-07-02) 10 | 11 | ### 1.0.29 (2024-06-26) 12 | 13 | ### 1.0.28 (2024-06-13) 14 | 15 | ### 1.0.27 (2024-03-19) 16 | 17 | ### 1.0.26 (2023-12-19) 18 | 19 | ### 1.0.25 (2023-12-01) 20 | 21 | ### 1.0.24 (2023-10-24) 22 | 23 | ### 1.0.23 (2023-10-10) 24 | 25 | ### 1.0.22 (2023-09-06) 26 | 27 | ### 1.0.21 (2023-08-09) 28 | 29 | ### 1.0.20 (2023-05-13) 30 | 31 | ### 1.0.19 (2023-02-20) 32 | 33 | ### 1.0.18 (2023-02-20) 34 | 35 | ### 1.0.17 (2022-09-30) 36 | 37 | ### 1.0.16 (2022-09-06) 38 | 39 | ### 1.0.15 (2022-05-17) 40 | 41 | ### 1.0.14 (2022-04-11) 42 | 43 | ### 1.0.13 (2022-04-01) 44 | 45 | ### 1.0.12 (2022-03-22) 46 | 47 | ### 1.0.11 (2022-03-02) 48 | 49 | ### 1.0.10 (2022-03-02) 50 | 51 | ### 1.0.9 (2022-02-25) 52 | 53 | ### 1.0.8 (2022-02-24) 54 | 55 | ### 1.0.7 (2022-02-22) 56 | 57 | ### 1.0.6 (2022-02-14) 58 | 59 | ### 1.0.5 (2022-02-01) 60 | 61 | ### 1.0.4 (2022-01-27) 62 | 63 | ### 1.0.3 (2021-12-24) 64 | 65 | ### 1.0.2 (2021-12-21) 66 | 67 | ### 1.0.1 (2021-11-02) 68 | 69 | ## 1.0.0 (2021-10-28) 70 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2019 Microlink (microlink.io) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | microlink logo 3 | microlink logo 4 |
5 |
6 |
7 | 8 | [![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy) 9 | 10 | > High resilient & reliable URLs processing queue. 11 | 12 | ## Motivation 13 | 14 | When you are consuming an API (such as [Microlink API](https://docs.microlink.io/api/#introduction) or using any other API) you need to assume it can be a wide resource and some of them can eventually fail for many and varied reasons: rate limit, timeout, proxy rotation, etc. 15 | 16 | Some of these errors can regret simply retrying the original request after waiting a prudent period of time 17 | 18 | In any case, you want to have the guaranteed the URL will be successfully processed in an indeterministic moment in the future. 19 | 20 | This package presents a small but powerful architecture for processing URLs with guarantees. 21 | 22 | ## Architecture 23 | 24 | For getting guarantees that your requests are successfully processed, we are going to use [bull](https://github.com/OptimalBits/bull), a lightweight FIFO queue backed on redis. 25 | 26 | The requests (called **jobs**) will remain in the queue until a consumer get it and verify is has been successfully processed. 27 | 28 | ## Sender 29 | 30 | > npm run start:sender 31 | 32 | The sender is is who sends the work to process the consumers. 33 | 34 | It's exposed using a HTTP server. 35 | 36 | For sending a job into the queue, just send it using a `GET` 37 | 38 | ```bash 39 | $ curl http://localhost:3000\?url\=http://microlink.com\&video 40 | ``` 41 | 42 | If the job has been added successfully, you will have a `201 Created` and the `job.id` back. 43 | 44 | ```bash 45 | HTTP/1.1 201 Created 46 | { 47 | "id": "7" 48 | } 49 | ``` 50 | 51 | You can provide two customizable things as query parameters: 52 | 53 | - **priority** (optional): Priority value. ranges from 1 (highest priority) to [MAX_INT](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER) (lowest priority). 54 | - **delay** (optional): An amount of miliseconds to wait until this job can be processed. 55 | 56 | Any other query parameter provided will be passed as part of the job. 57 | 58 | ## Receiver 59 | 60 | > npm run start:receiver 61 | 62 | The receivers will process the job pushed by the consumer. 63 | 64 | They are a pool of processes, waiting on idle until they have something to consume. 65 | 66 | ``` 67 | $ npm run start:receiver 68 | 69 | receiver=0 status=listening 70 | receiver=1 status=listening 71 | receiver=2 status=listening 72 | receiver=3 status=listening 73 | status=processing id=13 74 | receiver=0 status=processed id=13 status=success 75 | ``` 76 | 77 | by default, You are going to have one receiver per physical CPU core. Since we delegated on [farm](https://github.com/Kikobeats/farm-cli#farm-cli), this is easily customizable. 78 | 79 | ## License 80 | 81 | **queue** © [microlink.io](https://microlink.io), released under the [MIT](https://github.com/microlinkhq/queue/blob/master/LICENSE.md) License.
82 | Authored and maintained by [Kiko Beats](https://kikobeats.com) with help from [contributors](https://github.com/microlinkhq/queue/contributors). 83 | 84 | > [microlink.io](https://microlink.io) · GitHub [microlink.io](https://github.com/microlinkhq) · Twitter [@microlinkhq](https://twitter.com/microlinkhq) 85 | -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@microlinkhq/queue", 3 | "description": "High resilient queue for processing URLs", 4 | "repository": "https://github.com/microlinkhq/queue", 5 | "logo": "https://microlink.io/logo-trim.jpg", 6 | "keywords": ["microlink", "queue"], 7 | "addons": [ 8 | "scheduler:standard", 9 | "heroku-redis:hobby-dev", 10 | "papertrail:choklad" 11 | ], 12 | "env": { 13 | "NODE_ENV": "production" 14 | }, 15 | "formation": { 16 | "web": { 17 | "command": "npm start:sender" 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "queue", 3 | "description": "High resilient queue for processing URLs", 4 | "homepage": "https://nicedoc.io/microlinkhq/queue", 5 | "version": "1.0.32", 6 | "main": "src/receiver.js", 7 | "author": { 8 | "email": "hello@microlink.io", 9 | "name": "microlink.io", 10 | "url": "https://microlink.io" 11 | }, 12 | "contributors": [ 13 | { 14 | "name": "Kiko", 15 | "email": "josefrancisco.verdu@gmail.com" 16 | } 17 | ], 18 | "repository": { 19 | "type": "git", 20 | "url": "git+https://github.com/microlinkhq/queue.git" 21 | }, 22 | "bugs": { 23 | "url": "https://github.com/microlinkhq/queue/issues" 24 | }, 25 | "keywords": [ 26 | "microlink", 27 | "queue" 28 | ], 29 | "dependencies": { 30 | "@microlink/mql": "~0.14.0", 31 | "bull": "~4.16.0", 32 | "debug-logfmt": "~1.2.0", 33 | "send-http": "~1.0.2", 34 | "to-query": "~1.6.1" 35 | }, 36 | "devDependencies": { 37 | "@commitlint/cli": "latest", 38 | "@commitlint/config-conventional": "latest", 39 | "conventional-github-releaser": "latest", 40 | "finepack": "latest", 41 | "git-authors-cli": "latest", 42 | "nano-staged": "latest", 43 | "npm-check-updates": "latest", 44 | "prettier-standard": "latest", 45 | "simple-git-hooks": "latest", 46 | "standard": "latest", 47 | "standard-markdown": "latest", 48 | "standard-version": "latest" 49 | }, 50 | "engines": { 51 | "node": ">= 8" 52 | }, 53 | "files": [ 54 | "src" 55 | ], 56 | "scripts": { 57 | "clean": "rm -rf node_modules", 58 | "contributors": "(npx git-authors-cli && npx finepack && git add package.json && git commit -m 'build: contributors' --no-verify) || true", 59 | "lint": "standard-markdown README.md && standard", 60 | "postrelease": "npm run release:tags && npm run release:github", 61 | "prerelease": "npm run update:check && npm run contributors", 62 | "pretest": "npm run lint", 63 | "pretty": "prettier-standard index.js {core,test,bin,scripts}/**/*.js --single-quote --print-width 100", 64 | "release": "standard-version -a", 65 | "release:github": "conventional-github-releaser -p angular", 66 | "release:tags": "git push --follow-tags origin HEAD:master", 67 | "start:receiver": "DEBUG=queue* node src/receiver.js", 68 | "start:sender": "DEBUG=queue* node src/sender.js", 69 | "test": "exit 0", 70 | "update": "ncu -u", 71 | "update:check": "ncu -- --error-level 2" 72 | }, 73 | "private": true, 74 | "license": "MIT", 75 | "commitlint": { 76 | "extends": [ 77 | "@commitlint/config-conventional" 78 | ] 79 | }, 80 | "nano-staged": { 81 | "*.js,!*.min.js,": [ 82 | "prettier-standard" 83 | ], 84 | "*.md": [ 85 | "standard-markdown" 86 | ], 87 | "package.json": [ 88 | "finepack" 89 | ] 90 | }, 91 | "simple-git-hooks": { 92 | "commit-msg": "npx commitlint --edit", 93 | "pre-commit": "npx nano-staged" 94 | }, 95 | "standard-version": { 96 | "scripts": { 97 | "prechangelog": "git-authors-cli" 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/constants.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { REDIS_URL } = process.env 4 | 5 | if (!REDIS_URL) { 6 | throw new TypeError("You need to provide redis connection as 'REDIS_URL'.") 7 | } 8 | 9 | const PORT = process.env.PORT || process.env.port || 3000 10 | 11 | const CONCURRENCY = process.env.CONCURRENCY || 1 12 | 13 | module.exports = { 14 | CONCURRENCY, 15 | REDIS_URL, 16 | PORT 17 | } 18 | -------------------------------------------------------------------------------- /src/queue.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const Queue = require('bull') 4 | 5 | const { name } = require('../package.json') 6 | 7 | const { REDIS_URL } = require('./constants') 8 | 9 | module.exports = new Queue(name, REDIS_URL) 10 | -------------------------------------------------------------------------------- /src/receiver.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('debug-logfmt')('queue:receiver') 4 | const mql = require('@microlink/mql') 5 | 6 | const queue = require('./queue') 7 | const { CONCURRENCY } = require('./constants') 8 | 9 | debug('status=listening') 10 | 11 | queue.process(CONCURRENCY, async ({ id, data }) => { 12 | debug({ state: 'processing', id }) 13 | 14 | const { url, ...opts } = data 15 | const { status } = await mql(url, opts) 16 | 17 | debug({ state: 'processed', id, url, status }) 18 | 19 | if (status === 'success') return Promise.resolve() 20 | return Promise.reject(new Error(status)) 21 | }) 22 | -------------------------------------------------------------------------------- /src/sender.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('debug-logfmt')('queue:sender') 4 | const { createServer } = require('http') 5 | const toQuery = require('to-query')() 6 | const send = require('send-http') 7 | 8 | const { PORT } = require('./constants') 9 | const queue = require('./queue') 10 | 11 | const isStatic = req => req.url.startsWith('/favicon.ico') || req.url.startsWith('/robots.txt') 12 | 13 | const server = createServer(async (req, res) => { 14 | if (isStatic(req)) return send(res, 204) 15 | const query = toQuery(req.url) 16 | if (!query.url) return send(res, 200) 17 | const { priority, delay } = query 18 | const { id } = await queue.add(query, { priority, delay }) 19 | return send(res, 201, { id }) 20 | }) 21 | 22 | server.listen(PORT, () => { 23 | debug(`Listening at http://localhost:${PORT}`) 24 | }) 25 | --------------------------------------------------------------------------------