├── .eslintrc ├── tests ├── .eslintrc └── index-test.js ├── .gitattributes ├── .gitignore ├── nwb.config.js ├── .travis.yml ├── CONTRIBUTING.md ├── package.json ├── LICENSE ├── src └── index.js └── README.md /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "react-tools" 3 | } 4 | -------------------------------------------------------------------------------- /tests/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "mocha": true 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /coverage 2 | /demo/dist 3 | /es 4 | /lib 5 | /node_modules 6 | /umd 7 | npm-debug.log* 8 | -------------------------------------------------------------------------------- /nwb.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | type: 'web-module', 3 | npm: { 4 | esModules: true, 5 | umd: {} 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /tests/index-test.js: -------------------------------------------------------------------------------- 1 | import expect from 'expect' 2 | 3 | import message from 'src/index' 4 | 5 | describe('Module template', () => { 6 | it('displays a welcome message', () => { 7 | expect(message).toContain('Welcome to swimmer') 8 | }) 9 | }) 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | 3 | language: node_js 4 | node_js: 5 | - 8 6 | 7 | before_install: 8 | - npm install codecov.io coveralls 9 | 10 | after_success: 11 | - cat ./coverage/lcov.info | ./node_modules/codecov.io/bin/codecov.io.js 12 | - cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js 13 | 14 | branches: 15 | only: 16 | - master 17 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Prerequisites 2 | 3 | [Node.js](http://nodejs.org/) >= v4 must be installed. 4 | 5 | ## Installation 6 | 7 | - Running `npm install` in the module's root directory will install everything you need for development. 8 | 9 | ## Running Tests 10 | 11 | - `npm test` will run the tests once. 12 | 13 | - `npm run test:coverage` will run the tests and produce a coverage report in `coverage/`. 14 | 15 | - `npm run test:watch` will run the tests on every change. 16 | 17 | ## Building 18 | 19 | - `npm run build` will build the module for publishing to npm. 20 | 21 | - `npm run clean` will delete built resources. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "swimmer", 3 | "version": "1.3.1", 4 | "description": "An async task pooling and throttling utility for javascript.", 5 | "main": "lib/index.js", 6 | "module": "es/index.js", 7 | "files": [ 8 | "es", 9 | "lib", 10 | "umd" 11 | ], 12 | "scripts": { 13 | "build": "nwb build-web-module", 14 | "clean": "nwb clean-module", 15 | "test": "nwb test", 16 | "test:coverage": "nwb test --coverage", 17 | "test:watch": "nwb test --server", 18 | "prepublishOnly": "yarn build" 19 | }, 20 | "dependencies": { 21 | "babel-runtime": "^6.26.0" 22 | }, 23 | "devDependencies": { 24 | "eslint-config-react-tools": "^1.1.1", 25 | "nwb": "0.21.x" 26 | }, 27 | "homepage": "https://github.com/tannerlinsley/swimmer", 28 | "license": "MIT", 29 | "repository": "https://github.com/tannerlinsley/swimmer.git" 30 | } 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Tanner Linsley 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | const defaultConfig = { 2 | concurrency: 5, 3 | started: true, 4 | tasks: [] 5 | } 6 | 7 | export function createPool (config = defaultConfig) { 8 | const { concurrency, started, tasks } = { 9 | ...defaultConfig, 10 | ...config 11 | } 12 | 13 | let onSettles = [] 14 | let onErrors = [] 15 | let onSuccesses = [] 16 | let running = started 17 | let active = [] 18 | let pending = tasks 19 | let currentConcurrency = concurrency 20 | 21 | const tick = () => { 22 | if (!running) { 23 | return 24 | } 25 | if (!pending.length && !active.length) { 26 | onSettles.forEach(d => d()) 27 | return 28 | } 29 | while (active.length < currentConcurrency && pending.length) { 30 | const nextFn = pending.shift() 31 | active.push(nextFn) 32 | /* eslint-disable no-loop-func */ 33 | ;(async () => { 34 | let success = false 35 | let res 36 | let error 37 | try { 38 | res = await nextFn() 39 | success = true 40 | } catch (e) { 41 | error = e 42 | } 43 | active = active.filter(d => d !== nextFn) 44 | if (success) { 45 | nextFn.resolve(res) 46 | onSuccesses.forEach(d => d(res, nextFn)) 47 | } else { 48 | nextFn.reject(error) 49 | onErrors.forEach(d => d(error, nextFn)) 50 | } 51 | tick() 52 | })() 53 | /* eslint-enable no-loop-func */ 54 | } 55 | } 56 | 57 | const api = { 58 | add: (fn, { priority } = {}) => 59 | new Promise((resolve, reject) => { 60 | if (priority) { 61 | pending.unShift(fn) 62 | } else { 63 | pending.push(fn) 64 | } 65 | fn.resolve = resolve 66 | fn.reject = reject 67 | tick() 68 | }), 69 | throttle: n => { 70 | currentConcurrency = n 71 | }, 72 | onSettled: cb => { 73 | onSettles.push(cb) 74 | return () => { 75 | onSettles = onSettles.filter(d => d !== cb) 76 | } 77 | }, 78 | onError: cb => { 79 | onErrors.push(cb) 80 | return () => { 81 | onErrors = onErrors.filter(d => d !== cb) 82 | } 83 | }, 84 | onSuccess: cb => { 85 | onSuccesses.push(cb) 86 | return () => { 87 | onSuccesses = onSuccesses.filter(d => d !== cb) 88 | } 89 | }, 90 | stop: () => { 91 | running = false 92 | }, 93 | start: () => { 94 | running = true 95 | tick() 96 | }, 97 | clear: () => { 98 | pending = [] 99 | }, 100 | getActive: () => active, 101 | getPending: () => pending, 102 | getAll: () => [...active, ...pending], 103 | isRunning: () => running, 104 | isSettled: () => !running && !active.length && !pending.length 105 | } 106 | 107 | return api 108 | } 109 | 110 | export function poolAll (tasks, concurrency) { 111 | return new Promise((resolve, reject) => { 112 | const pool = createPool({ 113 | concurrency 114 | }) 115 | const results = [] 116 | pool.onSettled(() => { 117 | resolve(results) 118 | }) 119 | pool.onError(err => { 120 | reject(err) 121 | }) 122 | tasks.forEach((task, i) => { 123 | pool.add(async () => { 124 | const res = await task() 125 | results[i] = res 126 | return res 127 | }) 128 | }) 129 | pool.start() 130 | }) 131 | } 132 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🏊‍ swimmer 2 | 3 | [![David Dependancy Status](https://david-dm.org/tannerlinsley/swimmer.svg)](https://david-dm.org/tannerlinsley/swimmer) 4 | [![npm package v](https://img.shields.io/npm/v/swimmer.svg)](https://www.npmjs.org/package/swimmer) 5 | [![npm package dm](https://img.shields.io/npm/dm/swimmer.svg)](https://npmjs.com/package/swimmer) 6 | [![Join the community on Slack](https://img.shields.io/badge/slack-react--chat-blue.svg)](https://react-chat-signup.herokuapp.com/) 7 | [![Github Stars](https://img.shields.io/github/stars/tannerlinsley/swimmer.svg?style=social&label=Star)](https://github.com/tannerlinsley/swimmer) 8 | [![Twitter Follow](https://img.shields.io/twitter/follow/nozzleio.svg?style=social&label=Follow)](https://twitter.com/nozzleio) 9 | 10 | 11 | An async task pooling and throttling utility for javascript. 12 | 13 | ## Features 14 | - 🚀 3kb and zero dependencies 15 | - 🔥 ES6 and async/await ready 16 | - 😌 Simple to use! 17 | 18 | ## Interactive Demo 19 | - [CodeSandbox](https://codesandbox.io/s/mq2j7jq39x?expanddevtools=1&hidenavigation=1) 20 | 21 | ## Installation 22 | ```bash 23 | $ yarn add swimmer 24 | # or 25 | $ npm i swimmer --save 26 | ``` 27 | 28 | ## UMD 29 | ``` 30 | https://unpkg.com/swimmer/umd/swimmer.min.js 31 | ``` 32 | 33 | ## Inline Pooling 34 | Inline Pooling is great for: 35 | - Throttling intensive tasks in a serial fashion 36 | - Usage with async/await and promises. 37 | - Ensuring that all tasks succeed. 38 | 39 | ```javascript 40 | import { poolAll } from 'swimmer' 41 | 42 | const urls = [...] 43 | 44 | const doIntenseTasks = async () => { 45 | try { 46 | const res = await poolAll( 47 | urls.map(task => 48 | () => fetch(url) // Return an array of functions that return a promise 49 | ), 50 | 10 // Set the concurrency limit 51 | ) 52 | } catch (err, task) { 53 | // If an error is encountered, the entire pool stops and the error is thrown 54 | console.log(`Encountered an error with task: ${task}`) 55 | throw err 56 | } 57 | 58 | // If no errors are thrown, you get your results! 59 | console.log(res) // [result, result, result, result] 60 | } 61 | ``` 62 | 63 | ## Custom Pooling 64 | Custom pools are great for: 65 | - Non serial 66 | - Reusable pools 67 | - Handling errors gracefully 68 | - Task management and retry 69 | - Variable throttle speed, pausing, resuming of tasks 70 | 71 | ```javascript 72 | import { createPool } from 'swimmer' 73 | 74 | const urls = [...] 75 | const otherUrls = [...] 76 | 77 | // Create a new pool with a throttle speed and some default tasks 78 | const pool = createPool({ 79 | concurrency: 5, 80 | tasks: urls.map(url => () => fetch(url)) 81 | }) 82 | 83 | // Subscribe to errors 84 | pool.onError((err, task) => { 85 | console.warn(err) 86 | console.log(`Encountered an error with task ${task}. Resubmitting to pool!`) 87 | pool.add(task) 88 | }) 89 | 90 | // Subscribe to successes 91 | pool.onSuccess((res, task) => { 92 | console.log(`Task Complete. Result: ${res}`) 93 | }) 94 | 95 | // Subscribe to settle 96 | pool.onSettled(() => console.log("Pool is empty. All tasks are finished!")) 97 | 98 | const doIntenseTasks = async () => { 99 | // Add more tasks to the pool. 100 | tasks.forEach( 101 | url => pool.add( 102 | () => fetch(url) 103 | ) 104 | ) 105 | 106 | // Increase the concurrency to 10! This can also be done while it's running. 107 | pool.throttle(10) 108 | 109 | // Pause the pool 110 | pool.stop() 111 | 112 | // Start the pool again! 113 | pool.start() 114 | 115 | // Add a single task and WAIT for it's completion/failure 116 | try { 117 | const res = await pool.add(() => fetch('http://custom.com/asset.json')) 118 | console.log('A custom asset!', res) 119 | } catch (err) { 120 | console.log('Darn! An error...') 121 | throw err 122 | } 123 | 124 | // Then clear the pool. Any running tasks will attempt to finished. 125 | pool.clear() 126 | } 127 | ``` 128 | 129 | ### API 130 | Swimmer exports two functions: 131 | - `poolAll` - creates an inline async/await/promise compatible pool 132 | - Arguments 133 | - `Array[Function => Promise]` - An array of functions that return a promise. 134 | - `Int` - The concurrency limit for this pool. 135 | - Returns 136 | - A `Promise` that resolves when all tasks are complete, or throws an error if one of them fails. 137 | - Example: 138 | - `createPool` - creates an custom pool 139 | - Arguments 140 | - `Object{}` - An optional configuration object for this pool 141 | - `concurrency: Int (default: 5)` - The concurrency limit for this pool. 142 | - `started: Boolean (default: true)` - Whether the pool should be started by default or not. 143 | - `tasks: Array[Function => Promise]` - An array of functions that return a promise. These tasks will be preloaded into the pool. 144 | - Returns 145 | - `Object{}` 146 | - `add(() => Promise, config{})` - Adds a task to the pool. Optionally pass a config object 147 | - `config.priority` - Set this option to `true` to queue this task in front of all other `pending` tasks. 148 | - Returns a promise that resolves/rejects with the eventual response from this task 149 | - `start()` - Starts the pool. 150 | - `stop()` - Stops the pool. 151 | - `throttle(Int)` - Sets a new concurrency rate for the pool. 152 | - `clear()` - Clears all pending tasks from the pool. 153 | - `getActive()` - Returns all active tasks. 154 | - `getPending()` - Returns all pending tasks. 155 | - `getAll()` - Returns all tasks. 156 | - `isRunning()` - Returns `true` if the pool is running. 157 | - `isSettled()` - Returns `true` if the pool is settled. 158 | - `onSuccess((result, task) => {})` - Registers an onSuccess callback. 159 | - `onError((error, task) => {})` - Registers an onError callback. 160 | - `onSettled(() => {})` - Registers an onSettled callback. 161 | 162 | ## Tip of the year 163 | Make sure you are passing an array of `thunks`. A thunk is a function that returns your task, not your task itself. If you pass an array of tasks that have already been fired off then it's too late for Swimmer to manage them :( 164 | 165 | ## Contributing 166 | 167 | We are always looking for people to help us grow `swimmer`'s capabilities and examples. If you have an issue, feature request, or pull request, let us know! 168 | 169 | ## License 170 | 171 | Swimmer uses the MIT license. For more information on this license, [click here](https://github.com/tannerlinsley/swimmer/blob/master/LICENSE). 172 | 173 | [build-badge]: https://img.shields.io/travis/tannerlinsley/swimmer/master.png?style=flat-square 174 | [build]: https://travis-ci.org/tannerlinsley/swimmer 175 | 176 | [npm-badge]: https://img.shields.io/npm/v/npm-package.png?style=flat-square 177 | [npm]: https://www.npmjs.org/package/swimmer 178 | 179 | [coveralls-badge]: https://img.shields.io/coveralls/tannerlinsley/swimmer/master.png?style=flat-square 180 | [coveralls]: https://coveralls.io/github/tannerlinsley/swimmer 181 | --------------------------------------------------------------------------------