├── tslint.json ├── renovate.json ├── .gitignore ├── .editorconfig ├── src ├── utils.ts ├── pambdajs.ts ├── utils │ ├── slice.ts │ ├── slice.spec.ts │ └── fork.ts ├── worker │ ├── childProcessWorker.js │ └── childProcessWorker.spec.ts └── orchestrator │ ├── Orchestrator.spec.ts │ └── Orchestrator.ts ├── test └── pambdajs.test.ts ├── tsconfig.json ├── .github └── workflows │ ├── test.yml │ └── release.yml ├── CONTRIBUTING.md ├── LICENSE ├── tools ├── gh-pages-publish.ts └── semantic-release-prepare.ts ├── rollup.config.ts ├── code-of-conduct.md ├── README.md └── package.json /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "tslint-config-standard", 4 | "tslint-config-prettier" 5 | ] 6 | } -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base" 4 | ], 5 | "baseBranches": [ 6 | "master" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | .nyc_output 4 | .DS_Store 5 | *.log 6 | .vscode 7 | .idea 8 | dist 9 | compiled 10 | .awcache 11 | .rpt2_cache 12 | docs 13 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | #root = true 2 | 3 | [*] 4 | indent_style = space 5 | end_of_line = lf 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | max_line_length = 100 10 | indent_size = 2 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | export const functionToSource = (cb: Function, env: any) => { 2 | return ` 3 | process.on( 4 | "message", 5 | function(e) { 6 | global.env = ${env}; 7 | process.send( 8 | JSON.stringify( 9 | (${cb.toString()})(JSON.parse(e).data) 10 | ) 11 | ); 12 | })` 13 | } 14 | -------------------------------------------------------------------------------- /test/pambdajs.test.ts: -------------------------------------------------------------------------------- 1 | import { init } from '../src/pambdajs' 2 | import { Orchestrator } from '../src/orchestrator/Orchestrator' 3 | 4 | /** 5 | * Dummy test 6 | */ 7 | describe('Dummy test', () => { 8 | it('works if true is truthy', () => { 9 | expect(true).toBeTruthy() 10 | }) 11 | 12 | it('class can be instantiable', () => { 13 | expect(init()).toBeInstanceOf(Orchestrator) 14 | }) 15 | }) 16 | -------------------------------------------------------------------------------- /src/pambdajs.ts: -------------------------------------------------------------------------------- 1 | import { Orchestrator } from './orchestrator/Orchestrator' 2 | import { forkAProcess } from './utils/fork' 3 | import { cpus } from 'os' 4 | import { resolve } from 'path' 5 | 6 | export const fork = forkAProcess 7 | export const init = (processCount?: number): Orchestrator => { 8 | const theProcessCount = processCount ? processCount : cpus ? cpus().length : 2 9 | const workerPath = resolve(__dirname, './worker/childProcessWorker.js') 10 | return new Orchestrator(theProcessCount, workerPath) 11 | } 12 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "moduleResolution": "node", 4 | "target": "es5", 5 | "module":"es2015", 6 | "lib": ["es2015", "es2016", "es2017", "dom"], 7 | "strict": true, 8 | "sourceMap": true, 9 | "declaration": true, 10 | "allowSyntheticDefaultImports": true, 11 | "experimentalDecorators": true, 12 | "emitDecoratorMetadata": true, 13 | "declarationDir": "dist/types", 14 | "outDir": "dist", 15 | "allowJs": true, 16 | "typeRoots": [ 17 | "node_modules/@types" 18 | ] 19 | }, 20 | "include": [ 21 | "src" 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | ci: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | strategy: 11 | matrix: 12 | node-version: [14.x] 13 | # node-version: [10.x, 12.x, 14.x] 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Use Node.js ${{ matrix.node-version }} 18 | uses: actions/setup-node@v1 19 | with: 20 | node-version: ${{ matrix.node-version }} 21 | - run: npm ci 22 | - run: npm test 23 | - run: npm install -g codecov && codecov 24 | env: 25 | CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} 26 | -------------------------------------------------------------------------------- /src/utils/slice.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * split date to different parts 3 | * return a 2 dimension array 4 | * @param data 5 | * @param count 6 | */ 7 | export const slice = (data: any[], count: number): Array => { 8 | if (data.length <=count) { 9 | return data.map( 10 | d => [d] 11 | ) 12 | } else { 13 | const sliceRange = Math.floor(data.length / count) 14 | const slices = [] 15 | for (let i = 0; i < count; i++) { 16 | const startIndex = i * sliceRange 17 | let endIndex = startIndex + sliceRange 18 | if (i === count-1) { 19 | endIndex = data.length 20 | } 21 | slices.push(data.slice(startIndex, endIndex)) 22 | } 23 | return slices 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/utils/slice.spec.ts: -------------------------------------------------------------------------------- 1 | import { slice } from './slice' 2 | 3 | describe('test slice', () => { 4 | const data1 = ['a', 'b', 'c', 'D', 'e'] 5 | const count = 4 6 | 7 | it('5 elements', async () => { 8 | const r = slice(data1, count) 9 | expect(r).toEqual( 10 | [ 11 | ['a'], 12 | ['b'], 13 | ['c'], 14 | ['D', 'e'] 15 | ] 16 | ) 17 | }) 18 | 19 | it('slice with only 2 elements ', async () => { 20 | const r = slice(data1.slice(0, 2), count) 21 | expect(r).toEqual( 22 | [ 23 | ['a'], 24 | ['b'] 25 | ] 26 | ) 27 | }) 28 | it('slice to 2 parts ', async () => { 29 | const r = slice(data1, 2) 30 | expect(r).toEqual( 31 | [ 32 | ['a', 'b'], 33 | ['c', 'D', 'e'] 34 | ] 35 | ) 36 | }) 37 | }) 38 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | We're really glad you're reading this, because we need volunteer developers to help this project come to fruition. 👏 2 | 3 | ## Instructions 4 | 5 | These steps will guide you through contributing to this project: 6 | 7 | - Fork the repo 8 | - Clone it and install dependencies 9 | 10 | git clone https://github.com/YOUR-USERNAME/typescript-library-starter 11 | npm install 12 | 13 | Keep in mind that after running `npm install` the git repo is reset. So a good way to cope with this is to have a copy of the folder to push the changes, and the other to try them. 14 | 15 | Make and commit your changes. Make sure the commands npm run build and npm run test:prod are working. 16 | 17 | Finally send a [GitHub Pull Request](https://github.com/alexjoverm/typescript-library-starter/compare?expand=1) with a clear list of what you've done (read more [about pull requests](https://help.github.com/articles/about-pull-requests/)). Make sure all of your commits are atomic (one feature per commit). 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2020 tim-hub 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /src/utils/fork.ts: -------------------------------------------------------------------------------- 1 | import { ChildProcess, fork } from 'child_process' 2 | // import logger from '../logger' 3 | 4 | export enum WORK_TYPE { 5 | MAP = 'MAP', 6 | FILTER = 'FILTER', 7 | REDUCE = 'REDUCE', 8 | } 9 | 10 | export const forkAProcess = ( 11 | modulePath: string, 12 | pureFunction: Function, 13 | childIndex: number, 14 | customProcessID: string = '', 15 | workType: WORK_TYPE = WORK_TYPE.MAP 16 | ): ChildProcess => { 17 | let fun = pureFunction 18 | const childProcess = fork(modulePath, [customProcessID], { 19 | env: { 20 | INDEX: childIndex.toString(), 21 | PROCESS_ID: customProcessID, 22 | FUNCTION: fun.toString(), 23 | FUNCTION_NAME: fun.name, 24 | LOG_LEVEL: process.env.LOG_LEVEL, 25 | WORK_TYPE: workType, 26 | }, 27 | silent: false, // false stdin, stdout, and stderr of the child will be inherited from the parent 28 | }) 29 | 30 | childProcess.on('close', (code: string) => { 31 | // logger.debug('process quit with code ' + code) 32 | }) 33 | return childProcess 34 | } 35 | -------------------------------------------------------------------------------- /tools/gh-pages-publish.ts: -------------------------------------------------------------------------------- 1 | const { cd, exec, echo, touch } = require("shelljs") 2 | const { readFileSync } = require("fs") 3 | const url = require("url") 4 | 5 | let repoUrl 6 | let pkg = JSON.parse(readFileSync("package.json") as any) 7 | if (typeof pkg.repository === "object") { 8 | if (!pkg.repository.hasOwnProperty("url")) { 9 | throw new Error("URL does not exist in repository section") 10 | } 11 | repoUrl = pkg.repository.url 12 | } else { 13 | repoUrl = pkg.repository 14 | } 15 | 16 | let parsedUrl = url.parse(repoUrl) 17 | let repository = (parsedUrl.host || "") + (parsedUrl.path || "") 18 | let ghToken = process.env.GH_TOKEN 19 | let ghEmail = process.env.GH_EMAIL 20 | 21 | echo("Deploying docs!!!") 22 | cd("docs") 23 | touch(".nojekyll") 24 | exec("git init") 25 | exec("git add .") 26 | exec('git config user.name "tim-hub"') 27 | exec('git config user.email ' + `"${ghEmail}"`) 28 | exec('git commit -m "docs(docs): update gh-pages"') 29 | exec( 30 | `git push --force --quiet "https://${ghToken}@${repository}" master:gh-pages` 31 | ) 32 | echo("Docs deployed!!") 33 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions 3 | 4 | name: Release 5 | 6 | on: 7 | push: 8 | branches: [ master, next, alpha, beta, next-major ] 9 | 10 | jobs: 11 | release: 12 | 13 | runs-on: ubuntu-latest 14 | 15 | strategy: 16 | matrix: 17 | node-version: [14.x] 18 | 19 | steps: 20 | - uses: actions/checkout@v2 21 | - name: Use Node.js ${{ matrix.node-version }} 22 | uses: actions/setup-node@v1 23 | with: 24 | node-version: ${{ matrix.node-version }} 25 | - run: npm ci 26 | - run: npm run build --if-present 27 | - name: Release 28 | env: 29 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 30 | GH_TOKEN: ${{ secrets.GH_TOKEN }} 31 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 32 | run: npx semantic-release 33 | - name: Upload Artifacts 34 | uses: actions/upload-artifact@v2 35 | with: 36 | name: dist 37 | path: | 38 | dist 39 | coverage 40 | -------------------------------------------------------------------------------- /rollup.config.ts: -------------------------------------------------------------------------------- 1 | import resolve from 'rollup-plugin-node-resolve' 2 | import commonjs from 'rollup-plugin-commonjs' 3 | import sourceMaps from 'rollup-plugin-sourcemaps' 4 | import camelCase from 'lodash.camelcase' 5 | import typescript from 'rollup-plugin-typescript2' 6 | import json from 'rollup-plugin-json' 7 | 8 | const pkg = require('./package.json') 9 | 10 | const libraryName = 'pambdajs' 11 | 12 | export default { 13 | input: `src/${libraryName}.ts`, 14 | output: [ 15 | { file: pkg.main, name: camelCase(libraryName), format: 'umd', sourcemap: true }, 16 | { file: pkg.module, format: 'es', sourcemap: true }, 17 | ], 18 | // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') 19 | external: [], 20 | watch: { 21 | include: 'src/**', 22 | }, 23 | plugins: [ 24 | // Allow json resolution 25 | json(), 26 | // Compile TypeScript files 27 | typescript({ useTsconfigDeclarationDir: true }), 28 | // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) 29 | commonjs(), 30 | // Allow node_modules resolution, so you can use 'external' to control 31 | // which external modules to include in the bundle 32 | // https://github.com/rollup/rollup-plugin-node-resolve#usage 33 | resolve(), 34 | 35 | // Resolve source maps to the original source 36 | sourceMaps(), 37 | ], 38 | } 39 | -------------------------------------------------------------------------------- /src/worker/childProcessWorker.js: -------------------------------------------------------------------------------- 1 | /** 2 | * I am a hard working worker, give me your CPU processor, lol 3 | * todo, minimise this function for better performance 4 | * @returns {{data: any[], index: string}} 5 | */ 6 | 7 | const log = (...x) => { 8 | if (process.env.LOG_LEVEL === 'debug') { 9 | console.log(...x) 10 | } 11 | } 12 | 13 | const childProcessWorker = (data) => { 14 | // 15 | // if (process.argv[2]) { 16 | // log(`Child Process ${process.argv[2]} executed`) 17 | // } else { 18 | // log('Child Process executed') 19 | // } 20 | 21 | const theFunction = process.env.FUNCTION 22 | const theFunctionName = process.env.FUNCTION_NAME ? process.env.FUNCTION_NAME : 'fun' 23 | const theIndex = process.env.INDEX 24 | const theWorkType = process.env.WORK_TYPE 25 | 26 | let evalString = `const params = ${JSON.stringify(data)};` 27 | + `const ${theFunctionName} = ` + theFunction ; 28 | if (theWorkType === 'FILTER') { 29 | evalString += `; Array.prototype.filter.call(params, ${theFunctionName});`; 30 | } else if (theWorkType === 'REDUCE') { 31 | evalString += `; Array.prototype.reduce.call(params, ${theFunctionName});` 32 | } else { 33 | // by default MAP 34 | evalString += `; Array.prototype.map.call(params,${theFunctionName});` 35 | } 36 | 37 | const result = eval(evalString) 38 | return { 39 | index: theIndex, 40 | data: result 41 | } 42 | } 43 | 44 | // receive message from master process 45 | process.on('message', async (message) => { 46 | const result = childProcessWorker(message.data) 47 | // send response to master process 48 | process.send(result) 49 | }) 50 | 51 | 52 | -------------------------------------------------------------------------------- /tools/semantic-release-prepare.ts: -------------------------------------------------------------------------------- 1 | const path = require("path") 2 | const { fork } = require("child_process") 3 | const colors = require("colors") 4 | 5 | const { readFileSync, writeFileSync } = require("fs") 6 | const pkg = JSON.parse( 7 | readFileSync(path.resolve(__dirname, "..", "package.json")) 8 | ) 9 | 10 | pkg.scripts.prepush = "npm run test:prod && npm run build" 11 | pkg.scripts.commitmsg = "commitlint -E HUSKY_GIT_PARAMS" 12 | 13 | writeFileSync( 14 | path.resolve(__dirname, "..", "package.json"), 15 | JSON.stringify(pkg, null, 2) 16 | ) 17 | 18 | // Call husky to set up the hooks 19 | fork(path.resolve(__dirname, "..", "node_modules", "husky", "lib", "installer", 'bin'), ['install']) 20 | 21 | console.log() 22 | console.log(colors.green("Done!!")) 23 | console.log() 24 | 25 | if (pkg.repository.url.trim()) { 26 | console.log(colors.cyan("Now run:")) 27 | console.log(colors.cyan(" npm install -g semantic-release-cli")) 28 | console.log(colors.cyan(" semantic-release-cli setup")) 29 | console.log() 30 | console.log( 31 | colors.cyan('Important! Answer NO to "Generate travis.yml" question') 32 | ) 33 | console.log() 34 | console.log( 35 | colors.gray( 36 | 'Note: Make sure "repository.url" in your package.json is correct before' 37 | ) 38 | ) 39 | } else { 40 | console.log( 41 | colors.red( 42 | 'First you need to set the "repository.url" property in package.json' 43 | ) 44 | ) 45 | console.log(colors.cyan("Then run:")) 46 | console.log(colors.cyan(" npm install -g semantic-release-cli")) 47 | console.log(colors.cyan(" semantic-release-cli setup")) 48 | console.log() 49 | console.log( 50 | colors.cyan('Important! Answer NO to "Generate travis.yml" question') 51 | ) 52 | } 53 | 54 | console.log() 55 | -------------------------------------------------------------------------------- /src/orchestrator/Orchestrator.spec.ts: -------------------------------------------------------------------------------- 1 | import { Orchestrator } from './Orchestrator' 2 | import * as os from 'os' 3 | 4 | describe('to test orchestrator', () => { 5 | let orch: Orchestrator 6 | const data1 = ['a', 'b', 'c', 'D', 'e'] 7 | 8 | beforeAll(() => { 9 | orch = new Orchestrator(4) 10 | }) 11 | 12 | it('orchestrate/slice should work well', () => { 13 | const orch1 = new Orchestrator(4) 14 | expect(orch1.processCount).toEqual(4) 15 | 16 | const orch2 = new Orchestrator() 17 | expect(orch2.processCount).toEqual(os.cpus().length) 18 | }) 19 | 20 | it('test map function with a function with name', async () => { 21 | // function names are tested in worker 22 | function upper(a: string) { 23 | return a.toUpperCase() 24 | } 25 | 26 | const data = data1 27 | const r = await orch.map(upper, data) 28 | expect(r).toEqual(data.map((d) => upper(d))) 29 | }) 30 | 31 | it('map with one element', async () => { 32 | function upper(a: string) { 33 | return a.toUpperCase() 34 | } 35 | 36 | const data = ['a'] 37 | const r = await orch.map(upper, data) 38 | expect(r).toEqual(data.map((d) => upper(d))) 39 | }) 40 | 41 | it('map with 0 element', async () => { 42 | function upper(a: string) { 43 | return a.toUpperCase() 44 | } 45 | 46 | const data: any[] = [] 47 | const r = await orch.map(upper, data) 48 | expect(r).toEqual(data.map((d) => upper(d))) 49 | }) 50 | 51 | it('test filter function ', async () => { 52 | const uppperFilter = (a: string) => { 53 | return a.toUpperCase() === a 54 | } 55 | const data = data1 56 | const r = await orch.filter(uppperFilter, data) 57 | expect(r).toEqual(data.filter(uppperFilter)) 58 | }) 59 | 60 | it('filter with one element', async () => { 61 | const uppperFilter = (a: string) => { 62 | return a.toUpperCase() === a 63 | } 64 | const data = ['a'] 65 | const r = await orch.filter(uppperFilter, data) 66 | expect(r).toEqual(data.filter(uppperFilter)) 67 | 68 | const data1 = ['A'] 69 | const r1 = await orch.filter(uppperFilter, data1) 70 | expect(r1).toEqual(data1.filter(uppperFilter)) 71 | }) 72 | 73 | it('filter with 0 element', async () => { 74 | const uppperFilter = (a: string) => { 75 | return a.toUpperCase() === a 76 | } 77 | const data: any[] = [] 78 | const r = await orch.filter(uppperFilter, data) 79 | expect(r).toEqual(data.filter(uppperFilter)) 80 | }) 81 | }) 82 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at alexjovermorales@gmail.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [http://contributor-covenant.org/version/1/4][version] 72 | 73 | [homepage]: http://contributor-covenant.org 74 | [version]: http://contributor-covenant.org/version/1/4/ 75 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PambdaJS - Multi-Process Has Never Been This Easy 2 | [![npm version](https://badge.fury.io/js/pambdajs.svg)](https://www.npmjs.com/package/pambdajs) ![TEST Passing](https://github.com/tim-hub/pambdajs/workflows/Test/badge.svg) [![codecov](https://codecov.io/gh/tim-hub/pambdajs/branch/master/graph/badge.svg)](https://codecov.io/gh/tim-hub/pambdajs) 3 | > Parallelized Lambda, a wrapper to help run lambda/anonymous function in parallel easily. 4 | 5 | PambdaJS is to orchestrate different child process to finish one heavy iteration work. 6 | - make multi process programming easily 7 | - gain better performance through empowering multi core CPU 8 | 9 | **Start to use PambdaJS** 10 | 11 | ```javascript 12 | import pambda from 'pambdajs'; 13 | const p = await pambda.init(5); 14 | 15 | data = [1,2,3]; 16 | const heavyWork = (x) => { 17 | return x+x; 18 | } 19 | // original single process way 20 | const singleProcess = () => { 21 | data.map(heavyWork); 22 | } 23 | 24 | // PambdaJS multi process way 25 | const pambdaProcess = async () => { 26 | await p.map(heavyWork, data); 27 | } 28 | ``` 29 | 30 | ## Features 31 | Run your own anonymous functions in Parallel. 32 | - map 33 | - filter 34 | - reduce 35 | 36 | ## Limit 37 | The lambda function itself has to be a pure function, 38 | and it has to use only node build-in packages. 39 | 40 | for example: 41 | ```javascript 42 | // good 43 | const heavyWork = (x) => { 44 | return x+x; 45 | } 46 | ``` 47 | ```javascript 48 | // bad 49 | const sumItSelf = (x) => { 50 | return x + x; 51 | } 52 | 53 | const heavyWork = (x) => { 54 | return sumItSelf(x) 55 | } 56 | ``` 57 | 58 | ## Performance 59 | - PambdaJS can save up to 55% of processing time. (Diagram ↓) 60 | 61 | ![PambdaJS performance](https://i.imgur.com/F2HfHwF.png) 62 | As you can see the above. 63 | - The best case to repeat summing 10k number 100k times, 64 | is to spawn 5 child process, and **it saves more than half of time than single process.** 65 | 66 | - Besides, for simple work, multi process does not help at all. 67 | So make sure usig PambdaJS for heavy work only. ([more diagrams](https://pambdajs-performance.vercel.app/)) 68 | 69 | ## FAQ 70 | - Why not use child process directly? 71 | > Yeah, why not. child_process is the native module of Node, if you seek better flexibility, use child_process. 72 | However, if you want to make your life easier, trust me use PambdaJS, 73 | you do not need to worry about the message between child and parent process anymore. 74 | 75 | - Does it run on browser as well? 76 | > No, PambdaJS is a tool for Node environment, not for browser, 77 | because browser does not support multi-process, 78 | but you can try [web worker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) which is multi-thread solution. 79 | >or Gpu.js which is using GPU instead of CPU for getting better performance. 80 | 81 | 82 | ## Reference 83 | 84 | - [What is Lambda Function](https://stackoverflow.com/questions/16501/what-is-a-lambda-function) 85 | - Other solutions for better performance 86 | - [Child Process](https://nodejs.org/api/child_process.html) 87 | - [GPU.js](https://gpu.rocks/) 88 | - [Web Worker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) for multi threading. **We all love MDN!!!** 89 | - Based on Web Worker/ Multi Thread 90 | - [ncpu](https://github.com/zy445566/ncpu) 91 | - microsoft [napajs](https://github.com/microsoft/napajs) 92 | - Relative Repos 93 | - [PambdaJS Playground](https://github.com/tim-hub/pambdajs-playground) - the place to use pambdajs to run some simple sample work 94 | - [PambdaJS Performance Analyse](https://github.com/tim-hub/pambdajs-performance) - using Chart.js to generate the diagrams 95 | - [Code of conduct](./code-of-conduct.md) 96 | - [Contributing](./CONTRIBUTING.md) 97 | -------------------------------------------------------------------------------- /src/worker/childProcessWorker.spec.ts: -------------------------------------------------------------------------------- 1 | import * as path from 'path' 2 | import { forkAProcess, WORK_TYPE } from '../utils/fork' 3 | 4 | /** 5 | * similar to orchestrator fork and spawn function, here to use for testing child worker js 6 | * @param fun 7 | * @param thePath 8 | * @param data 9 | * @param workType 10 | */ 11 | const testFork = async ( 12 | fun: Function, 13 | thePath: string, 14 | data: any[], 15 | workType: WORK_TYPE = WORK_TYPE.MAP 16 | ) => { 17 | const thePureFunction = fun 18 | const process0 = forkAProcess(thePath, thePureFunction, 0, '0', workType) 19 | return new Promise((resolve, reject) => { 20 | process0.send({ data: data }, (error) => { 21 | if (error) { 22 | reject(error) 23 | } 24 | }) 25 | process0.on('message', async (message, senderHandler) => { 26 | console.log(`Result received from worked ${message.data}`) 27 | if (!process0.killed) { 28 | process.kill(process0.pid) 29 | } 30 | resolve(message) 31 | }) 32 | }) 33 | } 34 | 35 | describe('work in paraller properly', () => { 36 | const data1 = ['a', 'b', 'c', 'D', 'e'] 37 | const count = 4 38 | 39 | const thePath = path.resolve(__dirname, './childProcessWorker.js') 40 | 41 | it('test to fork a child process to work properly', async () => { 42 | const upper = (a: string) => { 43 | return a.toUpperCase() 44 | } 45 | 46 | const r = await testFork(upper, thePath, data1) 47 | // @ts-ignore 48 | const data = r.data 49 | // @ts-ignore 50 | expect(parseInt(r.index, 10)).toBeGreaterThanOrEqual(0) 51 | expect(data).toEqual(data1.map((d) => upper(d))) 52 | }) 53 | 54 | it('test to fork a child process to work properly with old way function without name', async () => { 55 | const upper = function (a: string) { 56 | return a.toUpperCase() 57 | } 58 | const r = await testFork(upper, thePath, data1) 59 | // @ts-ignore 60 | const data = r.data 61 | // @ts-ignore 62 | expect(parseInt(r.index, 10)).toBeGreaterThanOrEqual(0) 63 | expect(data).toEqual(data1.map((d) => upper(d))) 64 | }) 65 | 66 | it('test to fork a child process to work properly with old way function with name', async () => { 67 | function upper(a: string) { 68 | return a.toUpperCase() 69 | } 70 | 71 | const r = await testFork(upper, thePath, data1) 72 | // @ts-ignore 73 | const data = r.data 74 | // @ts-ignore 75 | expect(parseInt(r.index, 10)).toBeGreaterThanOrEqual(0) 76 | expect(data).toEqual(data1.map((d) => upper(d))) 77 | }) 78 | 79 | it('test to fork a child process to work properly with anonymous function', async () => { 80 | const r = await testFork( 81 | (a: string) => { 82 | return a.toUpperCase() 83 | }, 84 | thePath, 85 | data1 86 | ) 87 | // @ts-ignore 88 | const data = r.data 89 | // @ts-ignore 90 | expect(parseInt(r.index, 10)).toBeGreaterThanOrEqual(0) 91 | expect(data).toEqual(data1.map((d) => d.toUpperCase())) 92 | }) 93 | 94 | it('filter, test to fork a child process', async () => { 95 | const upperFilter = (a: string) => { 96 | return a.toUpperCase() === a 97 | } 98 | const r = await testFork(upperFilter, thePath, data1, WORK_TYPE.FILTER) 99 | // @ts-ignore 100 | const data = r.data 101 | // @ts-ignore 102 | expect(parseInt(r.index, 10)).toBeGreaterThanOrEqual(0) 103 | expect(data).toEqual(data1.filter(upperFilter)) 104 | }) 105 | 106 | it('reduce, test to fork a child process', async () => { 107 | const reduceFun = (a: string, b: string) => { 108 | return b.toLowerCase() === b ? a + b : a 109 | } 110 | const r = await testFork(reduceFun, thePath, data1, WORK_TYPE.REDUCE) 111 | // @ts-ignore 112 | const data = r.data 113 | // @ts-ignore 114 | expect(parseInt(r.index, 10)).toBeGreaterThanOrEqual(0) 115 | 116 | const result = data1.reduce(reduceFun) 117 | expect(data).toEqual(result) 118 | }) 119 | }) 120 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pambdajs", 3 | "version": "0.0.0-development", 4 | "description": "", 5 | "keywords": [ 6 | "multi-processing", 7 | "child_process" 8 | ], 9 | "main": "dist/index.umd.js", 10 | "module": "dist/index.es5.js", 11 | "typings": "dist/types/index.d.ts", 12 | "repository.url": "https://github.com:tim-hub/pambdajs", 13 | "files": [ 14 | "dist" 15 | ], 16 | "author": "tim-hub@github", 17 | "repository": { 18 | "type": "git", 19 | "url": "git@github.com:tim-hub/pambdajs.git" 20 | }, 21 | "license": "MIT", 22 | "engines": { 23 | "node": ">=6.0.0" 24 | }, 25 | "scripts": { 26 | "lint": "tslint --project tsconfig.json -t codeFrame 'src/**/*.ts' 'test/**/*.ts'", 27 | "prebuild": "rimraf dist", 28 | "build": "tsc --module commonjs && rollup -c rollup.config.ts && typedoc --out docs --target es6 --theme minimal --mode file src", 29 | "start": "rollup -c rollup.config.ts -w", 30 | "test": "jest", 31 | "test:watch": "jest --watch", 32 | "test:prod": "npm run lint && npm run test -- --no-cache", 33 | "deploy-docs": "ts-node tools/gh-pages-publish", 34 | "commit": "git-cz", 35 | "semantic-release": "semantic-release", 36 | "semantic-release-prepare": "ts-node tools/semantic-release-prepare", 37 | "precommit": "lint-staged", 38 | "travis-deploy-once": "travis-deploy-once", 39 | "prepush": "echo 'npm run test && npm run build'", 40 | "commitmsg": "commitlint -E HUSKY_GIT_PARAMS" 41 | }, 42 | "lint-staged": { 43 | "{src,test}/**/*.ts": [ 44 | "prettier --write", 45 | "git add" 46 | ] 47 | }, 48 | "config": { 49 | "commitizen": { 50 | "path": "node_modules/cz-conventional-changelog" 51 | } 52 | }, 53 | "jest": { 54 | "transform": { 55 | ".(ts|tsx)": "ts-jest" 56 | }, 57 | "testEnvironment": "node", 58 | "testRegex": "(/__tests__/.*|\\.(test|spec))\\.(ts|tsx|js)$", 59 | "moduleFileExtensions": [ 60 | "ts", 61 | "tsx", 62 | "js" 63 | ], 64 | "coveragePathIgnorePatterns": [ 65 | "/node_modules/", 66 | "/test/" 67 | ], 68 | "coverageThreshold": { 69 | "global": { 70 | "branches": 60, 71 | "functions": 80, 72 | "lines": 80, 73 | "statements": 80 74 | } 75 | }, 76 | "coverageDirectory": "./coverage/", 77 | "collectCoverage": true, 78 | "modulePathIgnorePatterns": [ 79 | "/dist/", 80 | "/docs/" 81 | ] 82 | }, 83 | "prettier": { 84 | "semi": false, 85 | "singleQuote": true 86 | }, 87 | "commitlint": { 88 | "extends": [ 89 | "@commitlint/config-conventional" 90 | ] 91 | }, 92 | "devDependencies": { 93 | "@commitlint/cli": "7.6.1", 94 | "@commitlint/config-conventional": "7.6.0", 95 | "@types/jest": "26.0.13", 96 | "@types/node": "10.17.29", 97 | "@types/winston": "2.4.4", 98 | "colors": "1.4.0", 99 | "commitizen": "3.1.2", 100 | "coveralls": "3.1.0", 101 | "cross-env": "7.0.2", 102 | "cz-conventional-changelog": "2.1.0", 103 | "husky": "1.3.1", 104 | "jest": "26.4.2", 105 | "jest-config": "26.4.2", 106 | "lint-staged": "8.2.1", 107 | "lodash.camelcase": "4.3.0", 108 | "prettier": "2.1.1", 109 | "prompt": "1.0.0", 110 | "replace-in-file": "3.4.4", 111 | "rimraf": "2.7.1", 112 | "rollup": "0.67.4", 113 | "rollup-plugin-commonjs": "9.3.4", 114 | "rollup-plugin-json": "3.1.0", 115 | "rollup-plugin-node-resolve": "3.4.0", 116 | "rollup-plugin-sourcemaps": "0.4.2", 117 | "rollup-plugin-typescript2": "0.27.1", 118 | "semantic-release": "17.1.1", 119 | "shelljs": "0.8.4", 120 | "travis-deploy-once": "5.0.11", 121 | "ts-jest": "23.10.5", 122 | "ts-node": "8.10.2", 123 | "tslint": "5.20.1", 124 | "tslint-config-prettier": "1.18.0", 125 | "tslint-config-standard": "8.0.1", 126 | "typedoc": "0.19.1", 127 | "typescript": "4.0.2" 128 | }, 129 | "dependencies": {}, 130 | "release": { 131 | "branches": [ 132 | "+([0-9])?(.{+([0-9]),x}).x", 133 | "master", 134 | "next", 135 | "next-major", 136 | { 137 | "name": "beta", 138 | "prerelease": true 139 | }, 140 | { 141 | "name": "alpha", 142 | "prerelease": true 143 | } 144 | ] 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/orchestrator/Orchestrator.ts: -------------------------------------------------------------------------------- 1 | import * as os from 'os' 2 | import { ChildProcess } from 'child_process' 3 | import { slice } from '../utils/slice' 4 | import * as path from 'path' 5 | import { forkAProcess, WORK_TYPE } from '../utils/fork' 6 | 7 | // import logger from '../logger' 8 | 9 | interface IResult { 10 | index: number 11 | data: any 12 | } 13 | 14 | export class Orchestrator { 15 | modulePath: string = path.resolve(__dirname, '../worker/childProcessWorker.js') 16 | processCount: number = os.cpus().length 17 | 18 | constructor(processCount = os.cpus().length, workerPath?: string) { 19 | this.processCount = processCount 20 | 21 | if (workerPath) { 22 | this.modulePath = workerPath 23 | } 24 | // logger.debug( 25 | // `start a new orchestrator **${this.modulePath}** with process count at ` + this.processCount 26 | // ) 27 | } 28 | 29 | public async map(pureFunction: Function, data: any[]): Promise { 30 | if (data?.length <= 0) { 31 | return data 32 | } 33 | const slices = this.slice(data) 34 | // logger.debug('slices', slices) 35 | const resultParts = await this._map(pureFunction, slices) 36 | const results: any[] = [] 37 | resultParts.forEach((p) => { 38 | results.push(...p) 39 | }) 40 | return results 41 | } 42 | 43 | public async filter(pureFunction: Function, data: any[]): Promise { 44 | if (data?.length <= 0) { 45 | return data 46 | } 47 | const slices = this.slice(data) 48 | const resultParts = await this._filter(pureFunction, slices) 49 | const results: any[] = [] 50 | resultParts.forEach((p) => { 51 | results.push(...p) 52 | }) 53 | return results 54 | } 55 | 56 | /** 57 | * 58 | * @param pureFunction 59 | * @param data is a 2 dimension of array with same amount of process count 60 | * @private 61 | */ 62 | private async _map(pureFunction: Function, data: Array): Promise { 63 | const results = new Array(data.length) 64 | await Promise.all( 65 | data.map(async (partOfData, i) => { 66 | const r = await this.spawn(i, pureFunction, partOfData) 67 | // keep the result in order 68 | results[r.index] = r.data 69 | return r 70 | }) 71 | ) 72 | // logger.debug(results, 'results of _map') 73 | return results 74 | } 75 | 76 | /** 77 | * 78 | * @param pureFunction 79 | * @param data is a 2 dimension of array with same amount of process count 80 | * @private 81 | */ 82 | private async _filter(pureFunction: Function, data: Array): Promise { 83 | const results: any[] = [] 84 | await Promise.all( 85 | data.map(async (partOfData, i) => { 86 | const r = await this.spawn(i, pureFunction, partOfData, WORK_TYPE.FILTER) 87 | results.push(r.data) 88 | return r 89 | }) 90 | ) 91 | return results 92 | } 93 | 94 | /** 95 | * slice data for each process worker 96 | * return a 2 dimension array 97 | * @param data 98 | */ 99 | private slice(data: any[]): Array { 100 | return slice(data, this.processCount) 101 | } 102 | 103 | /** 104 | * to spawn a child process worker 105 | * @param i 106 | * @param pureFunction 107 | * @param data 108 | */ 109 | private async spawn( 110 | i: number, 111 | pureFunction: Function, 112 | data: any[], 113 | workType: WORK_TYPE = WORK_TYPE.MAP 114 | ): Promise { 115 | return new Promise((resolve, reject) => { 116 | const child = this.fork(pureFunction, i, i.toString(), workType) 117 | child.send({ data: data }, (error) => { 118 | if (error) { 119 | reject(error) 120 | } 121 | }) 122 | // listen for messages from forked process 123 | child.on('message', async (message, senderHandler) => { 124 | // logger.debug(`Result received from worked ${message.data}`) 125 | if (!child.killed) { 126 | process.kill(child.pid) 127 | } 128 | resolve(message) 129 | }) 130 | }) 131 | } 132 | 133 | private fork( 134 | pureFunction: Function, 135 | childIndex: number, 136 | customProcessID: string = '', 137 | workType: WORK_TYPE = WORK_TYPE.MAP 138 | ): ChildProcess { 139 | return forkAProcess(this.modulePath, pureFunction, childIndex, customProcessID, workType) 140 | } 141 | } 142 | --------------------------------------------------------------------------------