├── .eslintrc.json ├── .github └── workflows │ ├── release.yml │ └── test.yml ├── .gitignore ├── .npmignore ├── .npmrc ├── .prettierrc.json ├── LICENSE ├── README.md ├── config ├── tsconfig.base.json └── tsconfig.build.json ├── jest.json ├── package.json ├── renovate.json ├── src ├── Etl.ts ├── extractors │ └── JsonExtractor.ts ├── index.ts ├── interfaces │ ├── Extractor.ts │ ├── GeneralTransformer.ts │ ├── Loader.ts │ └── Transformer.ts ├── loaders │ └── ConsoleLoader.ts └── transformers │ ├── MapTransformer.ts │ └── MatchMergeTransformer.ts ├── test ├── .testdata │ ├── json-extractor.array.json │ ├── json-extractor.object.json │ └── match-merge.json ├── Etl.spec.ts ├── JsonExtractor.spec.ts ├── MapTransformer.spec.ts └── MatchMergeTransformer.spec.ts └── tsconfig.json /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["@smartive/eslint-config"] 3 | } 4 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release npm package 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | release: 10 | name: Release 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@main 14 | - uses: actions/setup-node@v3 15 | with: 16 | node-version: '18.x' 17 | - run: npm install 18 | - run: npm run build 19 | - name: semantic release 20 | uses: cycjimmy/semantic-release-action@v3 21 | env: 22 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 23 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 24 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Unit Tests 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | branches: [master, develop] 8 | jobs: 9 | test: 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | node-version: [16.x, 18.x] 14 | steps: 15 | - uses: actions/checkout@v3 16 | - name: Use Node.js ${{ matrix.node-version }} 17 | uses: actions/setup-node@v3 18 | with: 19 | node-version: ${{ matrix.node-version }} 20 | - name: Run Tests 21 | run: | 22 | npm install 23 | npm test 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Dependency directory 7 | node_modules 8 | 9 | # Optional npm cache directory 10 | .npm 11 | 12 | # Optional REPL history 13 | .node_repl_history 14 | 15 | # Typescript stuff 16 | coverage 17 | dist 18 | 19 | package-lock.json 20 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Dependency directory 7 | node_modules 8 | 9 | # Optional npm cache directory 10 | .npm 11 | 12 | # Optional REPL history 13 | .node_repl_history 14 | 15 | # Typescript stuff 16 | build/ 17 | coverage/ 18 | config/ 19 | tsconfig.json 20 | tslint.json 21 | 22 | # Typescript files (but not the definitions) 23 | *.ts 24 | !*.d.ts 25 | *.js.map 26 | 27 | # Testfiles 28 | jest.json 29 | test/ 30 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | package-lock=false 2 | -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | "@smartive/prettier-config" 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Christoph Bühler 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # proc-that 2 | 3 | proc(ess)-that - easy extendable etl tool for nodejs written in typescript. 4 | 5 | Basically instantiate the `Etl` class and add extractors (which pull data from a datasource), transformers (which process the extracted data) and loaders (they load the results into a sink). 6 | 7 | A basic, hypothetic example could be: "Load data from a JSON array, snake_case all properties and store those objects into a mongoDB." 8 | 9 | The package is written in `typescript` but can be used in plain javascript as well 10 | 11 | ##### A bunch of badges 12 | 13 | [![Build Status](https://travis-ci.org/smartive/proc-that.svg?maxAge=3600)](https://travis-ci.org/smartive/proc-that) 14 | [![Build Status](https://ci.appveyor.com/api/projects/status/wm7ydpf62e9518h8?svg=true)](https://ci.appveyor.com/project/buehler/proc-that) 15 | [![npm](https://img.shields.io/npm/v/proc-that.svg?maxAge=3600)](https://www.npmjs.com/package/proc-that) 16 | [![Coverage status](https://img.shields.io/coveralls/smartive/proc-that.svg?maxAge=3600)](https://coveralls.io/github/smartive/proc-that) 17 | [![license](https://img.shields.io/github/license/smartive/proc-that.svg?maxAge=2592000)](https://github.com/smartive/proc-that) 18 | [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release) 19 | [![Greenkeeper badge](https://badges.greenkeeper.io/smartive/proc-that.svg)](https://greenkeeper.io/) 20 | 21 | ## Usage 22 | 23 | ```typescript 24 | import { Etl } from "proc-that"; 25 | 26 | new Etl() 27 | .addExtractor(/* class that implements Extractor */) 28 | .addTransformer(/* class that implements Transformer */) 29 | .addLoader(/* class that implements Loader */) 30 | .start() 31 | .subscribe(progress, error, success); 32 | ``` 33 | 34 | After all objects are extracted, transformed and loaded, the `.start()` observable completes and the process is finished. 35 | 36 | Below is a list if extractors and loaders that are already implemented. Feel free to implement your own extractor / transformer / loader and contribute it to this list with a PR. 37 | 38 | ## Extractors 39 | 40 | | Name | Description | Link | 41 | | -------------------------- | --------------------------------- | ---------------------------------------------------- | 42 | | `proc-that-rest-extractor` | Extract objects from GET requests | https://github.com/smartive/proc-that-rest-extractor | 43 | 44 | ## Loaders 45 | 46 | | Name | Description | Link | 47 | | -------------------------- | ------------------------------------------- | ---------------------------------------------------- | 48 | | `proc-that-elastic-loader` | Load transformed objects into elasticsearch | https://github.com/smartive/proc-that-elastic-loader | 49 | 50 | ## Implement your own 51 | 52 | To ease up implementing your own extractors / transformers or loaders, just create a new repository and install `proc-that` as a dev-dependency. This package contains the needed definition files for the interfaces you need to create the extensions. 53 | -------------------------------------------------------------------------------- /config/tsconfig.base.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es6", 4 | "module": "commonjs", 5 | "moduleResolution": "node", 6 | "removeComments": false, 7 | "outDir": "../dist", 8 | "rootDir": "../src", 9 | "declaration": true, 10 | "sourceMap": false, 11 | "importHelpers": true, 12 | "strictNullChecks": true, 13 | "experimentalDecorators": true, 14 | "emitDecoratorMetadata": true, 15 | "noUnusedLocals": true, 16 | "noUnusedParameters": true, 17 | "lib": [ 18 | "es2015", 19 | "dom" 20 | ] 21 | }, 22 | "include": [ 23 | "../src/**/*" 24 | ], 25 | "exclude": [ 26 | "../node_modules", 27 | "../build" 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /config/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.base.json", 3 | } 4 | -------------------------------------------------------------------------------- /jest.json: -------------------------------------------------------------------------------- 1 | { 2 | "collectCoverage": true, 3 | "mapCoverage": true, 4 | "transform": { 5 | "^.+\\.tsx?$": "ts-jest" 6 | }, 7 | "testMatch": ["**/test/**/*.spec.ts"], 8 | "testPathIgnorePatterns": ["/node_modules/"], 9 | "moduleFileExtensions": ["ts", "tsx", "js", "json"] 10 | } 11 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "proc-that", 3 | "version": "0.0.0-development", 4 | "description": "proc(ess)-that - easy extendable etl tool for nodejs written in typesript", 5 | "main": "dist/index.js", 6 | "typings": "dist/index.d.js", 7 | "scripts": { 8 | "clean": "del-cli ./dist ./coverage", 9 | "build": "npm run clean && tsc -p ./config/tsconfig.build.json", 10 | "develop": "npm run clean && tsc -p .", 11 | "lint": "npm run lint:ts && npm run prettier", 12 | "lint:fix": "npm run lint:ts:fix && npm run prettier:fix", 13 | "lint:ts": "eslint --max-warnings=-1", 14 | "lint:ts:fix": "eslint --max-warnings=-1 --fix", 15 | "prettier": "prettier --config .prettierrc.json --list-different \"./**/*.{ts,tsx}\"", 16 | "prettier:fix": "prettier --config .prettierrc.json --list-different \"./**/*.{ts,tsx}\" --write", 17 | "test": "npm run lint && npm run clean && jest -c ./jest.json", 18 | "test:watch": "npm run clean && jest -c ./jest.json --watch" 19 | }, 20 | "keywords": [ 21 | "etl", 22 | "node", 23 | "typescript" 24 | ], 25 | "engines": { 26 | "node": ">=16" 27 | }, 28 | "repository": { 29 | "type": "git", 30 | "url": "https://github.com/smartive/proc-that.git" 31 | }, 32 | "bugs": "https://github.com/smartive/proc-that/issues", 33 | "author": "Christoph Bühler ", 34 | "license": "MIT", 35 | "devDependencies": { 36 | "@smartive/eslint-config": "^3.1.1", 37 | "@smartive/prettier-config": "^3.0.0", 38 | "@types/jest": "^29.2.4", 39 | "del-cli": "^5.0.0", 40 | "eslint": "^8.30.0", 41 | "jest": "^29.3.1", 42 | "prettier": "^2.8.1", 43 | "ts-jest": "^29.0.3", 44 | "tsutils": "^3.21.0", 45 | "typescript": "^4.9.4" 46 | }, 47 | "dependencies": { 48 | "@types/node": "^18.11.17", 49 | "rxjs": "^7.8.0", 50 | "tslib": "^2.4.1" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "local>smartive/renovate-config", 5 | ":automergeDisabled", 6 | ":automergeMinor", 7 | ":preserveSemverRanges", 8 | ":widenPeerDependencies", 9 | ":maintainLockFilesWeekly" 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /src/Etl.ts: -------------------------------------------------------------------------------- 1 | import { EMPTY, merge, mergeMap, Observable, tap, throwError } from 'rxjs'; 2 | 3 | import { Extractor } from './interfaces/Extractor'; 4 | import { GeneralTransformer } from './interfaces/GeneralTransformer'; 5 | import { Loader } from './interfaces/Loader'; 6 | import { Transformer } from './interfaces/Transformer'; 7 | import { MapTransformer } from './transformers/MapTransformer'; 8 | 9 | export enum EtlState { 10 | Running, 11 | Stopped, 12 | Error, 13 | } 14 | 15 | /** 16 | * ETL Class. Instantiate one and add as many extractors, transformers and loaders as you want. 17 | * Then start the whole process with ".start()". 18 | * 19 | * This processor is modular, you can find other implemented loaders and extractors in the README 20 | */ 21 | export class Etl { 22 | private _extractors: Extractor[] = []; 23 | private _generalTransformers: GeneralTransformer[] = []; 24 | private _transformers: Transformer[] = []; 25 | private _loaders: Loader[] = []; 26 | private _state: EtlState = EtlState.Stopped; 27 | private _context: any = null; 28 | 29 | public constructor(context?: any) { 30 | this.setContext(context); 31 | } 32 | 33 | public get extractors(): Extractor[] { 34 | return this._extractors; 35 | } 36 | 37 | public get generalTransformers(): GeneralTransformer[] { 38 | return this._generalTransformers; 39 | } 40 | 41 | public get transformers(): Transformer[] { 42 | return this._transformers; 43 | } 44 | 45 | public get loaders(): Loader[] { 46 | return this._loaders; 47 | } 48 | 49 | public get state(): EtlState { 50 | return this._state; 51 | } 52 | 53 | public setContext(context: any): this { 54 | if (this._state !== EtlState.Stopped) { 55 | this._state = EtlState.Error; 56 | throw new Error('Tried to set context on invalid state.'); 57 | } 58 | this._context = context; 59 | return this; 60 | } 61 | 62 | public addExtractor(extract: Extractor): Etl { 63 | this._extractors.push(extract); 64 | return this; 65 | } 66 | 67 | public addGeneralTransformer(transformer: GeneralTransformer): Etl { 68 | this._generalTransformers.push(transformer); 69 | return this; 70 | } 71 | 72 | public addTransformer(transformer: Transformer): Etl { 73 | this.addGeneralTransformer(new MapTransformer(transformer)); 74 | this._transformers.push(transformer); 75 | return this; 76 | } 77 | 78 | public addLoader(loader: Loader): Etl { 79 | this._loaders.push(loader); 80 | return this; 81 | } 82 | 83 | /** 84 | * Starts the etl process. First, all extractors are run in parallel and deliver their results into an observable. 85 | * Once the buffer gets a result, it transfers all objects through the transformers (one by one). 86 | * After that, the transformed results are run through all loaders in parallel. 87 | * 88 | * @returns {Observable} Observable that completes when the process is finished, 89 | * during the "next" process step you get update on how many are processed yet. 90 | * Throws when any step produces an error. 91 | */ 92 | public start(observable: Observable = EMPTY): Observable { 93 | this._state = EtlState.Running; 94 | 95 | const o: Observable = merge(observable, ...this._extractors.map((extractor) => extractor.read(this._context))); 96 | 97 | return this._generalTransformers 98 | .reduce((observable, transformer) => transformer.process(observable, this._context), o) 99 | .pipe(mergeMap((object) => merge(...this._loaders.map((loader) => loader.write(object, this._context))))) 100 | .pipe( 101 | tap({ 102 | error: (err) => { 103 | this._state = EtlState.Error; 104 | return throwError(() => err); 105 | }, 106 | complete: () => { 107 | this._state = EtlState.Stopped; 108 | }, 109 | }) 110 | ); 111 | } 112 | 113 | /** 114 | * Resets the whole Etl object. Deletes all modifiers and resets the state. 115 | */ 116 | public reset(): void { 117 | this._extractors = []; 118 | this._transformers = []; 119 | this._loaders = []; 120 | this._state = EtlState.Stopped; 121 | this._context = null; 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/extractors/JsonExtractor.ts: -------------------------------------------------------------------------------- 1 | import { resolve } from 'path'; 2 | import { from, Observable, throwError } from 'rxjs'; 3 | 4 | import { Extractor } from '../interfaces/Extractor'; 5 | 6 | /** 7 | * Extractor that reads a JSON file at a given filepath. The path is resolved relatively to the running tasks root dir. 8 | */ 9 | export class JsonExtractor implements Extractor { 10 | private filePath: string; 11 | 12 | constructor(filePath: string) { 13 | this.filePath = resolve(process.cwd(), filePath); 14 | } 15 | 16 | public read(): Observable { 17 | try { 18 | const content = require(this.filePath); 19 | if (!(content instanceof Array) && content.constructor !== Array) { 20 | return from([content]); 21 | } 22 | return from(content); 23 | } catch (e) { 24 | return throwError(() => e); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './Etl'; 2 | export * from './extractors/JsonExtractor'; 3 | export * from './loaders/ConsoleLoader'; 4 | export * from './transformers/MapTransformer'; 5 | export * from './transformers/MatchMergeTransformer'; 6 | export * from './interfaces/Extractor'; 7 | export * from './interfaces/Transformer'; 8 | export * from './interfaces/Loader'; 9 | -------------------------------------------------------------------------------- /src/interfaces/Extractor.ts: -------------------------------------------------------------------------------- 1 | import { Observable } from 'rxjs'; 2 | 3 | /** 4 | * Extractor interface. Only provides "read()" method that returns an observable with the result. 5 | * 6 | * @export 7 | * @interface Extractor 8 | */ 9 | export interface Extractor { 10 | read(context?: any): Observable; 11 | } 12 | -------------------------------------------------------------------------------- /src/interfaces/GeneralTransformer.ts: -------------------------------------------------------------------------------- 1 | import { Observable } from 'rxjs'; 2 | 3 | /** 4 | * GeneralTransformer interface. Provides a "process(observable)" method that processes an observable. 5 | * Represents a stage in the ETL pipeline. 6 | * 7 | * @export 8 | * @interface GeneralTransformer 9 | */ 10 | export interface GeneralTransformer { 11 | process(observable: Observable, context?: any): Observable; 12 | } 13 | -------------------------------------------------------------------------------- /src/interfaces/Loader.ts: -------------------------------------------------------------------------------- 1 | import { Observable } from 'rxjs'; 2 | 3 | /** 4 | * Loader interface. Provides ".write(obj)" method that returns an observable with the loaded value. 5 | */ 6 | export interface Loader { 7 | write(object: any, context?: any): Observable; 8 | } 9 | -------------------------------------------------------------------------------- /src/interfaces/Transformer.ts: -------------------------------------------------------------------------------- 1 | import { Observable } from 'rxjs'; 2 | 3 | /** 4 | * Transformer interface. Only provides ".process(obj)" that returns an Observable with 5 | * the new result (array will be flattend). 6 | */ 7 | export interface Transformer { 8 | process(object: any, context?: any): Observable; 9 | } 10 | -------------------------------------------------------------------------------- /src/loaders/ConsoleLoader.ts: -------------------------------------------------------------------------------- 1 | import { Observable, of } from 'rxjs'; 2 | 3 | import { Loader } from '../interfaces/Loader'; 4 | 5 | /** 6 | * Loader that outputs everything to the console. 7 | * 8 | * @export 9 | * @class ConsoleLoader 10 | * @implements {Loader} 11 | */ 12 | export class ConsoleLoader implements Loader { 13 | public write(object: any): Observable { 14 | console.log(object); 15 | return of(object); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/transformers/MapTransformer.ts: -------------------------------------------------------------------------------- 1 | import { mergeMap, Observable } from 'rxjs'; 2 | 3 | import { GeneralTransformer } from '../interfaces/GeneralTransformer'; 4 | import { Transformer } from '../interfaces/Transformer'; 5 | 6 | export class MapTransformer implements GeneralTransformer { 7 | constructor(private transformer: Transformer) {} 8 | 9 | public process(observable: Observable, context?: any): Observable { 10 | return observable.pipe(mergeMap((o) => this.transformer.process(o, context))); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/transformers/MatchMergeTransformer.ts: -------------------------------------------------------------------------------- 1 | import { from, mergeMap, Observable, reduce } from 'rxjs'; 2 | 3 | import { GeneralTransformer } from '../interfaces/GeneralTransformer'; 4 | 5 | export abstract class MatchMergeTransformer implements GeneralTransformer { 6 | public process(observable: Observable, context?: any): Observable { 7 | const matchMerge = (merged: any[], o2: any) => { 8 | return this.matchMerge(merged, o2, context); 9 | }; 10 | return observable.pipe(reduce(matchMerge, [])).pipe(mergeMap((v) => from(v))); 11 | } 12 | 13 | protected abstract match(o1: any, o2: any, context?: any): boolean; 14 | 15 | protected abstract merge(o1: any, o2: any, context?: any): any; 16 | 17 | private matchMerge(merged: any[], o2: any, context?: any): any[] { 18 | for (let i = 0; i < merged.length; i++) { 19 | if (this.match(merged[i], o2, context)) { 20 | const o1 = merged.splice(i, 1)[0]; 21 | o2 = this.merge(o1, o2, context); 22 | // Try to merge the merged element with the remaining elements, 23 | // starting from the current position 24 | i--; 25 | } 26 | } 27 | merged.push(o2); 28 | return merged; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /test/.testdata/json-extractor.array.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "objId": 1, 4 | "name": "foobar" 5 | }, 6 | { 7 | "objId": 2, 8 | "name": "hello world" 9 | }, 10 | { 11 | "objId": 3, 12 | "name": "third test" 13 | } 14 | ] -------------------------------------------------------------------------------- /test/.testdata/json-extractor.object.json: -------------------------------------------------------------------------------- 1 | { 2 | "foo": "bar", 3 | "hello": "world" 4 | } -------------------------------------------------------------------------------- /test/.testdata/match-merge.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "location": "A", 4 | "things": ["a"] 5 | }, 6 | { 7 | "location": "B", 8 | "things": ["b"] 9 | }, 10 | { 11 | "location": "A", 12 | "things": ["c"] 13 | }, 14 | { 15 | "location": "B", 16 | "things": ["d"] 17 | } 18 | ] -------------------------------------------------------------------------------- /test/Etl.spec.ts: -------------------------------------------------------------------------------- 1 | import { from, of, reduce, throwError } from 'rxjs'; 2 | 3 | import { Etl, EtlState, Extractor, JsonExtractor, Loader, MatchMergeTransformer, Transformer } from '../src'; 4 | 5 | describe('Etl', () => { 6 | let etl: Etl; 7 | const extractor: Extractor = new JsonExtractor('./test/.testdata/json-extractor.object.json'); 8 | const arrayExtractor: Extractor = new JsonExtractor('./test/.testdata/json-extractor.array.json'); 9 | const matchMergeExtractor: Extractor = new JsonExtractor('./test/.testdata/match-merge.json'); 10 | let o; 11 | let dummyExtractor: Extractor; 12 | let dummyTransformer: Transformer; 13 | let dummyLoader: Loader; 14 | 15 | beforeEach(() => { 16 | etl = new Etl(); 17 | 18 | o = { _id: '001' }; 19 | 20 | dummyExtractor = { 21 | read: () => of(o), 22 | }; 23 | dummyExtractor.read = jest.fn(dummyExtractor.read); 24 | 25 | dummyTransformer = { 26 | process: (o) => of(o), 27 | }; 28 | dummyTransformer.process = jest.fn(dummyTransformer.process); 29 | 30 | dummyLoader = { 31 | write: (o) => of(o), 32 | }; 33 | dummyLoader.write = jest.fn(dummyLoader.write); 34 | }); 35 | 36 | it('should initialize with correct default params', () => { 37 | expect(etl.state).toBe(EtlState.Stopped); 38 | expect(etl.extractors.length).toBe(0); 39 | expect(etl.transformers.length).toBe(0); 40 | expect(etl.loaders.length).toBe(0); 41 | }); 42 | 43 | it('should reset correctly', () => { 44 | etl.addExtractor({ 45 | read: function () { 46 | return of(null); 47 | }, 48 | }); 49 | 50 | expect(etl.extractors.length).toBe(1); 51 | etl.reset(); 52 | expect(etl.extractors.length).toBe(0); 53 | }); 54 | 55 | it('should pass context down the pipeline', (done) => { 56 | const context = 1; 57 | etl = new Etl(context); 58 | etl 59 | .addExtractor(dummyExtractor) 60 | .addTransformer(dummyTransformer) 61 | .addLoader(dummyLoader) 62 | .start() 63 | .subscribe({ 64 | complete: () => { 65 | expect((dummyExtractor.read as any).mock.calls[0]).toContain(context); 66 | expect((dummyTransformer.process as any).mock.calls[0]).toContain(context); 67 | expect((dummyLoader.write as any).mock.calls[0]).toContain(context); 68 | done(); 69 | }, 70 | }); 71 | }); 72 | 73 | it('should pass newly set context down the pipeline', (done) => { 74 | const context = 1; 75 | etl 76 | .addExtractor(dummyExtractor) 77 | .addTransformer(dummyTransformer) 78 | .addLoader(dummyLoader) 79 | .setContext(context) 80 | .start() 81 | .subscribe({ 82 | complete: () => { 83 | expect((dummyExtractor.read as any).mock.calls[0]).toContain(context); 84 | expect((dummyTransformer.process as any).mock.calls[0]).toContain(context); 85 | expect((dummyLoader.write as any).mock.calls[0]).toContain(context); 86 | done(); 87 | }, 88 | }); 89 | }); 90 | 91 | it('should process simple object', (done) => { 92 | etl 93 | .addExtractor(extractor) 94 | .addLoader(dummyLoader) 95 | .start() 96 | .subscribe({ 97 | complete: () => { 98 | expect((dummyLoader.write as any).mock.calls).toHaveLength(1); 99 | expect((dummyLoader.write as any).mock.calls[0][0]).toMatchObject({ 100 | foo: 'bar', 101 | hello: 'world', 102 | }); 103 | done(); 104 | }, 105 | }); 106 | }); 107 | 108 | it('should process simple array', (done) => { 109 | etl 110 | .addExtractor(arrayExtractor) 111 | .addLoader(dummyLoader) 112 | .start() 113 | .subscribe({ 114 | complete: () => { 115 | expect((dummyLoader.write as any).mock.calls).toHaveLength(3); 116 | expect((dummyLoader.write as any).mock.calls[0][0]).toMatchObject({ 117 | objId: 1, 118 | name: 'foobar', 119 | }); 120 | expect((dummyLoader.write as any).mock.calls[1][0]).toMatchObject({ 121 | objId: 2, 122 | name: 'hello world', 123 | }); 124 | expect((dummyLoader.write as any).mock.calls[2][0]).toMatchObject({ 125 | objId: 3, 126 | name: 'third test', 127 | }); 128 | done(); 129 | }, 130 | }); 131 | }); 132 | 133 | it('should call error on extractor error', (done) => { 134 | etl 135 | .addExtractor({ 136 | read: () => throwError(() => new Error('test')), 137 | }) 138 | .addLoader(dummyLoader) 139 | .start() 140 | .subscribe({ 141 | error: () => { 142 | done(); 143 | }, 144 | complete: () => { 145 | done(new Error('did not throw')); 146 | }, 147 | }); 148 | }); 149 | 150 | it('should call error on loader error', (done) => { 151 | etl 152 | .addExtractor(extractor) 153 | .addLoader({ 154 | write: () => throwError(() => new Error('test')), 155 | }) 156 | .start() 157 | .subscribe({ 158 | error: () => { 159 | done(); 160 | }, 161 | complete: () => { 162 | done(new Error('did not throw')); 163 | }, 164 | }); 165 | }); 166 | 167 | it('should call error on transformer error', (done) => { 168 | etl 169 | .addExtractor(extractor) 170 | .addLoader(dummyLoader) 171 | .addTransformer({ 172 | process: () => throwError(() => new Error('test')), 173 | }) 174 | .start() 175 | .subscribe({ 176 | error: () => { 177 | done(); 178 | }, 179 | complete: () => { 180 | done(new Error('did not throw')); 181 | }, 182 | }); 183 | }); 184 | 185 | it('should process simple object with transformer', (done) => { 186 | const spy = jest.fn(); 187 | etl 188 | .addExtractor(extractor) 189 | .addLoader(dummyLoader) 190 | .addTransformer({ 191 | process: (o) => of(o), 192 | }) 193 | .start() 194 | .subscribe({ 195 | next: spy, 196 | error: () => { 197 | done(new Error('did throw')); 198 | }, 199 | complete: () => { 200 | expect(spy.mock.calls.length).toBe(1); 201 | done(); 202 | }, 203 | }); 204 | }); 205 | 206 | it('should process simple array with transformer (flat)', (done) => { 207 | const spy = jest.fn(); 208 | etl 209 | .addExtractor(arrayExtractor) 210 | .addLoader(dummyLoader) 211 | .addTransformer({ 212 | process: (o) => from([o, o]), 213 | }) 214 | .start() 215 | .subscribe({ 216 | next: spy, 217 | error: () => { 218 | done(new Error('did throw')); 219 | }, 220 | complete: () => { 221 | expect(spy.mock.calls.length).toBe(6); 222 | done(); 223 | }, 224 | }); 225 | }); 226 | 227 | it('should process a general transformer', (done) => { 228 | const spy = jest.fn(); 229 | etl 230 | .addExtractor(arrayExtractor) 231 | .addLoader(dummyLoader) 232 | .addGeneralTransformer({ 233 | process: (o) => o.pipe(reduce((x, y) => x + y.objId, 0)), 234 | }) 235 | .start() 236 | .subscribe({ 237 | next: spy, 238 | error: () => { 239 | done(new Error('did throw')); 240 | }, 241 | complete: () => { 242 | expect(spy.mock.calls.length).toBe(1); 243 | expect(spy.mock.calls[0][0]).toBe(6); 244 | done(); 245 | }, 246 | }); 247 | }); 248 | 249 | it('should process a match-merge transformer', (done) => { 250 | const spy = jest.fn(); 251 | 252 | class TestMatchTransformer extends MatchMergeTransformer { 253 | match(o1, o2) { 254 | return o1.location === o2.location; 255 | } 256 | 257 | merge(o1, o2) { 258 | return { 259 | location: o1.location, 260 | things: [...o1.things, ...o2.things], 261 | }; 262 | } 263 | } 264 | 265 | etl 266 | .addExtractor(matchMergeExtractor) 267 | .addLoader(dummyLoader) 268 | .addGeneralTransformer(new TestMatchTransformer()) 269 | .start() 270 | .subscribe({ 271 | next: spy, 272 | error: () => { 273 | done(new Error('did throw')); 274 | }, 275 | complete: () => { 276 | expect(spy.mock.calls.length).toBe(2); 277 | expect(spy.mock.calls[0][0]).toMatchObject({ 278 | location: 'A', 279 | things: ['a', 'c'], 280 | }); 281 | expect(spy.mock.calls[1][0]).toMatchObject({ 282 | location: 'B', 283 | things: ['b', 'd'], 284 | }); 285 | done(); 286 | }, 287 | }); 288 | }); 289 | 290 | it('should pipe inital observable', (done) => { 291 | const context = 1; 292 | etl = new Etl(context); 293 | etl 294 | .addTransformer(dummyTransformer) 295 | .addLoader(dummyLoader) 296 | .start(of('hi')) 297 | .subscribe({ 298 | complete: () => { 299 | expect((dummyTransformer.process as any).mock.calls[0]).toContain('hi'); 300 | expect((dummyLoader.write as any).mock.calls[0]).toContain('hi'); 301 | done(); 302 | }, 303 | }); 304 | }); 305 | }); 306 | -------------------------------------------------------------------------------- /test/JsonExtractor.spec.ts: -------------------------------------------------------------------------------- 1 | import { join } from 'path'; 2 | 3 | import { JsonExtractor } from '../src'; 4 | 5 | describe('JsonExtractor', () => { 6 | it('should return an observable', () => { 7 | const ext = new JsonExtractor('./test/.testdata/json-extractor.object.json'); 8 | expect(ext.read()).toBeInstanceOf(Object); 9 | }); 10 | 11 | it('should get correct path', () => { 12 | const ext = new JsonExtractor('hello'); 13 | const anyExt: any = ext; 14 | const result = join(process.cwd(), 'hello'); 15 | expect(anyExt.filePath).toBe(result); 16 | }); 17 | 18 | it('should receive a json object', (done) => { 19 | const ext = new JsonExtractor('./test/.testdata/json-extractor.object.json'); 20 | ext.read().subscribe({ 21 | next: (obj) => { 22 | expect(obj).toMatchObject({ 23 | foo: 'bar', 24 | hello: 'world', 25 | }); 26 | done(); 27 | }, 28 | }); 29 | }); 30 | 31 | it('should receive a json array', (done) => { 32 | const ext = new JsonExtractor('./test/.testdata/json-extractor.array.json'); 33 | const spy = jest.fn(); 34 | ext.read().subscribe({ 35 | next: spy, 36 | complete: () => { 37 | expect(spy.mock.calls.length).toBe(3); 38 | done(); 39 | }, 40 | }); 41 | }); 42 | 43 | it('should throw on not found file', (done) => { 44 | const ext = new JsonExtractor('404.json'); 45 | ext.read().subscribe({ 46 | next: () => { 47 | done(new Error('did not throw')); 48 | }, 49 | error: () => { 50 | done(); 51 | }, 52 | }); 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /test/MapTransformer.spec.ts: -------------------------------------------------------------------------------- 1 | import { from, of } from 'rxjs'; 2 | 3 | import { MapTransformer } from '../src'; 4 | 5 | describe('MapTransformer', () => { 6 | it('should return an observable', () => { 7 | const spy = jest.fn(); 8 | 9 | const subt = { 10 | process(o) { 11 | return of(o); 12 | }, 13 | }; 14 | 15 | subt.process = jest.fn(subt.process); 16 | 17 | const t = new MapTransformer(subt); 18 | 19 | t.process(from([1]), 2).subscribe(spy, null, () => { 20 | expect(spy.mock.calls.length).toBe(1); 21 | expect((subt.process as any).mock.calls.length).toBe(1); 22 | }); 23 | }); 24 | }); 25 | -------------------------------------------------------------------------------- /test/MatchMergeTransformer.spec.ts: -------------------------------------------------------------------------------- 1 | import { from } from 'rxjs'; 2 | 3 | import { MatchMergeTransformer } from '../src/transformers/MatchMergeTransformer'; 4 | 5 | describe('MatchMergeTransformer', () => { 6 | it('should return an observable', () => { 7 | const spy = jest.fn(); 8 | 9 | class TestMatchMergeTransformer extends MatchMergeTransformer { 10 | match(o1, o2) { 11 | return o1 === o2; 12 | } 13 | 14 | merge(o1) { 15 | return o1; 16 | } 17 | } 18 | 19 | const t = new TestMatchMergeTransformer(); 20 | 21 | t.process(from([1, 2, 3, 2, 3])).subscribe({ 22 | next: spy, 23 | complete: () => { 24 | expect(spy.mock.calls.length).toBe(3); 25 | }, 26 | }); 27 | }); 28 | }); 29 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./config/tsconfig.base.json", 3 | "compilerOptions": { 4 | "watch": true, 5 | "sourceMap": true 6 | } 7 | } 8 | --------------------------------------------------------------------------------