├── .prettierrc ├── .husky └── pre-commit ├── src ├── client.ts ├── server.ts ├── models │ ├── utils │ │ └── index.ts │ ├── worker │ │ └── index.ts │ ├── list │ │ └── index.ts │ ├── object │ │ └── index.ts │ ├── emitter │ │ └── index.ts │ └── stream │ │ └── index.ts ├── mocks │ ├── nextList.json │ └── prevList.json ├── index.ts └── lib │ ├── stream-list-diff │ ├── client │ │ ├── worker │ │ │ ├── web-worker.ts │ │ │ └── utils.ts │ │ └── index.ts │ ├── server │ │ ├── worker │ │ │ ├── node-worker.ts │ │ │ └── utils.ts │ │ ├── index.ts │ │ ├── stream-list-diff.test.ts │ │ └── stream-list-diff.worker.test.ts │ └── utils.ts │ ├── utils │ ├── index.ts │ └── utils.test.ts │ ├── list-diff │ ├── index.ts │ └── list-diff.test.ts │ └── object-diff │ ├── index.ts │ └── object-diff.test.ts ├── .github ├── FUNDING.yml └── workflows │ ├── cd.yml │ └── ci.yml ├── .prettierignore ├── .gitignore ├── jest.setup.ts ├── eslint.config.mjs ├── scripts └── transpile-node-worker.js ├── jest.config.ts ├── tsconfig.json ├── tsup.config.ts ├── package.json └── README.md /.prettierrc: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | npm run test && npm run tsc && npm run lint -------------------------------------------------------------------------------- /src/client.ts: -------------------------------------------------------------------------------- 1 | export { streamListDiff } from "./lib/stream-list-diff/client"; 2 | -------------------------------------------------------------------------------- /src/server.ts: -------------------------------------------------------------------------------- 1 | export { streamListDiff } from "./lib/stream-list-diff/server"; 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [DoneDeal0] 4 | -------------------------------------------------------------------------------- /src/models/utils/index.ts: -------------------------------------------------------------------------------- 1 | export type isEqualOptions = { 2 | ignoreArrayOrder?: boolean; 3 | }; 4 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | dist 2 | .github 3 | .husky 4 | jest.config.js 5 | package-lock.json 6 | README.md 7 | tsconfig.json 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | dist 3 | .eslintcache 4 | .DS_Store 5 | # Ignore generated worker files 6 | src/lib/stream-list-diff/server/worker/node-worker.cjs -------------------------------------------------------------------------------- /src/mocks/nextList.json: -------------------------------------------------------------------------------- 1 | [ 2 | { "id": 1, "name": "Item 1" }, 3 | { "id": 2, "name": "Item Two" }, 4 | { "id": 3, "name": "Item 3" }, 5 | { "id": 5, "name": "Item 5" } 6 | ] -------------------------------------------------------------------------------- /src/mocks/prevList.json: -------------------------------------------------------------------------------- 1 | [ 2 | { "id": 1, "name": "Item 1" }, 3 | { "id": 2, "name": "Item 2" }, 4 | { "id": 3, "name": "Item 3" }, 5 | { "id": 4, "name": "Item 4" } 6 | ] -------------------------------------------------------------------------------- /jest.setup.ts: -------------------------------------------------------------------------------- 1 | import { TextEncoder, TextDecoder } from "util"; 2 | 3 | global.TextEncoder = TextEncoder; 4 | //@ts-expect-error - the TextDecoder is valid 5 | global.TextDecoder = TextDecoder; 6 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export { getObjectDiff } from "./lib/object-diff"; 2 | export { getListDiff } from "./lib/list-diff"; 3 | export { isEqual, isObject } from "./lib/utils"; 4 | export * from "./models/list"; 5 | export * from "./models/object"; 6 | export * from "./models/stream"; 7 | -------------------------------------------------------------------------------- /eslint.config.mjs: -------------------------------------------------------------------------------- 1 | import pluginJs from "@eslint/js"; 2 | import tseslint from "typescript-eslint"; 3 | 4 | export default [ 5 | { files: ["**/*.{js,mjs,cjs,ts}"] }, 6 | { ignores: ["dist", "jest.config.js", "src/lib/stream-list-diff/server/worker/node-worker.cjs"] }, 7 | { settings: { react: { version: "detect" } } }, 8 | pluginJs.configs.recommended, 9 | ...tseslint.configs.recommended, 10 | ]; 11 | -------------------------------------------------------------------------------- /src/models/worker/index.ts: -------------------------------------------------------------------------------- 1 | import { StreamEvent, StreamListDiff } from "@models/stream"; 2 | 3 | export enum WorkerEvent { 4 | Message = "message", 5 | Error = "error", 6 | } 7 | 8 | type WorkerData> = { 9 | chunk: StreamListDiff[]; 10 | error: string; 11 | event: StreamEvent; 12 | }; 13 | 14 | type WorkerMessage> = { 15 | data: WorkerData; 16 | }; 17 | 18 | export type WebWorkerMessage> = 19 | WorkerMessage; 20 | 21 | export type NodeWorkerMessage> = 22 | WorkerData; 23 | -------------------------------------------------------------------------------- /scripts/transpile-node-worker.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-unused-vars */ 2 | /* eslint-disable no-undef */ 3 | import { execSync } from "child_process"; 4 | import { existsSync } from "fs" 5 | 6 | // The src/lib/stream-list-diff/server/node-worker.ts file needs to be transpiled to a .cjs file to be used in the tests. 7 | const workerFile = "src/lib/stream-list-diff/server/worker/node-worker" 8 | 9 | try { 10 | if(!existsSync(`${workerFile}.cjs`)){ 11 | execSync(`npx esbuild ${workerFile}.ts --bundle --platform=node --format=cjs --outfile=${workerFile}.cjs`, { 12 | stdio: "inherit", 13 | }); 14 | } 15 | } catch (_) { 16 | process.exit(1); 17 | } -------------------------------------------------------------------------------- /jest.config.ts: -------------------------------------------------------------------------------- 1 | import type { Config } from "jest"; 2 | 3 | const config: Config = { 4 | transform: { 5 | "^.+\\.(ts|js)$": [ 6 | "@swc/jest", 7 | { 8 | jsc: { 9 | baseUrl: ".", 10 | parser: { 11 | syntax: "typescript", 12 | tsx: true, 13 | dynamicImport: true, 14 | }, 15 | paths: { 16 | "@mocks/*": ["./src/mocks/*"], 17 | "@models/*": ["./src/models/*"], 18 | "@lib/*": ["./src/lib/*"], 19 | }, 20 | target: "esnext", 21 | }, 22 | }, 23 | ], 24 | }, 25 | testEnvironment: "node", 26 | setupFilesAfterEnv: ["/jest.setup.ts"], 27 | }; 28 | 29 | export default config; 30 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "exclude": ["node_modules", "dist"], 3 | "include": ["src"], 4 | "compilerOptions": { 5 | "declaration": true, 6 | "declarationDir": "./dist", 7 | "lib": ["esnext", "dom"], 8 | "module": "esnext", 9 | "target": "esnext", 10 | "moduleResolution": "node", 11 | "noUnusedParameters": true, 12 | "esModuleInterop": true, 13 | "noImplicitAny": true, 14 | "outDir": "./dist", 15 | "strict": true, 16 | "resolveJsonModule": true, 17 | "allowSyntheticDefaultImports": true, 18 | "skipLibCheck": true , 19 | "baseUrl": ".", 20 | "paths": { 21 | "@lib/*": ["./src/lib/*"], 22 | "@mocks/*": ["./src/mocks/*"], 23 | "@models/*": ["./src/models/*"] 24 | } 25 | }, 26 | } 27 | -------------------------------------------------------------------------------- /src/models/list/index.ts: -------------------------------------------------------------------------------- 1 | export const DEFAULT_LIST_DIFF_OPTIONS = { 2 | showOnly: [], 3 | referenceProperty: undefined, 4 | considerMoveAsUpdate: false, 5 | ignoreArrayOrder: false, 6 | }; 7 | 8 | export enum ListStatus { 9 | ADDED = "added", 10 | EQUAL = "equal", 11 | DELETED = "deleted", 12 | UPDATED = "updated", 13 | MOVED = "moved", 14 | } 15 | 16 | export enum ListType { 17 | PREV = "prevList", 18 | NEXT = "nextList", 19 | } 20 | 21 | export type ListDiffOptions = { 22 | showOnly?: `${ListStatus}`[]; 23 | referenceProperty?: string; 24 | considerMoveAsUpdate?: boolean; 25 | ignoreArrayOrder?: boolean; 26 | }; 27 | 28 | export type ListDiff = { 29 | type: "list"; 30 | status: `${ListStatus}`; 31 | diff: { 32 | value: unknown; 33 | prevIndex: number | null; 34 | newIndex: number | null; 35 | indexDiff: number | null; 36 | status: ListStatus; 37 | }[]; 38 | }; 39 | -------------------------------------------------------------------------------- /src/lib/stream-list-diff/client/worker/web-worker.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ListStreamOptions, 3 | ReferenceProperty, 4 | StreamEvent, 5 | } from "@models/stream"; 6 | import { workerDiff } from "./utils"; 7 | 8 | self.onmessage = async >( 9 | event: MessageEvent<{ 10 | prevList: File | T[]; 11 | nextList: File | T[]; 12 | referenceProperty: ReferenceProperty; 13 | options: ListStreamOptions; 14 | }>, 15 | ) => { 16 | const { prevList, nextList, referenceProperty, options } = event.data; 17 | const listener = workerDiff(prevList, nextList, referenceProperty, options); 18 | 19 | listener.on(StreamEvent.Data, (chunk) => { 20 | self.postMessage({ event: StreamEvent.Data, chunk }); 21 | }); 22 | 23 | listener.on(StreamEvent.Finish, () => { 24 | self.postMessage({ event: StreamEvent.Finish }); 25 | }); 26 | 27 | listener.on(StreamEvent.Error, (error) => { 28 | self.postMessage({ event: StreamEvent.Error, error: error.message }); 29 | }); 30 | }; 31 | -------------------------------------------------------------------------------- /src/models/object/index.ts: -------------------------------------------------------------------------------- 1 | export enum ObjectStatus { 2 | ADDED = "added", 3 | EQUAL = "equal", 4 | DELETED = "deleted", 5 | UPDATED = "updated", 6 | } 7 | 8 | export enum Granularity { 9 | BASIC = "basic", 10 | DEEP = "deep", 11 | } 12 | 13 | export type ObjectData = Record | undefined | null; 14 | 15 | export type ObjectDiffOptions = { 16 | ignoreArrayOrder?: boolean; 17 | showOnly?: { 18 | statuses: `${ObjectStatus}`[]; 19 | granularity?: `${Granularity}`; 20 | }; 21 | }; 22 | 23 | export const DEFAULT_OBJECT_DIFF_OPTIONS = { 24 | ignoreArrayOrder: false, 25 | showOnly: { statuses: [], granularity: Granularity.BASIC }, 26 | }; 27 | 28 | /** recursive diff in case of subproperties */ 29 | export type Diff = { 30 | property: string; 31 | previousValue: unknown; 32 | currentValue: unknown; 33 | status: `${ObjectStatus}`; 34 | diff?: Diff[]; 35 | }; 36 | 37 | export type ObjectDiff = { 38 | type: "object"; 39 | status: `${ObjectStatus}`; 40 | diff: Diff[]; 41 | }; 42 | -------------------------------------------------------------------------------- /src/models/emitter/index.ts: -------------------------------------------------------------------------------- 1 | import { StreamListDiff } from "@models/stream"; 2 | 3 | export type Listener = (...args: T) => void; 4 | 5 | export type EmitterEvents> = { 6 | data: [StreamListDiff[]]; 7 | error: [Error]; 8 | finish: []; 9 | }; 10 | 11 | export type IEmitter> = EventEmitter<{ 12 | data: [StreamListDiff[]]; 13 | error: [Error]; 14 | finish: []; 15 | }>; 16 | 17 | export class EventEmitter> { 18 | private events: Record[]> = {}; 19 | 20 | on(event: E, listener: Listener): this { 21 | if (!this.events[event as string]) { 22 | this.events[event as string] = []; 23 | } 24 | this.events[event as string].push(listener as Listener); 25 | return this; 26 | } 27 | 28 | emit(event: E, ...args: Events[E]): void { 29 | if (this.events[event as string]) { 30 | this.events[event as string].forEach((listener) => listener(...args)); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /tsup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig, Options } from "tsup"; 2 | 3 | const sharedConfig: Options = { 4 | dts: true, 5 | splitting: true, 6 | clean: true, 7 | treeshake: true, 8 | shims: true, 9 | minify: true, 10 | }; 11 | 12 | export default defineConfig([ 13 | { 14 | entry: ["src/index.ts"], 15 | format: ["cjs", "esm"], 16 | ...sharedConfig, 17 | platform: "neutral", 18 | name: "MAIN", 19 | }, 20 | { 21 | entry: ["src/client.ts"], 22 | format: ["esm"], 23 | ...sharedConfig, 24 | platform: "browser", 25 | name: "CLIENT", 26 | }, 27 | { 28 | entry: ["src/lib/stream-list-diff/client/worker/web-worker.ts"], 29 | format: ["esm"], 30 | ...sharedConfig, 31 | splitting: false, 32 | platform: "browser", 33 | name: "WEB WORKER", 34 | }, 35 | { 36 | entry: ["src/server.ts"], 37 | format: ["cjs"], 38 | ...sharedConfig, 39 | platform: "node", 40 | name: "SERVER", 41 | }, 42 | { 43 | entry: ["src/lib/stream-list-diff/server/worker/node-worker.ts"], 44 | format: ["cjs"], 45 | ...sharedConfig, 46 | splitting: false, 47 | shims: false, 48 | platform: "node", 49 | name: "NODEJS WORKER", 50 | }, 51 | ]); 52 | -------------------------------------------------------------------------------- /src/lib/stream-list-diff/server/worker/node-worker.ts: -------------------------------------------------------------------------------- 1 | import { parentPort } from "worker_threads"; 2 | import { 3 | FilePath, 4 | ListStreamOptions, 5 | ReferenceProperty, 6 | StreamEvent, 7 | } from "@models/stream"; 8 | import { WorkerEvent } from "@models/worker"; 9 | import { workerDiff } from "./utils"; 10 | 11 | parentPort?.on( 12 | WorkerEvent.Message, 13 | async >(event: { 14 | prevList: FilePath | T[]; 15 | nextList: FilePath | T[]; 16 | referenceProperty: ReferenceProperty; 17 | options: ListStreamOptions; 18 | }) => { 19 | const { prevList, nextList, referenceProperty, options } = event; 20 | 21 | const listener = workerDiff(prevList, nextList, referenceProperty, options); 22 | 23 | listener.on(StreamEvent.Data, (chunk) => { 24 | parentPort?.postMessage({ event: StreamEvent.Data, chunk }); 25 | }); 26 | 27 | listener.on(StreamEvent.Finish, () => { 28 | parentPort?.postMessage({ event: StreamEvent.Finish }); 29 | }); 30 | 31 | listener.on(StreamEvent.Error, (error) => { 32 | parentPort?.postMessage({ 33 | event: StreamEvent.Error, 34 | error: error.message, 35 | }); 36 | }); 37 | }, 38 | ); 39 | -------------------------------------------------------------------------------- /.github/workflows/cd.yml: -------------------------------------------------------------------------------- 1 | name: CD Pipeline 2 | 3 | on: 4 | workflow_run: 5 | workflows: ["CI Pipeline"] 6 | types: 7 | - completed 8 | branches: 9 | - master 10 | 11 | jobs: 12 | publish: 13 | if: ${{ github.event.workflow_run.conclusion == 'success' }} 14 | runs-on: ubuntu-latest 15 | permissions: 16 | contents: write 17 | issues: write 18 | pull-requests: write 19 | id-token: write 20 | 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v4 24 | with: 25 | fetch-depth: 0 26 | 27 | - name: Setup Node.js 28 | uses: actions/setup-node@v4 29 | with: 30 | node-version: "20" 31 | cache: "npm" 32 | registry-url: https://registry.npmjs.org 33 | cache-dependency-path: package-lock.json 34 | env: 35 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 36 | 37 | - name: Install dependencies 38 | run: npm install 39 | 40 | - name: Build the project 41 | run: npm run build 42 | 43 | - name: Release 44 | env: 45 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 46 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 47 | run: npx semantic-release 48 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI Pipeline 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - name: Checkout code 17 | uses: actions/checkout@v4 18 | 19 | - name: Setup Node.js 20 | uses: actions/setup-node@v4 21 | with: 22 | node-version: "20" 23 | cache: "npm" 24 | registry-url: https://registry.npmjs.org 25 | cache-dependency-path: package-lock.json 26 | 27 | - name: Retrieve dependencies 28 | id: "modules_cache" 29 | uses: actions/cache@v4 30 | with: 31 | path: node_modules 32 | key: node-modules-${{ runner.os }}-${{ hashFiles('package-lock.json') }} 33 | restore-keys: | 34 | node-modules-${{ runner.os }}- 35 | 36 | - name: Install dependencies 37 | if: steps.modules_cache.outputs.cache-hit != 'true' 38 | run: npm install 39 | 40 | - name: Run typescript 41 | run: npm run tsc 42 | 43 | - name: Run linter 44 | run: npm run lint 45 | 46 | - name: Run tests 47 | run: npm run test 48 | 49 | - name: Build the project 50 | run: npm run build 51 | -------------------------------------------------------------------------------- /src/lib/utils/index.ts: -------------------------------------------------------------------------------- 1 | import { isEqualOptions } from "@models/utils"; 2 | 3 | /** 4 | * Returns true if two data are equal 5 | * @param {unknown} a - The original data. 6 | * @param {unknown} b - The data to compare. 7 | * @param {isEqualOptions} options - The options to compare the data. 8 | * @returns boolean 9 | */ 10 | export function isEqual( 11 | a: unknown, 12 | b: unknown, 13 | options: isEqualOptions = { ignoreArrayOrder: false }, 14 | ): boolean { 15 | if (typeof a !== typeof b) return false; 16 | if (Array.isArray(a) && Array.isArray(b)) { 17 | if (a.length !== b.length) { 18 | return false; 19 | } 20 | if (options.ignoreArrayOrder) { 21 | return a.every((v) => 22 | b.some((nextV) => JSON.stringify(nextV) === JSON.stringify(v)), 23 | ); 24 | } 25 | return a.every((v, i) => JSON.stringify(v) === JSON.stringify(b[i])); 26 | } 27 | if (typeof a === "object") { 28 | return JSON.stringify(a) === JSON.stringify(b); 29 | } 30 | return a === b; 31 | } 32 | 33 | /** 34 | * Returns true if the provided value is an object 35 | * @param {unknown} value - The data to check. 36 | * @returns value is Record 37 | */ 38 | export function isObject(value: unknown): value is Record { 39 | return !!value && typeof value === "object" && !Array.isArray(value); 40 | } 41 | -------------------------------------------------------------------------------- /src/models/stream/index.ts: -------------------------------------------------------------------------------- 1 | import { EmitterEvents, Listener } from "@models/emitter"; 2 | import { ListStatus } from "@models/list"; 3 | 4 | export const READABLE_STREAM_ALERT = `Warning: using Readable streams may impact workers' performance since they need to be converted to arrays. 5 | Consider using arrays or files for optimal performance. Alternatively, you can turn the 'useWorker' option off. 6 | To disable this warning, set 'showWarnings' to false in production.`; 7 | 8 | export const DEFAULT_LIST_STREAM_OPTIONS: ListStreamOptions = { 9 | chunksSize: 0, 10 | useWorker: true, 11 | showWarnings: true, 12 | }; 13 | 14 | export enum StreamEvent { 15 | Data = "data", 16 | Finish = "finish", 17 | Error = "error", 18 | } 19 | 20 | export type StreamListDiff> = { 21 | currentValue: T | null; 22 | previousValue: T | null; 23 | prevIndex: number | null; 24 | newIndex: number | null; 25 | indexDiff: number | null; 26 | status: `${ListStatus}`; 27 | }; 28 | 29 | export type ReferenceProperty> = keyof T; 30 | 31 | export type StreamReferences> = Map< 32 | ReferenceProperty, 33 | { prevIndex: number; nextIndex?: number } 34 | >; 35 | 36 | export type DataBuffer> = Map< 37 | ReferenceProperty, 38 | { 39 | data: T | null; 40 | index: number | null; 41 | } 42 | >; 43 | 44 | export type ListStreamOptions = { 45 | chunksSize?: number; // 0 by default. 46 | showOnly?: `${ListStatus}`[]; 47 | considerMoveAsUpdate?: boolean; 48 | useWorker?: boolean; // true by default 49 | showWarnings?: boolean; // true by default 50 | }; 51 | 52 | export type FilePath = string; 53 | 54 | export interface StreamListener> { 55 | on>( 56 | event: E, 57 | listener: Listener[E]>, 58 | ): this; 59 | } 60 | -------------------------------------------------------------------------------- /src/lib/stream-list-diff/utils.ts: -------------------------------------------------------------------------------- 1 | import { isObject } from "@lib/utils"; 2 | import { IEmitter } from "@models/emitter"; 3 | import { ListType } from "@models/list"; 4 | import { 5 | ListStreamOptions, 6 | ReferenceProperty, 7 | StreamEvent, 8 | StreamListDiff, 9 | } from "@models/stream"; 10 | 11 | export function isValidChunkSize( 12 | chunksSize: ListStreamOptions["chunksSize"], 13 | ): boolean { 14 | if (!chunksSize) return true; 15 | const sign = String(Math.sign(chunksSize)); 16 | return sign !== "-1" && sign !== "NaN"; 17 | } 18 | 19 | export function isDataValid>( 20 | data: T, 21 | referenceProperty: ReferenceProperty, 22 | listType: ListType, 23 | ): { isValid: boolean; message?: string } { 24 | if (!isObject(data)) { 25 | return { 26 | isValid: false, 27 | message: `Your ${listType} must only contain valid objects. Found '${data}'`, 28 | }; 29 | } 30 | if (!Object.hasOwn(data, referenceProperty)) { 31 | return { 32 | isValid: false, 33 | message: `The reference property '${String(referenceProperty)}' is not available in all the objects of your ${listType}.`, 34 | }; 35 | } 36 | return { 37 | isValid: true, 38 | message: "", 39 | }; 40 | } 41 | 42 | export function outputDiffChunk>( 43 | emitter: IEmitter, 44 | ) { 45 | let chunks: StreamListDiff[] = []; 46 | 47 | function handleDiffChunk( 48 | chunk: StreamListDiff, 49 | options: ListStreamOptions, 50 | ): void { 51 | const showChunk = options?.showOnly 52 | ? options?.showOnly.includes(chunk.status) 53 | : true; 54 | if (!showChunk) { 55 | return; 56 | } 57 | if ((options.chunksSize as number) > 0) { 58 | chunks.push(chunk); 59 | if (chunks.length >= (options.chunksSize as number)) { 60 | const output = chunks; 61 | chunks = []; 62 | return emitter.emit(StreamEvent.Data, output); 63 | } else { 64 | return; 65 | } 66 | } 67 | return emitter.emit(StreamEvent.Data, [chunk]); 68 | } 69 | 70 | function releaseLastChunks() { 71 | if (chunks.length > 0) { 72 | const output = chunks; 73 | chunks = []; 74 | return emitter.emit(StreamEvent.Data, output); 75 | } 76 | } 77 | 78 | return { 79 | handleDiffChunk, 80 | releaseLastChunks, 81 | }; 82 | } 83 | -------------------------------------------------------------------------------- /src/lib/stream-list-diff/client/worker/utils.ts: -------------------------------------------------------------------------------- 1 | import { IEmitter, EmitterEvents, EventEmitter } from "@models/emitter"; 2 | import { 3 | ListStreamOptions, 4 | READABLE_STREAM_ALERT, 5 | ReferenceProperty, 6 | StreamEvent, 7 | StreamListener, 8 | } from "@models/stream"; 9 | import { WebWorkerMessage } from "@models/worker"; 10 | import { generateStream } from ".."; 11 | 12 | export function workerDiff>( 13 | prevList: File | T[], 14 | nextList: File | T[], 15 | referenceProperty: ReferenceProperty, 16 | options: ListStreamOptions, 17 | ): StreamListener { 18 | const emitter = new EventEmitter>(); 19 | setTimeout( 20 | () => 21 | generateStream(prevList, nextList, referenceProperty, options, emitter), 22 | 0, 23 | ); 24 | return emitter as StreamListener; 25 | } 26 | 27 | async function getArrayFromStream( 28 | readableStream: ReadableStream, 29 | showWarnings: boolean = true, 30 | ): Promise { 31 | if (showWarnings) { 32 | console.warn(READABLE_STREAM_ALERT); 33 | } 34 | const reader = readableStream.getReader(); 35 | const chunks: T[] = []; 36 | let result; 37 | while (!(result = await reader.read()).done) { 38 | chunks.push(result.value); 39 | } 40 | return chunks; 41 | } 42 | 43 | export async function generateWorker>( 44 | prevList: ReadableStream | File | T[], 45 | nextList: ReadableStream | File | T[], 46 | referenceProperty: ReferenceProperty, 47 | options: ListStreamOptions, 48 | emitter: IEmitter, 49 | ) { 50 | try { 51 | if (prevList instanceof ReadableStream) { 52 | prevList = await getArrayFromStream(prevList, options?.showWarnings); 53 | } 54 | if (nextList instanceof ReadableStream) { 55 | nextList = await getArrayFromStream(nextList, options?.showWarnings); 56 | } 57 | const worker = new Worker(new URL("./web-worker.js", import.meta.url), { 58 | type: "module", 59 | }); 60 | worker.postMessage({ prevList, nextList, referenceProperty, options }); 61 | worker.onmessage = (e: WebWorkerMessage) => { 62 | const { event, chunk, error } = e.data; 63 | if (event === StreamEvent.Data) { 64 | emitter.emit(StreamEvent.Data, chunk); 65 | } else if (event === StreamEvent.Finish) { 66 | emitter.emit(StreamEvent.Finish); 67 | worker.terminate(); 68 | } else if (event === StreamEvent.Error) { 69 | emitter.emit(StreamEvent.Error, new Error(error)); 70 | worker.terminate(); 71 | } 72 | }; 73 | worker.onerror = (err: ErrorEvent) => 74 | emitter.emit(StreamEvent.Error, new Error(err.message)); 75 | } catch (err) { 76 | return emitter.emit(StreamEvent.Error, err as Error); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/lib/stream-list-diff/server/worker/utils.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import { once, Readable } from "stream"; 3 | import { Worker } from "worker_threads"; 4 | import { IEmitter, EmitterEvents, EventEmitter } from "@models/emitter"; 5 | import { 6 | FilePath, 7 | ListStreamOptions, 8 | READABLE_STREAM_ALERT, 9 | ReferenceProperty, 10 | StreamEvent, 11 | StreamListener, 12 | } from "@models/stream"; 13 | import { NodeWorkerMessage, WorkerEvent } from "@models/worker"; 14 | import { generateStream } from ".."; 15 | 16 | async function getArrayFromStream( 17 | stream: Readable, 18 | showWarnings: boolean = true, 19 | ): Promise { 20 | if (showWarnings) { 21 | console.warn(READABLE_STREAM_ALERT); 22 | } 23 | const data: T[] = []; 24 | stream.on(StreamEvent.Data, (chunk) => data.push(chunk)); 25 | await once(stream, "end"); 26 | return data; 27 | } 28 | 29 | export async function generateWorker>( 30 | prevList: Readable | FilePath | T[], 31 | nextList: Readable | FilePath | T[], 32 | referenceProperty: ReferenceProperty, 33 | options: ListStreamOptions, 34 | emitter: IEmitter, 35 | ) { 36 | try { 37 | if (prevList instanceof Readable) { 38 | prevList = await getArrayFromStream(prevList, options?.showWarnings); 39 | } 40 | if (nextList instanceof Readable) { 41 | nextList = await getArrayFromStream(nextList, options?.showWarnings); 42 | } 43 | const worker = new Worker(path.resolve(__dirname, "./node-worker.cjs")); 44 | worker.postMessage({ prevList, nextList, referenceProperty, options }); 45 | worker.on(WorkerEvent.Message, (e: NodeWorkerMessage) => { 46 | const { event, chunk, error } = e; 47 | if (event === StreamEvent.Data) { 48 | emitter.emit(StreamEvent.Data, chunk); 49 | } else if (event === StreamEvent.Finish) { 50 | emitter.emit(StreamEvent.Finish); 51 | worker.terminate(); 52 | } else if (event === StreamEvent.Error) { 53 | emitter.emit(StreamEvent.Error, new Error(error)); 54 | worker.terminate(); 55 | } 56 | }); 57 | worker.on(WorkerEvent.Error, (err) => 58 | emitter.emit(StreamEvent.Error, new Error(err.message)), 59 | ); 60 | } catch (err) { 61 | return emitter.emit(StreamEvent.Error, err as Error); 62 | } 63 | } 64 | 65 | export function workerDiff>( 66 | prevList: FilePath | T[], 67 | nextList: FilePath | T[], 68 | referenceProperty: ReferenceProperty, 69 | options: ListStreamOptions, 70 | ): StreamListener { 71 | const emitter = new EventEmitter>(); 72 | setTimeout( 73 | () => 74 | generateStream(prevList, nextList, referenceProperty, options, emitter), 75 | 0, 76 | ); 77 | return emitter as StreamListener; 78 | } 79 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@donedeal0/superdiff", 3 | "version": "3.1.2", 4 | "type": "module", 5 | "description": "SuperDiff compares two arrays or objects and returns a full diff of their differences", 6 | "main": "dist/index.js", 7 | "module": "dist/index.js", 8 | "types": "dist/index.d.ts", 9 | "declaration": true, 10 | "files": ["dist"], 11 | "exports": { 12 | ".": "./dist/index.js", 13 | "./client": "./dist/client.js", 14 | "./server": "./dist/server.cjs" 15 | }, 16 | "author": "DoneDeal0", 17 | "license": "ISC", 18 | "repository": { 19 | "type": "git", 20 | "url": "git+https://github.com/DoneDeal0/superdiff" 21 | }, 22 | "bugs": { 23 | "url": "https://github.com/DoneDeal0/superdiff/issues" 24 | }, 25 | "funding": { 26 | "type": "github", 27 | "url": "https://github.com/sponsors/DoneDeal0" 28 | }, 29 | "readme": "./README.md", 30 | "release": { 31 | "branches": [ 32 | "master" 33 | ], 34 | "verifyConditions": [ 35 | "@semantic-release/github", 36 | "@semantic-release/npm" 37 | ], 38 | "prepare": [], 39 | "publish": [ 40 | "@semantic-release/npm", 41 | "@semantic-release/github" 42 | ], 43 | "plugins": [ 44 | "@semantic-release/commit-analyzer", 45 | "@semantic-release/release-notes-generator", 46 | "@semantic-release/github", 47 | [ 48 | "@semantic-release/npm", 49 | { 50 | "npmPublish": true 51 | } 52 | ] 53 | ] 54 | }, 55 | "keywords": [ 56 | "data diff", 57 | "comparison", 58 | "comparison-tool", 59 | "object-comparison", 60 | "array-comparison", 61 | "object-diff", 62 | "objectdifference", 63 | "object-difference", 64 | "object", 65 | "diff", 66 | "deep-diff", 67 | "json-diff", 68 | "files diff", 69 | "json", 70 | "file", 71 | "isobject", 72 | "comparison", 73 | "compare", 74 | "stream", 75 | "streaming", 76 | "isequal", 77 | "chunks" 78 | ], 79 | "scripts": { 80 | "build": "tsup", 81 | "format": "npx prettier . --write", 82 | "lint:dead-code": "npx -p typescript@latest -p knip knip", 83 | "lint": "eslint --cache --max-warnings=0 --fix", 84 | "prepare": "husky", 85 | "transpile": "node scripts/transpile-node-worker.js", 86 | "test": "npm run transpile && jest", 87 | "tsc": "tsc --noEmit --incremental" 88 | }, 89 | "devDependencies": { 90 | "@eslint/js": "^9.21.0", 91 | "@semantic-release/exec": "^7.0.3", 92 | "@semantic-release/git": "^10.0.1", 93 | "@semantic-release/github": "^11.0.0", 94 | "@semantic-release/npm": "^12.0.1", 95 | "@swc/core": "^1.10.18", 96 | "@swc/jest": "^0.2.37", 97 | "@types/jest": "^29.5.14", 98 | "blob-polyfill": "^9.0.20240710", 99 | "eslint": "^9.21.0", 100 | "husky": "^9.1.7", 101 | "jest": "^29.7.0", 102 | "jest-environment-jsdom": "^29.7.0", 103 | "jsdom": "^26.0.0", 104 | "prettier": "^3.5.2", 105 | "swc-loader": "^0.2.6", 106 | "ts-node": "^10.9.2", 107 | "tsup": "^8.3.6", 108 | "typescript": "^5.7.3", 109 | "typescript-eslint": "^8.24.1", 110 | "web-streams-polyfill": "^4.1.0" 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/lib/utils/utils.test.ts: -------------------------------------------------------------------------------- 1 | import { isEqual, isObject } from "."; 2 | 3 | describe("isEqual", () => { 4 | it("return true if data are the same", () => { 5 | expect(isEqual(null, null)).toBeTruthy(); 6 | expect(isEqual(undefined, undefined)).toBeTruthy(); 7 | expect(isEqual("hello", "hello")).toBeTruthy(); 8 | expect(isEqual(57, 57)).toBeTruthy(); 9 | expect(isEqual(["hello", "world"], ["hello", "world"])).toBeTruthy(); 10 | expect( 11 | isEqual( 12 | [ 13 | { name: "joe", age: 99 }, 14 | { name: "nina", age: 23 }, 15 | ], 16 | [ 17 | { name: "joe", age: 99 }, 18 | { name: "nina", age: 23 }, 19 | ], 20 | ), 21 | ).toBeTruthy(); 22 | }); 23 | it("return false if data are different", () => { 24 | expect(isEqual(null, "hello")).toBeFalsy(); 25 | expect(isEqual("hello", undefined)).toBeFalsy(); 26 | expect(isEqual("hello", "howdy")).toBeFalsy(); 27 | expect(isEqual(57, 51)).toBeFalsy(); 28 | expect(isEqual(["hello", "world"], ["howdy", "world"])).toBeFalsy(); 29 | expect( 30 | isEqual( 31 | [ 32 | { name: "joe", age: 99 }, 33 | { name: "nina", age: 23 }, 34 | ], 35 | [ 36 | { name: "joe", age: 98 }, 37 | { name: "nina", age: 23 }, 38 | ], 39 | ), 40 | ).toBeFalsy(); 41 | expect(isEqual(["psg"], ["psg", "nantes"])).toBeFalsy(); 42 | expect(isEqual(null, ["hello", "world"])).toBeFalsy(); 43 | expect(isEqual(["hello", "world"], null)).toBeFalsy(); 44 | }); 45 | it("return true if ignoreArrayOrder option is activated and arrays contains the same values regardless of their positions", () => { 46 | expect( 47 | isEqual(["hello", "world"], ["world", "hello"], { 48 | ignoreArrayOrder: true, 49 | }), 50 | ).toBeTruthy(); 51 | expect( 52 | isEqual([44, 45, "world"], [45, "world", 44], { ignoreArrayOrder: true }), 53 | ).toBeTruthy(); 54 | expect( 55 | isEqual( 56 | [ 57 | { name: "joe", age: 88 }, 58 | { name: "nina", isCool: true }, 59 | ], 60 | [ 61 | { name: "nina", isCool: true }, 62 | { name: "joe", age: 88 }, 63 | ], 64 | { 65 | ignoreArrayOrder: true, 66 | }, 67 | ), 68 | ).toBeTruthy(); 69 | expect( 70 | isEqual([true, 55, "hello"], ["hello", 55, true], { 71 | ignoreArrayOrder: true, 72 | }), 73 | ).toBeTruthy(); 74 | }); 75 | it("return false if ignoreArrayOrder option is activated but the arrays don't contain the same values", () => { 76 | expect( 77 | isEqual(["hello"], ["world", "hello"], { 78 | ignoreArrayOrder: true, 79 | }), 80 | ).toBeFalsy(); 81 | expect( 82 | isEqual([44, 47, "world"], [45, "world", 44], { ignoreArrayOrder: true }), 83 | ).toBeFalsy(); 84 | expect( 85 | isEqual( 86 | [ 87 | { name: "joey", age: 88 }, 88 | { name: "nina", isCool: true }, 89 | ], 90 | [ 91 | { name: "nina", isCool: true }, 92 | { name: "joe", age: 88 }, 93 | ], 94 | { 95 | ignoreArrayOrder: true, 96 | }, 97 | ), 98 | ).toBeFalsy(); 99 | expect( 100 | isEqual([false, 55, "hello"], ["hello", 55, true], { 101 | ignoreArrayOrder: true, 102 | }), 103 | ).toBeFalsy(); 104 | }); 105 | }); 106 | 107 | describe("isObject", () => { 108 | it("return true if the value has nested values", () => { 109 | expect(isObject({ name: "joe" })).toBeTruthy(); 110 | expect(isObject({ user: { name: "joe" } })).toBeTruthy(); 111 | }); 112 | it("return false if the value doesn't have nested values", () => { 113 | expect(isObject("joe")).toBeFalsy(); 114 | expect(isObject(56)).toBeFalsy(); 115 | expect(isObject(true)).toBeFalsy(); 116 | expect(isObject(null)).toBeFalsy(); 117 | expect(isObject(undefined)).toBeFalsy(); 118 | expect(isObject(["hello", "world"])).toBeFalsy(); 119 | }); 120 | }); 121 | -------------------------------------------------------------------------------- /src/lib/list-diff/index.ts: -------------------------------------------------------------------------------- 1 | import { isEqual, isObject } from "@lib/utils"; 2 | import { 3 | DEFAULT_LIST_DIFF_OPTIONS, 4 | ListStatus, 5 | ListDiff, 6 | ListDiffOptions, 7 | } from "@models/list"; 8 | 9 | function getLeanDiff( 10 | diff: ListDiff["diff"], 11 | showOnly = [] as ListDiffOptions["showOnly"], 12 | ): ListDiff["diff"] { 13 | return diff.filter((value) => showOnly?.includes(value.status)); 14 | } 15 | 16 | function formatSingleListDiff( 17 | listData: T[], 18 | status: ListStatus, 19 | options: ListDiffOptions = { showOnly: [] }, 20 | ): ListDiff { 21 | const diff = listData.map((data, i) => ({ 22 | value: data, 23 | prevIndex: status === ListStatus.ADDED ? null : i, 24 | newIndex: status === ListStatus.ADDED ? i : null, 25 | indexDiff: null, 26 | status, 27 | })); 28 | if (options.showOnly && options.showOnly.length > 0) { 29 | return { 30 | type: "list", 31 | status, 32 | diff: diff.filter((value) => options.showOnly?.includes(value.status)), 33 | }; 34 | } 35 | return { 36 | type: "list", 37 | status, 38 | diff, 39 | }; 40 | } 41 | 42 | function getListStatus(listDiff: ListDiff["diff"]): ListStatus { 43 | return listDiff.some((value) => value.status !== ListStatus.EQUAL) 44 | ? ListStatus.UPDATED 45 | : ListStatus.EQUAL; 46 | } 47 | 48 | function isReferencedObject( 49 | value: unknown, 50 | referenceProperty: ListDiffOptions["referenceProperty"], 51 | ): value is Record { 52 | if (isObject(value) && !!referenceProperty) { 53 | return Object.hasOwn(value, referenceProperty); 54 | } 55 | return false; 56 | } 57 | 58 | /** 59 | * Returns the diff between two arrays 60 | * @param {Array} prevList - The original array. 61 | * @param {Array} nextList - The new array. 62 | * @param {ListOptions} options - Options to refine your output. 63 | - `showOnly` gives you the option to return only the values whose status you are interested in (e.g. `["added", "equal"]`). 64 | - `referenceProperty` will consider an object to be updated instead of added or deleted if one of its properties remains stable, such as its `id`. This option has no effect on other datatypes. 65 | * @returns ListDiff 66 | */ 67 | export const getListDiff = ( 68 | prevList: T[] | undefined | null, 69 | nextList: T[] | undefined | null, 70 | options: ListDiffOptions = DEFAULT_LIST_DIFF_OPTIONS, 71 | ): ListDiff => { 72 | if (!prevList && !nextList) { 73 | return { 74 | type: "list", 75 | status: ListStatus.EQUAL, 76 | diff: [], 77 | }; 78 | } 79 | if (!prevList) { 80 | return formatSingleListDiff(nextList as T[], ListStatus.ADDED, options); 81 | } 82 | if (!nextList) { 83 | return formatSingleListDiff(prevList as T[], ListStatus.DELETED, options); 84 | } 85 | const diff: ListDiff["diff"] = []; 86 | const prevIndexMatches = new Set(); 87 | 88 | nextList.forEach((nextValue, i) => { 89 | const prevIndex = prevList.findIndex((prevValue, prevIdx) => { 90 | if (prevIndexMatches.has(prevIdx)) { 91 | return false; 92 | } 93 | if (isReferencedObject(prevValue, options.referenceProperty)) { 94 | if (isObject(nextValue)) { 95 | return isEqual( 96 | prevValue[options.referenceProperty as string], 97 | nextValue[options.referenceProperty as string], 98 | ); 99 | } 100 | return false; 101 | } 102 | return isEqual(prevValue, nextValue); 103 | }); 104 | if (prevIndex > -1) { 105 | prevIndexMatches.add(prevIndex); 106 | } 107 | const indexDiff = prevIndex === -1 ? null : i - prevIndex; 108 | if (indexDiff === 0 || options.ignoreArrayOrder) { 109 | let nextStatus = ListStatus.EQUAL; 110 | if (isReferencedObject(nextValue, options.referenceProperty)) { 111 | if (!isEqual(prevList[prevIndex], nextValue)) { 112 | nextStatus = ListStatus.UPDATED; 113 | } 114 | } 115 | return diff.push({ 116 | value: nextValue, 117 | prevIndex, 118 | newIndex: i, 119 | indexDiff, 120 | status: nextStatus, 121 | }); 122 | } 123 | if (prevIndex === -1) { 124 | return diff.push({ 125 | value: nextValue, 126 | prevIndex: null, 127 | newIndex: i, 128 | indexDiff, 129 | status: ListStatus.ADDED, 130 | }); 131 | } 132 | return diff.push({ 133 | value: nextValue, 134 | prevIndex, 135 | newIndex: i, 136 | indexDiff, 137 | status: options.considerMoveAsUpdate 138 | ? ListStatus.UPDATED 139 | : ListStatus.MOVED, 140 | }); 141 | }); 142 | 143 | prevList.forEach((prevValue, i) => { 144 | if (!prevIndexMatches.has(i)) { 145 | return diff.push({ 146 | value: prevValue, 147 | prevIndex: i, 148 | newIndex: null, 149 | indexDiff: null, 150 | status: ListStatus.DELETED, 151 | }); 152 | } 153 | }); 154 | if (options.showOnly && options?.showOnly?.length > 0) { 155 | return { 156 | type: "list", 157 | status: getListStatus(diff), 158 | diff: getLeanDiff(diff, options.showOnly), 159 | }; 160 | } 161 | return { 162 | type: "list", 163 | status: getListStatus(diff), 164 | diff, 165 | }; 166 | }; 167 | -------------------------------------------------------------------------------- /src/lib/object-diff/index.ts: -------------------------------------------------------------------------------- 1 | import { isEqual, isObject } from "@lib/utils"; 2 | import { 3 | Granularity, 4 | ObjectStatus, 5 | ObjectData, 6 | ObjectDiff, 7 | ObjectDiffOptions, 8 | Diff, 9 | DEFAULT_OBJECT_DIFF_OPTIONS, 10 | } from "@models/object"; 11 | 12 | function getLeanDiff( 13 | diff: ObjectDiff["diff"], 14 | showOnly: ObjectDiffOptions["showOnly"] = DEFAULT_OBJECT_DIFF_OPTIONS.showOnly, 15 | ): ObjectDiff["diff"] { 16 | const { statuses, granularity } = showOnly; 17 | const res: ObjectDiff["diff"] = []; 18 | for (let i = 0; i < diff.length; i++) { 19 | const value = diff[i]; 20 | if (granularity === Granularity.DEEP && value.diff) { 21 | const leanDiff = getLeanDiff(value.diff, showOnly); 22 | if (leanDiff.length > 0) { 23 | res.push({ ...value, diff: leanDiff }); 24 | } 25 | } else if (statuses.includes(value.status)) { 26 | res.push(value); 27 | } 28 | } 29 | return res; 30 | } 31 | 32 | function getObjectStatus(diff: ObjectDiff["diff"]): ObjectStatus { 33 | return diff.some((property) => property.status !== ObjectStatus.EQUAL) 34 | ? ObjectStatus.UPDATED 35 | : ObjectStatus.EQUAL; 36 | } 37 | 38 | function formatSingleObjectDiff( 39 | data: ObjectData, 40 | status: ObjectStatus, 41 | options: ObjectDiffOptions = DEFAULT_OBJECT_DIFF_OPTIONS, 42 | ): ObjectDiff { 43 | if (!data) { 44 | return { 45 | type: "object", 46 | status: ObjectStatus.EQUAL, 47 | diff: [], 48 | }; 49 | } 50 | const diff: ObjectDiff["diff"] = []; 51 | 52 | for (const [property, value] of Object.entries(data)) { 53 | if (isObject(value)) { 54 | const subPropertiesDiff: Diff[] = []; 55 | for (const [subProperty, subValue] of Object.entries(value)) { 56 | subPropertiesDiff.push({ 57 | property: subProperty, 58 | previousValue: status === ObjectStatus.ADDED ? undefined : subValue, 59 | currentValue: status === ObjectStatus.ADDED ? subValue : undefined, 60 | status, 61 | }); 62 | } 63 | diff.push({ 64 | property, 65 | previousValue: 66 | status === ObjectStatus.ADDED ? undefined : data[property], 67 | currentValue: status === ObjectStatus.ADDED ? value : undefined, 68 | status, 69 | diff: subPropertiesDiff, 70 | }); 71 | } else { 72 | diff.push({ 73 | property, 74 | previousValue: 75 | status === ObjectStatus.ADDED ? undefined : data[property], 76 | currentValue: status === ObjectStatus.ADDED ? value : undefined, 77 | status, 78 | }); 79 | } 80 | } 81 | 82 | if (options.showOnly && options.showOnly.statuses.length > 0) { 83 | return { 84 | type: "object", 85 | status, 86 | diff: getLeanDiff(diff, options.showOnly), 87 | }; 88 | } 89 | return { 90 | type: "object", 91 | status, 92 | diff, 93 | }; 94 | } 95 | 96 | function getValueStatus( 97 | previousValue: unknown, 98 | nextValue: unknown, 99 | options?: ObjectDiffOptions, 100 | ): ObjectStatus { 101 | if (isEqual(previousValue, nextValue, options)) { 102 | return ObjectStatus.EQUAL; 103 | } 104 | return ObjectStatus.UPDATED; 105 | } 106 | 107 | function getDiff( 108 | previousValue: Record | undefined = {}, 109 | nextValue: Record, 110 | options?: ObjectDiffOptions, 111 | ): Diff[] { 112 | const diff: Diff[] = []; 113 | const allKeys = new Set([ 114 | ...Object.keys(previousValue), 115 | ...Object.keys(nextValue), 116 | ]); 117 | 118 | for (const property of allKeys) { 119 | const prevSubValue = previousValue[property]; 120 | const nextSubValue = nextValue[property]; 121 | if (!(property in nextValue)) { 122 | diff.push({ 123 | property, 124 | previousValue: prevSubValue, 125 | currentValue: undefined, 126 | status: ObjectStatus.DELETED, 127 | }); 128 | continue; 129 | } 130 | if (!(property in previousValue)) { 131 | diff.push({ 132 | property, 133 | previousValue: undefined, 134 | currentValue: nextSubValue, 135 | status: ObjectStatus.ADDED, 136 | }); 137 | continue; 138 | } 139 | if (isObject(nextSubValue) && isObject(prevSubValue)) { 140 | const subDiff = getDiff(prevSubValue, nextSubValue, options); 141 | const isUpdated = subDiff.some( 142 | (entry) => entry.status !== ObjectStatus.EQUAL, 143 | ); 144 | diff.push({ 145 | property, 146 | previousValue: prevSubValue, 147 | currentValue: nextSubValue, 148 | status: isUpdated ? ObjectStatus.UPDATED : ObjectStatus.EQUAL, 149 | ...(isUpdated && { diff: subDiff }), 150 | }); 151 | } else { 152 | const status = getValueStatus(prevSubValue, nextSubValue, options); 153 | diff.push({ 154 | property, 155 | previousValue: prevSubValue, 156 | currentValue: nextSubValue, 157 | status, 158 | }); 159 | } 160 | } 161 | return diff; 162 | } 163 | 164 | /** 165 | * Returns the diff between two objects 166 | * @param {ObjectData} prevData - The original object. 167 | * @param {ObjectData} nextData - The new object. 168 | * @param {ObjectOptions} options - Options to refine your output. 169 | - `showOnly`: returns only the values whose status you are interested in. It takes two parameters: `statuses` and `granularity` 170 | `statuses` are the status you want to see in the output (e.g. `["added", "equal"]`) 171 | `granularity` can be either `basic` (to return only the main properties whose status matches your query) or `deep` (to return the main properties if some of their subproperties' status match your request. The subproperties are filtered accordingly). 172 | - `ignoreArrayOrder` if set to `true`, `["hello", "world"]` and `["world", "hello"]` will be treated as `equal`, because the two arrays have the same value, just not in the same order. 173 | * @returns ObjectDiff 174 | */ 175 | export function getObjectDiff( 176 | prevData: ObjectData, 177 | nextData: ObjectData, 178 | options: ObjectDiffOptions = DEFAULT_OBJECT_DIFF_OPTIONS, 179 | ): ObjectDiff { 180 | if (!prevData && !nextData) { 181 | return { 182 | type: "object", 183 | status: ObjectStatus.EQUAL, 184 | diff: [], 185 | }; 186 | } 187 | if (!prevData) { 188 | return formatSingleObjectDiff(nextData, ObjectStatus.ADDED, options); 189 | } 190 | if (!nextData) { 191 | return formatSingleObjectDiff(prevData, ObjectStatus.DELETED, options); 192 | } 193 | const diff: ObjectDiff["diff"] = getDiff(prevData, nextData, options); 194 | const status = getObjectStatus(diff); 195 | const showLeanDiff = (options?.showOnly?.statuses?.length || 0) > 0; 196 | return { 197 | type: "object", 198 | status, 199 | diff: showLeanDiff ? getLeanDiff(diff, options.showOnly) : diff, 200 | }; 201 | } 202 | -------------------------------------------------------------------------------- /src/lib/stream-list-diff/server/index.ts: -------------------------------------------------------------------------------- 1 | import { createReadStream } from "fs"; 2 | import { Readable, Transform } from "stream"; 3 | import { Worker } from "worker_threads"; 4 | import { IEmitter, EmitterEvents, EventEmitter } from "@models/emitter"; 5 | import { ListStatus, ListType } from "@models/list"; 6 | import { 7 | DataBuffer, 8 | DEFAULT_LIST_STREAM_OPTIONS, 9 | FilePath, 10 | ListStreamOptions, 11 | ReferenceProperty, 12 | StreamEvent, 13 | StreamListener, 14 | } from "@models/stream"; 15 | import { isDataValid, isValidChunkSize, outputDiffChunk } from "../utils"; 16 | import { generateWorker } from "./worker/utils"; 17 | 18 | async function getDiffChunks>( 19 | prevStream: Readable, 20 | nextStream: Readable, 21 | referenceProperty: ReferenceProperty, 22 | emitter: IEmitter, 23 | options: ListStreamOptions = DEFAULT_LIST_STREAM_OPTIONS, 24 | ): Promise { 25 | if (!isValidChunkSize(options?.chunksSize)) { 26 | return emitter.emit( 27 | StreamEvent.Error, 28 | new Error( 29 | `The chunk size can't be negative. You entered the value '${options.chunksSize}'`, 30 | ), 31 | ); 32 | } 33 | const { handleDiffChunk, releaseLastChunks } = outputDiffChunk(emitter); 34 | const prevDataBuffer: DataBuffer = new Map(); 35 | const nextDataBuffer: DataBuffer = new Map(); 36 | let currentPrevIndex = 0; 37 | let currentNextIndex = 0; 38 | 39 | async function processPrevStreamChunk(chunk: T) { 40 | const { isValid, message } = isDataValid( 41 | chunk, 42 | referenceProperty, 43 | ListType.PREV, 44 | ); 45 | if (!isValid) { 46 | emitter.emit(StreamEvent.Error, new Error(message)); 47 | emitter.emit(StreamEvent.Finish); 48 | return; 49 | } 50 | const ref = chunk[referenceProperty] as ReferenceProperty; 51 | const relatedChunk = nextDataBuffer.get(ref); 52 | 53 | if (relatedChunk) { 54 | nextDataBuffer.delete(ref); 55 | const isDataEqual = 56 | JSON.stringify(chunk) === JSON.stringify(relatedChunk.data); 57 | const indexDiff = (relatedChunk.index as number) - currentPrevIndex; 58 | if (isDataEqual) { 59 | handleDiffChunk( 60 | { 61 | previousValue: chunk, 62 | currentValue: relatedChunk.data, 63 | prevIndex: currentPrevIndex, 64 | newIndex: relatedChunk.index, 65 | indexDiff, 66 | status: 67 | indexDiff === 0 68 | ? ListStatus.EQUAL 69 | : options.considerMoveAsUpdate 70 | ? ListStatus.UPDATED 71 | : ListStatus.MOVED, 72 | }, 73 | options, 74 | ); 75 | } else { 76 | handleDiffChunk( 77 | { 78 | previousValue: chunk, 79 | currentValue: relatedChunk.data, 80 | prevIndex: currentPrevIndex, 81 | newIndex: relatedChunk.index, 82 | indexDiff, 83 | status: ListStatus.UPDATED, 84 | }, 85 | options, 86 | ); 87 | } 88 | } else { 89 | prevDataBuffer.set(ref, { data: chunk, index: currentPrevIndex }); 90 | } 91 | currentPrevIndex++; 92 | } 93 | 94 | async function processNextStreamChunk(chunk: T) { 95 | const { isValid, message } = isDataValid( 96 | chunk, 97 | referenceProperty, 98 | ListType.NEXT, 99 | ); 100 | if (!isValid) { 101 | emitter.emit(StreamEvent.Error, new Error(message)); 102 | emitter.emit(StreamEvent.Finish); 103 | return; 104 | } 105 | const ref = chunk[referenceProperty] as ReferenceProperty; 106 | const relatedChunk = prevDataBuffer.get(ref); 107 | 108 | if (relatedChunk) { 109 | prevDataBuffer.delete(ref); 110 | const isDataEqual = 111 | JSON.stringify(chunk) === JSON.stringify(relatedChunk.data); 112 | const indexDiff = currentNextIndex - (relatedChunk.index as number); 113 | if (isDataEqual) { 114 | handleDiffChunk( 115 | { 116 | previousValue: relatedChunk.data, 117 | currentValue: chunk, 118 | prevIndex: relatedChunk.index, 119 | newIndex: currentNextIndex, 120 | indexDiff, 121 | status: 122 | indexDiff === 0 123 | ? ListStatus.EQUAL 124 | : options.considerMoveAsUpdate 125 | ? ListStatus.UPDATED 126 | : ListStatus.MOVED, 127 | }, 128 | options, 129 | ); 130 | } else { 131 | handleDiffChunk( 132 | { 133 | previousValue: relatedChunk.data, 134 | currentValue: chunk, 135 | prevIndex: relatedChunk.index, 136 | newIndex: currentNextIndex, 137 | indexDiff, 138 | status: ListStatus.UPDATED, 139 | }, 140 | options, 141 | ); 142 | } 143 | } else { 144 | nextDataBuffer.set(ref, { data: chunk, index: currentNextIndex }); 145 | } 146 | currentNextIndex++; 147 | } 148 | 149 | const prevStreamReader = async () => { 150 | for await (const chunk of prevStream) { 151 | await processPrevStreamChunk(chunk); 152 | } 153 | }; 154 | 155 | const nextStreamReader = async () => { 156 | for await (const chunk of nextStream) { 157 | await processNextStreamChunk(chunk); 158 | } 159 | }; 160 | await Promise.all([prevStreamReader(), nextStreamReader()]); 161 | 162 | for (const [key, chunk] of prevDataBuffer.entries()) { 163 | handleDiffChunk( 164 | { 165 | previousValue: chunk.data, 166 | currentValue: null, 167 | prevIndex: chunk.index, 168 | newIndex: null, 169 | indexDiff: null, 170 | status: ListStatus.DELETED, 171 | }, 172 | options, 173 | ); 174 | prevDataBuffer.delete(key); 175 | } 176 | for (const [key, chunk] of nextDataBuffer.entries()) { 177 | handleDiffChunk( 178 | { 179 | previousValue: null, 180 | currentValue: chunk.data, 181 | prevIndex: null, 182 | newIndex: chunk.index, 183 | indexDiff: null, 184 | status: ListStatus.ADDED, 185 | }, 186 | options, 187 | ); 188 | nextDataBuffer.delete(key); 189 | } 190 | releaseLastChunks(); 191 | return emitter.emit(StreamEvent.Finish); 192 | } 193 | 194 | function getValidStream( 195 | input: Readable | FilePath | T[], 196 | listType: ListType, 197 | ): Readable { 198 | if (input instanceof Readable) { 199 | return input; 200 | } 201 | 202 | if (Array.isArray(input)) { 203 | return Readable.from(input, { objectMode: true }); 204 | } 205 | 206 | if (typeof input === "string") { 207 | return createReadStream(input, { encoding: "utf8" }).pipe( 208 | new Transform({ 209 | objectMode: true, 210 | transform(chunk, _, callback) { 211 | try { 212 | const data: T = JSON.parse(chunk.toString()); 213 | if (Array.isArray(data)) { 214 | for (let i = 0; i < data.length; i++) { 215 | this.push(data[i]); 216 | } 217 | } else { 218 | this.push(data); 219 | } 220 | callback(); 221 | } catch (err) { 222 | callback(err as Error); 223 | } 224 | }, 225 | }), 226 | ); 227 | } 228 | throw new Error(`Invalid ${listType}. Expected Readable, Array, or File.`); 229 | } 230 | 231 | export async function generateStream>( 232 | prevList: Readable | FilePath | T[], 233 | nextList: Readable | FilePath | T[], 234 | referenceProperty: ReferenceProperty, 235 | options: ListStreamOptions, 236 | emitter: IEmitter, 237 | ): Promise { 238 | try { 239 | const [prevStream, nextStream] = await Promise.all([ 240 | getValidStream(prevList, ListType.PREV), 241 | getValidStream(nextList, ListType.NEXT), 242 | ]); 243 | getDiffChunks(prevStream, nextStream, referenceProperty, emitter, options); 244 | } catch (err) { 245 | return emitter.emit(StreamEvent.Error, err as Error); 246 | } 247 | } 248 | 249 | /** 250 | * Streams the diff of two object lists 251 | * @param {Readable | FilePath | Record[]} prevList - The original object list. 252 | * @param {Readable | FilePath | Record[]} nextList - The new object list. 253 | * @param {string} referenceProperty - A common property in all the objects of your lists (e.g. `id`) 254 | * @param {ListStreamOptions} options - Options to refine your output. 255 | - `chunksSize`: the number of object diffs returned by each streamed chunk. (e.g. `0` = 1 object diff by chunk, `10` = 10 object diffs by chunk). 256 | - `showOnly`: returns only the values whose status you are interested in. (e.g. `["added", "equal"]`). 257 | - `considerMoveAsUpdate`: if set to `true` a `moved` object will be considered as `updated`. 258 | - `useWorker`: if set to `true`, the diff will be run in a worker. Recommended for maximum performance, `true` by default. 259 | - `showWarnings`: if set to `true`, potential warnings will be displayed in the console. 260 | * @returns StreamListener 261 | */ 262 | export function streamListDiff>( 263 | prevList: Readable | FilePath | T[], 264 | nextList: Readable | FilePath | T[], 265 | referenceProperty: ReferenceProperty, 266 | options: ListStreamOptions = DEFAULT_LIST_STREAM_OPTIONS, 267 | ): StreamListener { 268 | const emitter = new EventEmitter>(); 269 | if (typeof Worker === "undefined" || !options.useWorker) { 270 | setTimeout( 271 | () => 272 | generateStream(prevList, nextList, referenceProperty, options, emitter), 273 | 0, 274 | ); 275 | } else { 276 | generateWorker(prevList, nextList, referenceProperty, options, emitter); 277 | } 278 | return emitter as StreamListener; 279 | } 280 | -------------------------------------------------------------------------------- /src/lib/stream-list-diff/client/index.ts: -------------------------------------------------------------------------------- 1 | import { IEmitter, EmitterEvents, EventEmitter } from "@models/emitter"; 2 | import { 3 | DataBuffer, 4 | DEFAULT_LIST_STREAM_OPTIONS, 5 | ListStreamOptions, 6 | ReferenceProperty, 7 | StreamEvent, 8 | StreamListener, 9 | } from "@models/stream"; 10 | import { ListStatus, ListType } from "@models/list"; 11 | import { isDataValid, isValidChunkSize, outputDiffChunk } from "../utils"; 12 | import { generateWorker } from "./worker/utils"; 13 | 14 | async function getDiffChunks>( 15 | prevStream: ReadableStream, 16 | nextStream: ReadableStream, 17 | referenceProperty: ReferenceProperty, 18 | emitter: IEmitter, 19 | options: ListStreamOptions = DEFAULT_LIST_STREAM_OPTIONS, 20 | ): Promise { 21 | if (!isValidChunkSize(options?.chunksSize)) { 22 | return emitter.emit( 23 | StreamEvent.Error, 24 | new Error( 25 | `The chunk size can't be negative. You entered the value '${options.chunksSize}'`, 26 | ), 27 | ); 28 | } 29 | 30 | const prevList = prevStream.getReader(); 31 | const nextList = nextStream.getReader(); 32 | const { handleDiffChunk, releaseLastChunks } = outputDiffChunk(emitter); 33 | const prevDataBuffer: DataBuffer = new Map(); 34 | const nextDataBuffer: DataBuffer = new Map(); 35 | let currentPrevIndex = 0; 36 | let currentNextIndex = 0; 37 | 38 | async function processPrevStreamChunk(chunk: T) { 39 | const { isValid, message } = isDataValid( 40 | chunk, 41 | referenceProperty, 42 | ListType.PREV, 43 | ); 44 | if (!isValid) { 45 | emitter.emit(StreamEvent.Error, new Error(message)); 46 | emitter.emit(StreamEvent.Finish); 47 | return; 48 | } 49 | const ref = chunk[referenceProperty] as ReferenceProperty; 50 | const relatedChunk = nextDataBuffer.get(ref); 51 | 52 | if (relatedChunk) { 53 | nextDataBuffer.delete(ref); 54 | const isDataEqual = 55 | JSON.stringify(chunk) === JSON.stringify(relatedChunk.data); 56 | const indexDiff = (relatedChunk.index as number) - currentPrevIndex; 57 | if (isDataEqual) { 58 | handleDiffChunk( 59 | { 60 | previousValue: chunk, 61 | currentValue: relatedChunk.data, 62 | prevIndex: currentPrevIndex, 63 | newIndex: relatedChunk.index, 64 | indexDiff, 65 | status: 66 | indexDiff === 0 67 | ? ListStatus.EQUAL 68 | : options.considerMoveAsUpdate 69 | ? ListStatus.UPDATED 70 | : ListStatus.MOVED, 71 | }, 72 | options, 73 | ); 74 | } else { 75 | handleDiffChunk( 76 | { 77 | previousValue: chunk, 78 | currentValue: relatedChunk.data, 79 | prevIndex: currentPrevIndex, 80 | newIndex: relatedChunk.index, 81 | indexDiff, 82 | status: ListStatus.UPDATED, 83 | }, 84 | options, 85 | ); 86 | } 87 | } else { 88 | prevDataBuffer.set(ref, { data: chunk, index: currentPrevIndex }); 89 | } 90 | currentPrevIndex++; 91 | } 92 | 93 | async function processNextStreamChunk(chunk: T) { 94 | const { isValid, message } = isDataValid( 95 | chunk, 96 | referenceProperty, 97 | ListType.NEXT, 98 | ); 99 | if (!isValid) { 100 | emitter.emit(StreamEvent.Error, new Error(message)); 101 | emitter.emit(StreamEvent.Finish); 102 | return; 103 | } 104 | const ref = chunk[referenceProperty] as ReferenceProperty; 105 | const relatedChunk = prevDataBuffer.get(ref); 106 | 107 | if (relatedChunk) { 108 | prevDataBuffer.delete(ref); 109 | const isDataEqual = 110 | JSON.stringify(chunk) === JSON.stringify(relatedChunk.data); 111 | const indexDiff = currentNextIndex - (relatedChunk.index as number); 112 | if (isDataEqual) { 113 | handleDiffChunk( 114 | { 115 | previousValue: relatedChunk.data, 116 | currentValue: chunk, 117 | prevIndex: relatedChunk.index, 118 | newIndex: currentNextIndex, 119 | indexDiff, 120 | status: 121 | indexDiff === 0 122 | ? ListStatus.EQUAL 123 | : options.considerMoveAsUpdate 124 | ? ListStatus.UPDATED 125 | : ListStatus.MOVED, 126 | }, 127 | options, 128 | ); 129 | } else { 130 | handleDiffChunk( 131 | { 132 | previousValue: relatedChunk.data, 133 | currentValue: chunk, 134 | prevIndex: relatedChunk.index, 135 | newIndex: currentNextIndex, 136 | indexDiff, 137 | status: ListStatus.UPDATED, 138 | }, 139 | options, 140 | ); 141 | } 142 | } else { 143 | nextDataBuffer.set(ref, { data: chunk, index: currentNextIndex }); 144 | } 145 | currentNextIndex++; 146 | } 147 | 148 | const readStream = async ( 149 | reader: ReadableStreamDefaultReader, 150 | processChunk: (chunk: T) => Promise, 151 | ) => { 152 | let result; 153 | while (!(result = await reader.read()).done) { 154 | await processChunk(result.value); 155 | } 156 | }; 157 | 158 | await Promise.all([ 159 | readStream(prevList, async (chunk) => { 160 | await processPrevStreamChunk(chunk); 161 | }), 162 | readStream(nextList, async (chunk) => { 163 | await processNextStreamChunk(chunk); 164 | }), 165 | ]); 166 | 167 | for (const [key, chunk] of prevDataBuffer.entries()) { 168 | handleDiffChunk( 169 | { 170 | previousValue: chunk.data, 171 | currentValue: null, 172 | prevIndex: chunk.index, 173 | newIndex: null, 174 | indexDiff: null, 175 | status: ListStatus.DELETED, 176 | }, 177 | options, 178 | ); 179 | prevDataBuffer.delete(key); 180 | } 181 | for (const [key, chunk] of nextDataBuffer.entries()) { 182 | handleDiffChunk( 183 | { 184 | previousValue: null, 185 | currentValue: chunk.data, 186 | prevIndex: null, 187 | newIndex: chunk.index, 188 | indexDiff: null, 189 | status: ListStatus.ADDED, 190 | }, 191 | options, 192 | ); 193 | nextDataBuffer.delete(key); 194 | } 195 | 196 | releaseLastChunks(); 197 | return emitter.emit(StreamEvent.Finish); 198 | } 199 | 200 | async function getValidClientStream>( 201 | input: ReadableStream | T[] | File, 202 | listType: ListType, 203 | ): Promise> { 204 | if (input instanceof ReadableStream) { 205 | return input; 206 | } 207 | let nextInput = input; 208 | if (input instanceof File) { 209 | const fileText = await input.text(); 210 | try { 211 | nextInput = JSON.parse(fileText); 212 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 213 | } catch (_: unknown) { 214 | throw new Error(`Your ${listType} is not a valid JSON array.`); 215 | } 216 | 217 | if (!Array.isArray(nextInput)) { 218 | throw new Error(`Your ${listType} is not a JSON array.`); 219 | } 220 | } 221 | 222 | if (Array.isArray(nextInput)) { 223 | return new ReadableStream({ 224 | start(controller) { 225 | nextInput.forEach((item) => controller.enqueue(item)); 226 | controller.close(); 227 | }, 228 | }); 229 | } 230 | 231 | throw new Error( 232 | `Invalid ${listType}. Expected ReadableStream, Array, or File.`, 233 | ); 234 | } 235 | 236 | export async function generateStream>( 237 | prevList: ReadableStream | File | T[], 238 | nextList: ReadableStream | File | T[], 239 | referenceProperty: ReferenceProperty, 240 | options: ListStreamOptions, 241 | emitter: IEmitter, 242 | ): Promise { 243 | try { 244 | const [prevStream, nextStream] = await Promise.all([ 245 | getValidClientStream(prevList, ListType.PREV), 246 | getValidClientStream(nextList, ListType.NEXT), 247 | ]); 248 | 249 | getDiffChunks(prevStream, nextStream, referenceProperty, emitter, options); 250 | } catch (err) { 251 | return emitter.emit(StreamEvent.Error, err as Error); 252 | } 253 | } 254 | 255 | /** 256 | * Streams the diff of two object lists 257 | * @param {ReadableStream | File | Record[]} prevList - The original object list. 258 | * @param {ReadableStream | File | Record[]} nextList - The new object list. 259 | * @param {string} referenceProperty - A common property in all the objects of your lists (e.g. `id`) 260 | * @param {ListStreamOptions} options - Options to refine your output. 261 | - `chunksSize`: the number of object diffs returned by each streamed chunk. (e.g. `0` = 1 object diff by chunk, `10` = 10 object diffs by chunk). 262 | - `showOnly`: returns only the values whose status you are interested in. (e.g. `["added", "equal"]`). 263 | - `considerMoveAsUpdate`: if set to `true` a `moved` object will be considered as `updated`. 264 | - `useWorker`: if set to `true`, the diff will be run in a worker. Recommended for maximum performance, `true` by default. 265 | - `showWarnings`: if set to `true`, potential warnings will be displayed in the console. 266 | * @returns StreamListener 267 | */ 268 | export function streamListDiff>( 269 | prevList: ReadableStream | File | T[], 270 | nextList: ReadableStream | File | T[], 271 | referenceProperty: ReferenceProperty, 272 | options: ListStreamOptions = DEFAULT_LIST_STREAM_OPTIONS, 273 | ): StreamListener { 274 | const emitter = new EventEmitter>(); 275 | 276 | if (typeof Worker === "undefined" || !options.useWorker) { 277 | setTimeout( 278 | () => 279 | generateStream(prevList, nextList, referenceProperty, options, emitter), 280 | 0, 281 | ); 282 | } else { 283 | generateWorker(prevList, nextList, referenceProperty, options, emitter); 284 | } 285 | 286 | return emitter as StreamListener; 287 | } 288 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | superdiff-logo 2 | 3 | 4 | [![CI](https://github.com/DoneDeal0/superdiff/actions/workflows/ci.yml/badge.svg)](https://github.com/DoneDeal0/superdiff/actions/workflows/ci.yml) 5 | [![CD](https://github.com/DoneDeal0/superdiff/actions/workflows/cd.yml/badge.svg)](https://github.com/DoneDeal0/superdiff/actions/workflows/cd.yml) 6 | ![NPM Downloads](https://img.shields.io/npm/dy/%40donedeal0%2Fsuperdiff?logo=npm) 7 | ![GitHub Tag](https://img.shields.io/github/v/tag/DoneDeal0/superdiff?label=latest%20release) 8 | 9 |
10 | 11 | # WHAT IS IT? 12 | 13 | This library compares two arrays or objects and returns a full diff of their differences. 14 | 15 | ℹ️ The documentation is also available on our [website](https://superdiff.gitbook.io/donedeal0-superdiff)! 16 | 17 |
18 | 19 | ## WHY YOU SHOULD USE THIS LIBRARY 20 | 21 | Most existing solutions return a confusing diff format that often requires extra parsing. They are also limited to object comparison. 22 | 23 | **Superdiff** provides a complete and readable diff for both arrays **and** objects. Plus, it supports stream and file inputs for handling large datasets efficiently, is battle-tested, has zero dependencies, and is super fast. 24 | 25 | Import. Enjoy. 👍 26 | 27 |
28 | 29 | ## DONORS 30 | 31 | I am grateful to the generous donors of **Superdiff**! 32 | 33 |
AlexisAnzieu 36 | omonk 37 | sneko 38 | 39 |
40 | 41 |
42 | 43 | ## FEATURES 44 | 45 | **Superdiff** exports 5 functions: 46 | 47 | ```ts 48 | // Returns a complete diff of two objects 49 | getObjectDiff(prevObject, nextObject) 50 | 51 | // Returns a complete diff of two arrays 52 | getListDiff(prevList, nextList) 53 | 54 | // Streams the diff of two object lists, ideal for large lists and maximum performance 55 | streamListDiff(prevList, nextList, referenceProperty) 56 | 57 | // Checks whether two values are equal 58 | isEqual(dataA, dataB) 59 | 60 | // Checks whether a value is an object 61 | isObject(data) 62 | ``` 63 |
64 | 65 | ### getObjectDiff() 66 | 67 | ```js 68 | import { getObjectDiff } from "@donedeal0/superdiff"; 69 | ``` 70 | 71 | Compares two objects and returns a diff for each value and its possible subvalues. Supports deeply nested objects of any value type. 72 | 73 | #### FORMAT 74 | 75 | **Input** 76 | 77 | ```ts 78 | prevData: Record; 79 | nextData: Record; 80 | options?: { 81 | ignoreArrayOrder?: boolean, // false by default, 82 | showOnly?: { 83 | statuses: ("added" | "deleted" | "updated" | "equal")[], // [] by default 84 | granularity?: "basic" | "deep" // "basic" by default 85 | } 86 | } 87 | ``` 88 | 89 | - `prevData`: the original object. 90 | - `nextData`: the new object. 91 | - `options` 92 | - `ignoreArrayOrder`: if set to `true`, `["hello", "world"]` and `["world", "hello"]` will be treated as `equal`, because the two arrays contain the same values, just in a different order. 93 | - `showOnly`: returns only the values whose status you are interested in. It takes two parameters: 94 | 95 | - `statuses`: status you want to see in the output (e.g. `["added", "equal"]`) 96 | - `granularity`: 97 | - `basic` returns only the main properties whose status matches your query. 98 | - `deep` can return main properties if some of their subproperties' status match your request. The subproperties are filtered accordingly. 99 | 100 | **Output** 101 | 102 | ```ts 103 | type ObjectDiff = { 104 | type: "object"; 105 | status: "added" | "deleted" | "equal" | "updated"; 106 | diff: Diff[]; 107 | }; 108 | 109 | type Diff = { 110 | property: string; 111 | previousValue: unknown; 112 | currentValue: unknown; 113 | status: "added" | "deleted" | "equal" | "updated"; 114 | // recursive diff in case of subproperties 115 | diff?: Diff[]; 116 | }; 117 | ``` 118 | #### USAGE 119 | 120 | **Input** 121 | 122 | ```diff 123 | getObjectDiff( 124 | { 125 | id: 54, 126 | user: { 127 | name: "joe", 128 | - member: true, 129 | - hobbies: ["golf", "football"], 130 | age: 66, 131 | }, 132 | }, 133 | { 134 | id: 54, 135 | user: { 136 | name: "joe", 137 | + member: false, 138 | + hobbies: ["golf", "chess"], 139 | age: 66, 140 | }, 141 | } 142 | ); 143 | ``` 144 | 145 | **Output** 146 | 147 | ```diff 148 | { 149 | type: "object", 150 | + status: "updated", 151 | diff: [ 152 | { 153 | property: "id", 154 | previousValue: 54, 155 | currentValue: 54, 156 | status: "equal", 157 | }, 158 | { 159 | property: "user", 160 | previousValue: { 161 | name: "joe", 162 | member: true, 163 | hobbies: ["golf", "football"], 164 | age: 66, 165 | }, 166 | currentValue: { 167 | name: "joe", 168 | member: false, 169 | hobbies: ["golf", "chess"], 170 | age: 66, 171 | }, 172 | + status: "updated", 173 | diff: [ 174 | { 175 | property: "name", 176 | previousValue: "joe", 177 | currentValue: "joe", 178 | status: "equal", 179 | }, 180 | + { 181 | + property: "member", 182 | + previousValue: true, 183 | + currentValue: false, 184 | + status: "updated", 185 | + }, 186 | + { 187 | + property: "hobbies", 188 | + previousValue: ["golf", "football"], 189 | + currentValue: ["golf", "chess"], 190 | + status: "updated", 191 | + }, 192 | { 193 | property: "age", 194 | previousValue: 66, 195 | currentValue: 66, 196 | status: "equal", 197 | }, 198 | ], 199 | }, 200 | ], 201 | } 202 | ``` 203 |
204 | 205 | ### getListDiff() 206 | 207 | ```js 208 | import { getListDiff } from "@donedeal0/superdiff"; 209 | ``` 210 | 211 | Compares two arrays and returns a diff for each entry. Supports duplicate values, primitive values and objects. 212 | 213 | #### FORMAT 214 | 215 | **Input** 216 | 217 | ```ts 218 | prevList: T[]; 219 | nextList: T[]; 220 | options?: { 221 | showOnly?: ("added" | "deleted" | "moved" | "updated" | "equal")[], // [] by default 222 | referenceProperty?: string, // "" by default 223 | ignoreArrayOrder?: boolean, // false by default, 224 | considerMoveAsUpdate?: boolean // false by default 225 | } 226 | ``` 227 | - `prevList`: the original list. 228 | - `nextList`: the new list. 229 | - `options` 230 | - `showOnly` gives you the option to return only the values whose status you are interested in (e.g. `["added", "equal"]`). 231 | - `referenceProperty` will consider an object to be `updated` rather than `added` or `deleted` if one of its properties remains stable, such as its `id`. This option has no effect on other datatypes. 232 | - `ignoreArrayOrder`: if set to `true`, `["hello", "world"]` and `["world", "hello"]` will be treated as `equal`, because the two arrays contain the same values, just in a different order. 233 | - `considerMoveAsUpdate`: if set to `true` a `moved` value will be considered as `updated`. 234 | 235 | **Output** 236 | 237 | ```ts 238 | type ListDiff = { 239 | type: "list"; 240 | status: "added" | "deleted" | "equal" | "moved" | "updated"; 241 | diff: { 242 | value: unknown; 243 | prevIndex: number | null; 244 | newIndex: number | null; 245 | indexDiff: number | null; 246 | status: "added" | "deleted" | "equal" | "moved" | "updated"; 247 | }[]; 248 | }; 249 | ``` 250 | #### USAGE 251 | 252 | **Input** 253 | 254 | ```diff 255 | getListDiff( 256 | - ["mbappe", "mendes", "verratti", "ruiz"], 257 | + ["mbappe", "messi", "ruiz"] 258 | ); 259 | ``` 260 | 261 | **Output** 262 | 263 | ```diff 264 | { 265 | type: "list", 266 | + status: "updated", 267 | diff: [ 268 | { 269 | value: "mbappe", 270 | prevIndex: 0, 271 | newIndex: 0, 272 | indexDiff: 0, 273 | status: "equal", 274 | }, 275 | - { 276 | - value: "mendes", 277 | - prevIndex: 1, 278 | - newIndex: null, 279 | - indexDiff: null, 280 | - status: "deleted", 281 | - }, 282 | - { 283 | - value: "verratti", 284 | - prevIndex: 2, 285 | - newIndex: null, 286 | - indexDiff: null, 287 | - status: "deleted", 288 | - }, 289 | + { 290 | + value: "messi", 291 | + prevIndex: null, 292 | + newIndex: 1, 293 | + indexDiff: null, 294 | + status: "added", 295 | + }, 296 | + { 297 | + value: "ruiz", 298 | + prevIndex: 3, 299 | + newIndex: 2, 300 | + indexDiff: -1, 301 | + status: "moved", 302 | }, 303 | ], 304 | } 305 | ``` 306 |
307 | 308 | ### streamListDiff() 309 | 310 | ```js 311 | // If you are in a server environment 312 | import { streamListDiff } from "@donedeal0/superdiff/server"; 313 | // If you are in a browser environment 314 | import { streamListDiff } from "@donedeal0/superdiff/client"; 315 | ``` 316 | 317 | Streams the diff of two object lists, ideal for large lists and maximum performance. 318 | 319 | ℹ️ `streamListDiff` requires ESM support for browser usage. It will work out of the box if you use a modern bundler (Webpack, Rollup) or JavaScript framework (Next.js, Vue.js). 320 | 321 | #### FORMAT 322 | 323 | **Input** 324 | 325 | #### Server 326 | 327 | > In a server environment, `Readable` refers to Node.js streams, and `FilePath` refers to the path of a file (e.g., `./list.json`). Examples are provided in the #usage section below. 328 | 329 | ```ts 330 | prevList: Readable | FilePath | Record[], 331 | nextList: Readable | FilePath | Record[], 332 | referenceProperty: keyof Record, 333 | options: { 334 | showOnly?: ("added" | "deleted" | "moved" | "updated" | "equal")[], // [] by default 335 | chunksSize?: number, // 0 by default 336 | considerMoveAsUpdate?: boolean; // false by default 337 | useWorker?: boolean; // true by default 338 | showWarnings?: boolean; // true by default 339 | } 340 | ``` 341 | 342 | #### Browser 343 | 344 | > In a browser environment, `ReadableStream` refers to the browser's streaming API, and `File` refers to an uploaded or local file. Examples are provided in the #usage section below. 345 | 346 | ```ts 347 | prevList: ReadableStream> | File | Record[], 348 | nextList: ReadableStream> | File | Record[], 349 | referenceProperty: keyof Record, 350 | options: { 351 | showOnly?: ("added" | "deleted" | "moved" | "updated" | "equal")[], // [] by default 352 | chunksSize?: number, // 0 by default 353 | considerMoveAsUpdate?: boolean; // false by default 354 | useWorker?: boolean; // true by default 355 | showWarnings?: boolean; // true by default 356 | 357 | } 358 | ``` 359 | 360 | - `prevList`: the original object list. 361 | - `nextList`: the new object list. 362 | - `referenceProperty`: a property common to all objects in your lists (e.g. `id`). 363 | - `options` 364 | - `chunksSize` the number of object diffs returned by each streamed chunk. (e.g. `0` = 1 object diff per chunk, `10` = 10 object diffs per chunk). 365 | - `showOnly` gives you the option to return only the values whose status you are interested in (e.g. `["added", "equal"]`). 366 | - `considerMoveAsUpdate`: if set to `true` a `moved` value will be considered as `updated`. 367 | - `useWorker`: if set to `true`, the diff will be run in a worker for maximum performance. Only recommended for large lists (e.g. +100,000 items). 368 | - `showWarnings`: if set to `true`, potential warnings will be displayed in the console. 369 | 370 | > ⚠️ Warning: using Readable streams may impact workers' performance since they need to be converted to arrays. Consider using arrays or files for optimal performance. Alternatively, you can turn the `useWorker` option off. 371 | 372 | **Output** 373 | 374 | The objects diff are grouped into arrays - called `chunks` - and are consumed thanks to an event listener. You have access to 3 events: 375 | - `data`: to be notified when a new chunk of object diffs is available. 376 | - `finish`: to be notified when the stream is finished. 377 | - `error`: to be notified if an error occurs during the stream. 378 | 379 | ```ts 380 | interface StreamListener { 381 | on(event: "data", listener: (chunk: StreamListDiff[]) => void); 382 | on(event: "finish", listener: () => void); 383 | on(event: "error", listener: (error: Error) => void); 384 | } 385 | 386 | type StreamListDiff> = { 387 | currentValue: T | null; 388 | previousValue: T | null; 389 | prevIndex: number | null; 390 | newIndex: number | null; 391 | indexDiff: number | null; 392 | status: "added" | "deleted" | "moved" | "updated" | "equal"; 393 | }; 394 | ``` 395 | 396 | #### USAGE 397 | 398 | **Input** 399 | 400 | You can send streams, file paths, or arrays as input: 401 | 402 | > If you are in a server environment 403 | 404 | ```ts 405 | // for a simple array 406 | const stream = [{ id: 1, name: "hello" }] 407 | // for a large array 408 | const stream = Readable.from(list, { objectMode: true }); 409 | // for a local file 410 | const stream = path.resolve(__dirname, "./list.json"); 411 | 412 | ``` 413 | 414 | > If you are in a browser environment 415 | 416 | ```ts 417 | // for a simple array 418 | const stream = [{ id: 1, name: "hello" }] 419 | // for a large array 420 | const stream = new ReadableStream({ 421 | start(controller) { 422 | list.forEach((value) => controller.enqueue(value)); 423 | controller.close(); 424 | }, 425 | }); 426 | // for a local file 427 | const stream = new File([JSON.stringify(file)], "file.json", { type: "application/json" }); 428 | // for a file input 429 | const stream = e.target.files[0]; 430 | 431 | ``` 432 | > Example 433 | 434 | ```diff 435 | const diff = streamListDiff( 436 | [ 437 | - { id: 1, name: "Item 1" }, 438 | { id: 2, name: "Item 2" }, 439 | { id: 3, name: "Item 3" } 440 | ], 441 | [ 442 | + { id: 0, name: "Item 0" }, 443 | { id: 2, name: "Item 2" }, 444 | + { id: 3, name: "Item Three" }, 445 | ], 446 | "id", 447 | { chunksSize: 2 } 448 | ); 449 | ``` 450 | 451 | **Output** 452 | 453 | ```diff 454 | diff.on("data", (chunk) => { 455 | // first chunk received (2 object diffs) 456 | [ 457 | + { 458 | + previousValue: null, 459 | + currentValue: { id: 0, name: 'Item 0' }, 460 | + prevIndex: null, 461 | + newIndex: 0, 462 | + indexDiff: null, 463 | + status: 'added' 464 | + }, 465 | - { 466 | - previousValue: { id: 1, name: 'Item 1' }, 467 | - currentValue: null, 468 | - prevIndex: 0, 469 | - newIndex: null, 470 | - indexDiff: null, 471 | - status: 'deleted' 472 | - } 473 | ] 474 | // second chunk received (2 object diffs) 475 | [ 476 | { 477 | previousValue: { id: 2, name: 'Item 2' }, 478 | currentValue: { id: 2, name: 'Item 2' }, 479 | prevIndex: 1, 480 | newIndex: 1, 481 | indexDiff: 0, 482 | status: 'equal' 483 | }, 484 | + { 485 | + previousValue: { id: 3, name: 'Item 3' }, 486 | + currentValue: { id: 3, name: 'Item Three' }, 487 | + prevIndex: 2, 488 | + newIndex: 2, 489 | + indexDiff: 0, 490 | + status: 'updated' 491 | + }, 492 | ] 493 | }); 494 | 495 | diff.on("finish", () => console.log("Your data has been processed. The full diff is available.")) 496 | diff.on("error", (err) => console.log(err)) 497 | ``` 498 | 499 |
500 | 501 | ### isEqual() 502 | 503 | ```js 504 | import { isEqual } from "@donedeal0/superdiff"; 505 | ``` 506 | 507 | Tests whether two values are equal. 508 | 509 | #### FORMAT 510 | 511 | **Input** 512 | 513 | ```ts 514 | a: unknown, 515 | b: unknown, 516 | options: { 517 | ignoreArrayOrder: boolean; // false by default 518 | }, 519 | ``` 520 | - `a`: the value to be compared to the value `b`. 521 | - `b`: the value to be compared to the value `a`. 522 | - `ignoreArrayOrder`: if set to `true`, `["hello", "world"]` and `["world", "hello"]` will be treated as `equal`, because the two arrays contain the same values, just in a different order. 523 | 524 | #### USAGE 525 | 526 | 527 | ```ts 528 | isEqual( 529 | [ 530 | { name: "joe", age: 99 }, 531 | { name: "nina", age: 23 }, 532 | ], 533 | [ 534 | { name: "joe", age: 98 }, 535 | { name: "nina", age: 23 }, 536 | ], 537 | ); 538 | ``` 539 | 540 | **Output** 541 | 542 | ```ts 543 | false; 544 | ``` 545 |
546 | 547 | ### isObject() 548 | 549 | ```js 550 | import { isObject } from "@donedeal0/superdiff"; 551 | ``` 552 | 553 | Tests whether a value is an object. 554 | 555 | #### FORMAT 556 | 557 | **Input** 558 | 559 | ```ts 560 | value: unknown; 561 | ``` 562 | 563 | - `value`: the value whose type will be checked. 564 | 565 | #### USAGE 566 | 567 | **Input** 568 | 569 | ```ts 570 | isObject(["hello", "world"]); 571 | ``` 572 | 573 | **Output** 574 | 575 | ```ts 576 | false; 577 | ``` 578 | 579 |
580 | 581 | ### ℹ️ More examples are available in the source code tests. 582 | 583 | 584 |
585 | 586 | ## CREDITS 587 | 588 | DoneDeal0 589 | 590 | ## SUPPORT 591 | 592 | If you or your company uses **Superdiff**, please show your support by becoming a sponsor! Your name and company logo will be displayed on the `README.md`. Premium support is also available. https://github.com/sponsors/DoneDeal0 593 | 594 |
595 | 596 | sponsor 597 | 598 |
599 | 600 | ## CONTRIBUTING 601 | 602 | Issues and pull requests are welcome! 603 | -------------------------------------------------------------------------------- /src/lib/list-diff/list-diff.test.ts: -------------------------------------------------------------------------------- 1 | import { ListStatus } from "@models/list"; 2 | import { getListDiff } from "."; 3 | 4 | describe("getListDiff", () => { 5 | it("returns an empty diff if no lists are provided", () => { 6 | expect(getListDiff(null, null)).toStrictEqual({ 7 | type: "list", 8 | status: "equal", 9 | diff: [], 10 | }); 11 | }); 12 | it("consider previous list as completely deleted if no next list is provided", () => { 13 | expect( 14 | getListDiff(["mbappe", "mendes", "verratti", "ruiz"], null), 15 | ).toStrictEqual({ 16 | type: "list", 17 | status: "deleted", 18 | diff: [ 19 | { 20 | value: "mbappe", 21 | prevIndex: 0, 22 | newIndex: null, 23 | indexDiff: null, 24 | status: "deleted", 25 | }, 26 | { 27 | value: "mendes", 28 | prevIndex: 1, 29 | newIndex: null, 30 | indexDiff: null, 31 | status: "deleted", 32 | }, 33 | { 34 | value: "verratti", 35 | prevIndex: 2, 36 | newIndex: null, 37 | indexDiff: null, 38 | status: "deleted", 39 | }, 40 | { 41 | value: "ruiz", 42 | prevIndex: 3, 43 | newIndex: null, 44 | indexDiff: null, 45 | status: "deleted", 46 | }, 47 | ], 48 | }); 49 | }); 50 | it("consider next list as completely added if no previous list is provided", () => { 51 | expect( 52 | getListDiff(null, ["mbappe", "mendes", "verratti", "ruiz"]), 53 | ).toStrictEqual({ 54 | type: "list", 55 | status: "added", 56 | diff: [ 57 | { 58 | value: "mbappe", 59 | prevIndex: null, 60 | newIndex: 0, 61 | indexDiff: null, 62 | status: "added", 63 | }, 64 | { 65 | value: "mendes", 66 | prevIndex: null, 67 | newIndex: 1, 68 | indexDiff: null, 69 | status: "added", 70 | }, 71 | { 72 | value: "verratti", 73 | prevIndex: null, 74 | newIndex: 2, 75 | indexDiff: null, 76 | status: "added", 77 | }, 78 | { 79 | value: "ruiz", 80 | prevIndex: null, 81 | newIndex: 3, 82 | indexDiff: null, 83 | status: "added", 84 | }, 85 | ], 86 | }); 87 | }); 88 | it("detects changes between string lists", () => { 89 | expect( 90 | getListDiff( 91 | ["mbappe", "mendes", "verratti", "ruiz"], 92 | ["mbappe", "messi", "ruiz"], 93 | ), 94 | ).toStrictEqual({ 95 | type: "list", 96 | status: "updated", 97 | diff: [ 98 | { 99 | value: "mbappe", 100 | prevIndex: 0, 101 | newIndex: 0, 102 | indexDiff: 0, 103 | status: "equal", 104 | }, 105 | { 106 | value: "messi", 107 | prevIndex: null, 108 | newIndex: 1, 109 | indexDiff: null, 110 | status: "added", 111 | }, 112 | { 113 | value: "ruiz", 114 | prevIndex: 3, 115 | newIndex: 2, 116 | indexDiff: -1, 117 | status: "moved", 118 | }, 119 | { 120 | value: "mendes", 121 | prevIndex: 1, 122 | newIndex: null, 123 | indexDiff: null, 124 | status: "deleted", 125 | }, 126 | { 127 | value: "verratti", 128 | prevIndex: 2, 129 | newIndex: null, 130 | indexDiff: null, 131 | status: "deleted", 132 | }, 133 | ], 134 | }); 135 | }); 136 | it("detects changes between number lists", () => { 137 | expect(getListDiff([54, 234, 76, 0], [54, 200, 0])).toStrictEqual({ 138 | type: "list", 139 | status: "updated", 140 | diff: [ 141 | { 142 | value: 54, 143 | prevIndex: 0, 144 | newIndex: 0, 145 | indexDiff: 0, 146 | status: "equal", 147 | }, 148 | { 149 | value: 200, 150 | prevIndex: null, 151 | newIndex: 1, 152 | indexDiff: null, 153 | status: "added", 154 | }, 155 | { 156 | value: 0, 157 | prevIndex: 3, 158 | newIndex: 2, 159 | indexDiff: -1, 160 | status: "moved", 161 | }, 162 | { 163 | value: 234, 164 | prevIndex: 1, 165 | newIndex: null, 166 | indexDiff: null, 167 | status: "deleted", 168 | }, 169 | { 170 | value: 76, 171 | prevIndex: 2, 172 | newIndex: null, 173 | indexDiff: null, 174 | status: "deleted", 175 | }, 176 | ], 177 | }); 178 | }); 179 | it("detects changes between object lists", () => { 180 | expect( 181 | getListDiff( 182 | [ 183 | { name: "joe", age: 87 }, 184 | { name: "nina", age: 23 }, 185 | { name: "paul", age: 32 }, 186 | ], 187 | [ 188 | { name: "paul", age: 32 }, 189 | { name: "joe", age: 88 }, 190 | { name: "nina", age: 23 }, 191 | ], 192 | ), 193 | ).toStrictEqual({ 194 | type: "list", 195 | status: "updated", 196 | diff: [ 197 | { 198 | value: { name: "paul", age: 32 }, 199 | prevIndex: 2, 200 | newIndex: 0, 201 | indexDiff: -2, 202 | status: "moved", 203 | }, 204 | { 205 | value: { name: "joe", age: 88 }, 206 | prevIndex: null, 207 | newIndex: 1, 208 | indexDiff: null, 209 | status: "added", 210 | }, 211 | { 212 | value: { name: "nina", age: 23 }, 213 | prevIndex: 1, 214 | newIndex: 2, 215 | indexDiff: 1, 216 | status: "moved", 217 | }, 218 | { 219 | value: { name: "joe", age: 87 }, 220 | prevIndex: 0, 221 | newIndex: null, 222 | indexDiff: null, 223 | status: "deleted", 224 | }, 225 | ], 226 | }); 227 | }); 228 | it("detects changes between lists containing duplicated values", () => { 229 | expect( 230 | getListDiff(["mbappe", "messi"], ["mbappe", "mbappe", "messi"]), 231 | ).toStrictEqual({ 232 | type: "list", 233 | status: "updated", 234 | diff: [ 235 | { 236 | value: "mbappe", 237 | prevIndex: 0, 238 | newIndex: 0, 239 | indexDiff: 0, 240 | status: "equal", 241 | }, 242 | { 243 | value: "mbappe", 244 | prevIndex: null, 245 | newIndex: 1, 246 | indexDiff: null, 247 | status: "added", 248 | }, 249 | { 250 | value: "messi", 251 | prevIndex: 1, 252 | newIndex: 2, 253 | indexDiff: 1, 254 | status: "moved", 255 | }, 256 | ], 257 | }); 258 | expect( 259 | getListDiff( 260 | ["mbappe", "messi", "messi", "mbappe"], 261 | ["mbappe", "messi", "messi"], 262 | ), 263 | ).toStrictEqual({ 264 | type: "list", 265 | status: "updated", 266 | diff: [ 267 | { 268 | value: "mbappe", 269 | prevIndex: 0, 270 | newIndex: 0, 271 | indexDiff: 0, 272 | status: "equal", 273 | }, 274 | { 275 | value: "messi", 276 | prevIndex: 1, 277 | newIndex: 1, 278 | indexDiff: 0, 279 | status: "equal", 280 | }, 281 | { 282 | value: "messi", 283 | prevIndex: 2, 284 | newIndex: 2, 285 | indexDiff: 0, 286 | status: "equal", 287 | }, 288 | { 289 | value: "mbappe", 290 | prevIndex: 3, 291 | newIndex: null, 292 | indexDiff: null, 293 | status: "deleted", 294 | }, 295 | ], 296 | }); 297 | expect( 298 | getListDiff( 299 | [ 300 | false, 301 | true, 302 | true, 303 | undefined, 304 | "hello", 305 | { name: "joe", age: 88 }, 306 | false, 307 | 13, 308 | ], 309 | [ 310 | false, 311 | false, 312 | true, 313 | undefined, 314 | "hello", 315 | { name: "joe", age: 88 }, 316 | false, 317 | { name: "joe", age: 88 }, 318 | ], 319 | ), 320 | ).toStrictEqual({ 321 | type: "list", 322 | status: "updated", 323 | diff: [ 324 | { 325 | value: false, 326 | prevIndex: 0, 327 | newIndex: 0, 328 | indexDiff: 0, 329 | status: "equal", 330 | }, 331 | { 332 | value: false, 333 | prevIndex: 6, 334 | newIndex: 1, 335 | indexDiff: -5, 336 | status: "moved", 337 | }, 338 | 339 | { 340 | value: true, 341 | prevIndex: 1, 342 | newIndex: 2, 343 | indexDiff: 1, 344 | status: "moved", 345 | }, 346 | { 347 | value: undefined, 348 | prevIndex: 3, 349 | newIndex: 3, 350 | indexDiff: 0, 351 | status: "equal", 352 | }, 353 | { 354 | value: "hello", 355 | prevIndex: 4, 356 | newIndex: 4, 357 | indexDiff: 0, 358 | status: "equal", 359 | }, 360 | { 361 | value: { name: "joe", age: 88 }, 362 | prevIndex: 5, 363 | newIndex: 5, 364 | indexDiff: 0, 365 | status: "equal", 366 | }, 367 | { 368 | value: false, 369 | prevIndex: null, 370 | newIndex: 6, 371 | indexDiff: null, 372 | status: "added", 373 | }, 374 | { 375 | value: { name: "joe", age: 88 }, 376 | prevIndex: null, 377 | newIndex: 7, 378 | indexDiff: null, 379 | status: "added", 380 | }, 381 | { 382 | value: true, 383 | prevIndex: 2, 384 | newIndex: null, 385 | indexDiff: null, 386 | status: "deleted", 387 | }, 388 | { 389 | value: 13, 390 | prevIndex: 7, 391 | newIndex: null, 392 | indexDiff: null, 393 | status: "deleted", 394 | }, 395 | ], 396 | }); 397 | }); 398 | it("showOnly added and deleted values", () => { 399 | expect( 400 | getListDiff( 401 | [ 402 | false, 403 | true, 404 | true, 405 | undefined, 406 | "hello", 407 | { name: "joe", age: 88 }, 408 | false, 409 | 13, 410 | ], 411 | [ 412 | false, 413 | false, 414 | true, 415 | undefined, 416 | "hello", 417 | { name: "joe", age: 88 }, 418 | false, 419 | { name: "joe", age: 88 }, 420 | ], 421 | { showOnly: [ListStatus.ADDED, ListStatus.DELETED] }, 422 | ), 423 | ).toStrictEqual({ 424 | type: "list", 425 | status: "updated", 426 | diff: [ 427 | { 428 | value: false, 429 | prevIndex: null, 430 | newIndex: 6, 431 | indexDiff: null, 432 | status: "added", 433 | }, 434 | { 435 | value: { name: "joe", age: 88 }, 436 | prevIndex: null, 437 | newIndex: 7, 438 | indexDiff: null, 439 | status: "added", 440 | }, 441 | { 442 | value: true, 443 | prevIndex: 2, 444 | newIndex: null, 445 | indexDiff: null, 446 | status: "deleted", 447 | }, 448 | { 449 | value: 13, 450 | prevIndex: 7, 451 | newIndex: null, 452 | indexDiff: null, 453 | status: "deleted", 454 | }, 455 | ], 456 | }); 457 | }); 458 | it("returns an empty diff if no property match the required statuses output", () => { 459 | expect(getListDiff(null, null)).toStrictEqual({ 460 | type: "list", 461 | status: "equal", 462 | diff: [], 463 | }); 464 | expect( 465 | getListDiff(["mbappe", "mendes", "verratti", "ruiz"], null, { 466 | showOnly: [ListStatus.MOVED, ListStatus.UPDATED], 467 | }), 468 | ).toStrictEqual({ 469 | type: "list", 470 | status: "deleted", 471 | diff: [], 472 | }); 473 | }); 474 | it("returns all values if their status match the required statuses", () => { 475 | expect( 476 | getListDiff(null, ["mbappe", "mendes", "verratti", "ruiz"], { 477 | showOnly: [ListStatus.ADDED], 478 | }), 479 | ).toStrictEqual({ 480 | type: "list", 481 | status: "added", 482 | diff: [ 483 | { 484 | value: "mbappe", 485 | prevIndex: null, 486 | newIndex: 0, 487 | indexDiff: null, 488 | status: "added", 489 | }, 490 | { 491 | value: "mendes", 492 | prevIndex: null, 493 | newIndex: 1, 494 | indexDiff: null, 495 | status: "added", 496 | }, 497 | { 498 | value: "verratti", 499 | prevIndex: null, 500 | newIndex: 2, 501 | indexDiff: null, 502 | status: "added", 503 | }, 504 | { 505 | value: "ruiz", 506 | prevIndex: null, 507 | newIndex: 3, 508 | indexDiff: null, 509 | status: "added", 510 | }, 511 | ], 512 | }); 513 | }); 514 | it("consider object updated if a reference property is given and this property hasn't changed", () => { 515 | expect( 516 | getListDiff( 517 | [ 518 | "hello", 519 | { id: 37, isCool: true, hobbies: ["golf", "ski"] }, 520 | { id: 38, isCool: false, hobbies: ["football"] }, 521 | undefined, 522 | { id: 8, age: 77 }, 523 | { id: 55, character: { strength: 66 } }, 524 | ], 525 | [ 526 | { id: 8, age: 77 }, 527 | { id: 37, isCool: false, hobbies: ["golf", "ski"] }, 528 | { id: 38, isCool: false, hobbies: ["football"] }, 529 | undefined, 530 | { id: 99, character: { strength: 69 } }, 531 | ], 532 | { 533 | referenceProperty: "id", 534 | }, 535 | ), 536 | ).toStrictEqual({ 537 | type: "list", 538 | status: "updated", 539 | diff: [ 540 | { 541 | value: { id: 8, age: 77 }, 542 | prevIndex: 4, 543 | newIndex: 0, 544 | indexDiff: -4, 545 | status: "moved", 546 | }, 547 | { 548 | value: { id: 37, isCool: false, hobbies: ["golf", "ski"] }, 549 | prevIndex: 1, 550 | newIndex: 1, 551 | indexDiff: 0, 552 | status: "updated", 553 | }, 554 | { 555 | value: { id: 38, isCool: false, hobbies: ["football"] }, 556 | prevIndex: 2, 557 | newIndex: 2, 558 | indexDiff: 0, 559 | status: "equal", 560 | }, 561 | { 562 | value: undefined, 563 | prevIndex: 3, 564 | newIndex: 3, 565 | indexDiff: 0, 566 | status: "equal", 567 | }, 568 | { 569 | value: { id: 99, character: { strength: 69 } }, 570 | prevIndex: null, 571 | newIndex: 4, 572 | indexDiff: null, 573 | status: "added", 574 | }, 575 | { 576 | value: "hello", 577 | prevIndex: 0, 578 | newIndex: null, 579 | indexDiff: null, 580 | status: "deleted", 581 | }, 582 | { 583 | value: { id: 55, character: { strength: 66 } }, 584 | prevIndex: 5, 585 | newIndex: null, 586 | indexDiff: null, 587 | status: "deleted", 588 | }, 589 | ], 590 | }); 591 | }); 592 | it("consider moved values as updated if the considerMoveAsUpdate option is true", () => { 593 | expect( 594 | getListDiff(["mbappe", "messi"], ["mbappe", "mbappe", "messi"], { 595 | considerMoveAsUpdate: true, 596 | }), 597 | ).toStrictEqual({ 598 | type: "list", 599 | status: "updated", 600 | diff: [ 601 | { 602 | value: "mbappe", 603 | prevIndex: 0, 604 | newIndex: 0, 605 | indexDiff: 0, 606 | status: "equal", 607 | }, 608 | { 609 | value: "mbappe", 610 | prevIndex: null, 611 | newIndex: 1, 612 | indexDiff: null, 613 | status: "added", 614 | }, 615 | { 616 | value: "messi", 617 | prevIndex: 1, 618 | newIndex: 2, 619 | indexDiff: 1, 620 | status: "updated", 621 | }, 622 | ], 623 | }); 624 | expect( 625 | getListDiff( 626 | [ 627 | "hello", 628 | { id: 37, isCool: true, hobbies: ["golf", "ski"] }, 629 | { id: 38, isCool: false, hobbies: ["football"] }, 630 | undefined, 631 | { id: 8, age: 77 }, 632 | { id: 55, character: { strength: 66 } }, 633 | ], 634 | [ 635 | { id: 8, age: 77 }, 636 | { id: 37, isCool: false, hobbies: ["golf", "ski"] }, 637 | { id: 38, isCool: false, hobbies: ["football"] }, 638 | undefined, 639 | { id: 99, character: { strength: 69 } }, 640 | ], 641 | { 642 | referenceProperty: "id", 643 | considerMoveAsUpdate: true, 644 | }, 645 | ), 646 | ).toStrictEqual({ 647 | type: "list", 648 | status: "updated", 649 | diff: [ 650 | { 651 | value: { id: 8, age: 77 }, 652 | prevIndex: 4, 653 | newIndex: 0, 654 | indexDiff: -4, 655 | status: "updated", 656 | }, 657 | { 658 | value: { id: 37, isCool: false, hobbies: ["golf", "ski"] }, 659 | prevIndex: 1, 660 | newIndex: 1, 661 | indexDiff: 0, 662 | status: "updated", 663 | }, 664 | { 665 | value: { id: 38, isCool: false, hobbies: ["football"] }, 666 | prevIndex: 2, 667 | newIndex: 2, 668 | indexDiff: 0, 669 | status: "equal", 670 | }, 671 | { 672 | value: undefined, 673 | prevIndex: 3, 674 | newIndex: 3, 675 | indexDiff: 0, 676 | status: "equal", 677 | }, 678 | { 679 | value: { id: 99, character: { strength: 69 } }, 680 | prevIndex: null, 681 | newIndex: 4, 682 | indexDiff: null, 683 | status: "added", 684 | }, 685 | { 686 | value: "hello", 687 | prevIndex: 0, 688 | newIndex: null, 689 | indexDiff: null, 690 | status: "deleted", 691 | }, 692 | { 693 | value: { id: 55, character: { strength: 66 } }, 694 | prevIndex: 5, 695 | newIndex: null, 696 | indexDiff: null, 697 | status: "deleted", 698 | }, 699 | ], 700 | }); 701 | }); 702 | it("consider moved values as equal if they have not changed and ignoreArrayOrder option is true", () => { 703 | expect( 704 | getListDiff( 705 | [ 706 | { id: 3, name: "nina", hobbies: ["swiming"] }, 707 | { id: 1, name: "joe", hobbies: ["golf", "fishing"] }, 708 | { id: 2, name: "jack", hobbies: ["coding"] }, 709 | ], 710 | [ 711 | { id: 1, name: "joe", hobbies: ["golf", "fishing"] }, 712 | { id: 2, name: "jack", hobbies: ["coding"] }, 713 | { id: 3, name: "nina", hobbies: ["swiming"] }, 714 | ], 715 | { 716 | ignoreArrayOrder: true, 717 | }, 718 | ), 719 | ).toStrictEqual({ 720 | type: "list", 721 | status: "equal", 722 | diff: [ 723 | { 724 | value: { id: 1, name: "joe", hobbies: ["golf", "fishing"] }, 725 | prevIndex: 1, 726 | newIndex: 0, 727 | indexDiff: -1, 728 | status: "equal", 729 | }, 730 | { 731 | value: { id: 2, name: "jack", hobbies: ["coding"] }, 732 | prevIndex: 2, 733 | newIndex: 1, 734 | indexDiff: -1, 735 | status: "equal", 736 | }, 737 | { 738 | value: { id: 3, name: "nina", hobbies: ["swiming"] }, 739 | prevIndex: 0, 740 | newIndex: 2, 741 | indexDiff: 2, 742 | status: "equal", 743 | }, 744 | ], 745 | }); 746 | }); 747 | it("consider moved values as updated if they have changed and ignoreArrayOrder option is true", () => { 748 | expect( 749 | getListDiff( 750 | [ 751 | { id: 3, name: "nina", hobbies: ["swiming"] }, 752 | { id: 1, name: "joseph", hobbies: ["golf", "fishing"] }, 753 | { id: 2, name: "jack", hobbies: ["coding"] }, 754 | ], 755 | [ 756 | { id: 1, name: "joe", hobbies: ["golf", "fishing"] }, 757 | { id: 2, name: "jack", hobbies: ["coding"] }, 758 | { id: 3, name: "nina", hobbies: ["swiming"] }, 759 | ], 760 | { 761 | ignoreArrayOrder: true, 762 | referenceProperty: "id", 763 | }, 764 | ), 765 | ).toStrictEqual({ 766 | type: "list", 767 | status: "updated", 768 | diff: [ 769 | { 770 | value: { id: 1, name: "joe", hobbies: ["golf", "fishing"] }, 771 | prevIndex: 1, 772 | newIndex: 0, 773 | indexDiff: -1, 774 | status: "updated", 775 | }, 776 | { 777 | value: { id: 2, name: "jack", hobbies: ["coding"] }, 778 | prevIndex: 2, 779 | newIndex: 1, 780 | indexDiff: -1, 781 | status: "equal", 782 | }, 783 | { 784 | value: { id: 3, name: "nina", hobbies: ["swiming"] }, 785 | prevIndex: 0, 786 | newIndex: 2, 787 | indexDiff: 2, 788 | status: "equal", 789 | }, 790 | ], 791 | }); 792 | }); 793 | }); 794 | -------------------------------------------------------------------------------- /src/lib/object-diff/object-diff.test.ts: -------------------------------------------------------------------------------- 1 | import { Granularity, ObjectStatus } from "@models/object"; 2 | import { getObjectDiff } from "."; 3 | 4 | describe("getObjectDiff", () => { 5 | it("returns an empty diff if no objects are provided", () => { 6 | expect(getObjectDiff(null, null)).toStrictEqual({ 7 | type: "object", 8 | status: "equal", 9 | diff: [], 10 | }); 11 | }); 12 | it("consider previous object as completely deleted if no next object is provided", () => { 13 | expect( 14 | getObjectDiff( 15 | { name: "joe", age: 54, hobbies: ["golf", "football"] }, 16 | null, 17 | ), 18 | ).toStrictEqual({ 19 | type: "object", 20 | status: "deleted", 21 | diff: [ 22 | { 23 | property: "name", 24 | previousValue: "joe", 25 | currentValue: undefined, 26 | status: "deleted", 27 | }, 28 | { 29 | property: "age", 30 | previousValue: 54, 31 | currentValue: undefined, 32 | status: "deleted", 33 | }, 34 | { 35 | property: "hobbies", 36 | previousValue: ["golf", "football"], 37 | currentValue: undefined, 38 | status: "deleted", 39 | }, 40 | ], 41 | }); 42 | }); 43 | it("consider previous object as completely deleted if no next object is provided, and return an empty diff if showOnly doesn't require deleted values", () => { 44 | expect( 45 | getObjectDiff( 46 | { 47 | name: "joe", 48 | age: 54, 49 | hobbies: ["golf", "football"], 50 | }, 51 | null, 52 | { 53 | showOnly: { 54 | statuses: [ObjectStatus.ADDED], 55 | granularity: Granularity.DEEP, 56 | }, 57 | }, 58 | ), 59 | ).toStrictEqual({ 60 | type: "object", 61 | status: "deleted", 62 | diff: [], 63 | }); 64 | }); 65 | it("consider next object as completely added if no previous object is provided", () => { 66 | expect( 67 | getObjectDiff(null, { 68 | name: "joe", 69 | age: 54, 70 | hobbies: ["golf", "football"], 71 | }), 72 | ).toStrictEqual({ 73 | type: "object", 74 | status: "added", 75 | diff: [ 76 | { 77 | property: "name", 78 | previousValue: undefined, 79 | currentValue: "joe", 80 | status: "added", 81 | }, 82 | { 83 | property: "age", 84 | previousValue: undefined, 85 | currentValue: 54, 86 | status: "added", 87 | }, 88 | { 89 | property: "hobbies", 90 | previousValue: undefined, 91 | currentValue: ["golf", "football"], 92 | status: "added", 93 | }, 94 | ], 95 | }); 96 | }); 97 | it("consider objects as equal if no changes are detected", () => { 98 | expect( 99 | getObjectDiff( 100 | { 101 | age: 66, 102 | member: false, 103 | promoCode: null, 104 | city: undefined, 105 | hobbies: ["golf", "football"], 106 | options: { vegan: undefined, phone: null }, 107 | }, 108 | { 109 | age: 66, 110 | member: false, 111 | promoCode: null, 112 | city: undefined, 113 | hobbies: ["golf", "football"], 114 | options: { vegan: undefined, phone: null }, 115 | }, 116 | ), 117 | ).toStrictEqual({ 118 | type: "object", 119 | status: "equal", 120 | diff: [ 121 | { 122 | property: "age", 123 | previousValue: 66, 124 | currentValue: 66, 125 | status: "equal", 126 | }, 127 | { 128 | property: "member", 129 | previousValue: false, 130 | currentValue: false, 131 | status: "equal", 132 | }, 133 | { 134 | property: "promoCode", 135 | previousValue: null, 136 | currentValue: null, 137 | status: "equal", 138 | }, 139 | { 140 | property: "city", 141 | previousValue: undefined, 142 | currentValue: undefined, 143 | status: "equal", 144 | }, 145 | { 146 | property: "hobbies", 147 | previousValue: ["golf", "football"], 148 | currentValue: ["golf", "football"], 149 | status: "equal", 150 | }, 151 | { 152 | property: "options", 153 | previousValue: { vegan: undefined, phone: null }, 154 | currentValue: { vegan: undefined, phone: null }, 155 | status: "equal", 156 | }, 157 | ], 158 | }); 159 | }); 160 | it("detects changed between two objects", () => { 161 | expect( 162 | getObjectDiff( 163 | { 164 | id: 54, 165 | type: "sport", 166 | user: { 167 | name: "joe", 168 | member: true, 169 | hobbies: ["golf", "football"], 170 | age: 66, 171 | }, 172 | }, 173 | { 174 | id: 54, 175 | country: "us", 176 | user: { 177 | name: "joe", 178 | member: false, 179 | hobbies: ["golf", "chess"], 180 | nickname: "super joe", 181 | }, 182 | }, 183 | ), 184 | ).toStrictEqual({ 185 | type: "object", 186 | status: "updated", 187 | diff: [ 188 | { 189 | property: "id", 190 | previousValue: 54, 191 | currentValue: 54, 192 | status: "equal", 193 | }, 194 | { 195 | property: "type", 196 | previousValue: "sport", 197 | currentValue: undefined, 198 | status: "deleted", 199 | }, 200 | { 201 | property: "user", 202 | previousValue: { 203 | name: "joe", 204 | member: true, 205 | hobbies: ["golf", "football"], 206 | age: 66, 207 | }, 208 | currentValue: { 209 | name: "joe", 210 | member: false, 211 | hobbies: ["golf", "chess"], 212 | nickname: "super joe", 213 | }, 214 | status: "updated", 215 | diff: [ 216 | { 217 | property: "name", 218 | previousValue: "joe", 219 | currentValue: "joe", 220 | status: "equal", 221 | }, 222 | { 223 | property: "member", 224 | previousValue: true, 225 | currentValue: false, 226 | status: "updated", 227 | }, 228 | { 229 | property: "hobbies", 230 | previousValue: ["golf", "football"], 231 | currentValue: ["golf", "chess"], 232 | status: "updated", 233 | }, 234 | { 235 | property: "age", 236 | previousValue: 66, 237 | currentValue: undefined, 238 | status: "deleted", 239 | }, 240 | { 241 | property: "nickname", 242 | previousValue: undefined, 243 | currentValue: "super joe", 244 | status: "added", 245 | }, 246 | ], 247 | }, 248 | { 249 | property: "country", 250 | previousValue: undefined, 251 | currentValue: "us", 252 | status: "added", 253 | }, 254 | ], 255 | }); 256 | }); 257 | it("detects changed between two deep nested objects", () => { 258 | expect( 259 | getObjectDiff( 260 | { 261 | id: 54, 262 | user: { 263 | name: "joe", 264 | data: { 265 | member: true, 266 | hobbies: { 267 | football: ["psg"], 268 | rugby: ["france"], 269 | }, 270 | }, 271 | }, 272 | }, 273 | { 274 | id: 54, 275 | user: { 276 | name: "joe", 277 | data: { 278 | member: true, 279 | hobbies: { 280 | football: ["psg", "nantes"], 281 | golf: ["st andrews"], 282 | }, 283 | }, 284 | }, 285 | }, 286 | ), 287 | ).toStrictEqual({ 288 | type: "object", 289 | status: "updated", 290 | diff: [ 291 | { 292 | property: "id", 293 | previousValue: 54, 294 | currentValue: 54, 295 | status: "equal", 296 | }, 297 | { 298 | property: "user", 299 | previousValue: { 300 | name: "joe", 301 | data: { 302 | member: true, 303 | hobbies: { 304 | football: ["psg"], 305 | rugby: ["france"], 306 | }, 307 | }, 308 | }, 309 | currentValue: { 310 | name: "joe", 311 | data: { 312 | member: true, 313 | hobbies: { 314 | football: ["psg", "nantes"], 315 | golf: ["st andrews"], 316 | }, 317 | }, 318 | }, 319 | status: "updated", 320 | diff: [ 321 | { 322 | property: "name", 323 | previousValue: "joe", 324 | currentValue: "joe", 325 | status: "equal", 326 | }, 327 | { 328 | property: "data", 329 | previousValue: { 330 | member: true, 331 | hobbies: { 332 | football: ["psg"], 333 | rugby: ["france"], 334 | }, 335 | }, 336 | currentValue: { 337 | member: true, 338 | hobbies: { 339 | football: ["psg", "nantes"], 340 | golf: ["st andrews"], 341 | }, 342 | }, 343 | status: "updated", 344 | diff: [ 345 | { 346 | property: "member", 347 | previousValue: true, 348 | currentValue: true, 349 | status: "equal", 350 | }, 351 | { 352 | property: "hobbies", 353 | previousValue: { 354 | football: ["psg"], 355 | rugby: ["france"], 356 | }, 357 | currentValue: { 358 | football: ["psg", "nantes"], 359 | golf: ["st andrews"], 360 | }, 361 | status: "updated", 362 | diff: [ 363 | { 364 | property: "football", 365 | previousValue: ["psg"], 366 | currentValue: ["psg", "nantes"], 367 | status: "updated", 368 | }, 369 | { 370 | property: "rugby", 371 | previousValue: ["france"], 372 | currentValue: undefined, 373 | status: "deleted", 374 | }, 375 | { 376 | property: "golf", 377 | previousValue: undefined, 378 | currentValue: ["st andrews"], 379 | status: "added", 380 | }, 381 | ], 382 | }, 383 | ], 384 | }, 385 | ], 386 | }, 387 | ], 388 | }); 389 | }); 390 | it("detects changed between two objects BUT doesn't care about array order as long as all values are preserved when ignoreArrayOrder option is activated", () => { 391 | expect( 392 | getObjectDiff( 393 | { 394 | id: 54, 395 | type: "sport", 396 | user: { 397 | name: "joe", 398 | member: true, 399 | hobbies: ["golf", "football"], 400 | age: 66, 401 | }, 402 | }, 403 | { 404 | id: 54, 405 | country: "us", 406 | user: { 407 | name: "joe", 408 | member: false, 409 | hobbies: ["football", "golf"], 410 | nickname: "super joe", 411 | }, 412 | }, 413 | { ignoreArrayOrder: true }, 414 | ), 415 | ).toStrictEqual({ 416 | type: "object", 417 | status: "updated", 418 | diff: [ 419 | { 420 | property: "id", 421 | previousValue: 54, 422 | currentValue: 54, 423 | status: "equal", 424 | }, 425 | { 426 | property: "type", 427 | previousValue: "sport", 428 | currentValue: undefined, 429 | status: "deleted", 430 | }, 431 | 432 | { 433 | property: "user", 434 | previousValue: { 435 | name: "joe", 436 | member: true, 437 | hobbies: ["golf", "football"], 438 | age: 66, 439 | }, 440 | currentValue: { 441 | name: "joe", 442 | member: false, 443 | hobbies: ["football", "golf"], 444 | nickname: "super joe", 445 | }, 446 | status: "updated", 447 | diff: [ 448 | { 449 | property: "name", 450 | previousValue: "joe", 451 | currentValue: "joe", 452 | status: "equal", 453 | }, 454 | { 455 | property: "member", 456 | previousValue: true, 457 | currentValue: false, 458 | status: "updated", 459 | }, 460 | { 461 | property: "hobbies", 462 | previousValue: ["golf", "football"], 463 | currentValue: ["football", "golf"], 464 | status: "equal", 465 | }, 466 | { 467 | property: "age", 468 | previousValue: 66, 469 | currentValue: undefined, 470 | status: "deleted", 471 | }, 472 | { 473 | property: "nickname", 474 | previousValue: undefined, 475 | currentValue: "super joe", 476 | status: "added", 477 | }, 478 | ], 479 | }, 480 | { 481 | property: "country", 482 | previousValue: undefined, 483 | currentValue: "us", 484 | status: "added", 485 | }, 486 | ], 487 | }); 488 | }); 489 | it("shows only main added values", () => { 490 | expect( 491 | getObjectDiff( 492 | { 493 | id: 54, 494 | type: "sport", 495 | user: { 496 | name: "joe", 497 | member: true, 498 | hobbies: ["golf", "football"], 499 | age: 66, 500 | }, 501 | }, 502 | { 503 | id: 54, 504 | country: "us", 505 | user: { 506 | name: "joe", 507 | member: false, 508 | hobbies: ["golf", "chess"], 509 | nickname: "super joe", 510 | }, 511 | }, 512 | { showOnly: { statuses: [ObjectStatus.ADDED] } }, 513 | ), 514 | ).toStrictEqual({ 515 | type: "object", 516 | status: "updated", 517 | diff: [ 518 | { 519 | property: "country", 520 | previousValue: undefined, 521 | currentValue: "us", 522 | status: "added", 523 | }, 524 | ], 525 | }); 526 | }); 527 | it("shows only added and deleted values in nested objects", () => { 528 | expect( 529 | getObjectDiff( 530 | { 531 | id: 54, 532 | type: "sport", 533 | user: { 534 | name: "joe", 535 | member: true, 536 | hobbies: ["golf", "football"], 537 | age: 66, 538 | }, 539 | }, 540 | { 541 | id: 54, 542 | country: "us", 543 | user: { 544 | name: "joe", 545 | member: false, 546 | hobbies: ["golf", "chess"], 547 | nickname: "super joe", 548 | }, 549 | }, 550 | { 551 | showOnly: { 552 | statuses: [ObjectStatus.ADDED, ObjectStatus.DELETED], 553 | granularity: Granularity.DEEP, 554 | }, 555 | }, 556 | ), 557 | ).toStrictEqual({ 558 | type: "object", 559 | status: "updated", 560 | diff: [ 561 | { 562 | property: "type", 563 | previousValue: "sport", 564 | currentValue: undefined, 565 | status: "deleted", 566 | }, 567 | { 568 | property: "user", 569 | previousValue: { 570 | name: "joe", 571 | member: true, 572 | hobbies: ["golf", "football"], 573 | age: 66, 574 | }, 575 | currentValue: { 576 | name: "joe", 577 | member: false, 578 | hobbies: ["golf", "chess"], 579 | nickname: "super joe", 580 | }, 581 | status: "updated", 582 | diff: [ 583 | { 584 | property: "age", 585 | previousValue: 66, 586 | currentValue: undefined, 587 | status: "deleted", 588 | }, 589 | { 590 | property: "nickname", 591 | previousValue: undefined, 592 | currentValue: "super joe", 593 | status: "added", 594 | }, 595 | ], 596 | }, 597 | { 598 | property: "country", 599 | previousValue: undefined, 600 | currentValue: "us", 601 | status: "added", 602 | }, 603 | ], 604 | }); 605 | }); 606 | it("shows only updated values in deeply nested objects", () => { 607 | expect( 608 | getObjectDiff( 609 | { 610 | id: 54, 611 | user: { 612 | name: "joe", 613 | data: { 614 | member: true, 615 | hobbies: { 616 | football: ["psg"], 617 | rugby: ["france"], 618 | }, 619 | }, 620 | }, 621 | }, 622 | { 623 | id: 54, 624 | user: { 625 | name: "joe", 626 | data: { 627 | member: true, 628 | hobbies: { 629 | football: ["psg", "nantes"], 630 | golf: ["st andrews"], 631 | }, 632 | }, 633 | }, 634 | }, 635 | { 636 | showOnly: { 637 | statuses: [ObjectStatus.UPDATED], 638 | granularity: Granularity.DEEP, 639 | }, 640 | }, 641 | ), 642 | ).toStrictEqual({ 643 | type: "object", 644 | status: "updated", 645 | diff: [ 646 | { 647 | property: "user", 648 | previousValue: { 649 | name: "joe", 650 | data: { 651 | member: true, 652 | hobbies: { 653 | football: ["psg"], 654 | rugby: ["france"], 655 | }, 656 | }, 657 | }, 658 | currentValue: { 659 | name: "joe", 660 | data: { 661 | member: true, 662 | hobbies: { 663 | football: ["psg", "nantes"], 664 | golf: ["st andrews"], 665 | }, 666 | }, 667 | }, 668 | status: "updated", 669 | diff: [ 670 | { 671 | property: "data", 672 | previousValue: { 673 | member: true, 674 | hobbies: { 675 | football: ["psg"], 676 | rugby: ["france"], 677 | }, 678 | }, 679 | currentValue: { 680 | member: true, 681 | hobbies: { 682 | football: ["psg", "nantes"], 683 | golf: ["st andrews"], 684 | }, 685 | }, 686 | status: "updated", 687 | diff: [ 688 | { 689 | property: "hobbies", 690 | previousValue: { 691 | football: ["psg"], 692 | rugby: ["france"], 693 | }, 694 | currentValue: { 695 | football: ["psg", "nantes"], 696 | golf: ["st andrews"], 697 | }, 698 | status: "updated", 699 | diff: [ 700 | { 701 | property: "football", 702 | previousValue: ["psg"], 703 | currentValue: ["psg", "nantes"], 704 | status: "updated", 705 | }, 706 | ], 707 | }, 708 | ], 709 | }, 710 | ], 711 | }, 712 | ], 713 | }); 714 | }); 715 | it("shows only added values in deeply nested objects", () => { 716 | expect( 717 | getObjectDiff( 718 | { 719 | id: 54, 720 | user: { 721 | name: "joe", 722 | data: { 723 | member: true, 724 | hobbies: { 725 | rugby: ["france"], 726 | }, 727 | }, 728 | }, 729 | }, 730 | { 731 | id: 54, 732 | user: { 733 | name: "joe", 734 | data: { 735 | member: true, 736 | hobbies: { 737 | football: ["psg", "nantes"], 738 | golf: ["st andrews"], 739 | }, 740 | }, 741 | }, 742 | }, 743 | { 744 | showOnly: { 745 | statuses: [ObjectStatus.ADDED], 746 | granularity: Granularity.DEEP, 747 | }, 748 | }, 749 | ), 750 | ).toStrictEqual({ 751 | type: "object", 752 | status: "updated", 753 | diff: [ 754 | { 755 | property: "user", 756 | previousValue: { 757 | name: "joe", 758 | data: { 759 | member: true, 760 | hobbies: { 761 | rugby: ["france"], 762 | }, 763 | }, 764 | }, 765 | currentValue: { 766 | name: "joe", 767 | data: { 768 | member: true, 769 | hobbies: { 770 | football: ["psg", "nantes"], 771 | golf: ["st andrews"], 772 | }, 773 | }, 774 | }, 775 | status: "updated", 776 | diff: [ 777 | { 778 | property: "data", 779 | previousValue: { 780 | member: true, 781 | hobbies: { 782 | rugby: ["france"], 783 | }, 784 | }, 785 | currentValue: { 786 | member: true, 787 | hobbies: { 788 | football: ["psg", "nantes"], 789 | golf: ["st andrews"], 790 | }, 791 | }, 792 | status: "updated", 793 | diff: [ 794 | { 795 | property: "hobbies", 796 | previousValue: { 797 | rugby: ["france"], 798 | }, 799 | currentValue: { 800 | football: ["psg", "nantes"], 801 | golf: ["st andrews"], 802 | }, 803 | status: "updated", 804 | diff: [ 805 | { 806 | property: "football", 807 | previousValue: undefined, 808 | currentValue: ["psg", "nantes"], 809 | status: "added", 810 | }, 811 | { 812 | property: "golf", 813 | previousValue: undefined, 814 | currentValue: ["st andrews"], 815 | status: "added", 816 | }, 817 | ], 818 | }, 819 | ], 820 | }, 821 | ], 822 | }, 823 | ], 824 | }); 825 | }); 826 | it("returns an empty diff if no property match the required statuses output", () => { 827 | expect( 828 | getObjectDiff( 829 | null, 830 | { 831 | name: "joe", 832 | age: 54, 833 | hobbies: ["golf", "football"], 834 | }, 835 | { 836 | showOnly: { 837 | statuses: [ObjectStatus.DELETED], 838 | granularity: Granularity.DEEP, 839 | }, 840 | }, 841 | ), 842 | ).toStrictEqual({ 843 | type: "object", 844 | status: "added", 845 | diff: [], 846 | }); 847 | }); 848 | it("returns all values if their status match the required statuses", () => { 849 | expect( 850 | getObjectDiff( 851 | { name: "joe", age: 54, hobbies: ["golf", "football"] }, 852 | null, 853 | { showOnly: { statuses: [ObjectStatus.DELETED] } }, 854 | ), 855 | ).toStrictEqual({ 856 | type: "object", 857 | status: "deleted", 858 | diff: [ 859 | { 860 | property: "name", 861 | previousValue: "joe", 862 | currentValue: undefined, 863 | status: "deleted", 864 | }, 865 | { 866 | property: "age", 867 | previousValue: 54, 868 | currentValue: undefined, 869 | status: "deleted", 870 | }, 871 | { 872 | property: "hobbies", 873 | previousValue: ["golf", "football"], 874 | currentValue: undefined, 875 | status: "deleted", 876 | }, 877 | ], 878 | }); 879 | }); 880 | it("detects changes when comparing an array value property to a non-array value property", () => { 881 | expect( 882 | getObjectDiff( 883 | { 884 | name: "joe", 885 | age: 55, 886 | hobbies: ["golf", "football"], 887 | }, 888 | { 889 | name: "joe", 890 | age: 55, 891 | hobbies: null, 892 | }, 893 | ), 894 | ).toStrictEqual({ 895 | type: "object", 896 | status: "updated", 897 | diff: [ 898 | { 899 | currentValue: "joe", 900 | previousValue: "joe", 901 | property: "name", 902 | status: "equal", 903 | }, 904 | { 905 | currentValue: 55, 906 | previousValue: 55, 907 | property: "age", 908 | status: "equal", 909 | }, 910 | { 911 | currentValue: null, 912 | previousValue: ["golf", "football"], 913 | property: "hobbies", 914 | status: "updated", 915 | }, 916 | ], 917 | }); 918 | }); 919 | it("detects changes when comparing a non-array value property to an array value property", () => { 920 | expect( 921 | getObjectDiff( 922 | { 923 | name: "joe", 924 | age: 55, 925 | hobbies: null, 926 | }, 927 | { 928 | name: "joe", 929 | age: 55, 930 | hobbies: ["golf", "football"], 931 | }, 932 | ), 933 | ).toStrictEqual({ 934 | type: "object", 935 | status: "updated", 936 | diff: [ 937 | { 938 | currentValue: "joe", 939 | previousValue: "joe", 940 | property: "name", 941 | status: "equal", 942 | }, 943 | { 944 | currentValue: 55, 945 | previousValue: 55, 946 | property: "age", 947 | status: "equal", 948 | }, 949 | { 950 | currentValue: ["golf", "football"], 951 | previousValue: null, 952 | property: "hobbies", 953 | status: "updated", 954 | }, 955 | ], 956 | }); 957 | }); 958 | }); 959 | -------------------------------------------------------------------------------- /src/lib/stream-list-diff/server/stream-list-diff.test.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import { Readable } from "stream"; 3 | import { ListStatus } from "@models/list"; 4 | import { StreamListDiff } from "@models/stream"; 5 | import { streamListDiff } from "."; 6 | 7 | describe("data emission", () => { 8 | it("emits 'data' event and consider the all the nextList added if no prevList is provided", (done) => { 9 | const nextList = [ 10 | { id: 1, name: "Item 1" }, 11 | { id: 2, name: "Item 2" }, 12 | ]; 13 | const diff = streamListDiff([], nextList, "id", { 14 | chunksSize: 2, 15 | useWorker: false, 16 | }); 17 | 18 | const expectedChunks = [ 19 | { 20 | previousValue: null, 21 | currentValue: { id: 1, name: "Item 1" }, 22 | prevIndex: null, 23 | newIndex: 0, 24 | indexDiff: null, 25 | status: ListStatus.ADDED, 26 | }, 27 | { 28 | previousValue: null, 29 | currentValue: { id: 2, name: "Item 2" }, 30 | prevIndex: null, 31 | newIndex: 1, 32 | indexDiff: null, 33 | status: ListStatus.ADDED, 34 | }, 35 | ]; 36 | let chunkCount = 0; 37 | diff.on("data", (chunk) => { 38 | expect(chunk).toStrictEqual(expectedChunks); 39 | chunkCount++; 40 | }); 41 | diff.on("finish", () => { 42 | expect(chunkCount).toBe(1); 43 | done(); 44 | }); 45 | }); 46 | it("emits 'data' event and consider the all the prevList deleted if no nextList is provided", (done) => { 47 | const prevList = [ 48 | { id: 1, name: "Item 1" }, 49 | { id: 2, name: "Item 2" }, 50 | ]; 51 | const diff = streamListDiff(prevList, [], "id", { 52 | chunksSize: 2, 53 | useWorker: false, 54 | }); 55 | 56 | const expectedChunks = [ 57 | { 58 | previousValue: { id: 1, name: "Item 1" }, 59 | currentValue: null, 60 | prevIndex: 0, 61 | newIndex: null, 62 | indexDiff: null, 63 | status: ListStatus.DELETED, 64 | }, 65 | { 66 | previousValue: { id: 2, name: "Item 2" }, 67 | currentValue: null, 68 | prevIndex: 1, 69 | newIndex: null, 70 | indexDiff: null, 71 | status: ListStatus.DELETED, 72 | }, 73 | ]; 74 | let chunkCount = 0; 75 | diff.on("data", (chunk) => { 76 | expect(chunk).toStrictEqual(expectedChunks); 77 | chunkCount++; 78 | }); 79 | diff.on("error", (err) => console.error(err)); 80 | diff.on("finish", () => { 81 | expect(chunkCount).toBe(1); 82 | done(); 83 | }); 84 | }); 85 | it("emits 'data' event with one object diff by chunk if chunkSize is 0 or undefined", (done) => { 86 | const prevList = [ 87 | { id: 1, name: "Item 1" }, 88 | { id: 2, name: "Item 2" }, 89 | ]; 90 | const nextList = [ 91 | { id: 2, name: "Item 2" }, 92 | { id: 3, name: "Item 3" }, 93 | ]; 94 | const diff = streamListDiff(prevList, nextList, "id", { useWorker: false }); 95 | 96 | const expectedChunks = [ 97 | [ 98 | { 99 | previousValue: { id: 2, name: "Item 2" }, 100 | currentValue: { id: 2, name: "Item 2" }, 101 | prevIndex: 1, 102 | newIndex: 0, 103 | indexDiff: -1, 104 | status: ListStatus.MOVED, 105 | }, 106 | ], 107 | [ 108 | { 109 | previousValue: { id: 1, name: "Item 1" }, 110 | currentValue: null, 111 | prevIndex: 0, 112 | newIndex: null, 113 | indexDiff: null, 114 | status: ListStatus.DELETED, 115 | }, 116 | ], 117 | [ 118 | { 119 | previousValue: null, 120 | currentValue: { id: 3, name: "Item 3" }, 121 | prevIndex: null, 122 | newIndex: 1, 123 | indexDiff: null, 124 | status: ListStatus.ADDED, 125 | }, 126 | ], 127 | ]; 128 | 129 | let chunkCount = 0; 130 | 131 | diff.on("data", (chunk) => { 132 | expect(chunk).toStrictEqual(expectedChunks[chunkCount]); 133 | chunkCount++; 134 | }); 135 | diff.on("finish", () => { 136 | expect(chunkCount).toBe(3); 137 | done(); 138 | }); 139 | }); 140 | it("emits 'data' event with 5 object diff by chunk and return the last object diff in a one entry chunk at the end", (done) => { 141 | const prevList = [ 142 | { id: 1, name: "Item 1" }, 143 | { id: 2, name: "Item 2" }, 144 | { id: 3, name: "Item 3" }, 145 | { id: 4, name: "Item 4" }, 146 | { id: 5, name: "Item 5" }, 147 | { id: 6, name: "Item 6" }, 148 | { id: 7, name: "Item 7" }, 149 | { id: 8, name: "Item 8" }, 150 | { id: 9, name: "Item 9" }, 151 | { id: 10, name: "Item 10" }, 152 | ]; 153 | const nextList = [ 154 | { id: 1, name: "Item 1" }, 155 | { id: 2, name: "Item Two" }, 156 | { id: 3, name: "Item 3" }, 157 | { id: 5, name: "Item 5" }, 158 | { id: 6, name: "Item Six" }, 159 | { id: 7, name: "Item 7" }, 160 | { id: 10, name: "Item 10" }, 161 | { id: 11, name: "Item 11" }, 162 | { id: 9, name: "Item 9" }, 163 | { id: 8, name: "Item 8" }, 164 | ]; 165 | const diff = streamListDiff(prevList, nextList, "id", { 166 | chunksSize: 5, 167 | useWorker: false, 168 | }); 169 | 170 | const expectedChunks = [ 171 | [ 172 | { 173 | previousValue: { id: 1, name: "Item 1" }, 174 | currentValue: { id: 1, name: "Item 1" }, 175 | prevIndex: 0, 176 | newIndex: 0, 177 | indexDiff: 0, 178 | status: ListStatus.EQUAL, 179 | }, 180 | { 181 | previousValue: { id: 2, name: "Item 2" }, 182 | currentValue: { id: 2, name: "Item Two" }, 183 | prevIndex: 1, 184 | newIndex: 1, 185 | indexDiff: 0, 186 | status: ListStatus.UPDATED, 187 | }, 188 | { 189 | previousValue: { id: 3, name: "Item 3" }, 190 | currentValue: { id: 3, name: "Item 3" }, 191 | prevIndex: 2, 192 | newIndex: 2, 193 | indexDiff: 0, 194 | status: ListStatus.EQUAL, 195 | }, 196 | { 197 | previousValue: { id: 5, name: "Item 5" }, 198 | currentValue: { id: 5, name: "Item 5" }, 199 | prevIndex: 4, 200 | newIndex: 3, 201 | indexDiff: -1, 202 | status: ListStatus.MOVED, 203 | }, 204 | { 205 | previousValue: { id: 6, name: "Item 6" }, 206 | currentValue: { id: 6, name: "Item Six" }, 207 | prevIndex: 5, 208 | newIndex: 4, 209 | indexDiff: -1, 210 | status: ListStatus.UPDATED, 211 | }, 212 | ], 213 | [ 214 | { 215 | previousValue: { id: 7, name: "Item 7" }, 216 | currentValue: { id: 7, name: "Item 7" }, 217 | prevIndex: 6, 218 | newIndex: 5, 219 | indexDiff: -1, 220 | status: ListStatus.MOVED, 221 | }, 222 | { 223 | previousValue: { id: 9, name: "Item 9" }, 224 | currentValue: { id: 9, name: "Item 9" }, 225 | prevIndex: 8, 226 | newIndex: 8, 227 | indexDiff: 0, 228 | status: ListStatus.EQUAL, 229 | }, 230 | { 231 | previousValue: { id: 10, name: "Item 10" }, 232 | currentValue: { id: 10, name: "Item 10" }, 233 | prevIndex: 9, 234 | newIndex: 6, 235 | indexDiff: -3, 236 | status: ListStatus.MOVED, 237 | }, 238 | { 239 | previousValue: { id: 8, name: "Item 8" }, 240 | currentValue: { id: 8, name: "Item 8" }, 241 | prevIndex: 7, 242 | newIndex: 9, 243 | indexDiff: 2, 244 | status: ListStatus.MOVED, 245 | }, 246 | { 247 | previousValue: { id: 4, name: "Item 4" }, 248 | currentValue: null, 249 | prevIndex: 3, 250 | newIndex: null, 251 | indexDiff: null, 252 | status: ListStatus.DELETED, 253 | }, 254 | ], 255 | [ 256 | { 257 | previousValue: null, 258 | currentValue: { id: 11, name: "Item 11" }, 259 | prevIndex: null, 260 | newIndex: 7, 261 | indexDiff: null, 262 | status: ListStatus.ADDED, 263 | }, 264 | ], 265 | ]; 266 | 267 | let chunkCount = 0; 268 | 269 | diff.on("data", (chunk) => { 270 | expect(chunk).toStrictEqual(expectedChunks[chunkCount]); 271 | chunkCount++; 272 | }); 273 | 274 | diff.on("finish", () => { 275 | expect(chunkCount).toBe(3); 276 | done(); 277 | }); 278 | }); 279 | it("emits 'data' event with all the objects diff in a single chunk if the chunkSize is bigger than the provided lists ", (done) => { 280 | const prevList = [ 281 | { id: 1, name: "Item 1" }, 282 | { id: 2, name: "Item 2" }, 283 | { id: 3, name: "Item 3" }, 284 | { id: 4, name: "Item 4" }, 285 | ]; 286 | 287 | const nextList = [ 288 | { id: 1, name: "Item 1" }, 289 | { id: 2, name: "Item Two" }, 290 | { id: 3, name: "Item 3" }, 291 | { id: 5, name: "Item 5" }, 292 | ]; 293 | 294 | const diff = streamListDiff(prevList, nextList, "id", { 295 | chunksSize: 5, 296 | useWorker: false, 297 | }); 298 | 299 | const expectedChunks = [ 300 | { 301 | previousValue: { id: 1, name: "Item 1" }, 302 | currentValue: { id: 1, name: "Item 1" }, 303 | prevIndex: 0, 304 | newIndex: 0, 305 | indexDiff: 0, 306 | status: ListStatus.EQUAL, 307 | }, 308 | { 309 | previousValue: { id: 2, name: "Item 2" }, 310 | currentValue: { id: 2, name: "Item Two" }, 311 | prevIndex: 1, 312 | newIndex: 1, 313 | indexDiff: 0, 314 | status: ListStatus.UPDATED, 315 | }, 316 | { 317 | previousValue: { id: 3, name: "Item 3" }, 318 | currentValue: { id: 3, name: "Item 3" }, 319 | prevIndex: 2, 320 | newIndex: 2, 321 | indexDiff: 0, 322 | status: ListStatus.EQUAL, 323 | }, 324 | { 325 | previousValue: { id: 4, name: "Item 4" }, 326 | currentValue: null, 327 | prevIndex: 3, 328 | newIndex: null, 329 | indexDiff: null, 330 | status: ListStatus.DELETED, 331 | }, 332 | { 333 | previousValue: null, 334 | currentValue: { id: 5, name: "Item 5" }, 335 | prevIndex: null, 336 | newIndex: 3, 337 | indexDiff: null, 338 | status: ListStatus.ADDED, 339 | }, 340 | ]; 341 | 342 | let chunkCount = 0; 343 | diff.on("data", (chunk) => { 344 | expect(chunk).toStrictEqual(expectedChunks); 345 | chunkCount++; 346 | }); 347 | diff.on("error", (err) => console.error(err)); 348 | diff.on("finish", () => { 349 | expect(chunkCount).toBe(1); 350 | done(); 351 | }); 352 | }); 353 | it("emits 'data' event with moved objects considered as updated if considerMoveAsUpdate is true", (done) => { 354 | const prevList = [ 355 | { id: 1, name: "Item 1" }, 356 | { id: 2, name: "Item 2" }, 357 | { id: 3, name: "Item 3" }, 358 | { id: 4, name: "Item 4" }, 359 | ]; 360 | const nextList = [ 361 | { id: 2, name: "Item Two" }, 362 | { id: 1, name: "Item 1" }, 363 | { id: 3, name: "Item 3" }, 364 | { id: 5, name: "Item 5" }, 365 | ]; 366 | const diff = streamListDiff(prevList, nextList, "id", { 367 | chunksSize: 5, 368 | considerMoveAsUpdate: true, 369 | useWorker: false, 370 | }); 371 | 372 | const expectedChunks = [ 373 | { 374 | previousValue: { id: 2, name: "Item 2" }, 375 | currentValue: { id: 2, name: "Item Two" }, 376 | prevIndex: 1, 377 | newIndex: 0, 378 | indexDiff: -1, 379 | status: ListStatus.UPDATED, 380 | }, 381 | { 382 | previousValue: { id: 1, name: "Item 1" }, 383 | currentValue: { id: 1, name: "Item 1" }, 384 | prevIndex: 0, 385 | newIndex: 1, 386 | indexDiff: 1, 387 | status: ListStatus.UPDATED, 388 | }, 389 | { 390 | previousValue: { id: 3, name: "Item 3" }, 391 | currentValue: { id: 3, name: "Item 3" }, 392 | prevIndex: 2, 393 | newIndex: 2, 394 | indexDiff: 0, 395 | status: ListStatus.EQUAL, 396 | }, 397 | { 398 | previousValue: { id: 4, name: "Item 4" }, 399 | currentValue: null, 400 | prevIndex: 3, 401 | newIndex: null, 402 | indexDiff: null, 403 | status: ListStatus.DELETED, 404 | }, 405 | { 406 | previousValue: null, 407 | currentValue: { id: 5, name: "Item 5" }, 408 | prevIndex: null, 409 | newIndex: 3, 410 | indexDiff: null, 411 | status: ListStatus.ADDED, 412 | }, 413 | ]; 414 | 415 | let chunkCount = 0; 416 | diff.on("data", (chunk) => { 417 | expect(chunk).toStrictEqual(expectedChunks); 418 | chunkCount++; 419 | }); 420 | 421 | diff.on("finish", () => { 422 | expect(chunkCount).toBe(1); 423 | done(); 424 | }); 425 | }); 426 | it("emits 'data' event only with objects diff whose status match with showOnly's", (done) => { 427 | const prevList = [ 428 | { id: 1, name: "Item 1" }, 429 | { id: 2, name: "Item 2" }, 430 | { id: 3, name: "Item 3" }, 431 | { id: 4, name: "Item 4" }, 432 | ]; 433 | const nextList = [ 434 | { id: 2, name: "Item Two" }, 435 | { id: 1, name: "Item 1" }, 436 | { id: 3, name: "Item 3" }, 437 | { id: 5, name: "Item 5" }, 438 | ]; 439 | const diff = streamListDiff(prevList, nextList, "id", { 440 | chunksSize: 5, 441 | showOnly: ["added", "deleted"], 442 | useWorker: false, 443 | }); 444 | 445 | const expectedChunks = [ 446 | { 447 | previousValue: { id: 4, name: "Item 4" }, 448 | currentValue: null, 449 | prevIndex: 3, 450 | newIndex: null, 451 | indexDiff: null, 452 | status: ListStatus.DELETED, 453 | }, 454 | { 455 | previousValue: null, 456 | currentValue: { id: 5, name: "Item 5" }, 457 | prevIndex: null, 458 | newIndex: 3, 459 | indexDiff: null, 460 | status: ListStatus.ADDED, 461 | }, 462 | ]; 463 | 464 | let chunkCount = 0; 465 | diff.on("data", (chunk) => { 466 | expect(chunk).toStrictEqual(expectedChunks); 467 | chunkCount++; 468 | }); 469 | 470 | diff.on("finish", () => { 471 | expect(chunkCount).toBe(1); 472 | done(); 473 | }); 474 | }); 475 | it("emits 'data' event with deep nested objects diff", (done) => { 476 | const prevList = [ 477 | { 478 | id: 1, 479 | name: "Item 1", 480 | user: { role: "admin", hobbies: ["golf", "football"] }, 481 | }, 482 | { id: 2, name: "Item 2" }, 483 | { id: 3, name: "Item 3", user: { role: "admin", hobbies: ["rugby"] } }, 484 | { 485 | id: 4, 486 | name: "Item 4", 487 | user: { role: "reader", hobbies: ["video games", "fishing"] }, 488 | }, 489 | { id: 5, name: "Item 5" }, 490 | { id: 6, name: "Item 6", user: { role: "root", hobbies: ["coding"] } }, 491 | { id: 7, name: "Item 7" }, 492 | { id: 8, name: "Item 8" }, 493 | { id: 9, name: "Item 9" }, 494 | { 495 | id: 10, 496 | name: "Item 10", 497 | user: { 498 | role: "root", 499 | hobbies: ["coding"], 500 | skills: { driving: true, diving: false }, 501 | }, 502 | }, 503 | ]; 504 | const nextList = [ 505 | { 506 | id: 1, 507 | name: "Item 1", 508 | user: { role: "admin", hobbies: ["golf", "football"] }, 509 | }, 510 | { id: 2, name: "Item Two" }, 511 | { id: 3, name: "Item 3", user: { role: "admin", hobbies: ["rugby"] } }, 512 | { id: 5, name: "Item 5" }, 513 | { id: 6, name: "Item 6", user: { role: "root", hobbies: ["farming"] } }, 514 | { id: 7, name: "Item 7" }, 515 | { 516 | id: 10, 517 | name: "Item 10", 518 | user: { 519 | role: "root", 520 | hobbies: ["coding"], 521 | skills: { driving: true, diving: false }, 522 | }, 523 | }, 524 | { id: 11, name: "Item 11" }, 525 | { id: 9, name: "Item 9" }, 526 | { id: 8, name: "Item 8" }, 527 | ]; 528 | const diff = streamListDiff(prevList, nextList, "id", { 529 | chunksSize: 5, 530 | useWorker: false, 531 | }); 532 | 533 | const expectedChunks = [ 534 | [ 535 | { 536 | previousValue: { 537 | id: 1, 538 | name: "Item 1", 539 | user: { role: "admin", hobbies: ["golf", "football"] }, 540 | }, 541 | currentValue: { 542 | id: 1, 543 | name: "Item 1", 544 | user: { role: "admin", hobbies: ["golf", "football"] }, 545 | }, 546 | prevIndex: 0, 547 | newIndex: 0, 548 | indexDiff: 0, 549 | status: ListStatus.EQUAL, 550 | }, 551 | { 552 | previousValue: { id: 2, name: "Item 2" }, 553 | currentValue: { id: 2, name: "Item Two" }, 554 | prevIndex: 1, 555 | newIndex: 1, 556 | indexDiff: 0, 557 | status: ListStatus.UPDATED, 558 | }, 559 | { 560 | previousValue: { 561 | id: 3, 562 | name: "Item 3", 563 | user: { role: "admin", hobbies: ["rugby"] }, 564 | }, 565 | currentValue: { 566 | id: 3, 567 | name: "Item 3", 568 | user: { role: "admin", hobbies: ["rugby"] }, 569 | }, 570 | prevIndex: 2, 571 | newIndex: 2, 572 | indexDiff: 0, 573 | status: ListStatus.EQUAL, 574 | }, 575 | { 576 | previousValue: { id: 5, name: "Item 5" }, 577 | currentValue: { id: 5, name: "Item 5" }, 578 | prevIndex: 4, 579 | newIndex: 3, 580 | indexDiff: -1, 581 | status: ListStatus.MOVED, 582 | }, 583 | { 584 | previousValue: { 585 | id: 6, 586 | name: "Item 6", 587 | user: { role: "root", hobbies: ["coding"] }, 588 | }, 589 | currentValue: { 590 | id: 6, 591 | name: "Item 6", 592 | user: { role: "root", hobbies: ["farming"] }, 593 | }, 594 | prevIndex: 5, 595 | newIndex: 4, 596 | indexDiff: -1, 597 | status: ListStatus.UPDATED, 598 | }, 599 | ], 600 | [ 601 | { 602 | previousValue: { id: 7, name: "Item 7" }, 603 | currentValue: { id: 7, name: "Item 7" }, 604 | prevIndex: 6, 605 | newIndex: 5, 606 | indexDiff: -1, 607 | status: ListStatus.MOVED, 608 | }, 609 | { 610 | previousValue: { id: 9, name: "Item 9" }, 611 | currentValue: { id: 9, name: "Item 9" }, 612 | prevIndex: 8, 613 | newIndex: 8, 614 | indexDiff: 0, 615 | status: ListStatus.EQUAL, 616 | }, 617 | { 618 | previousValue: { 619 | id: 10, 620 | name: "Item 10", 621 | user: { 622 | role: "root", 623 | hobbies: ["coding"], 624 | skills: { driving: true, diving: false }, 625 | }, 626 | }, 627 | currentValue: { 628 | id: 10, 629 | name: "Item 10", 630 | user: { 631 | role: "root", 632 | hobbies: ["coding"], 633 | skills: { driving: true, diving: false }, 634 | }, 635 | }, 636 | prevIndex: 9, 637 | newIndex: 6, 638 | indexDiff: -3, 639 | status: ListStatus.MOVED, 640 | }, 641 | { 642 | previousValue: { id: 8, name: "Item 8" }, 643 | currentValue: { id: 8, name: "Item 8" }, 644 | prevIndex: 7, 645 | newIndex: 9, 646 | indexDiff: 2, 647 | status: ListStatus.MOVED, 648 | }, 649 | { 650 | previousValue: { 651 | id: 4, 652 | name: "Item 4", 653 | user: { role: "reader", hobbies: ["video games", "fishing"] }, 654 | }, 655 | currentValue: null, 656 | prevIndex: 3, 657 | newIndex: null, 658 | indexDiff: null, 659 | status: ListStatus.DELETED, 660 | }, 661 | ], 662 | [ 663 | { 664 | previousValue: null, 665 | currentValue: { id: 11, name: "Item 11" }, 666 | prevIndex: null, 667 | newIndex: 7, 668 | indexDiff: null, 669 | status: ListStatus.ADDED, 670 | }, 671 | ], 672 | ]; 673 | 674 | let chunkCount = 0; 675 | 676 | diff.on("data", (chunk) => { 677 | expect(chunk).toStrictEqual(expectedChunks[chunkCount]); 678 | chunkCount++; 679 | }); 680 | 681 | diff.on("finish", () => { 682 | expect(chunkCount).toBe(3); 683 | done(); 684 | }); 685 | }); 686 | }); 687 | 688 | describe("input handling", () => { 689 | const prevList = [ 690 | { id: 1, name: "Item 1" }, 691 | { id: 2, name: "Item 2" }, 692 | { id: 3, name: "Item 3" }, 693 | { id: 4, name: "Item 4" }, 694 | ]; 695 | const nextList = [ 696 | { id: 1, name: "Item 1" }, 697 | { id: 2, name: "Item Two" }, 698 | { id: 3, name: "Item 3" }, 699 | { id: 5, name: "Item 5" }, 700 | ]; 701 | const expectedChunks = [ 702 | { 703 | previousValue: { id: 1, name: "Item 1" }, 704 | currentValue: { id: 1, name: "Item 1" }, 705 | prevIndex: 0, 706 | newIndex: 0, 707 | indexDiff: 0, 708 | status: ListStatus.EQUAL, 709 | }, 710 | { 711 | previousValue: { id: 2, name: "Item 2" }, 712 | currentValue: { id: 2, name: "Item Two" }, 713 | prevIndex: 1, 714 | newIndex: 1, 715 | indexDiff: 0, 716 | status: ListStatus.UPDATED, 717 | }, 718 | { 719 | previousValue: { id: 3, name: "Item 3" }, 720 | currentValue: { id: 3, name: "Item 3" }, 721 | prevIndex: 2, 722 | newIndex: 2, 723 | indexDiff: 0, 724 | status: ListStatus.EQUAL, 725 | }, 726 | { 727 | previousValue: { id: 4, name: "Item 4" }, 728 | currentValue: null, 729 | prevIndex: 3, 730 | newIndex: null, 731 | indexDiff: null, 732 | status: ListStatus.DELETED, 733 | }, 734 | { 735 | previousValue: null, 736 | currentValue: { id: 5, name: "Item 5" }, 737 | prevIndex: null, 738 | newIndex: 3, 739 | indexDiff: null, 740 | status: ListStatus.ADDED, 741 | }, 742 | ]; 743 | 744 | it("handles two readable streams", (done) => { 745 | const prevStream = Readable.from(prevList, { objectMode: true }); 746 | const nextStream = Readable.from(nextList, { objectMode: true }); 747 | 748 | const diff = streamListDiff(prevStream, nextStream, "id", { 749 | chunksSize: 5, 750 | useWorker: false, 751 | }); 752 | 753 | let chunkCount = 0; 754 | diff.on("data", (chunk) => { 755 | expect(chunk).toStrictEqual(expectedChunks); 756 | chunkCount++; 757 | }); 758 | diff.on("error", (err) => console.error(err)); 759 | diff.on("finish", () => { 760 | expect(chunkCount).toBe(1); 761 | done(); 762 | }); 763 | }); 764 | it("handles two local files", (done) => { 765 | const prevFile = path.resolve(__dirname, "../../../mocks/prevList.json"); 766 | const nextFile = path.resolve(__dirname, "../../../mocks/nextList.json"); 767 | 768 | const diff = streamListDiff(prevFile, nextFile, "id", { 769 | chunksSize: 5, 770 | useWorker: false, 771 | }); 772 | 773 | let chunkCount = 0; 774 | diff.on("data", (chunk) => { 775 | expect(chunk).toStrictEqual(expectedChunks); 776 | chunkCount++; 777 | }); 778 | diff.on("error", (err) => console.error(err)); 779 | diff.on("finish", () => { 780 | expect(chunkCount).toBe(1); 781 | done(); 782 | }); 783 | }); 784 | it("handles a readable stream against a local file", (done) => { 785 | const prevStream = Readable.from(prevList, { objectMode: true }); 786 | const nextFile = path.resolve(__dirname, "../../../mocks/nextList.json"); 787 | 788 | const diff = streamListDiff(prevStream, nextFile, "id", { 789 | chunksSize: 5, 790 | useWorker: false, 791 | }); 792 | 793 | let chunkCount = 0; 794 | diff.on("data", (chunk) => { 795 | expect(chunk).toStrictEqual(expectedChunks); 796 | chunkCount++; 797 | }); 798 | diff.on("error", (err) => console.error(err)); 799 | diff.on("finish", () => { 800 | expect(chunkCount).toBe(1); 801 | done(); 802 | }); 803 | }); 804 | it("handles a readable stream against an array", (done) => { 805 | const prevStream = Readable.from(prevList, { objectMode: true }); 806 | 807 | const diff = streamListDiff(prevStream, nextList, "id", { 808 | chunksSize: 5, 809 | useWorker: false, 810 | }); 811 | 812 | let chunkCount = 0; 813 | diff.on("data", (chunk) => { 814 | expect(chunk).toStrictEqual(expectedChunks); 815 | chunkCount++; 816 | }); 817 | diff.on("error", (err) => console.error(err)); 818 | diff.on("finish", () => { 819 | expect(chunkCount).toBe(1); 820 | done(); 821 | }); 822 | }); 823 | it("handles a local file against an array", (done) => { 824 | const prevFile = path.resolve(__dirname, "../../../mocks/prevList.json"); 825 | 826 | const diff = streamListDiff(prevFile, nextList, "id", { 827 | chunksSize: 5, 828 | useWorker: false, 829 | }); 830 | 831 | let chunkCount = 0; 832 | diff.on("data", (chunk) => { 833 | expect(chunk).toStrictEqual(expectedChunks); 834 | chunkCount++; 835 | }); 836 | diff.on("error", (err) => console.error(err)); 837 | diff.on("finish", () => { 838 | expect(chunkCount).toBe(1); 839 | done(); 840 | }); 841 | }); 842 | }); 843 | 844 | describe("finish event", () => { 845 | it("emits 'finish' event if no prevList nor nextList is provided", (done) => { 846 | const diff = streamListDiff([], [], "id", { useWorker: false }); 847 | diff.on("finish", () => done()); 848 | }); 849 | it("emits 'finish' event when all the chunks have been processed", (done) => { 850 | const prevList = [ 851 | { id: 1, name: "Item 1" }, 852 | { id: 2, name: "Item 2" }, 853 | ]; 854 | const nextList = [ 855 | { id: 2, name: "Item 2" }, 856 | { id: 3, name: "Item 3" }, 857 | ]; 858 | const diff = streamListDiff(prevList, nextList, "id", { useWorker: false }); 859 | diff.on("finish", () => done()); 860 | }); 861 | }); 862 | 863 | describe("error event", () => { 864 | test("emits 'error' event when prevList has invalid data", (done) => { 865 | const prevList = [ 866 | { id: 1, name: "Item 1" }, 867 | "hello", 868 | { id: 2, name: "Item 2" }, 869 | ]; 870 | const nextList = [ 871 | { id: 1, name: "Item 1" }, 872 | { id: 2, name: "Item 2" }, 873 | ]; 874 | 875 | // @ts-expect-error prevList is invalid by design for the test 876 | const diff = streamListDiff(prevList, nextList, "id", { useWorker: false }); 877 | 878 | diff.on("error", (err) => { 879 | expect(err["message"]).toEqual( 880 | `Your prevList must only contain valid objects. Found 'hello'`, 881 | ); 882 | done(); 883 | }); 884 | }); 885 | 886 | test("emits 'error' event when nextList has invalid data", (done) => { 887 | const prevList = [ 888 | { id: 1, name: "Item 1" }, 889 | { id: 2, name: "Item 2" }, 890 | ]; 891 | const nextList = [ 892 | { id: 1, name: "Item 1" }, 893 | "hello", 894 | { id: 2, name: "Item 2" }, 895 | ]; 896 | 897 | // @ts-expect-error nextList is invalid by design for the test 898 | const diff = streamListDiff(prevList, nextList, "id", { useWorker: false }); 899 | 900 | diff.on("error", (err) => { 901 | expect(err["message"]).toEqual( 902 | `Your nextList must only contain valid objects. Found 'hello'`, 903 | ); 904 | done(); 905 | }); 906 | }); 907 | 908 | test("emits 'error' event when all prevList ojects don't have the requested reference property", (done) => { 909 | const prevList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }]; 910 | const nextList = [ 911 | { id: 1, name: "Item 1" }, 912 | { id: 2, name: "Item 2" }, 913 | ]; 914 | 915 | const diff = streamListDiff(prevList, nextList, "id", { useWorker: false }); 916 | 917 | diff.on("error", (err) => { 918 | expect(err["message"]).toEqual( 919 | `The reference property 'id' is not available in all the objects of your prevList.`, 920 | ); 921 | done(); 922 | }); 923 | }); 924 | 925 | test("emits 'error' event when all nextList ojects don't have the requested reference property", (done) => { 926 | const prevList = [ 927 | { id: 1, name: "Item 1" }, 928 | { id: 2, name: "Item 2" }, 929 | ]; 930 | const nextList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }]; 931 | 932 | const diff = streamListDiff(prevList, nextList, "id", { useWorker: false }); 933 | 934 | diff.on("error", (err) => { 935 | expect(err["message"]).toEqual( 936 | `The reference property 'id' is not available in all the objects of your nextList.`, 937 | ); 938 | done(); 939 | }); 940 | }); 941 | 942 | test("emits 'error' event when the chunkSize option is negative", (done) => { 943 | const prevList = [ 944 | { id: 1, name: "Item 1" }, 945 | { id: 2, name: "Item 2" }, 946 | ]; 947 | const nextList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }]; 948 | 949 | const diff = streamListDiff(prevList, nextList, "id", { 950 | chunksSize: -3, 951 | useWorker: false, 952 | }); 953 | 954 | diff.on("error", (err) => { 955 | expect(err["message"]).toEqual( 956 | "The chunk size can't be negative. You entered the value '-3'", 957 | ); 958 | done(); 959 | }); 960 | }); 961 | 962 | test("emits 'error' event when the prevList is not a valid type", (done) => { 963 | const nextList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }]; 964 | 965 | // @ts-expect-error - prevList is invalid by design for the test 966 | const diff = streamListDiff({ name: "hello" }, nextList, "id", { 967 | useWorker: false, 968 | }); 969 | 970 | diff.on("error", (err) => { 971 | expect(err["message"]).toEqual( 972 | "Invalid prevList. Expected Readable, Array, or File.", 973 | ); 974 | done(); 975 | }); 976 | }); 977 | test("emits 'error' event when the nextList is not a valid type", (done) => { 978 | const prevList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }]; 979 | 980 | // @ts-expect-error - nextList is invalid by design for the test 981 | const diff = streamListDiff(prevList, null, "id", { useWorker: false }); 982 | 983 | diff.on("error", (err) => { 984 | expect(err["message"]).toEqual( 985 | "Invalid nextList. Expected Readable, Array, or File.", 986 | ); 987 | done(); 988 | }); 989 | }); 990 | }); 991 | 992 | const generateLargeDataset = (count: number) => { 993 | const data: Array<{ id: number; value: string }> = []; 994 | for (let i = 0; i < count; i++) { 995 | data.push({ id: i, value: `value-${i}` }); 996 | } 997 | return data; 998 | }; 999 | 1000 | describe("performance", () => { 1001 | it("process 100.000 in each stream", (done) => { 1002 | const numEntries = 100_000; 1003 | 1004 | const prevList = generateLargeDataset(numEntries); 1005 | const nextList = generateLargeDataset(numEntries); 1006 | 1007 | nextList[100].value = "updated-value-100"; // 1 updated entry 1008 | nextList[20_000].value = "updated-value-20000"; // Another updated entry 1009 | nextList.push({ id: numEntries, value: `new-value-${numEntries}` }); // 1 added entry 1010 | 1011 | const diffListener = streamListDiff<{ id: number; value: string }>( 1012 | prevList, 1013 | nextList, 1014 | "id", 1015 | { 1016 | chunksSize: 10_000, 1017 | }, 1018 | ); 1019 | 1020 | const diffs: StreamListDiff<{ id: number; value: string }>[] = []; 1021 | 1022 | diffListener.on("data", (chunk) => { 1023 | diffs.push(...chunk); 1024 | }); 1025 | 1026 | diffListener.on("finish", () => { 1027 | try { 1028 | const updatedEntries = diffs.filter((d) => d.status === "updated"); 1029 | const addedEntries = diffs.filter((d) => d.status === "added"); 1030 | const deletedEntries = diffs.filter((d) => d.status === "deleted"); 1031 | const equalEntries = diffs.filter((d) => d.status === "equal"); 1032 | 1033 | expect(updatedEntries.length).toBe(2); 1034 | expect(addedEntries.length).toBe(1); 1035 | expect(deletedEntries.length).toBe(0); 1036 | expect(equalEntries.length).toBe(99998); 1037 | done(); 1038 | } catch (err) { 1039 | done(err); 1040 | } 1041 | }); 1042 | 1043 | diffListener.on("error", (err) => done(err)); 1044 | }); 1045 | }); 1046 | -------------------------------------------------------------------------------- /src/lib/stream-list-diff/server/stream-list-diff.worker.test.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import { Readable } from "stream"; 3 | import { ListStatus } from "@models/list"; 4 | import { StreamListDiff } from "@models/stream"; 5 | import { streamListDiff } from "."; 6 | 7 | describe("data emission", () => { 8 | it("emits 'data' event and consider the all the nextList added if no prevList is provided", (done) => { 9 | const nextList = [ 10 | { id: 1, name: "Item 1" }, 11 | { id: 2, name: "Item 2" }, 12 | ]; 13 | const diff = streamListDiff([], nextList, "id", { 14 | chunksSize: 2, 15 | useWorker: true, 16 | }); 17 | 18 | const expectedChunks = [ 19 | { 20 | previousValue: null, 21 | currentValue: { id: 1, name: "Item 1" }, 22 | prevIndex: null, 23 | newIndex: 0, 24 | indexDiff: null, 25 | status: ListStatus.ADDED, 26 | }, 27 | { 28 | previousValue: null, 29 | currentValue: { id: 2, name: "Item 2" }, 30 | prevIndex: null, 31 | newIndex: 1, 32 | indexDiff: null, 33 | status: ListStatus.ADDED, 34 | }, 35 | ]; 36 | let chunkCount = 0; 37 | diff.on("data", (chunk) => { 38 | expect(chunk).toEqual(expectedChunks); 39 | chunkCount++; 40 | }); 41 | diff.on("finish", () => { 42 | expect(chunkCount).toBe(1); 43 | done(); 44 | }); 45 | }); 46 | it("emits 'data' event and consider the all the prevList deleted if no nextList is provided", (done) => { 47 | const prevList = [ 48 | { id: 1, name: "Item 1" }, 49 | { id: 2, name: "Item 2" }, 50 | ]; 51 | const diff = streamListDiff(prevList, [], "id", { 52 | chunksSize: 2, 53 | useWorker: true, 54 | }); 55 | 56 | const expectedChunks = [ 57 | { 58 | previousValue: { id: 1, name: "Item 1" }, 59 | currentValue: null, 60 | prevIndex: 0, 61 | newIndex: null, 62 | indexDiff: null, 63 | status: ListStatus.DELETED, 64 | }, 65 | { 66 | previousValue: { id: 2, name: "Item 2" }, 67 | currentValue: null, 68 | prevIndex: 1, 69 | newIndex: null, 70 | indexDiff: null, 71 | status: ListStatus.DELETED, 72 | }, 73 | ]; 74 | let chunkCount = 0; 75 | diff.on("data", (chunk) => { 76 | expect(chunk).toEqual(expectedChunks); 77 | chunkCount++; 78 | }); 79 | diff.on("error", (err) => console.error(err)); 80 | diff.on("finish", () => { 81 | expect(chunkCount).toBe(1); 82 | done(); 83 | }); 84 | }); 85 | it("emits 'data' event with one object diff by chunk if chunkSize is 0 or undefined", (done) => { 86 | const prevList = [ 87 | { id: 1, name: "Item 1" }, 88 | { id: 2, name: "Item 2" }, 89 | ]; 90 | const nextList = [ 91 | { id: 2, name: "Item 2" }, 92 | { id: 3, name: "Item 3" }, 93 | ]; 94 | const diff = streamListDiff(prevList, nextList, "id", { useWorker: true }); 95 | 96 | const expectedChunks = [ 97 | [ 98 | { 99 | previousValue: { id: 2, name: "Item 2" }, 100 | currentValue: { id: 2, name: "Item 2" }, 101 | prevIndex: 1, 102 | newIndex: 0, 103 | indexDiff: -1, 104 | status: ListStatus.MOVED, 105 | }, 106 | ], 107 | [ 108 | { 109 | previousValue: { id: 1, name: "Item 1" }, 110 | currentValue: null, 111 | prevIndex: 0, 112 | newIndex: null, 113 | indexDiff: null, 114 | status: ListStatus.DELETED, 115 | }, 116 | ], 117 | [ 118 | { 119 | previousValue: null, 120 | currentValue: { id: 3, name: "Item 3" }, 121 | prevIndex: null, 122 | newIndex: 1, 123 | indexDiff: null, 124 | status: ListStatus.ADDED, 125 | }, 126 | ], 127 | ]; 128 | 129 | let chunkCount = 0; 130 | 131 | diff.on("data", (chunk) => { 132 | expect(chunk).toEqual(expectedChunks[chunkCount]); 133 | chunkCount++; 134 | }); 135 | diff.on("finish", () => { 136 | expect(chunkCount).toBe(3); 137 | done(); 138 | }); 139 | }); 140 | it("emits 'data' event with 5 object diff by chunk and return the last object diff in a one entry chunk at the end", (done) => { 141 | const prevList = [ 142 | { id: 1, name: "Item 1" }, 143 | { id: 2, name: "Item 2" }, 144 | { id: 3, name: "Item 3" }, 145 | { id: 4, name: "Item 4" }, 146 | { id: 5, name: "Item 5" }, 147 | { id: 6, name: "Item 6" }, 148 | { id: 7, name: "Item 7" }, 149 | { id: 8, name: "Item 8" }, 150 | { id: 9, name: "Item 9" }, 151 | { id: 10, name: "Item 10" }, 152 | ]; 153 | const nextList = [ 154 | { id: 1, name: "Item 1" }, 155 | { id: 2, name: "Item Two" }, 156 | { id: 3, name: "Item 3" }, 157 | { id: 5, name: "Item 5" }, 158 | { id: 6, name: "Item Six" }, 159 | { id: 7, name: "Item 7" }, 160 | { id: 10, name: "Item 10" }, 161 | { id: 11, name: "Item 11" }, 162 | { id: 9, name: "Item 9" }, 163 | { id: 8, name: "Item 8" }, 164 | ]; 165 | const diff = streamListDiff(prevList, nextList, "id", { 166 | chunksSize: 5, 167 | useWorker: true, 168 | }); 169 | 170 | const expectedChunks = [ 171 | [ 172 | { 173 | previousValue: { id: 1, name: "Item 1" }, 174 | currentValue: { id: 1, name: "Item 1" }, 175 | prevIndex: 0, 176 | newIndex: 0, 177 | indexDiff: 0, 178 | status: ListStatus.EQUAL, 179 | }, 180 | { 181 | previousValue: { id: 2, name: "Item 2" }, 182 | currentValue: { id: 2, name: "Item Two" }, 183 | prevIndex: 1, 184 | newIndex: 1, 185 | indexDiff: 0, 186 | status: ListStatus.UPDATED, 187 | }, 188 | { 189 | previousValue: { id: 3, name: "Item 3" }, 190 | currentValue: { id: 3, name: "Item 3" }, 191 | prevIndex: 2, 192 | newIndex: 2, 193 | indexDiff: 0, 194 | status: ListStatus.EQUAL, 195 | }, 196 | { 197 | previousValue: { id: 5, name: "Item 5" }, 198 | currentValue: { id: 5, name: "Item 5" }, 199 | prevIndex: 4, 200 | newIndex: 3, 201 | indexDiff: -1, 202 | status: ListStatus.MOVED, 203 | }, 204 | { 205 | previousValue: { id: 6, name: "Item 6" }, 206 | currentValue: { id: 6, name: "Item Six" }, 207 | prevIndex: 5, 208 | newIndex: 4, 209 | indexDiff: -1, 210 | status: ListStatus.UPDATED, 211 | }, 212 | ], 213 | [ 214 | { 215 | previousValue: { id: 7, name: "Item 7" }, 216 | currentValue: { id: 7, name: "Item 7" }, 217 | prevIndex: 6, 218 | newIndex: 5, 219 | indexDiff: -1, 220 | status: ListStatus.MOVED, 221 | }, 222 | { 223 | previousValue: { id: 9, name: "Item 9" }, 224 | currentValue: { id: 9, name: "Item 9" }, 225 | prevIndex: 8, 226 | newIndex: 8, 227 | indexDiff: 0, 228 | status: ListStatus.EQUAL, 229 | }, 230 | { 231 | previousValue: { id: 10, name: "Item 10" }, 232 | currentValue: { id: 10, name: "Item 10" }, 233 | prevIndex: 9, 234 | newIndex: 6, 235 | indexDiff: -3, 236 | status: ListStatus.MOVED, 237 | }, 238 | { 239 | previousValue: { id: 8, name: "Item 8" }, 240 | currentValue: { id: 8, name: "Item 8" }, 241 | prevIndex: 7, 242 | newIndex: 9, 243 | indexDiff: 2, 244 | status: ListStatus.MOVED, 245 | }, 246 | { 247 | previousValue: { id: 4, name: "Item 4" }, 248 | currentValue: null, 249 | prevIndex: 3, 250 | newIndex: null, 251 | indexDiff: null, 252 | status: ListStatus.DELETED, 253 | }, 254 | ], 255 | [ 256 | { 257 | previousValue: null, 258 | currentValue: { id: 11, name: "Item 11" }, 259 | prevIndex: null, 260 | newIndex: 7, 261 | indexDiff: null, 262 | status: ListStatus.ADDED, 263 | }, 264 | ], 265 | ]; 266 | 267 | let chunkCount = 0; 268 | 269 | diff.on("data", (chunk) => { 270 | expect(chunk).toEqual(expectedChunks[chunkCount]); 271 | chunkCount++; 272 | }); 273 | 274 | diff.on("finish", () => { 275 | expect(chunkCount).toBe(3); 276 | done(); 277 | }); 278 | }); 279 | it("emits 'data' event with all the objects diff in a single chunk if the chunkSize is bigger than the provided lists ", (done) => { 280 | const prevList = [ 281 | { id: 1, name: "Item 1" }, 282 | { id: 2, name: "Item 2" }, 283 | { id: 3, name: "Item 3" }, 284 | { id: 4, name: "Item 4" }, 285 | ]; 286 | 287 | const nextList = [ 288 | { id: 1, name: "Item 1" }, 289 | { id: 2, name: "Item Two" }, 290 | { id: 3, name: "Item 3" }, 291 | { id: 5, name: "Item 5" }, 292 | ]; 293 | 294 | const diff = streamListDiff(prevList, nextList, "id", { 295 | chunksSize: 5, 296 | useWorker: true, 297 | }); 298 | 299 | const expectedChunks = [ 300 | { 301 | previousValue: { id: 1, name: "Item 1" }, 302 | currentValue: { id: 1, name: "Item 1" }, 303 | prevIndex: 0, 304 | newIndex: 0, 305 | indexDiff: 0, 306 | status: ListStatus.EQUAL, 307 | }, 308 | { 309 | previousValue: { id: 2, name: "Item 2" }, 310 | currentValue: { id: 2, name: "Item Two" }, 311 | prevIndex: 1, 312 | newIndex: 1, 313 | indexDiff: 0, 314 | status: ListStatus.UPDATED, 315 | }, 316 | { 317 | previousValue: { id: 3, name: "Item 3" }, 318 | currentValue: { id: 3, name: "Item 3" }, 319 | prevIndex: 2, 320 | newIndex: 2, 321 | indexDiff: 0, 322 | status: ListStatus.EQUAL, 323 | }, 324 | { 325 | previousValue: { id: 4, name: "Item 4" }, 326 | currentValue: null, 327 | prevIndex: 3, 328 | newIndex: null, 329 | indexDiff: null, 330 | status: ListStatus.DELETED, 331 | }, 332 | { 333 | previousValue: null, 334 | currentValue: { id: 5, name: "Item 5" }, 335 | prevIndex: null, 336 | newIndex: 3, 337 | indexDiff: null, 338 | status: ListStatus.ADDED, 339 | }, 340 | ]; 341 | 342 | let chunkCount = 0; 343 | diff.on("data", (chunk) => { 344 | expect(chunk).toEqual(expectedChunks); 345 | chunkCount++; 346 | }); 347 | diff.on("error", (err) => console.error(err)); 348 | diff.on("finish", () => { 349 | expect(chunkCount).toBe(1); 350 | done(); 351 | }); 352 | }); 353 | it("emits 'data' event with moved objects considered as updated if considerMoveAsUpdate is true", (done) => { 354 | const prevList = [ 355 | { id: 1, name: "Item 1" }, 356 | { id: 2, name: "Item 2" }, 357 | { id: 3, name: "Item 3" }, 358 | { id: 4, name: "Item 4" }, 359 | ]; 360 | const nextList = [ 361 | { id: 2, name: "Item Two" }, 362 | { id: 1, name: "Item 1" }, 363 | { id: 3, name: "Item 3" }, 364 | { id: 5, name: "Item 5" }, 365 | ]; 366 | const diff = streamListDiff(prevList, nextList, "id", { 367 | chunksSize: 5, 368 | considerMoveAsUpdate: true, 369 | useWorker: true, 370 | }); 371 | 372 | const expectedChunks = [ 373 | { 374 | previousValue: { id: 2, name: "Item 2" }, 375 | currentValue: { id: 2, name: "Item Two" }, 376 | prevIndex: 1, 377 | newIndex: 0, 378 | indexDiff: -1, 379 | status: ListStatus.UPDATED, 380 | }, 381 | { 382 | previousValue: { id: 1, name: "Item 1" }, 383 | currentValue: { id: 1, name: "Item 1" }, 384 | prevIndex: 0, 385 | newIndex: 1, 386 | indexDiff: 1, 387 | status: ListStatus.UPDATED, 388 | }, 389 | { 390 | previousValue: { id: 3, name: "Item 3" }, 391 | currentValue: { id: 3, name: "Item 3" }, 392 | prevIndex: 2, 393 | newIndex: 2, 394 | indexDiff: 0, 395 | status: ListStatus.EQUAL, 396 | }, 397 | { 398 | previousValue: { id: 4, name: "Item 4" }, 399 | currentValue: null, 400 | prevIndex: 3, 401 | newIndex: null, 402 | indexDiff: null, 403 | status: ListStatus.DELETED, 404 | }, 405 | { 406 | previousValue: null, 407 | currentValue: { id: 5, name: "Item 5" }, 408 | prevIndex: null, 409 | newIndex: 3, 410 | indexDiff: null, 411 | status: ListStatus.ADDED, 412 | }, 413 | ]; 414 | 415 | let chunkCount = 0; 416 | diff.on("data", (chunk) => { 417 | expect(chunk).toEqual(expectedChunks); 418 | chunkCount++; 419 | }); 420 | 421 | diff.on("finish", () => { 422 | expect(chunkCount).toBe(1); 423 | done(); 424 | }); 425 | }); 426 | it("emits 'data' event only with objects diff whose status match with showOnly's", (done) => { 427 | const prevList = [ 428 | { id: 1, name: "Item 1" }, 429 | { id: 2, name: "Item 2" }, 430 | { id: 3, name: "Item 3" }, 431 | { id: 4, name: "Item 4" }, 432 | ]; 433 | const nextList = [ 434 | { id: 2, name: "Item Two" }, 435 | { id: 1, name: "Item 1" }, 436 | { id: 3, name: "Item 3" }, 437 | { id: 5, name: "Item 5" }, 438 | ]; 439 | const diff = streamListDiff(prevList, nextList, "id", { 440 | chunksSize: 5, 441 | showOnly: ["added", "deleted"], 442 | useWorker: true, 443 | }); 444 | 445 | const expectedChunks = [ 446 | { 447 | previousValue: { id: 4, name: "Item 4" }, 448 | currentValue: null, 449 | prevIndex: 3, 450 | newIndex: null, 451 | indexDiff: null, 452 | status: ListStatus.DELETED, 453 | }, 454 | { 455 | previousValue: null, 456 | currentValue: { id: 5, name: "Item 5" }, 457 | prevIndex: null, 458 | newIndex: 3, 459 | indexDiff: null, 460 | status: ListStatus.ADDED, 461 | }, 462 | ]; 463 | 464 | let chunkCount = 0; 465 | diff.on("data", (chunk) => { 466 | expect(chunk).toEqual(expectedChunks); 467 | chunkCount++; 468 | }); 469 | 470 | diff.on("finish", () => { 471 | expect(chunkCount).toBe(1); 472 | done(); 473 | }); 474 | }); 475 | it("emits 'data' event with deep nested objects diff", (done) => { 476 | const prevList = [ 477 | { 478 | id: 1, 479 | name: "Item 1", 480 | user: { role: "admin", hobbies: ["golf", "football"] }, 481 | }, 482 | { id: 2, name: "Item 2" }, 483 | { id: 3, name: "Item 3", user: { role: "admin", hobbies: ["rugby"] } }, 484 | { 485 | id: 4, 486 | name: "Item 4", 487 | user: { role: "reader", hobbies: ["video games", "fishing"] }, 488 | }, 489 | { id: 5, name: "Item 5" }, 490 | { id: 6, name: "Item 6", user: { role: "root", hobbies: ["coding"] } }, 491 | { id: 7, name: "Item 7" }, 492 | { id: 8, name: "Item 8" }, 493 | { id: 9, name: "Item 9" }, 494 | { 495 | id: 10, 496 | name: "Item 10", 497 | user: { 498 | role: "root", 499 | hobbies: ["coding"], 500 | skills: { driving: true, diving: false }, 501 | }, 502 | }, 503 | ]; 504 | const nextList = [ 505 | { 506 | id: 1, 507 | name: "Item 1", 508 | user: { role: "admin", hobbies: ["golf", "football"] }, 509 | }, 510 | { id: 2, name: "Item Two" }, 511 | { id: 3, name: "Item 3", user: { role: "admin", hobbies: ["rugby"] } }, 512 | { id: 5, name: "Item 5" }, 513 | { id: 6, name: "Item 6", user: { role: "root", hobbies: ["farming"] } }, 514 | { id: 7, name: "Item 7" }, 515 | { 516 | id: 10, 517 | name: "Item 10", 518 | user: { 519 | role: "root", 520 | hobbies: ["coding"], 521 | skills: { driving: true, diving: false }, 522 | }, 523 | }, 524 | { id: 11, name: "Item 11" }, 525 | { id: 9, name: "Item 9" }, 526 | { id: 8, name: "Item 8" }, 527 | ]; 528 | const diff = streamListDiff(prevList, nextList, "id", { 529 | chunksSize: 5, 530 | useWorker: true, 531 | }); 532 | 533 | const expectedChunks = [ 534 | [ 535 | { 536 | previousValue: { 537 | id: 1, 538 | name: "Item 1", 539 | user: { role: "admin", hobbies: ["golf", "football"] }, 540 | }, 541 | currentValue: { 542 | id: 1, 543 | name: "Item 1", 544 | user: { role: "admin", hobbies: ["golf", "football"] }, 545 | }, 546 | prevIndex: 0, 547 | newIndex: 0, 548 | indexDiff: 0, 549 | status: ListStatus.EQUAL, 550 | }, 551 | { 552 | previousValue: { id: 2, name: "Item 2" }, 553 | currentValue: { id: 2, name: "Item Two" }, 554 | prevIndex: 1, 555 | newIndex: 1, 556 | indexDiff: 0, 557 | status: ListStatus.UPDATED, 558 | }, 559 | { 560 | previousValue: { 561 | id: 3, 562 | name: "Item 3", 563 | user: { role: "admin", hobbies: ["rugby"] }, 564 | }, 565 | currentValue: { 566 | id: 3, 567 | name: "Item 3", 568 | user: { role: "admin", hobbies: ["rugby"] }, 569 | }, 570 | prevIndex: 2, 571 | newIndex: 2, 572 | indexDiff: 0, 573 | status: ListStatus.EQUAL, 574 | }, 575 | { 576 | previousValue: { id: 5, name: "Item 5" }, 577 | currentValue: { id: 5, name: "Item 5" }, 578 | prevIndex: 4, 579 | newIndex: 3, 580 | indexDiff: -1, 581 | status: ListStatus.MOVED, 582 | }, 583 | { 584 | previousValue: { 585 | id: 6, 586 | name: "Item 6", 587 | user: { role: "root", hobbies: ["coding"] }, 588 | }, 589 | currentValue: { 590 | id: 6, 591 | name: "Item 6", 592 | user: { role: "root", hobbies: ["farming"] }, 593 | }, 594 | prevIndex: 5, 595 | newIndex: 4, 596 | indexDiff: -1, 597 | status: ListStatus.UPDATED, 598 | }, 599 | ], 600 | [ 601 | { 602 | previousValue: { id: 7, name: "Item 7" }, 603 | currentValue: { id: 7, name: "Item 7" }, 604 | prevIndex: 6, 605 | newIndex: 5, 606 | indexDiff: -1, 607 | status: ListStatus.MOVED, 608 | }, 609 | { 610 | previousValue: { id: 9, name: "Item 9" }, 611 | currentValue: { id: 9, name: "Item 9" }, 612 | prevIndex: 8, 613 | newIndex: 8, 614 | indexDiff: 0, 615 | status: ListStatus.EQUAL, 616 | }, 617 | { 618 | previousValue: { 619 | id: 10, 620 | name: "Item 10", 621 | user: { 622 | role: "root", 623 | hobbies: ["coding"], 624 | skills: { driving: true, diving: false }, 625 | }, 626 | }, 627 | currentValue: { 628 | id: 10, 629 | name: "Item 10", 630 | user: { 631 | role: "root", 632 | hobbies: ["coding"], 633 | skills: { driving: true, diving: false }, 634 | }, 635 | }, 636 | prevIndex: 9, 637 | newIndex: 6, 638 | indexDiff: -3, 639 | status: ListStatus.MOVED, 640 | }, 641 | { 642 | previousValue: { id: 8, name: "Item 8" }, 643 | currentValue: { id: 8, name: "Item 8" }, 644 | prevIndex: 7, 645 | newIndex: 9, 646 | indexDiff: 2, 647 | status: ListStatus.MOVED, 648 | }, 649 | { 650 | previousValue: { 651 | id: 4, 652 | name: "Item 4", 653 | user: { role: "reader", hobbies: ["video games", "fishing"] }, 654 | }, 655 | currentValue: null, 656 | prevIndex: 3, 657 | newIndex: null, 658 | indexDiff: null, 659 | status: ListStatus.DELETED, 660 | }, 661 | ], 662 | [ 663 | { 664 | previousValue: null, 665 | currentValue: { id: 11, name: "Item 11" }, 666 | prevIndex: null, 667 | newIndex: 7, 668 | indexDiff: null, 669 | status: ListStatus.ADDED, 670 | }, 671 | ], 672 | ]; 673 | 674 | let chunkCount = 0; 675 | 676 | diff.on("data", (chunk) => { 677 | expect(chunk).toEqual(expectedChunks[chunkCount]); 678 | chunkCount++; 679 | }); 680 | 681 | diff.on("finish", () => { 682 | expect(chunkCount).toBe(3); 683 | done(); 684 | }); 685 | }); 686 | }); 687 | 688 | describe("input handling", () => { 689 | const prevList = [ 690 | { id: 1, name: "Item 1" }, 691 | { id: 2, name: "Item 2" }, 692 | { id: 3, name: "Item 3" }, 693 | { id: 4, name: "Item 4" }, 694 | ]; 695 | const nextList = [ 696 | { id: 1, name: "Item 1" }, 697 | { id: 2, name: "Item Two" }, 698 | { id: 3, name: "Item 3" }, 699 | { id: 5, name: "Item 5" }, 700 | ]; 701 | const expectedChunks = [ 702 | { 703 | previousValue: { id: 1, name: "Item 1" }, 704 | currentValue: { id: 1, name: "Item 1" }, 705 | prevIndex: 0, 706 | newIndex: 0, 707 | indexDiff: 0, 708 | status: ListStatus.EQUAL, 709 | }, 710 | { 711 | previousValue: { id: 2, name: "Item 2" }, 712 | currentValue: { id: 2, name: "Item Two" }, 713 | prevIndex: 1, 714 | newIndex: 1, 715 | indexDiff: 0, 716 | status: ListStatus.UPDATED, 717 | }, 718 | { 719 | previousValue: { id: 3, name: "Item 3" }, 720 | currentValue: { id: 3, name: "Item 3" }, 721 | prevIndex: 2, 722 | newIndex: 2, 723 | indexDiff: 0, 724 | status: ListStatus.EQUAL, 725 | }, 726 | { 727 | previousValue: { id: 4, name: "Item 4" }, 728 | currentValue: null, 729 | prevIndex: 3, 730 | newIndex: null, 731 | indexDiff: null, 732 | status: ListStatus.DELETED, 733 | }, 734 | { 735 | previousValue: null, 736 | currentValue: { id: 5, name: "Item 5" }, 737 | prevIndex: null, 738 | newIndex: 3, 739 | indexDiff: null, 740 | status: ListStatus.ADDED, 741 | }, 742 | ]; 743 | 744 | it("handles two readable streams", (done) => { 745 | const prevStream = Readable.from(prevList, { objectMode: true }); 746 | const nextStream = Readable.from(nextList, { objectMode: true }); 747 | 748 | const diff = streamListDiff(prevStream, nextStream, "id", { 749 | chunksSize: 5, 750 | useWorker: true, 751 | showWarnings: false, 752 | }); 753 | 754 | let chunkCount = 0; 755 | diff.on("data", (chunk) => { 756 | expect(chunk).toEqual(expectedChunks); 757 | chunkCount++; 758 | }); 759 | diff.on("error", (err) => console.error(err)); 760 | diff.on("finish", () => { 761 | expect(chunkCount).toBe(1); 762 | done(); 763 | }); 764 | }); 765 | it("handles two local files", (done) => { 766 | const prevFile = path.resolve(__dirname, "../../../mocks/prevList.json"); 767 | const nextFile = path.resolve(__dirname, "../../../mocks/nextList.json"); 768 | 769 | const diff = streamListDiff(prevFile, nextFile, "id", { 770 | chunksSize: 5, 771 | useWorker: true, 772 | }); 773 | 774 | let chunkCount = 0; 775 | diff.on("data", (chunk) => { 776 | expect(chunk).toEqual(expectedChunks); 777 | chunkCount++; 778 | }); 779 | diff.on("error", (err) => console.error(err)); 780 | diff.on("finish", () => { 781 | expect(chunkCount).toBe(1); 782 | done(); 783 | }); 784 | }); 785 | it("handles a readable stream against a local file", (done) => { 786 | const prevStream = Readable.from(prevList, { objectMode: true }); 787 | const nextFile = path.resolve(__dirname, "../../../mocks/nextList.json"); 788 | 789 | const diff = streamListDiff(prevStream, nextFile, "id", { 790 | chunksSize: 5, 791 | useWorker: true, 792 | showWarnings: false, 793 | }); 794 | 795 | let chunkCount = 0; 796 | diff.on("data", (chunk) => { 797 | expect(chunk).toEqual(expectedChunks); 798 | chunkCount++; 799 | }); 800 | diff.on("error", (err) => console.error(err)); 801 | diff.on("finish", () => { 802 | expect(chunkCount).toBe(1); 803 | done(); 804 | }); 805 | }); 806 | it("handles a readable stream against an array", (done) => { 807 | const prevStream = Readable.from(prevList, { objectMode: true }); 808 | 809 | const diff = streamListDiff(prevStream, nextList, "id", { 810 | chunksSize: 5, 811 | useWorker: true, 812 | showWarnings: false, 813 | }); 814 | 815 | let chunkCount = 0; 816 | diff.on("data", (chunk) => { 817 | expect(chunk).toEqual(expectedChunks); 818 | chunkCount++; 819 | }); 820 | diff.on("error", (err) => console.error(err)); 821 | diff.on("finish", () => { 822 | expect(chunkCount).toBe(1); 823 | done(); 824 | }); 825 | }); 826 | it("handles a local file against an array", (done) => { 827 | const prevFile = path.resolve(__dirname, "../../../mocks/prevList.json"); 828 | 829 | const diff = streamListDiff(prevFile, nextList, "id", { 830 | chunksSize: 5, 831 | useWorker: true, 832 | }); 833 | 834 | let chunkCount = 0; 835 | diff.on("data", (chunk) => { 836 | expect(chunk).toEqual(expectedChunks); 837 | chunkCount++; 838 | }); 839 | diff.on("error", (err) => console.error(err)); 840 | diff.on("finish", () => { 841 | expect(chunkCount).toBe(1); 842 | done(); 843 | }); 844 | }); 845 | }); 846 | 847 | describe("finish event", () => { 848 | it("emits 'finish' event if no prevList nor nextList is provided", (done) => { 849 | const diff = streamListDiff([], [], "id", { useWorker: true }); 850 | diff.on("finish", () => done()); 851 | }); 852 | it("emits 'finish' event when all the chunks have been processed", (done) => { 853 | const prevList = [ 854 | { id: 1, name: "Item 1" }, 855 | { id: 2, name: "Item 2" }, 856 | ]; 857 | const nextList = [ 858 | { id: 2, name: "Item 2" }, 859 | { id: 3, name: "Item 3" }, 860 | ]; 861 | const diff = streamListDiff(prevList, nextList, "id", { useWorker: true }); 862 | diff.on("finish", () => done()); 863 | }); 864 | }); 865 | 866 | describe("error event", () => { 867 | test("emits 'error' event when prevList has invalid data", (done) => { 868 | const prevList = [ 869 | { id: 1, name: "Item 1" }, 870 | "hello", 871 | { id: 2, name: "Item 2" }, 872 | ]; 873 | const nextList = [ 874 | { id: 1, name: "Item 1" }, 875 | { id: 2, name: "Item 2" }, 876 | ]; 877 | 878 | // @ts-expect-error prevList is invalid by design for the test 879 | const diff = streamListDiff(prevList, nextList, "id", { useWorker: true }); 880 | 881 | diff.on("error", (err) => { 882 | expect(err["message"]).toEqual( 883 | `Your prevList must only contain valid objects. Found 'hello'`, 884 | ); 885 | done(); 886 | }); 887 | }); 888 | 889 | test("emits 'error' event when nextList has invalid data", (done) => { 890 | const prevList = [ 891 | { id: 1, name: "Item 1" }, 892 | { id: 2, name: "Item 2" }, 893 | ]; 894 | const nextList = [ 895 | { id: 1, name: "Item 1" }, 896 | "hello", 897 | { id: 2, name: "Item 2" }, 898 | ]; 899 | 900 | // @ts-expect-error nextList is invalid by design for the test 901 | const diff = streamListDiff(prevList, nextList, "id", { useWorker: true }); 902 | 903 | diff.on("error", (err) => { 904 | expect(err["message"]).toEqual( 905 | `Your nextList must only contain valid objects. Found 'hello'`, 906 | ); 907 | done(); 908 | }); 909 | }); 910 | 911 | test("emits 'error' event when all prevList ojects don't have the requested reference property", (done) => { 912 | const prevList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }]; 913 | const nextList = [ 914 | { id: 1, name: "Item 1" }, 915 | { id: 2, name: "Item 2" }, 916 | ]; 917 | 918 | const diff = streamListDiff(prevList, nextList, "id", { useWorker: true }); 919 | 920 | diff.on("error", (err) => { 921 | expect(err["message"]).toEqual( 922 | `The reference property 'id' is not available in all the objects of your prevList.`, 923 | ); 924 | done(); 925 | }); 926 | }); 927 | 928 | test("emits 'error' event when all nextList ojects don't have the requested reference property", (done) => { 929 | const prevList = [ 930 | { id: 1, name: "Item 1" }, 931 | { id: 2, name: "Item 2" }, 932 | ]; 933 | const nextList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }]; 934 | 935 | const diff = streamListDiff(prevList, nextList, "id", { useWorker: true }); 936 | 937 | diff.on("error", (err) => { 938 | expect(err["message"]).toEqual( 939 | `The reference property 'id' is not available in all the objects of your nextList.`, 940 | ); 941 | done(); 942 | }); 943 | }); 944 | 945 | test("emits 'error' event when the chunkSize option is negative", (done) => { 946 | const prevList = [ 947 | { id: 1, name: "Item 1" }, 948 | { id: 2, name: "Item 2" }, 949 | ]; 950 | const nextList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }]; 951 | 952 | const diff = streamListDiff(prevList, nextList, "id", { 953 | chunksSize: -3, 954 | useWorker: true, 955 | }); 956 | 957 | diff.on("error", (err) => { 958 | expect(err["message"]).toEqual( 959 | "The chunk size can't be negative. You entered the value '-3'", 960 | ); 961 | done(); 962 | }); 963 | }); 964 | 965 | test("emits 'error' event when the prevList is not a valid type", (done) => { 966 | const nextList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }]; 967 | 968 | // @ts-expect-error - prevList is invalid by design for the test 969 | const diff = streamListDiff({ name: "hello" }, nextList, "id", { 970 | useWorker: true, 971 | }); 972 | 973 | diff.on("error", (err) => { 974 | expect(err["message"]).toEqual( 975 | "Invalid prevList. Expected Readable, Array, or File.", 976 | ); 977 | done(); 978 | }); 979 | }); 980 | test("emits 'error' event when the nextList is not a valid type", (done) => { 981 | const prevList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }]; 982 | 983 | // @ts-expect-error - nextList is invalid by design for the test 984 | const diff = streamListDiff(prevList, null, "id", { useWorker: true }); 985 | 986 | diff.on("error", (err) => { 987 | expect(err["message"]).toEqual( 988 | "Invalid nextList. Expected Readable, Array, or File.", 989 | ); 990 | done(); 991 | }); 992 | }); 993 | }); 994 | 995 | const generateLargeDataset = (count: number) => { 996 | const data: Array<{ id: number; value: string }> = []; 997 | for (let i = 0; i < count; i++) { 998 | data.push({ id: i, value: `value-${i}` }); 999 | } 1000 | return data; 1001 | }; 1002 | 1003 | describe("performance", () => { 1004 | it("process 100.000 in each stream", (done) => { 1005 | const numEntries = 100_000; 1006 | 1007 | const prevList = generateLargeDataset(numEntries); 1008 | const nextList = generateLargeDataset(numEntries); 1009 | 1010 | nextList[100].value = "updated-value-100"; // 1 updated entry 1011 | nextList[20_000].value = "updated-value-20000"; // Another updated entry 1012 | nextList.push({ id: numEntries, value: `new-value-${numEntries}` }); // 1 added entry 1013 | 1014 | const diffListener = streamListDiff<{ id: number; value: string }>( 1015 | prevList, 1016 | nextList, 1017 | "id", 1018 | { 1019 | chunksSize: 10_000, 1020 | }, 1021 | ); 1022 | 1023 | const diffs: StreamListDiff<{ id: number; value: string }>[] = []; 1024 | 1025 | diffListener.on("data", (chunk) => { 1026 | diffs.push(...chunk); 1027 | }); 1028 | 1029 | diffListener.on("finish", () => { 1030 | try { 1031 | const updatedEntries = diffs.filter((d) => d.status === "updated"); 1032 | const addedEntries = diffs.filter((d) => d.status === "added"); 1033 | const deletedEntries = diffs.filter((d) => d.status === "deleted"); 1034 | const equalEntries = diffs.filter((d) => d.status === "equal"); 1035 | 1036 | expect(updatedEntries.length).toBe(2); 1037 | expect(addedEntries.length).toBe(1); 1038 | expect(deletedEntries.length).toBe(0); 1039 | expect(equalEntries.length).toBe(99998); 1040 | done(); 1041 | } catch (err) { 1042 | done(err); 1043 | } 1044 | }); 1045 | 1046 | diffListener.on("error", (err) => done(err)); 1047 | }); 1048 | }); 1049 | --------------------------------------------------------------------------------